diff options
Diffstat (limited to 'buildstream')
| -rw-r--r-- | buildstream/_scheduler/queues/buildqueue.py | 3 | ||||
| -rw-r--r-- | buildstream/_scheduler/queues/fetchqueue.py | 3 | ||||
| -rw-r--r-- | buildstream/_scheduler/queues/pullqueue.py | 3 | ||||
| -rw-r--r-- | buildstream/element.py | 34 | ||||
| -rw-r--r-- | buildstream/plugin.py | 6 | ||||
| -rw-r--r-- | buildstream/types.py | 177 |
6 files changed, 214 insertions, 12 deletions
diff --git a/buildstream/_scheduler/queues/buildqueue.py b/buildstream/_scheduler/queues/buildqueue.py index e63475f05..05e6f7a8b 100644 --- a/buildstream/_scheduler/queues/buildqueue.py +++ b/buildstream/_scheduler/queues/buildqueue.py @@ -35,9 +35,6 @@ class BuildQueue(Queue): return element._assemble() def status(self, element): - # state of dependencies may have changed, recalculate element state - element._update_state() - if not element._is_required(): # Artifact is not currently required but it may be requested later. # Keep it in the queue. diff --git a/buildstream/_scheduler/queues/fetchqueue.py b/buildstream/_scheduler/queues/fetchqueue.py index 114790c05..e121f6349 100644 --- a/buildstream/_scheduler/queues/fetchqueue.py +++ b/buildstream/_scheduler/queues/fetchqueue.py @@ -44,9 +44,6 @@ class FetchQueue(Queue): source._fetch() def status(self, element): - # state of dependencies may have changed, recalculate element state - element._update_state() - if not element._is_required(): # Artifact is not currently required but it may be requested later. # Keep it in the queue. diff --git a/buildstream/_scheduler/queues/pullqueue.py b/buildstream/_scheduler/queues/pullqueue.py index 2842c5e21..e4868953e 100644 --- a/buildstream/_scheduler/queues/pullqueue.py +++ b/buildstream/_scheduler/queues/pullqueue.py @@ -38,9 +38,6 @@ class PullQueue(Queue): raise SkipJob(self.action_name) def status(self, element): - # state of dependencies may have changed, recalculate element state - element._update_state() - if not element._is_required(): # Artifact is not currently required but it may be requested later. # Keep it in the queue. diff --git a/buildstream/element.py b/buildstream/element.py index dd1169f15..9851d042e 100644 --- a/buildstream/element.py +++ b/buildstream/element.py @@ -88,6 +88,7 @@ from ._variables import Variables from ._versions import BST_CORE_ARTIFACT_VERSION from ._exceptions import BstError, LoadError, LoadErrorReason, ImplError, ErrorDomain from .utils import UtilError +from .types import _UniquePriorityQueue from . import Plugin, Consistency from . import SandboxFlags from . import utils @@ -214,6 +215,8 @@ class Element(Plugin): self.__runtime_dependencies = [] # Direct runtime dependency Elements self.__build_dependencies = [] # Direct build dependency Elements + self.__reverse_dependencies = set() # Direct reverse dependency Elements + self.__ready_for_runtime = False # Wether the element has all its dependencies ready and has a cache key self.__sources = [] # List of Sources self.__weak_cache_key = None # Our cached weak cache key self.__strict_cache_key = None # Our cached cache key for strict builds @@ -924,9 +927,12 @@ class Element(Plugin): for meta_dep in meta.dependencies: dependency = Element._new_from_meta(meta_dep, artifacts) element.__runtime_dependencies.append(dependency) + dependency.__reverse_dependencies.add(element) + for meta_dep in meta.build_dependencies: dependency = Element._new_from_meta(meta_dep, artifacts) element.__build_dependencies.append(dependency) + dependency.__reverse_dependencies.add(element) return element @@ -1143,6 +1149,10 @@ class Element(Plugin): # Strong cache key could not be calculated yet return + if not self.__ready_for_runtime and self.__cache_key is not None: + self.__ready_for_runtime = all( + dep.__ready_for_runtime for dep in self.__runtime_dependencies) + # _get_display_key(): # # Returns cache keys for display purposes @@ -1245,7 +1255,7 @@ class Element(Plugin): self.__tracking_scheduled = False self.__tracking_done = True - self._update_state() + self.__update_state_recursively() # _track(): # @@ -1421,7 +1431,7 @@ class Element(Plugin): self.__assemble_scheduled = False self.__assemble_done = True - self._update_state() + self.__update_state_recursively() if self._get_workspace() and self._cached(): # @@ -1625,7 +1635,7 @@ class Element(Plugin): def _pull_done(self): self.__pull_done = True - self._update_state() + self.__update_state_recursively() def _pull_strong(self, *, progress=None): weak_key = self._get_cache_key(strength=_KeyStrength.WEAK) @@ -2504,6 +2514,24 @@ class Element(Plugin): return utils._deduplicate(keys) + # __update_state_recursively() + # + # Update the state of all reverse dependencies, recursively. + # + def __update_state_recursively(self): + queue = _UniquePriorityQueue() + queue.push(self._unique_id, self) + + while queue: + element = queue.pop() + + old_ready_for_runtime = element.__ready_for_runtime + element._update_state() + + if element.__ready_for_runtime != old_ready_for_runtime: + for rdep in element.__reverse_dependencies: + queue.push(rdep._unique_id, rdep) + def _overlap_error_detail(f, forbidden_overlap_elements, elements): if forbidden_overlap_elements: diff --git a/buildstream/plugin.py b/buildstream/plugin.py index f6a10384d..2c94c212c 100644 --- a/buildstream/plugin.py +++ b/buildstream/plugin.py @@ -178,6 +178,12 @@ class Plugin(): # Unique ID # # This id allows to uniquely identify a plugin. + # + # /!\ the unique id must be an increasing value /!\ + # This is because we are depending on it in buildstream.element.Element + # to give us a topological sort over all elements. + # Modifying how we handle ids here will modify the behavior of the + # Element's state handling. self._unique_id = next(self.__id_generator) # register ourself in the table containing all existing plugins diff --git a/buildstream/types.py b/buildstream/types.py new file mode 100644 index 000000000..d54bf0b6e --- /dev/null +++ b/buildstream/types.py @@ -0,0 +1,177 @@ +# +# Copyright (C) 2018 Bloomberg LP +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> +# Jim MacArthur <jim.macarthur@codethink.co.uk> +# Benjamin Schubert <bschubert15@bloomberg.net> + +""" +Foundation types +================ + +""" + +from enum import Enum +import heapq + + +class Scope(Enum): + """Defines the scope of dependencies to include for a given element + when iterating over the dependency graph in APIs like + :func:`Element.dependencies() <buildstream.element.Element.dependencies>` + """ + + ALL = 1 + """All elements which the given element depends on, following + all elements required for building. Including the element itself. + """ + + BUILD = 2 + """All elements required for building the element, including their + respective run dependencies. Not including the given element itself. + """ + + RUN = 3 + """All elements required for running the element. Including the element + itself. + """ + + NONE = 4 + """Just the element itself, no dependencies. + + *Since: 1.4* + """ + + +class Consistency(): + """Defines the various consistency states of a :class:`.Source`. + """ + + INCONSISTENT = 0 + """Inconsistent + + Inconsistent sources have no explicit reference set. They cannot + produce a cache key, be fetched or staged. They can only be tracked. + """ + + RESOLVED = 1 + """Resolved + + Resolved sources have a reference and can produce a cache key and + be fetched, however they cannot be staged. + """ + + CACHED = 2 + """Cached + + Sources have a cached unstaged copy in the source directory. + """ + + +class CoreWarnings(): + """CoreWarnings() + + Some common warnings which are raised by core functionalities within BuildStream are found in this class. + """ + + OVERLAPS = "overlaps" + """ + This warning will be produced when buildstream detects an overlap on an element + which is not whitelisted. See :ref:`Overlap Whitelist <public_overlap_whitelist>` + """ + + REF_NOT_IN_TRACK = "ref-not-in-track" + """ + This warning will be produced when a source is configured with a reference + which is found to be invalid based on the configured track + """ + + BAD_ELEMENT_SUFFIX = "bad-element-suffix" + """ + This warning will be produced when an element whose name does not end in .bst + is referenced either on the command line or by another element + """ + + BAD_CHARACTERS_IN_NAME = "bad-characters-in-name" + """ + This warning will be produces when filename for a target contains invalid + characters in its name. + """ + + +# _KeyStrength(): +# +# Strength of cache key +# +class _KeyStrength(Enum): + + # Includes strong cache keys of all build dependencies and their + # runtime dependencies. + STRONG = 1 + + # Includes names of direct build dependencies but does not include + # cache keys of dependencies. + WEAK = 2 + + +# _UniquePriorityQueue(): +# +# Implements a priority queue that adds only each key once. +# +# The queue will store and priority based on a tuple (key, item). +# +class _UniquePriorityQueue: + + def __init__(self): + self._items = set() + self._heap = [] + + # push(): + # + # Push a new item in the queue. + # + # If the item is already present in the queue as identified by the key, + # this is a noop. + # + # Args: + # key (hashable, comparable): unique key to use for checking for + # the object's existence and used for + # ordering + # item (any): item to push to the queue + # + def push(self, key, item): + if key not in self._items: + self._items.add(key) + heapq.heappush(self._heap, (key, item)) + + # pop(): + # + # Pop the next item from the queue, by priority order. + # + # Returns: + # (any): the next item + # + # Throw: + # IndexError: when the list is empty + # + def pop(self): + key, item = heapq.heappop(self._heap) + self._items.remove(key) + return item + + def __len__(self): + return len(self._heap) |