diff options
author | Tristan Maat <tristan.maat@codethink.co.uk> | 2019-11-18 13:39:26 +0000 |
---|---|---|
committer | Tristan Maat <tristan.maat@codethink.co.uk> | 2019-11-22 11:41:31 +0000 |
commit | 14e32a34f67df754d9146efafe9686bfe6c91e50 (patch) | |
tree | 0f5b5fbf879350dfe67613d1904a2b650676cd80 /src | |
parent | cc5b8e80a97eea56cbd1bfe749de800a00fd98cc (diff) | |
download | buildstream-14e32a34f67df754d9146efafe9686bfe6c91e50.tar.gz |
Schedule elements instead of "requiring" them
Diffstat (limited to 'src')
-rw-r--r-- | src/buildstream/_loader/loader.py | 3 | ||||
-rw-r--r-- | src/buildstream/_scheduler/queues/queue.py | 5 | ||||
-rw-r--r-- | src/buildstream/_stream.py | 4 | ||||
-rw-r--r-- | src/buildstream/element.py | 69 |
4 files changed, 11 insertions, 70 deletions
diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py index 3b721d6f2..b2e58b9e7 100644 --- a/src/buildstream/_loader/loader.py +++ b/src/buildstream/_loader/loader.py @@ -669,9 +669,6 @@ class Loader: # Optimization for junctions with a single local source basedir = sources[0]._get_local_path() else: - # Stage sources - element._set_required() - # Note: We use _KeyStrength.WEAK here because junctions # cannot have dependencies, therefore the keys are # equivalent. diff --git a/src/buildstream/_scheduler/queues/queue.py b/src/buildstream/_scheduler/queues/queue.py index 986ac6c0a..0dfc06d58 100644 --- a/src/buildstream/_scheduler/queues/queue.py +++ b/src/buildstream/_scheduler/queues/queue.py @@ -181,10 +181,7 @@ class Queue: # Obtain immediate element status for elt in elts: - if self._required_element_check and not elt._is_required(): - elt._set_required_callback(self._enqueue_element) - else: - self._enqueue_element(elt) + self._enqueue_element(elt) # dequeue() # diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index 067357384..0ce2bbf11 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -1264,10 +1264,10 @@ class Stream: # others are requested dynamically as needed. # This avoids pulling, fetching, or building unneeded build-only dependencies. for element in elements: - element._set_required() + element._schedule_assembly_when_necessary() else: for element in selected: - element._set_required() + element._schedule_assembly_when_necessary() return selected diff --git a/src/buildstream/element.py b/src/buildstream/element.py index 74fb1a056..86fd15f6f 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -290,7 +290,6 @@ class Element(Plugin): self.__batch_prepare_assemble_collect = None # type: Optional[str] # Callbacks - self.__required_callback = None # Callback to Queues self.__can_query_cache_callback = None # Callback to PullQueue/FetchQueue self.__buildable_callback = None # Callback to BuildQueue @@ -1252,7 +1251,7 @@ class Element(Plugin): # - _update_artifact_state() # - Computes the state of the element's artifact using the # cache key. - # - __schedule_assembly_when_necessary() + # - _schedule_assembly_when_necessary() # - Schedules assembly of an element, iff its current state # allows/necessitates it # - __update_cache_key_non_strict() @@ -1519,34 +1518,6 @@ class Element(Plugin): # Ensure deterministic owners of sources at build time vdirectory.set_deterministic_user() - # _set_required(): - # - # Mark this element and its runtime dependencies as required. - # This unblocks pull/fetch/build. - # - def _set_required(self): - if self.__required: - # Already done - return - - self.__required = True - - # Request artifacts of runtime dependencies - for dep in self.dependencies(Scope.RUN, recurse=False): - dep._set_required() - - # When an element becomes required, it must be assembled for - # the current pipeline. `__schedule_assembly_when_necessary()` - # will abort if some other state prevents it from being built, - # and changes to such states will cause re-scheduling, so this - # is safe. - self.__schedule_assembly_when_necessary() - - # Callback to the Queue - if self.__required_callback is not None: - self.__required_callback(self) - self.__required_callback = None - # _is_required(): # # Returns whether this element has been marked as required. @@ -1592,9 +1563,6 @@ class Element(Plugin): # We're not processing not processing and - # We're required for the current build - self._is_required() - and # We have figured out the state of our artifact self.__artifact and @@ -1602,12 +1570,12 @@ class Element(Plugin): not self._cached() ) - # __schedule_assembly_when_necessary(): + # _schedule_assembly_when_necessary(): # # This is called in the main process before the element is assembled # in a subprocess. # - def __schedule_assembly_when_necessary(self): + def _schedule_assembly_when_necessary(self): # FIXME: We could reduce the number of function calls a bit by # factoring this out of this method (and checking whether we # should schedule at the calling end). @@ -1621,7 +1589,7 @@ class Element(Plugin): # Requests artifacts of build dependencies for dep in self.dependencies(Scope.BUILD, recurse=False): - dep._set_required() + dep._schedule_assembly_when_necessary() # Once we schedule an element for assembly, we know that our # build dependencies have strong cache keys, so we can update @@ -1885,7 +1853,7 @@ class Element(Plugin): # We may not have actually pulled an artifact - the pull may # have failed. We might therefore need to schedule assembly. - self.__schedule_assembly_when_necessary() + self._schedule_assembly_when_necessary() # If we've finished pulling, an artifact might now exist # locally, so we might need to update a non-strict strong # cache key. @@ -2282,22 +2250,6 @@ class Element(Plugin): else: return True - # _set_required_callback() - # - # - # Notify the pull/fetch/build queue that the element is potentially - # ready to be processed. - # - # _Set the _required_callback - the _required_callback is invoked when an - # element is marked as required. This informs us that the element needs to - # either be pulled or fetched + built. - # - # Args: - # callback (callable) - The callback function - # - def _set_required_callback(self, callback): - self.__required_callback = callback - # _set_can_query_cache_callback() # # Notify the pull/fetch queue that the element is potentially @@ -2406,11 +2358,6 @@ class Element(Plugin): assert "_Element__buildable_callback" in state state["_Element__buildable_callback"] = None - # This callback is not even read in the child process, so delete it. - # If this assumption is invalidated, we will get an attribute error to - # let us know, and we will need to update accordingly. - del state["_Element__required_callback"] - return self.__meta_kind, state def _walk_artifact_files(self): @@ -3225,7 +3172,7 @@ class Element(Plugin): # to this element. # # If the state changes, this will subsequently call - # `self.__schedule_assembly_when_necessary()` to schedule assembly if it becomes + # `self._schedule_assembly_when_necessary()` to schedule assembly if it becomes # possible. # # Element.__update_cache_keys() must be called before this to have @@ -3241,7 +3188,7 @@ class Element(Plugin): if not context.get_strict() and not self.__artifact: # We've calculated the weak_key, so instantiate artifact instance member self.__artifact = Artifact(self, context, weak_key=self.__weak_cache_key) - self.__schedule_assembly_when_necessary() + self._schedule_assembly_when_necessary() if not self.__strict_cache_key: return @@ -3253,7 +3200,7 @@ class Element(Plugin): if context.get_strict(): self.__artifact = self.__strict_artifact - self.__schedule_assembly_when_necessary() + self._schedule_assembly_when_necessary() else: self.__update_cache_key_non_strict() |