diff options
author | Benjamin Schubert <bschubert15@bloomberg.net> | 2019-11-26 18:40:23 +0000 |
---|---|---|
committer | Benjamin Schubert <contact@benschubert.me> | 2020-01-16 14:36:35 +0000 |
commit | 93ebe4992c32b2588ae213a98c3e05d60c974a00 (patch) | |
tree | 0805dadc7f75ee208c46b08bada7b4ba13752acb /src | |
parent | c48e89d5da3a7b2b2e72d536aaea9ec182d1e1fd (diff) | |
download | buildstream-93ebe4992c32b2588ae213a98c3e05d60c974a00.tar.gz |
element.py: Remove _get_consistency and introduce explicit methods
This replaces the _get_consistency method by two methods:
`_has_all_sources_resolved` and `_has_all_sources_cached` which allows
a more fine grained control on what information is needed.
Diffstat (limited to 'src')
-rw-r--r-- | src/buildstream/_frontend/widget.py | 7 | ||||
-rw-r--r-- | src/buildstream/_loader/loader.py | 17 | ||||
-rw-r--r-- | src/buildstream/_pipeline.py | 6 | ||||
-rw-r--r-- | src/buildstream/_scheduler/queues/fetchqueue.py | 5 | ||||
-rw-r--r-- | src/buildstream/element.py | 28 |
5 files changed, 33 insertions, 30 deletions
diff --git a/src/buildstream/_frontend/widget.py b/src/buildstream/_frontend/widget.py index 44922cac8..fcc951d00 100644 --- a/src/buildstream/_frontend/widget.py +++ b/src/buildstream/_frontend/widget.py @@ -27,7 +27,7 @@ from ruamel import yaml import click from .profile import Profile -from .. import Consistency, Scope +from .. import Scope from .. import __version__ as bst_version from .._exceptions import ImplError from .._message import MessageType @@ -346,8 +346,7 @@ class LogLine(Widget): line = p.fmt_subst(line, "key", cache_key, fg="yellow", dim=dim_keys) line = p.fmt_subst(line, "full-key", full_key, fg="yellow", dim=dim_keys) - consistency = element._get_consistency() - if consistency == Consistency.INCONSISTENT: + if not element._has_all_sources_resolved(): line = p.fmt_subst(line, "state", "no reference", fg="red") else: if element.get_kind() == "junction": @@ -356,7 +355,7 @@ class LogLine(Widget): line = p.fmt_subst(line, "state", "failed", fg="red") elif element._cached_success(): line = p.fmt_subst(line, "state", "cached", fg="magenta") - elif consistency == Consistency.RESOLVED and not element._has_all_sources_in_source_cache(): + elif not element._has_all_sources_in_source_cache() and not element._has_all_sources_cached(): line = p.fmt_subst(line, "state", "fetch needed", fg="red") elif element._buildable(): line = p.fmt_subst(line, "state", "buildable", fg="green") diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py index 05c139bb2..531655c09 100644 --- a/src/buildstream/_loader/loader.py +++ b/src/buildstream/_loader/loader.py @@ -20,7 +20,6 @@ import os from .._exceptions import LoadError, LoadErrorReason -from .. import Consistency from .. import _yaml from ..element import Element from ..node import Node @@ -651,16 +650,9 @@ class Loader: self._loaders[filename] = loader return loader - # Handle the case where a subproject needs to be fetched - # - if element._get_consistency() >= Consistency.RESOLVED and not element._has_all_sources_in_source_cache(): - if ticker: - ticker(filename, "Fetching subproject") - self._fetch_subprojects([element]) - # Handle the case where a subproject has no ref # - elif element._get_consistency() == Consistency.INCONSISTENT: + if not element._has_all_sources_resolved(): detail = "Try tracking the junction element with `bst source track {}`".format(filename) raise LoadError( "{}Subproject has no ref for junction: {}".format(provenance_str, filename), @@ -668,6 +660,13 @@ class Loader: detail=detail, ) + # Handle the case where a subproject needs to be fetched + # + if not element._has_all_sources_in_source_cache(): + if ticker: + ticker(filename, "Fetching subproject") + self._fetch_subprojects([element]) + sources = list(element.sources()) if len(sources) == 1 and sources[0]._get_local_path(): # Optimization for junctions with a single local source diff --git a/src/buildstream/_pipeline.py b/src/buildstream/_pipeline.py index 8d0a3b411..1a98fccf5 100644 --- a/src/buildstream/_pipeline.py +++ b/src/buildstream/_pipeline.py @@ -29,7 +29,7 @@ from pyroaring import BitMap # pylint: disable=no-name-in-module from ._exceptions import PipelineError from ._message import Message, MessageType from ._profile import Topics, PROFILER -from . import Scope, Consistency +from . import Scope from ._project import ProjectRefStorage from .types import _PipelineSelection @@ -340,7 +340,7 @@ class Pipeline: inconsistent_workspaced = [] with self._context.messenger.timed_activity("Checking sources"): for element in elements: - if element._get_consistency() == Consistency.INCONSISTENT: + if not element._has_all_sources_resolved(): if element._get_workspace(): inconsistent_workspaced.append(element) else: @@ -375,7 +375,7 @@ class Pipeline: uncached = [] with self._context.messenger.timed_activity("Checking sources"): for element in elements: - if element._get_consistency() < Consistency.CACHED and not element._has_all_sources_in_source_cache(): + if not element._has_all_sources_in_source_cache() and not element._has_all_sources_cached(): uncached.append(element) if uncached: diff --git a/src/buildstream/_scheduler/queues/fetchqueue.py b/src/buildstream/_scheduler/queues/fetchqueue.py index 77c8b8c9c..8123b7611 100644 --- a/src/buildstream/_scheduler/queues/fetchqueue.py +++ b/src/buildstream/_scheduler/queues/fetchqueue.py @@ -18,9 +18,6 @@ # Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> # Jürg Billeter <juerg.billeter@codethink.co.uk> -# BuildStream toplevel imports -from ... import Consistency - # Local imports from . import Queue, QueueStatus from ..resources import ResourceType @@ -73,7 +70,7 @@ class FetchQueue(Queue): # Successful fetch, we must be CACHED or in the sourcecache if self._should_fetch_original: - assert element._get_consistency() == Consistency.CACHED + assert element._has_all_sources_cached() else: assert element._has_all_sources_in_source_cache() diff --git a/src/buildstream/element.py b/src/buildstream/element.py index 057b229e8..da62c9bb0 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -1072,13 +1072,6 @@ class Element(Plugin): cls.__instantiated_elements = {} cls.__redundant_source_refs = [] - # _get_consistency() - # - # Returns cached consistency state - # - def _get_consistency(self): - return self.__consistency - # _cached(): # # Returns: @@ -1169,7 +1162,7 @@ class Element(Plugin): # (bool): Whether this element can currently be built # def _buildable(self): - if self._get_consistency() < Consistency.CACHED and not self._has_all_sources_in_source_cache(): + if not (self._has_all_sources_in_source_cache() or self._has_all_sources_cached()): return False if not self.__assemble_scheduled: @@ -2178,13 +2171,28 @@ class Element(Plugin): self.__has_all_sources_in_source_cache = True return True + # _has_all_sources_resolved() + # + # Get whether all sources of the element are resolved + # + def _has_all_sources_resolved(self): + return self.__consistency >= Consistency.RESOLVED + + # _has_all_sources_cached() + # + # Get whether all the sources of the element have their own cached + # copy of their sources. + # + def _has_all_sources_cached(self): + return self.__consistency >= Consistency.CACHED + def _should_fetch(self, fetch_original=False): """ return bool of if we need to run the fetch stage for this element Args: fetch_original (bool): whether we need to original unstaged source """ - if (self._get_consistency() == Consistency.CACHED and fetch_original) or ( + if (self._has_all_sources_cached() and fetch_original) or ( self._has_all_sources_in_source_cache() and not fetch_original ): return False @@ -3018,7 +3026,7 @@ class Element(Plugin): # dependency has changed. # def __update_cache_keys(self): - if self._get_consistency() == Consistency.INCONSISTENT: + if not self._has_all_sources_resolved(): # Tracking may still be pending return |