summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorTristan van Berkom <tristan.vanberkom@codethink.co.uk>2020-08-10 19:41:10 +0900
committerTristan van Berkom <tristan.vanberkom@codethink.co.uk>2020-08-13 16:38:33 +0900
commit898fde9effcfc8dd632a6bdb731e3b73e438656c (patch)
tree5a3f2d18d95e58f39c82f955d5e36fb8b47f0b6c /src
parent830401ba6abbced060ceb5e833a5a76fea39a743 (diff)
downloadbuildstream-898fde9effcfc8dd632a6bdb731e3b73e438656c.tar.gz
Completely remove MetaElement
This dramatically affects the load process and removes one hoop we had to jump through, which is the creation of the extra intermediate MetaElement objects. This allows us to more easily carry state discovered by the Loader over to the Element constructor, as we need not add additional state to the intermediate MetaElement for this. Instead we have the Element initializer understand the LoadElement directly. Summary of changes: * _loader/metaelement.py: Removed * _loader/loadelement.py: Added some attributes previously required on MetaElement * _loader/loader.py: Removed _collect_element() and collect_element_no_deps(), removing the process of Loader.load() which translates LoadElements into MetaElements completely. * _loader/init.py: Export LoadElement, Dependency and Symbol types, stop exporting MetaElement * _loader/metasource.py: Now take the 'first_pass' parameter as an argument * _artifactelement.py: Use a virtual LoadElement instead of a virtual MetaElement to instantiate the ArtifactElement objects. * _pluginfactory/elementfactory.py: Adjust to now take a LoadElement * _project.py: Adjust Project.create_element() to now take a LoadElement, and call the new Element._new_from_load_element() instead of the old Element._new_from_meta() function * element.py: - Now export Element._new_from_load_element() instead of Element._new_from_meta() - Adjust the constructor to do the LoadElement toplevel node parsing instead of expecting members on the MetaElement object - Added __load_sources() which parses out and creates MetaSource objects for the sake of instantiating the element's Source objects. Consequently this simplifies the scenario where workspaces are involved. * source.py: Adjusted to use the new `first_pass` parameter to MetaSource when creating a duplicate clone.
Diffstat (limited to 'src')
-rw-r--r--src/buildstream/_artifactelement.py8
-rw-r--r--src/buildstream/_loader/__init__.py3
-rw-r--r--src/buildstream/_loader/loadelement.pyx45
-rw-r--r--src/buildstream/_loader/loader.py157
-rw-r--r--src/buildstream/_loader/metaelement.py73
-rw-r--r--src/buildstream/_loader/metasource.py4
-rw-r--r--src/buildstream/_pluginfactory/elementfactory.py8
-rw-r--r--src/buildstream/_project.py10
-rw-r--r--src/buildstream/element.py216
-rw-r--r--src/buildstream/source.py3
10 files changed, 204 insertions, 323 deletions
diff --git a/src/buildstream/_artifactelement.py b/src/buildstream/_artifactelement.py
index 4066cef06..44e52ea90 100644
--- a/src/buildstream/_artifactelement.py
+++ b/src/buildstream/_artifactelement.py
@@ -22,7 +22,8 @@ from typing import TYPE_CHECKING
from . import Element
from . import _cachekey
from ._exceptions import ArtifactElementError
-from ._loader.metaelement import MetaElement
+from ._loader import LoadElement
+from .node import Node
from .types import Scope
if TYPE_CHECKING:
@@ -49,10 +50,9 @@ class ArtifactElement(Element):
self._key = key
project = context.get_toplevel_project()
- meta = MetaElement(project, element) # NOTE element has no .bst suffix
- plugin_conf = None
+ load_element = LoadElement(Node.from_dict({}), element, project.loader) # NOTE element has no .bst suffix
- super().__init__(context, project, meta, plugin_conf)
+ super().__init__(context, project, load_element, None)
# _new_from_artifact_ref():
#
diff --git a/src/buildstream/_loader/__init__.py b/src/buildstream/_loader/__init__.py
index fd5cac2ae..a4be9cfe5 100644
--- a/src/buildstream/_loader/__init__.py
+++ b/src/buildstream/_loader/__init__.py
@@ -17,7 +17,8 @@
# Authors:
# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+from .types import Symbol
from .metasource import MetaSource
-from .metaelement import MetaElement
+from .loadelement import LoadElement, Dependency
from .loadcontext import LoadContext
from .loader import Loader
diff --git a/src/buildstream/_loader/loadelement.pyx b/src/buildstream/_loader/loadelement.pyx
index 6cd5b46b7..01334d124 100644
--- a/src/buildstream/_loader/loadelement.pyx
+++ b/src/buildstream/_loader/loadelement.pyx
@@ -184,9 +184,10 @@ cdef class LoadElement:
cdef readonly MappingNode node
cdef readonly str name
- cdef readonly full_name
- cdef public bint meta_done
+ cdef readonly str full_name
+ cdef readonly str kind
cdef int node_id
+ cdef readonly bint first_pass
cdef readonly object _loader
cdef readonly str link_target
cdef readonly ProvenanceInformation link_target_provenance
@@ -199,10 +200,10 @@ cdef class LoadElement:
#
# Public members
#
+ self.kind = None # The Element kind
self.node = node # The YAML node
self.name = filename # The element name
self.full_name = None # The element full name (with associated junction)
- self.meta_done = False # If the MetaElement for this LoadElement is done
self.node_id = _next_synthetic_counter()
self.link_target = None # The target of a link element
self.link_target_provenance = None # The provenance of the link target
@@ -233,13 +234,15 @@ cdef class LoadElement:
'build-depends', 'runtime-depends',
])
+ self.kind = node.get_str(Symbol.KIND, default=None)
+ self.first_pass = self.kind in ("junction", "link")
+
#
# If this is a link, resolve it right away and just
# store the link target and provenance
#
- if self.node.get_str(Symbol.KIND, default=None) == 'link':
- meta_element = self._loader.collect_element_no_deps(self)
- element = Element._new_from_meta(meta_element)
+ if self.kind == 'link':
+ element = Element._new_from_load_element(self)
element._initialize_state()
# Custom error for link dependencies, since we don't completely
@@ -254,6 +257,36 @@ cdef class LoadElement:
self.link_target = element.target
self.link_target_provenance = element.target_provenance
+ # We don't count progress for junction elements or link
+ # as they do not represent real elements in the build graph.
+ #
+ # We check for a `None` kind, to avoid reporting progress for
+ # the virtual toplevel element used to load the pipeline.
+ #
+ if self._loader.load_context.task and self.kind is not None and not self.first_pass:
+ self._loader.load_context.task.add_current_progress()
+
+ # provenance
+ #
+ # A property reporting the ProvenanceInformation of the element
+ #
+ @property
+ def provenance(self):
+ return self.node.get_provenance()
+
+ # project
+ #
+ # A property reporting the Project in which this element resides.
+ #
+ @property
+ def project(self):
+ return self._loader.project
+
+ # junction
+ #
+ # A property reporting the junction element accessing this
+ # element, if any.
+ #
@property
def junction(self):
return self._loader.project.junction
diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py
index b0f4a4a07..94ee9078b 100644
--- a/src/buildstream/_loader/loader.py
+++ b/src/buildstream/_loader/loader.py
@@ -32,8 +32,6 @@ from ._loader import valid_chars_name
from .types import Symbol
from . import loadelement
from .loadelement import LoadElement, Dependency, extract_depends_from_node
-from .metaelement import MetaElement
-from .metasource import MetaSource
from ..types import CoreWarnings, _KeyStrength
from .._message import Message, MessageType
@@ -41,8 +39,8 @@ from .._message import Message, MessageType
# Loader():
#
# The Loader class does the heavy lifting of parsing target
-# bst files and ultimately transforming them into a list of MetaElements
-# with their own MetaSources, ready for instantiation by the core.
+# bst files and ultimately transforming them into a list of LoadElements
+# ready for instantiation by the core.
#
# Args:
# project (Project): The toplevel Project object
@@ -112,7 +110,8 @@ class Loader:
#
# Raises: LoadError
#
- # Returns: The toplevel LoadElement
+ # Returns:
+ # (list): The corresponding LoadElement instances matching the `targets`
#
def load(self, targets):
@@ -154,7 +153,6 @@ class Loader:
with PROFILER.profile(Topics.CIRCULAR_CHECK, "_".join(targets)):
self._check_circular_deps(dummy_target)
- ret = []
#
# Sort direct dependencies of elements by their dependency ordering
#
@@ -167,18 +165,13 @@ class Loader:
with PROFILER.profile(Topics.SORT_DEPENDENCIES, element.name):
loadelement.sort_dependencies(element, visited_elements)
- # Finally, wrap what we have into LoadElements and return the target
- #
- ret.append(loader._collect_element(element))
-
self._clean_caches()
# Cache how many Elements have just been loaded
if self.load_context.task:
- # Workaround for task potentially being None (because no State object)
self.loaded = self.load_context.task.current_progress
- return ret
+ return target_elements
# get_loader():
#
@@ -252,83 +245,6 @@ class Loader:
for parent in self._alternative_parents:
yield from foreach_parent(parent)
- # collect_element_no_deps()
- #
- # Collect a single element, without its dependencies, into a meta_element
- #
- # NOTE: This is declared public in order to share with the LoadElement
- # and should not be used outside of that `_loader` module.
- #
- # Args:
- # element (LoadElement): The element for which to load a MetaElement
- # report_progress (bool): Whether to report progress for this element, this is
- # because we ignore junctions and links when counting
- # how many elements we load.
- #
- # Returns:
- # (MetaElement): A partially loaded MetaElement
- #
- def collect_element_no_deps(self, element, *, report_progress=False):
- # Return the already built one, if we already built it
- meta_element = self._meta_elements.get(element.name)
- if meta_element:
- return meta_element
-
- node = element.node
- elt_provenance = node.get_provenance()
- meta_sources = []
-
- element_kind = node.get_str(Symbol.KIND)
-
- # if there's a workspace for this element then just append a dummy workspace
- # metasource.
- workspace = self.load_context.context.get_workspaces().get_workspace(element.full_name)
- skip_workspace = True
- if workspace:
- workspace_node = {"kind": "workspace"}
- workspace_node["path"] = workspace.get_absolute_path()
- workspace_node["last_build"] = str(workspace.to_dict().get("last_build", ""))
- node[Symbol.SOURCES] = [workspace_node]
- skip_workspace = False
-
- sources = node.get_sequence(Symbol.SOURCES, default=[])
- for index, source in enumerate(sources):
- kind = source.get_str(Symbol.KIND)
- # the workspace source plugin cannot be used unless the element is workspaced
- if kind == "workspace" and skip_workspace:
- continue
-
- del source[Symbol.KIND]
-
- # Directory is optional
- directory = source.get_str(Symbol.DIRECTORY, default=None)
- if directory:
- del source[Symbol.DIRECTORY]
- meta_source = MetaSource(element.name, index, element_kind, kind, source, directory)
- meta_sources.append(meta_source)
-
- meta_element = MetaElement(
- self.project,
- element.name,
- element_kind,
- elt_provenance,
- meta_sources,
- node.get_mapping(Symbol.CONFIG, default={}),
- node.get_mapping(Symbol.VARIABLES, default={}),
- node.get_mapping(Symbol.ENVIRONMENT, default={}),
- node.get_str_list(Symbol.ENV_NOCACHE, default=[]),
- node.get_mapping(Symbol.PUBLIC, default={}),
- node.get_mapping(Symbol.SANDBOX, default={}),
- element_kind in ("junction", "link"),
- )
-
- # Cache it now, make sure it's already there before recursing
- self._meta_elements[element.name] = meta_element
- if self.load_context.task and report_progress:
- self.load_context.task.add_current_progress()
-
- return meta_element
-
###########################################
# Private Methods #
###########################################
@@ -553,52 +469,6 @@ class Loader:
check_elements.remove(this_element)
validated.add(this_element)
- # _collect_element()
- #
- # Collect the toplevel elements we have
- #
- # Args:
- # top_element (LoadElement): The element for which to load a MetaElement
- #
- # Returns:
- # (MetaElement): A fully loaded MetaElement
- #
- def _collect_element(self, top_element):
- element_queue = [top_element]
- meta_element_queue = [self.collect_element_no_deps(top_element, report_progress=True)]
-
- while element_queue:
- element = element_queue.pop()
- meta_element = meta_element_queue.pop()
-
- if element.meta_done:
- # This can happen if there are multiple top level targets
- # in which case, we simply skip over this element.
- continue
-
- for dep in element.dependencies:
-
- loader = dep.element._loader
- name = dep.element.name
-
- try:
- meta_dep = loader._meta_elements[name]
- except KeyError:
- meta_dep = loader.collect_element_no_deps(dep.element, report_progress=True)
- element_queue.append(dep.element)
- meta_element_queue.append(meta_dep)
-
- if dep.dep_type != "runtime":
- meta_element.build_dependencies.append(meta_dep)
- if dep.dep_type != "build":
- meta_element.dependencies.append(meta_dep)
- if dep.strict:
- meta_element.strict_dependencies.append(meta_dep)
-
- element.meta_done = True
-
- return self._meta_elements[top_element.name]
-
# _search_for_override():
#
# Search parent projects for an overridden subproject to replace this junction.
@@ -711,20 +581,9 @@ class Loader:
if not load_subprojects:
return None
- # meta junction element
- #
- # Note that junction elements are not allowed to have
- # dependencies, so disabling progress reporting here should
- # have no adverse effects - the junction element itself cannot
- # be depended on, so it would be confusing for its load to
- # show up in logs.
- #
- # Any task counting *inside* the junction will be handled by
- # its loader.
- meta_element = self.collect_element_no_deps(self._elements[filename])
- if meta_element.kind != "junction":
+ if load_element.kind != "junction":
raise LoadError(
- "{}{}: Expected junction but element kind is {}".format(provenance_str, filename, meta_element.kind),
+ "{}{}: Expected junction but element kind is {}".format(provenance_str, filename, load_element.kind),
LoadErrorReason.INVALID_DATA,
)
@@ -748,7 +607,7 @@ class Loader:
"{}: Dependencies are forbidden for 'junction' elements".format(p), LoadErrorReason.INVALID_JUNCTION
)
- element = Element._new_from_meta(meta_element)
+ element = Element._new_from_load_element(load_element)
element._initialize_state()
# Handle the case where a subproject has no ref
diff --git a/src/buildstream/_loader/metaelement.py b/src/buildstream/_loader/metaelement.py
deleted file mode 100644
index 1c1f6feb2..000000000
--- a/src/buildstream/_loader/metaelement.py
+++ /dev/null
@@ -1,73 +0,0 @@
-#
-# Copyright (C) 2016 Codethink Limited
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library. If not, see <http://www.gnu.org/licenses/>.
-#
-# Authors:
-# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
-
-from ..node import Node
-
-
-class MetaElement:
-
- # MetaElement()
- #
- # An abstract object holding data suitable for constructing an Element
- #
- # Args:
- # project: The project that contains the element
- # name: The resolved element name
- # kind: The element kind
- # provenance: The provenance of the element
- # sources: An array of MetaSource objects
- # config: The configuration data for the element
- # variables: The variables declared or overridden on this element
- # environment: The environment variables declared or overridden on this element
- # env_nocache: List of environment vars which should not be considered in cache keys
- # public: Public domain data dictionary
- # sandbox: Configuration specific to the sandbox environment
- # first_pass: The element is to be loaded with first pass configuration (junction)
- #
- def __init__(
- self,
- project,
- name,
- kind=None,
- provenance=None,
- sources=None,
- config=None,
- variables=None,
- environment=None,
- env_nocache=None,
- public=None,
- sandbox=None,
- first_pass=False,
- ):
- self.project = project
- self.name = name
- self.kind = kind
- self.provenance = provenance
- self.sources = sources
- self.config = config or Node.from_dict({})
- self.variables = variables or Node.from_dict({})
- self.environment = environment or Node.from_dict({})
- self.env_nocache = env_nocache or []
- self.public = public or Node.from_dict({})
- self.sandbox = sandbox or Node.from_dict({})
- self.build_dependencies = []
- self.dependencies = []
- self.strict_dependencies = []
- self.first_pass = first_pass
- self.is_junction = kind in ("junction", "link")
diff --git a/src/buildstream/_loader/metasource.py b/src/buildstream/_loader/metasource.py
index ddaa538f5..d49ae3d82 100644
--- a/src/buildstream/_loader/metasource.py
+++ b/src/buildstream/_loader/metasource.py
@@ -32,12 +32,12 @@ class MetaSource:
# config: The configuration data for the source
# first_pass: This source will be used with first project pass configuration (used for junctions).
#
- def __init__(self, element_name, element_index, element_kind, kind, config, directory):
+ def __init__(self, element_name, element_index, element_kind, kind, config, directory, first_pass):
self.element_name = element_name
self.element_index = element_index
self.element_kind = element_kind
self.kind = kind
self.config = config
self.directory = directory
- self.first_pass = False
+ self.first_pass = first_pass
self.provenance = config.get_provenance()
diff --git a/src/buildstream/_pluginfactory/elementfactory.py b/src/buildstream/_pluginfactory/elementfactory.py
index 9854c1a5c..7d94c2541 100644
--- a/src/buildstream/_pluginfactory/elementfactory.py
+++ b/src/buildstream/_pluginfactory/elementfactory.py
@@ -39,7 +39,7 @@ class ElementFactory(PluginFactory):
# Args:
# context (object): The Context object for processing
# project (object): The project object
- # meta (object): The loaded MetaElement
+ # load_element (object): The LoadElement
#
# Returns: A newly created Element object of the appropriate kind
#
@@ -47,7 +47,7 @@ class ElementFactory(PluginFactory):
# PluginError (if the kind lookup failed)
# LoadError (if the element itself took issue with the config)
#
- def create(self, context, project, meta):
- element_type, default_config = self.lookup(context.messenger, meta.kind, meta.provenance)
- element = element_type(context, project, meta, default_config)
+ def create(self, context, project, load_element):
+ element_type, default_config = self.lookup(context.messenger, load_element.kind, load_element.provenance)
+ element = element_type(context, project, load_element, default_config)
return element
diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py
index 3562ea5d4..87772995a 100644
--- a/src/buildstream/_project.py
+++ b/src/buildstream/_project.py
@@ -345,13 +345,13 @@ class Project:
# Instantiate and return an element
#
# Args:
- # meta (MetaElement): The loaded MetaElement
+ # load_element (LoadElement): The LoadElement
#
# Returns:
# (Element): A newly created Element object of the appropriate kind
#
- def create_element(self, meta):
- return self.element_factory.create(self._context, self, meta)
+ def create_element(self, load_element):
+ return self.element_factory.create(self._context, self, load_element)
# create_source()
#
@@ -426,13 +426,13 @@ class Project:
with self._context.messenger.simple_task("Loading elements", silent_nested=True) as task:
self.load_context.set_task(task)
- meta_elements = self.loader.load(targets)
+ load_elements = self.loader.load(targets)
self.load_context.set_task(None)
with self._context.messenger.simple_task("Resolving elements") as task:
if task:
task.set_maximum_progress(self.loader.loaded)
- elements = [Element._new_from_meta(meta, task) for meta in meta_elements]
+ elements = [Element._new_from_load_element(load_element, task) for load_element in load_elements]
Element._clear_meta_elements_cache()
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index f1e909bce..f7d49d57f 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -80,7 +80,7 @@ import copy
import warnings
from collections import OrderedDict
import contextlib
-from contextlib import contextmanager
+from contextlib import contextmanager, suppress
from functools import partial
from itertools import chain
import string
@@ -106,6 +106,7 @@ from .sandbox._sandboxremote import SandboxRemote
from .types import CoreWarnings, Scope, _CacheBuildTrees, _KeyStrength
from ._artifact import Artifact
from ._elementsources import ElementSources
+from ._loader import Symbol, MetaSource
from .storage.directory import Directory
from .storage._filebaseddirectory import FileBasedDirectory
@@ -120,7 +121,7 @@ if TYPE_CHECKING:
from .sandbox import Sandbox
from .source import Source
from ._context import Context
- from ._loader.metaelement import MetaElement
+ from ._loader import LoadElement
from ._project import Project
# pylint: enable=cyclic-import
@@ -157,8 +158,8 @@ class Element(Plugin):
# The defaults from the yaml file and project
__defaults = None
- # A hash of Element by MetaElement
- __instantiated_elements = {} # type: Dict[MetaElement, Element]
+ # A hash of Element by LoadElement
+ __instantiated_elements = {} # type: Dict[LoadElement, Element]
# A list of (source, ref) tuples which were redundantly specified
__redundant_source_refs = [] # type: List[Tuple[Source, SourceRef]]
@@ -197,15 +198,17 @@ class Element(Plugin):
"""Whether the element produces an artifact when built.
"""
- def __init__(self, context: "Context", project: "Project", meta: "MetaElement", plugin_conf: Dict[str, Any]):
+ def __init__(
+ self, context: "Context", project: "Project", load_element: "LoadElement", plugin_conf: Dict[str, Any]
+ ):
self.__cache_key_dict = None # Dict for cache key calculation
self.__cache_key = None # Our cached cache key
- super().__init__(meta.name, context, project, meta.provenance, "element")
+ super().__init__(load_element.name, context, project, load_element.provenance, "element")
# Ensure the project is fully loaded here rather than later on
- if not meta.is_junction:
+ if not load_element.first_pass:
project.ensure_fully_loaded()
self.project_name = self._get_project().name
@@ -270,44 +273,44 @@ class Element(Plugin):
self.__resolved_initial_state = False # Whether the initial state of the Element has been resolved
# Ensure we have loaded this class's defaults
- self.__init_defaults(project, plugin_conf, meta.kind, meta.is_junction)
+ self.__init_defaults(project, plugin_conf, load_element.kind, load_element.first_pass)
# Collect the composited variables and resolve them
- variables = self.__extract_variables(project, meta)
+ variables = self.__extract_variables(project, load_element)
variables["element-name"] = self.name
self.__variables = Variables(variables)
- if not meta.is_junction:
+ if not load_element.first_pass:
self.__variables.check()
# Collect the composited environment now that we have variables
- unexpanded_env = self.__extract_environment(project, meta)
+ unexpanded_env = self.__extract_environment(project, load_element)
self.__variables.expand(unexpanded_env)
self.__environment = unexpanded_env.strip_node_info()
# Collect the environment nocache blacklist list
- nocache = self.__extract_env_nocache(project, meta)
+ nocache = self.__extract_env_nocache(project, load_element)
self.__env_nocache = nocache
# Grab public domain data declared for this instance
- self.__public = self.__extract_public(meta)
+ self.__public = self.__extract_public(load_element)
self.__variables.expand(self.__public)
self.__dynamic_public = None
# Collect the composited element configuration and
# ask the element to configure itself.
- self.__config = self.__extract_config(meta)
+ self.__config = self.__extract_config(load_element)
self.__variables.expand(self.__config)
self._configure(self.__config)
# Extract remote execution URL
- if meta.is_junction:
+ if load_element.first_pass:
self.__remote_execution_specs = None
else:
self.__remote_execution_specs = project.remote_execution_specs
# Extract Sandbox config
- sandbox_config = self.__extract_sandbox_config(project, meta)
+ sandbox_config = self.__extract_sandbox_config(project, load_element)
self.__variables.expand(sandbox_config)
self.__sandbox_config = SandboxConfig(sandbox_config, context.platform)
@@ -874,61 +877,56 @@ class Element(Plugin):
# Private Methods used in BuildStream #
#############################################################
- # _new_from_meta():
+ # _new_from_load_element():
#
# Recursively instantiate a new Element instance, its sources
- # and its dependencies from a meta element.
+ # and its dependencies from a LoadElement.
+ #
+ # FIXME: Need to use an iterative algorithm here since recursion
+ # will limit project dependency depth.
#
# Args:
- # meta (MetaElement): The meta element
+ # load_element (LoadElement): The LoadElement
# task (Task): A task object to report progress to
#
# Returns:
# (Element): A newly created Element instance
#
@classmethod
- def _new_from_meta(cls, meta, task=None):
+ def _new_from_load_element(cls, load_element, task=None):
- if not meta.first_pass:
- meta.project.ensure_fully_loaded()
+ if not load_element.first_pass:
+ load_element.project.ensure_fully_loaded()
- if meta in cls.__instantiated_elements:
- return cls.__instantiated_elements[meta]
+ with suppress(KeyError):
+ return cls.__instantiated_elements[load_element]
- element = meta.project.create_element(meta)
- cls.__instantiated_elements[meta] = element
+ element = load_element.project.create_element(load_element)
+ cls.__instantiated_elements[load_element] = element
- # Instantiate sources and generate their keys
- for meta_source in meta.sources:
- meta_source.first_pass = meta.is_junction
- source = meta.project.create_source(meta_source, variables=element.__variables)
+ # Load the sources from the LoadElement
+ element.__load_sources(load_element)
- redundant_ref = source._load_ref()
+ # Instantiate dependencies
+ for dep in load_element.dependencies:
+ dependency = Element._new_from_load_element(dep.element, task)
- element.__sources.add_source(source)
+ if dep.dep_type != "runtime":
+ element.__build_dependencies.append(dependency)
+ dependency.__reverse_build_deps.add(element)
- # Collect redundant refs which occurred at load time
- if redundant_ref is not None:
- cls.__redundant_source_refs.append((source, redundant_ref))
+ if dep.dep_type != "build":
+ element.__runtime_dependencies.append(dependency)
+ dependency.__reverse_runtime_deps.add(element)
+
+ if dep.strict:
+ element.__strict_dependencies.append(dependency)
- # Instantiate dependencies
- for meta_dep in meta.dependencies:
- dependency = Element._new_from_meta(meta_dep, task)
- element.__runtime_dependencies.append(dependency)
- dependency.__reverse_runtime_deps.add(element)
no_of_runtime_deps = len(element.__runtime_dependencies)
element.__runtime_deps_without_strict_cache_key = no_of_runtime_deps
element.__runtime_deps_without_cache_key = no_of_runtime_deps
element.__runtime_deps_uncached = no_of_runtime_deps
- for meta_dep in meta.build_dependencies:
- dependency = Element._new_from_meta(meta_dep, task)
- element.__build_dependencies.append(dependency)
- dependency.__reverse_build_deps.add(element)
-
- if meta_dep in meta.strict_dependencies:
- element.__strict_dependencies.append(dependency)
-
no_of_build_deps = len(element.__build_dependencies)
element.__build_deps_without_strict_cache_key = no_of_build_deps
element.__build_deps_without_cache_key = no_of_build_deps
@@ -2203,6 +2201,63 @@ class Element(Plugin):
# Private Local Methods #
#############################################################
+ # __load_sources()
+ #
+ # Load the Source objects from the LoadElement
+ #
+ def __load_sources(self, load_element):
+ project = self._get_project()
+ workspace = self._get_workspace()
+ meta_sources = []
+
+ # If there's a workspace for this element then we just load a workspace
+ # source plugin instead of the real plugins
+ if workspace:
+ workspace_node = {"kind": "workspace"}
+ workspace_node["path"] = workspace.get_absolute_path()
+ workspace_node["last_build"] = str(workspace.to_dict().get("last_build", ""))
+ meta = MetaSource(
+ self.name,
+ 0,
+ self.get_kind(),
+ "workspace",
+ Node.from_dict(workspace_node),
+ None,
+ load_element.first_pass,
+ )
+ meta_sources.append(meta)
+ else:
+ sources = load_element.node.get_sequence(Symbol.SOURCES, default=[])
+ for index, source in enumerate(sources):
+ kind = source.get_scalar(Symbol.KIND)
+
+ # The workspace source plugin is only valid for internal use
+ if kind.as_str() == "workspace":
+ raise LoadError(
+ "{}: Invalid usage of workspace source kind".format(kind.get_provenance()),
+ LoadErrorReason.INVALID_DATA,
+ )
+ del source[Symbol.KIND]
+
+ # Directory is optional
+ directory = source.get_str(Symbol.DIRECTORY, default=None)
+ if directory:
+ del source[Symbol.DIRECTORY]
+ meta_source = MetaSource(
+ self.name, index, self.get_kind(), kind.as_str(), source, directory, load_element.first_pass
+ )
+ meta_sources.append(meta_source)
+
+ for meta_source in meta_sources:
+ source = project.create_source(meta_source, variables=self.__variables)
+ redundant_ref = source._load_ref()
+
+ self.__sources.add_source(source)
+
+ # Collect redundant refs which occurred at load time
+ if redundant_ref is not None:
+ self.__redundant_source_refs.append((source, redundant_ref))
+
# __get_dependency_refs()
#
# Retrieve the artifact refs of the element's dependencies
@@ -2433,13 +2488,13 @@ class Element(Plugin):
yield sandbox
@classmethod
- def __compose_default_splits(cls, project, defaults, is_junction):
+ def __compose_default_splits(cls, project, defaults, first_pass):
- element_public = defaults.get_mapping("public", default={})
+ element_public = defaults.get_mapping(Symbol.PUBLIC, default={})
element_bst = element_public.get_mapping("bst", default={})
element_splits = element_bst.get_mapping("split-rules", default={})
- if is_junction:
+ if first_pass:
splits = element_splits.clone()
else:
assert project._splits is not None
@@ -2450,10 +2505,10 @@ class Element(Plugin):
element_bst["split-rules"] = splits
element_public["bst"] = element_bst
- defaults["public"] = element_public
+ defaults[Symbol.PUBLIC] = element_public
@classmethod
- def __init_defaults(cls, project, plugin_conf, kind, is_junction):
+ def __init_defaults(cls, project, plugin_conf, kind, first_pass):
# Defaults are loaded once per class and then reused
#
if cls.__defaults is None:
@@ -2468,11 +2523,11 @@ class Element(Plugin):
raise e
# Special case; compose any element-wide split-rules declarations
- cls.__compose_default_splits(project, defaults, is_junction)
+ cls.__compose_default_splits(project, defaults, first_pass)
# Override the element's defaults with element specific
# overrides from the project.conf
- if is_junction:
+ if first_pass:
elements = project.first_pass_config.element_overrides
else:
elements = project.element_overrides
@@ -2488,29 +2543,30 @@ class Element(Plugin):
# creating sandboxes for this element
#
@classmethod
- def __extract_environment(cls, project, meta):
- default_env = cls.__defaults.get_mapping("environment", default={})
+ def __extract_environment(cls, project, load_element):
+ default_env = cls.__defaults.get_mapping(Symbol.ENVIRONMENT, default={})
+ element_env = load_element.node.get_mapping(Symbol.ENVIRONMENT, default={}) or Node.from_dict({})
- if meta.is_junction:
+ if load_element.first_pass:
environment = Node.from_dict({})
else:
environment = project.base_environment.clone()
default_env._composite(environment)
- meta.environment._composite(environment)
+ element_env._composite(environment)
environment._assert_fully_composited()
return environment
@classmethod
- def __extract_env_nocache(cls, project, meta):
- if meta.is_junction:
+ def __extract_env_nocache(cls, project, load_element):
+ if load_element.first_pass:
project_nocache = []
else:
project_nocache = project.base_env_nocache
- default_nocache = cls.__defaults.get_str_list("environment-nocache", default=[])
- element_nocache = meta.env_nocache
+ default_nocache = cls.__defaults.get_str_list(Symbol.ENV_NOCACHE, default=[])
+ element_nocache = load_element.node.get_str_list(Symbol.ENV_NOCACHE, default=[])
# Accumulate values from the element default, the project and the element
# itself to form a complete list of nocache env vars.
@@ -2523,16 +2579,17 @@ class Element(Plugin):
# substituting command strings to be run in the sandbox
#
@classmethod
- def __extract_variables(cls, project, meta):
- default_vars = cls.__defaults.get_mapping("variables", default={})
+ def __extract_variables(cls, project, load_element):
+ default_vars = cls.__defaults.get_mapping(Symbol.VARIABLES, default={})
+ element_vars = load_element.node.get_mapping(Symbol.VARIABLES, default={}) or Node.from_dict({})
- if meta.is_junction:
+ if load_element.first_pass:
variables = project.first_pass_config.base_variables.clone()
else:
variables = project.base_variables.clone()
default_vars._composite(variables)
- meta.variables._composite(variables)
+ element_vars._composite(variables)
variables._assert_fully_composited()
for var in ("project-name", "element-name", "max-jobs"):
@@ -2554,13 +2611,14 @@ class Element(Plugin):
# off to element.configure()
#
@classmethod
- def __extract_config(cls, meta):
+ def __extract_config(cls, load_element):
+ element_config = load_element.node.get_mapping(Symbol.CONFIG, default={}) or Node.from_dict({})
# The default config is already composited with the project overrides
- config = cls.__defaults.get_mapping("config", default={})
+ config = cls.__defaults.get_mapping(Symbol.CONFIG, default={})
config = config.clone()
- meta.config._composite(config)
+ element_config._composite(config)
config._assert_fully_composited()
return config
@@ -2568,18 +2626,20 @@ class Element(Plugin):
# Sandbox-specific configuration data, to be passed to the sandbox's constructor.
#
@classmethod
- def __extract_sandbox_config(cls, project, meta):
- if meta.is_junction:
+ def __extract_sandbox_config(cls, project, load_element):
+ element_sandbox = load_element.node.get_mapping(Symbol.SANDBOX, default={}) or Node.from_dict({})
+
+ if load_element.first_pass:
sandbox_config = Node.from_dict({})
else:
sandbox_config = project._sandbox.clone()
# The default config is already composited with the project overrides
- sandbox_defaults = cls.__defaults.get_mapping("sandbox", default={})
+ sandbox_defaults = cls.__defaults.get_mapping(Symbol.SANDBOX, default={})
sandbox_defaults = sandbox_defaults.clone()
sandbox_defaults._composite(sandbox_config)
- meta.sandbox._composite(sandbox_config)
+ element_sandbox._composite(sandbox_config)
sandbox_config._assert_fully_composited()
return sandbox_config
@@ -2588,14 +2648,16 @@ class Element(Plugin):
# elements may extend but whos defaults are defined in the project.
#
@classmethod
- def __extract_public(cls, meta):
- base_public = cls.__defaults.get_mapping("public", default={})
+ def __extract_public(cls, load_element):
+ element_public = load_element.node.get_mapping(Symbol.PUBLIC, default={}) or Node.from_dict({})
+
+ base_public = cls.__defaults.get_mapping(Symbol.PUBLIC, default={})
base_public = base_public.clone()
base_bst = base_public.get_mapping("bst", default={})
base_splits = base_bst.get_mapping("split-rules", default={})
- element_public = meta.public.clone()
+ element_public = element_public.clone()
element_bst = element_public.get_mapping("bst", default={})
element_splits = element_bst.get_mapping("split-rules", default={})
diff --git a/src/buildstream/source.py b/src/buildstream/source.py
index dc87c35de..ea77a6537 100644
--- a/src/buildstream/source.py
+++ b/src/buildstream/source.py
@@ -1213,10 +1213,9 @@ class Source(Plugin):
self.get_kind(),
self.__config,
self.__directory,
+ self.__first_pass,
)
- meta.first_pass = self.__first_pass
-
clone = source_kind(
context, project, meta, self.__variables, alias_override=(alias, uri), unique_id=self._unique_id
)