path: root/buildstream/_loader/loader.py

#
#  Copyright (C) 2018 Codethink Limited
#
#  This program is free software; you can redistribute it and/or
#  modify it under the terms of the GNU Lesser General Public
#  License as published by the Free Software Foundation; either
#  version 2 of the License, or (at your option) any later version.
#
#  This library is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
#  Lesser General Public License for more details.
#
#  You should have received a copy of the GNU Lesser General Public
#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
#
#  Authors:
#        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>

import os
from functools import cmp_to_key
from collections import Mapping, namedtuple
import tempfile
import shutil

from .._exceptions import LoadError, LoadErrorReason
from .. import Consistency
from .. import _yaml
from ..element import Element
from .._profile import Topics, profile_start, profile_end
from .._includes import Includes

from .types import Symbol, Dependency
from .loadelement import LoadElement
from . import MetaElement
from . import MetaSource
from ..types import CoreWarnings


# Loader():
#
# The Loader class does the heavy lifting of parsing target
# bst files and ultimately transforming them into a list of MetaElements
# with their own MetaSources, ready for instantiation by the core.
#
# Args:
#    context (Context): The Context object
#    project (Project): The toplevel Project object
#    parent (Loader): A parent Loader object, in the case this is a junctioned Loader
#    tempdir (str): A directory to cleanup with the Loader, given to the loader by a parent
#                   loader in the case that this loader is a subproject loader.
#
class Loader():

    def __init__(self, context, project, *, parent=None, tempdir=None):

        # Ensure we have an absolute path for the base directory
        basedir = project.element_path
        if not os.path.isabs(basedir):
            basedir = os.path.abspath(basedir)

        #
        # Public members
        #
        self.project = project   # The associated Project

        #
        # Private members
        #
        self._context = context
        self._options = project.options      # Project options (OptionPool)
        self._basedir = basedir              # Base project directory
        self._first_pass_options = project.first_pass_config.options  # Project options (OptionPool)
        self._tempdir = tempdir              # A directory to cleanup
        self._parent = parent                # The parent loader

        self._meta_elements = {}  # Dict of resolved meta elements by name
        self._elements = {}       # Dict of elements
        self._loaders = {}        # Dict of junction loaders

        self._includes = Includes(self, copy_tree=True)

    # load():
    #
    # Loads the project based on the parameters given to the constructor
    #
    # Args:
    #    rewritable (bool): Whether the loaded files should be rewritable
    #                       this is a bit more expensive due to deep copies
    #    ticker (callable): An optional function for tracking load progress
    #    targets (list of str): Target, element-path relative bst filenames in the project
    #    fetch_subprojects (bool): Whether to fetch subprojects while loading
    #
    # Raises: LoadError
    #
    # Returns: The toplevel LoadElement
    def load(self, targets, rewritable=False, ticker=None, fetch_subprojects=False):

        for filename in targets:
            if os.path.isabs(filename):
                # XXX Should this just be an assertion ?
                # Expect that the caller gives us the right thing at least ?
                raise LoadError(LoadErrorReason.INVALID_DATA,
                                "Target '{}' was not specified as a relative "
                                "path to the base project directory: {}"
                                .format(filename, self._basedir))

        # First pass, recursively load files and populate our table of LoadElements
        #
        deps = []

        for target in targets:
            profile_start(Topics.LOAD_PROJECT, target)
            junction, name, loader = self._parse_name(target, rewritable, ticker,
                                                      fetch_subprojects=fetch_subprojects)
            loader._load_file(name, rewritable, ticker, fetch_subprojects)
            deps.append(Dependency(target, provenance="[command line]"))
            profile_end(Topics.LOAD_PROJECT, target)

        #
        # Now that we've resolve the dependencies, scan them for circular dependencies
        #

        # Set up a dummy element that depends on all top-level targets
        # to resolve potential circular dependencies between them
        DummyTarget = namedtuple('DummyTarget', ['name', 'full_name', 'deps'])

        dummy = DummyTarget(name='', full_name='', deps=deps)
        self._elements[''] = dummy

        profile_key = "_".join(t for t in targets)
        profile_start(Topics.CIRCULAR_CHECK, profile_key)
        self._check_circular_deps('')
        profile_end(Topics.CIRCULAR_CHECK, profile_key)

        ret = []
        #
        # Sort direct dependencies of elements by their dependency ordering
        #
        for target in targets:
            profile_start(Topics.SORT_DEPENDENCIES, target)
            junction, name, loader = self._parse_name(target, rewritable, ticker,
                                                      fetch_subprojects=fetch_subprojects)
            loader._sort_dependencies(name)
            profile_end(Topics.SORT_DEPENDENCIES, target)
            # Finally, wrap what we have into LoadElements and return the target
            #
            ret.append(loader._collect_element(name))

        return ret

    # cleanup():
    #
    # Remove temporary checkout directories of subprojects
    #
    def cleanup(self):
        if self._parent and not self._tempdir:
            # already done
            return

        # recurse
        for loader in self._loaders.values():
            # value may be None with nested junctions without overrides
            if loader is not None:
                loader.cleanup()

        if not self._parent:
            # basedir of top-level loader is never a temporary directory
            return

        # safe guard to not accidentally delete directories outside builddir
        if self._tempdir.startswith(self._context.builddir + os.sep):
            if os.path.exists(self._tempdir):
                shutil.rmtree(self._tempdir)

    # get_element_for_dep():
    #
    # Gets a cached LoadElement by Dependency object
    #
    # This is used by LoadElement
    #
    # Args:
    #    dep (Dependency): The dependency to search for
    #
    # Returns:
    #    (LoadElement): The cached LoadElement
    #
    def get_element_for_dep(self, dep):
        loader = self._get_loader_for_dep(dep)
        return loader._elements[dep.name]

    ###########################################
    #            Private Methods              #
    ###########################################

    # _load_file():
    #
    # Recursively load bst files
    #
    # Args:
    #    filename (str): The element-path relative bst file
    #    rewritable (bool): Whether we should load in round trippable mode
    #    ticker (callable): A callback to report loaded filenames to the frontend
    #    fetch_subprojects (bool): Whether to fetch subprojects while loading
    #    provenance (Provenance): The location from where the file was referred to, or None
    #
    # Returns:
    #    (LoadElement): A loaded LoadElement
    #
    def _load_file(self, filename, rewritable, ticker, fetch_subprojects, provenance=None):

        # Silently ignore already loaded files
        if filename in self._elements:
            return self._elements[filename]

        # Call the ticker
        if ticker:
            ticker(filename)

        if self.project.junction and filename in self.project.junction.replacements:
            replacement = _yaml.node_get(self.project.junction.replacements, str, filename)
            elt = self._parent._load_file(replacement,
                                          rewritable, ticker, fetch_subprojects, provenance=provenance)
            self._elements[filename] = elt
            return elt

        # Load the data and process any conditional statements therein
        fullpath = os.path.join(self._basedir, filename)
        try:
            node = _yaml.load(fullpath, shortname=filename, copy_tree=rewritable, project=self.project)
        except LoadError as e:
            if e.reason == LoadErrorReason.MISSING_FILE:

                if self.project.junction:
                    message = "Could not find element '{}' in project referred to by junction element '{}'" \
                              .format(filename, self.project.junction.name)
                else:
                    message = "Could not find element '{}' in elements directory '{}'".format(filename, self._basedir)

                if provenance:
                    message = "{}: {}".format(provenance, message)

                # If we can't find the file, try to suggest plausible
                # alternatives by stripping the element-path from the given
                # filename, and verifying that it exists.
                detail = None
                elements_dir = os.path.relpath(self._basedir, self.project.directory)
                element_relpath = os.path.relpath(filename, elements_dir)
                if filename.startswith(elements_dir) and os.path.exists(os.path.join(self._basedir, element_relpath)):
                    detail = "Did you mean '{}'?".format(element_relpath)

                raise LoadError(LoadErrorReason.MISSING_FILE,
                                message, detail=detail) from e

            elif e.reason == LoadErrorReason.LOADING_DIRECTORY:
                # If a <directory>.bst file exists in the element path,
                # let's suggest this as a plausible alternative.
                message = str(e)
                if provenance:
                    message = "{}: {}".format(provenance, message)
                detail = None
                if os.path.exists(os.path.join(self._basedir, filename + '.bst')):
                    element_name = filename + '.bst'
                    detail = "Did you mean '{}'?\n".format(element_name)
                raise LoadError(LoadErrorReason.LOADING_DIRECTORY,
                                message, detail=detail) from e
            else:
                raise
        kind = _yaml.node_get(node, str, Symbol.KIND)
        if kind == "junction":
            self._first_pass_options.process_node(node)
        else:
            self.project.ensure_fully_loaded()

            self._includes.process(node)

            self._options.process_node(node)

        element = LoadElement(node, filename, self)

        self._elements[filename] = element

        # Load all dependency files for the new LoadElement
        for dep in element.deps:
            if dep.junction:
                self._load_file(dep.junction, rewritable, ticker, fetch_subprojects, dep.provenance)
                loader = self._get_loader(dep.junction, rewritable=rewritable, ticker=ticker,
                                          fetch_subprojects=fetch_subprojects)
            else:
                loader = self

            dep_element = loader._load_file(dep.name, rewritable, ticker,
                                            fetch_subprojects, dep.provenance)

            if _yaml.node_get(dep_element.node, str, Symbol.KIND) == 'junction':
                raise LoadError(LoadErrorReason.INVALID_DATA,
                                "{}: Cannot depend on junction"
                                .format(dep.provenance))

        return element

    # _check_circular_deps():
    #
    # Detect circular dependencies on LoadElements with
    # dependencies already resolved.
    #
    # Args:
    #    element_name (str): The element-path relative element name to check
    #
    # Raises:
    #    (LoadError): In case there was a circular dependency error
    #
    def _check_circular_deps(self, element_name, check_elements=None, validated=None):

        if check_elements is None:
            check_elements = {}
        if validated is None:
            validated = {}

        element = self._elements[element_name]
        if getattr(element, '_loader', self) != self:
            return element._loader._check_circular_deps(element.name, check_elements, validated)

        # element name must be unique across projects
        # to be usable as key for the check_elements and validated dicts
        element_name = element.full_name

        # Skip already validated branches
        if validated.get(element_name) is not None:
            return

        if check_elements.get(element_name) is not None:
            raise LoadError(LoadErrorReason.CIRCULAR_DEPENDENCY,
                            "Circular dependency detected for element: {}"
                            .format(element.name))

        # Push / Check each dependency / Pop
        check_elements[element_name] = True
        for dep in element.deps:
            loader = self._get_loader_for_dep(dep)
            loader._check_circular_deps(dep.name, check_elements, validated)
        del check_elements[element_name]

        # Eliminate duplicate paths
        validated[element_name] = True

    # _sort_dependencies():
    #
    # Sort dependencies of each element by their dependencies,
    # so that direct dependencies which depend on other direct
    # dependencies (directly or indirectly) appear later in the
    # list.
    #
    # This avoids the need for performing multiple topological
    # sorts throughout the build process.
    #
    # Args:
    #    element_name (str): The element-path relative element name to sort
    #
    def _sort_dependencies(self, element_name, visited=None):
        if visited is None:
            visited = {}

        element = self._elements[element_name]
        if getattr(element, '_loader', self) != self:
            return element._loader._sort_dependencies(element.name, visited)

        # element name must be unique across projects
        # to be usable as key for the visited dict
        element_name = element.full_name

        if visited.get(element_name) is not None:
            return

        for dep in element.deps:
            loader = self._get_loader_for_dep(dep)
            loader._sort_dependencies(dep.name, visited=visited)

        def dependency_cmp(dep_a, dep_b):
            element_a = self.get_element_for_dep(dep_a)
            element_b = self.get_element_for_dep(dep_b)

            # Sort on inter element dependency first
            if element_a.depends(element_b):
                return 1
            elif element_b.depends(element_a):
                return -1

            # If there are no inter element dependencies, place
            # runtime only dependencies last
            if dep_a.dep_type != dep_b.dep_type:
                if dep_a.dep_type == Symbol.RUNTIME:
                    return 1
                elif dep_b.dep_type == Symbol.RUNTIME:
                    return -1

            # All things being equal, string comparison.
            if dep_a.name > dep_b.name:
                return 1
            elif dep_a.name < dep_b.name:
                return -1

            # Sort local elements before junction elements
            # and use string comparison between junction elements
            if dep_a.junction and dep_b.junction:
                if dep_a.junction > dep_b.junction:
                    return 1
                elif dep_a.junction < dep_b.junction:
                    return -1
            elif dep_a.junction:
                return -1
            elif dep_b.junction:
                return 1

            # This wont ever happen
            return 0

        # Now dependency sort, we ensure that if any direct dependency
        # directly or indirectly depends on another direct dependency,
        # it is found later in the list.
        element.deps.sort(key=cmp_to_key(dependency_cmp))

        visited[element_name] = True

    # _collect_element()
    #
    # Collect the toplevel elements we have
    #
    # Args:
    #    element_name (str): The element-path relative element name to sort
    #
    # Returns:
    #    (MetaElement): A recursively loaded MetaElement
    #
    def _collect_element(self, element_name):

        element = self._elements[element_name]
        if element._loader != self:
            return element._loader._collect_element(element.name)

        # Return the already built one, if we already built it
        meta_element = self._meta_elements.get(element_name)
        if meta_element:
            return meta_element

        node = element.node
        elt_provenance = _yaml.node_get_provenance(node)
        meta_sources = []

        sources = _yaml.node_get(node, list, Symbol.SOURCES, default_value=[])
        element_kind = _yaml.node_get(node, str, Symbol.KIND)

        # Safe loop calling into _yaml.node_get() for each element ensures
        # we have good error reporting
        for i in range(len(sources)):
            source = _yaml.node_get(node, Mapping, Symbol.SOURCES, indices=[i])
            kind = _yaml.node_get(source, str, Symbol.KIND)
            del source[Symbol.KIND]

            # Directory is optional
            directory = _yaml.node_get(source, str, Symbol.DIRECTORY, default_value=None)
            if directory:
                del source[Symbol.DIRECTORY]

            index = sources.index(source)
            meta_source = MetaSource(element_name, index, element_kind, kind, source, directory)
            meta_sources.append(meta_source)

        meta_element = MetaElement(self.project, element_name, element_kind,
                                   elt_provenance, meta_sources,
                                   _yaml.node_get(node, Mapping, Symbol.CONFIG, default_value={}),
                                   _yaml.node_get(node, Mapping, Symbol.VARIABLES, default_value={}),
                                   _yaml.node_get(node, Mapping, Symbol.ENVIRONMENT, default_value={}),
                                   _yaml.node_get(node, list, Symbol.ENV_NOCACHE, default_value=[]),
                                   _yaml.node_get(node, Mapping, Symbol.PUBLIC, default_value={}),
                                   _yaml.node_get(node, Mapping, Symbol.SANDBOX, default_value={}),
                                   element_kind == 'junction')

        # Cache it now, make sure it's already there before recursing
        self._meta_elements[element_name] = meta_element

        # Descend
        for dep in element.deps:
            loader = self._get_loader_for_dep(dep)
            meta_dep = loader._collect_element(dep.name)
            if dep.dep_type != 'runtime':
                meta_element.build_dependencies.append(meta_dep)
            if dep.dep_type != 'build':
                meta_element.dependencies.append(meta_dep)
            if dep.strict:
                meta_element.strict_dependencies.append(meta_dep)

        return meta_element

    # _get_loader():
    #
    # Return loader for specified junction
    #
    # Args:
    #    filename (str): Junction name
    #    fetch_subprojects (bool): Whether to fetch subprojects while loading
    #
    # Raises: LoadError
    #
    # Returns: A Loader or None if specified junction does not exist
    def _get_loader(self, filename, *, rewritable=False, ticker=None, level=0, fetch_subprojects=False):
        # return previously determined result
        if filename in self._loaders:
            loader = self._loaders[filename]

            if loader is None:
                # do not allow junctions with the same name in different
                # subprojects
                raise LoadError(LoadErrorReason.CONFLICTING_JUNCTION,
                                "Conflicting junction {} in subprojects, define junction in {}"
                                .format(filename, self.project.name))

            return loader

        if self._parent:
            # junctions in the parent take precedence over junctions defined
            # in subprojects
            loader = self._parent._get_loader(filename, rewritable=rewritable, ticker=ticker,
                                              level=level + 1, fetch_subprojects=fetch_subprojects)
            if loader:
                self._loaders[filename] = loader
                return loader

        try:
            self._load_file(filename, rewritable, ticker, fetch_subprojects)
        except LoadError as e:
            if e.reason != LoadErrorReason.MISSING_FILE:
                # other load error
                raise

            if level == 0:
                # junction element not found in this or ancestor projects
                raise
            else:
                # mark junction as not available to allow detection of
                # conflicting junctions in subprojects
                self._loaders[filename] = None
                return None

        # meta junction element
        meta_element = self._collect_element(filename)
        if meta_element.kind != 'junction':
            raise LoadError(LoadErrorReason.INVALID_DATA,
                            "{}: Expected junction but element kind is {}".format(filename, meta_element.kind))

        element = Element._new_from_meta(meta_element, self._context.artifactcache)
        element._preflight()
        element._update_state()

        # Handle the case where a subproject needs to be fetched
        #
        if element._get_consistency() == Consistency.RESOLVED:
            if fetch_subprojects:
                sources = list(element.sources())
                for idx, source in enumerate(sources):
                    if ticker:
                        ticker(filename, 'Fetching subproject from {} source'.format(source.get_kind()))
                    if source._get_consistency() != Consistency.CACHED:
                        source._fetch(sources[0:idx])
            else:
                detail = "Try fetching the project with `bst fetch {}`".format(filename)
                raise LoadError(LoadErrorReason.SUBPROJECT_FETCH_NEEDED,
                                "Subproject fetch needed for junction: {}".format(filename),
                                detail=detail)

        # Handle the case where a subproject has no ref
        #
        elif element._get_consistency() == Consistency.INCONSISTENT:
            detail = "Try tracking the junction element with `bst track {}`".format(filename)
            raise LoadError(LoadErrorReason.SUBPROJECT_INCONSISTENT,
                            "Subproject has no ref for junction: {}".format(filename),
                            detail=detail)

        # Stage sources
        os.makedirs(self._context.builddir, exist_ok=True)
        basedir = tempfile.mkdtemp(prefix="{}-".format(element.normal_name), dir=self._context.builddir)
        element._stage_sources_at(basedir, mount_workspaces=False)

        # Load the project
        project_dir = os.path.join(basedir, element.path)
        try:
            from .._project import Project
            project = Project(project_dir, self._context, junction=element,
                              parent_loader=self, tempdir=basedir)
        except LoadError as e:
            if e.reason == LoadErrorReason.MISSING_PROJECT_CONF:
                raise LoadError(reason=LoadErrorReason.INVALID_JUNCTION,
                                message="Could not find the project.conf file for {}. "
                                        "Expecting a project at path '{}'"
                                .format(element, element.path or '.')) from e
            else:
                raise

        loader = project.loader
        self._loaders[filename] = loader

        return loader

    # _get_loader_for_dep():
    #
    # Gets the appropriate Loader for a Dependency object
    #
    # Args:
    #    dep (Dependency): A Dependency object
    #
    # Returns:
    #    (Loader): The Loader object to use for this Dependency
    #
    def _get_loader_for_dep(self, dep):
        if dep.junction:
            # junction dependency, delegate to appropriate loader
            return self._loaders[dep.junction]
        else:
            return self

    # _parse_name():
    #
    # Get junction and base name of element along with loader for the sub-project
    #
    # Args:
    #   name (str): Name of target
    #   rewritable (bool): Whether the loaded files should be rewritable
    #                      this is a bit more expensive due to deep copies
    #   ticker (callable): An optional function for tracking load progress
    #   fetch_subprojects (bool): Whether to fetch subprojects while loading
    #
    # Returns:
    #   (tuple): - (str): name of the junction element
    #            - (str): name of the element
    #            - (Loader): loader for sub-project
    #
    def _parse_name(self, name, rewritable, ticker, fetch_subprojects=False):
        # We allow to split only once since deep junctions names are forbidden.
        # Users who want to refer to elements in sub-sub-projects are required
        # to create junctions on the top level project.
        junction_path = name.rsplit(':', 1)
        if len(junction_path) == 1:
            return None, junction_path[-1], self
        else:
            self._load_file(junction_path[-2], rewritable, ticker, fetch_subprojects)
            loader = self._get_loader(junction_path[-2], rewritable=rewritable, ticker=ticker,
                                      fetch_subprojects=fetch_subprojects)
            return junction_path[-2], junction_path[-1], loader