#!/usr/bin/env python3
#
# Copyright (C) 2016 Codethink Limited
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see .
#
# Authors:
# Tristan Van Berkom
"""
Project
=======
The :class:`.Project` object holds all of the project settings from
the project configuration file including the project directory it
was loaded from.
"""
import os
import multiprocessing # for cpu_count()
from collections import Mapping
from ._yaml import CompositePolicy, CompositeTypeError, CompositeOverrideError
from . import utils
from . import _site
from . import _yaml
from . import _loader # For resolve_arch()
from ._profile import Topics, profile_start, profile_end
from . import LoadError, LoadErrorReason
BST_FORMAT_VERSION = 0
"""The base BuildStream format version
This version is bumped whenever enhancements are made
to the ``project.conf`` format or the format in general.
"""
BST_ARTIFACT_VERSION = 0
"""The base BuildStream artifact version
The artifact version changes whenever the cache key
calculation algorithm changes in an incompatible way
or if buildstream was changed in a way which can cause
the same cache key to produce something that is no longer
the same.
"""
# The separator we use for user specified aliases
_ALIAS_SEPARATOR = ':'
# Private object for dealing with project variants
#
class _ProjectVariant():
def __init__(self, data):
self.name = _yaml.node_get(data, str, 'variant')
self.data = data
del self.data['variant']
class Project():
"""Project Configuration
Args:
directory (str): The project directory
host_arch (str): Symbolic host machine architecture name
target_arch (str): Symbolic target machine architecture name
Raises:
:class:`.LoadError`
"""
def __init__(self, directory, host_arch, target_arch=None):
self.name = None
"""str: The project name"""
self.directory = os.path.abspath(directory)
"""str: The project directory"""
self.element_path = None
"""str: Absolute path to where elements are loaded from within the project"""
self._variables = {} # The default variables overridden with project wide overrides
self._environment = {} # The base sandbox environment
self._elements = {} # Element specific configurations
self._aliases = {} # Aliases dictionary
self.__workspaces = {} # Workspaces
self._plugin_source_paths = [] # Paths to custom sources
self._plugin_element_paths = [] # Paths to custom plugins
self._cache_key = None
self._variants = []
self._host_arch = host_arch
self._target_arch = target_arch or host_arch
self._source_format_versions = {}
self._element_format_versions = {}
profile_start(Topics.LOAD_PROJECT, self.directory.replace(os.sep, '-'))
self._unresolved_config = self._load_first_half()
profile_end(Topics.LOAD_PROJECT, self.directory.replace(os.sep, '-'))
def translate_url(self, url):
"""Translates the given url which may be specified with an alias
into a fully qualified url.
Args:
url (str): A url, which may be using an alias
Returns:
str: The fully qualified url, with aliases resolved
This method is provided for :class:`.Source` objects to resolve
fully qualified urls based on the shorthand which is allowed
to be specified in the YAML
"""
if url and _ALIAS_SEPARATOR in url:
url_alias, url_body = url.split(_ALIAS_SEPARATOR, 1)
alias_url = self._aliases.get(url_alias)
if alias_url:
url = alias_url + url_body
return url
# _load_first_half():
#
# Loads the project configuration file in the project directory
# and extracts some things.
#
# Raises: LoadError if there was a problem with the project.conf
#
def _load_first_half(self):
# Load builtin default
projectfile = os.path.join(self.directory, "project.conf")
config = _yaml.load(_site.default_project_config)
# Special variables which have a computed default value must
# be processed here before compositing any overrides
variables = _yaml.node_get(config, Mapping, 'variables')
variables['max-jobs'] = multiprocessing.cpu_count()
variables['bst-host-arch'] = self._host_arch
variables['bst-target-arch'] = self._target_arch
# This is kept around for compatibility with existing definitions,
# but we should probably remove it due to being ambiguous.
variables['bst-arch'] = self._host_arch
# Load project local config and override the builtin
project_conf = _yaml.load(projectfile)
_yaml.composite(config, project_conf, typesafe=True)
_yaml.validate_node(config, [
'required-versions',
'element-path', 'variables',
'environment', 'environment-nocache',
'split-rules', 'elements', 'plugins',
'aliases', 'name',
'arches', 'host-arches',
])
# Resolve arches keyword, project may have arch conditionals
_loader.resolve_arch(config, self._host_arch, self._target_arch)
# Resolve element base path
elt_path = _yaml.node_get(config, str, 'element-path')
self.element_path = os.path.join(self.directory, elt_path)
# Load variants
variants_node = _yaml.node_get(config, list, 'variants', default_value=[])
for variant_node in variants_node:
index = variants_node.index(variant_node)
variant_node = _yaml.node_get(config, Mapping, 'variants', indices=[index])
variant = _ProjectVariant(variant_node)
# Process arch conditionals on individual variants
_loader.resolve_arch(variant.data, self._host_arch, self._target_arch)
self._variants.append(variant)
if len(self._variants) == 1:
provenance = _yaml.node_get_provenance(config, key='variants')
raise LoadError(LoadErrorReason.INVALID_DATA,
"{}: Only one variant declared, a project "
"declaring variants must declare at least two"
.format(provenance))
# Workspace configurations
self.__workspaces = self._load_workspace_config()
return config
# _resolve():
#
# First resolves the project variant and then resolves the remaining
# properties of the project based on the final composition
#
# Raises: LoadError if there was a problem with the project.conf
#
def _resolve(self, variant_name):
# Apply the selected variant
#
variant = None
if variant_name:
variant = self._lookup_variant(variant_name)
elif self._variants:
variant = self._variants[0]
if variant:
provenance = _yaml.node_get_provenance(variant.data)
# Composite anything from the variant data into the element data
#
# Possibly this should not be typesafe, since branch names can
# possibly be strings or interpreted by YAML as integers (for
# numeric branch names)
#
try:
_yaml.composite_dict(self._unresolved_config, variant.data,
policy=CompositePolicy.ARRAY_APPEND,
typesafe=True)
except CompositeTypeError as e:
raise LoadError(
LoadErrorReason.ILLEGAL_COMPOSITE,
"%s: Variant '%s' specifies type '%s' for path '%s', expected '%s'" %
(str(provenance),
variant.name,
e.actual_type.__name__, e.path,
e.expected_type.__name__)) from e
# The project name
self.name = _yaml.node_get(self._unresolved_config, str, 'name')
# Version requirements
versions = _yaml.node_get(self._unresolved_config, Mapping, 'required-versions')
_yaml.validate_node(versions, ['project', 'elements', 'sources'])
# Assert project version first
format_version = _yaml.node_get(versions, int, 'project')
if BST_FORMAT_VERSION < format_version:
major, minor = utils.get_bst_version()
raise LoadError(
LoadErrorReason.UNSUPPORTED_PROJECT,
"Project requested format version {}, but BuildStream {}.{} only supports up until format version {}"
.format(format_version, major, minor, BST_FORMAT_VERSION))
# The source versions
source_versions = _yaml.node_get(versions, Mapping, 'sources', default_value={})
for key, _ in source_versions.items():
if key == _yaml.PROVENANCE_KEY:
continue
self._source_format_versions[key] = _yaml.node_get(source_versions, int, key)
# The element versions
element_versions = _yaml.node_get(versions, Mapping, 'elements', default_value={})
for key, _ in element_versions.items():
if key == _yaml.PROVENANCE_KEY:
continue
self._element_format_versions[key] = _yaml.node_get(element_versions, int, key)
# Load the plugin paths
plugins = _yaml.node_get(self._unresolved_config, Mapping, 'plugins', default_value={})
_yaml.validate_node(plugins, ['elements', 'sources'])
self._plugin_source_paths = [os.path.join(self.directory, path)
for path in self._extract_plugin_paths(plugins, 'sources')]
self._plugin_element_paths = [os.path.join(self.directory, path)
for path in self._extract_plugin_paths(plugins, 'elements')]
# Source url aliases
self._aliases = _yaml.node_get(self._unresolved_config, Mapping, 'aliases', default_value={})
# Load base variables
self._variables = _yaml.node_get(self._unresolved_config, Mapping, 'variables')
# Load sandbox configuration
self._environment = _yaml.node_get(self._unresolved_config, Mapping, 'environment')
self._env_nocache = _yaml.node_get(self._unresolved_config, list, 'environment-nocache')
# Load project split rules
self._splits = _yaml.node_get(self._unresolved_config, Mapping, 'split-rules')
# Element configurations
self._elements = _yaml.node_get(self._unresolved_config, Mapping, 'elements', default_value={})
def _lookup_variant(self, variant_name):
for variant in self._variants:
if variant.name == variant_name:
return variant
def _list_variants(self):
for variant in self._variants:
yield variant.name
# _workspaces()
#
# Generator function to enumerate workspaces.
#
# Yields:
# A tuple in the following format: (element, source, path).
def _workspaces(self):
for element in self.__workspaces:
if element == _yaml.PROVENANCE_KEY:
continue
for source in self.__workspaces[element]:
if source == _yaml.PROVENANCE_KEY:
continue
yield (element, int(source), self.__workspaces[element][source])
# _get_workspace()
#
# Get the path of the workspace source associated with the given
# element's source at the given index
#
# Args:
# element (str) - The element name
# index (int) - The source index
#
# Returns:
# None if no workspace is open, the path to the workspace
# otherwise
#
def _get_workspace(self, element, index):
try:
return self.__workspaces[element][index]
except KeyError:
return None
# _set_workspace()
#
# Set the path of the workspace associated with the given
# element's source at the given index
#
# Args:
# element (str) - The element name
# index (int) - The source index
# path (str) - The path to set the workspace to
#
def _set_workspace(self, element, index, path):
if element.name not in self.__workspaces:
self.__workspaces[element.name] = {}
self.__workspaces[element.name][index] = path
element._set_source_workspace(index, path)
# _delete_workspace()
#
# Remove the workspace from the workspace element. Note that this
# does *not* remove the workspace from the stored yaml
# configuration, call _save_workspace_config() afterwards.
#
# Args:
# element (str) - The element name
# index (int) - The source index
#
def _delete_workspace(self, element, index):
del self.__workspaces[element][index]
# Contains a provenance object
if len(self.__workspaces[element]) == 1:
del self.__workspaces[element]
# _load_workspace_config()
#
# Load the workspace configuration and return a node containing
# all open workspaces for the project
#
# Returns:
#
# A node containing a dict that assigns projects to their
# workspaces. For example:
#
# amhello.bst: {
# 0: /home/me/automake,
# 1: /home/me/amhello
# }
#
def _load_workspace_config(self):
os.makedirs(os.path.join(self.directory, ".bst"), exist_ok=True)
workspace_file = os.path.join(self.directory, ".bst", "workspaces.yml")
try:
open(workspace_file, "a").close()
except IOError as e:
raise LoadError(LoadErrorReason.MISSING_FILE,
"Could not load workspace config: {}".format(e)) from e
return _yaml.load(workspace_file)
# _save_workspace_config()
#
# Dump the current workspace element to the project configuration
# file. This makes any changes performed with _delete_workspace or
# _set_workspace permanent
#
def _save_workspace_config(self):
_yaml.dump(_yaml.node_sanitize(self.__workspaces),
os.path.join(self.directory, ".bst", "workspaces.yml"))
def _extract_plugin_paths(self, node, name):
if not node:
return
path_list = _yaml.node_get(node, list, name, default_value=[])
for i in range(len(path_list)):
path = _yaml.node_get(node, str, name, indices=[i])
yield path
# _get_cache_key():
#
# Returns the cache key, calculating it if necessary
#
# Returns:
# (str): A hex digest cache key for the Context
#
def _get_cache_key(self):
if self._cache_key is None:
# Anything that alters the build goes into the unique key
# (currently nothing here)
self._cache_key = utils._generate_key({})
return self._cache_key