diff options
author | Josh Smith <joshsmith@codethink.co.uk> | 2018-07-27 15:01:07 +0100 |
---|---|---|
committer | Josh Smith <joshsmith@codethink.co.uk> | 2018-07-27 15:01:07 +0100 |
commit | 8a96679a7fae6ce7d844a596131a61c8a5ad780c (patch) | |
tree | 1c40daa8aca9d3b891057b52d8618773c282c8f5 | |
parent | cb4693b2bb9818c1a62608b59177a36aee1f0eda (diff) | |
parent | bd4d03554f1537251df4ff190da38b979cf6e594 (diff) | |
download | buildstream-8a96679a7fae6ce7d844a596131a61c8a5ad780c.tar.gz |
Merge branch 'master' of gitlab.com:BuildStream/buildstream
-rw-r--r-- | NEWS | 4 | ||||
-rw-r--r-- | README.rst | 24 | ||||
-rw-r--r-- | buildstream/__init__.py | 2 | ||||
-rw-r--r-- | buildstream/_context.py | 2 | ||||
-rw-r--r-- | buildstream/_frontend/app.py | 3 | ||||
-rw-r--r-- | buildstream/_frontend/cli.py | 2 | ||||
-rw-r--r-- | buildstream/_loader/loader.py | 2 | ||||
-rw-r--r-- | buildstream/_project.py | 64 | ||||
-rw-r--r-- | buildstream/_versions.py | 2 | ||||
-rw-r--r-- | buildstream/plugins/sources/bzr.py | 11 | ||||
-rw-r--r-- | buildstream/plugins/sources/git.py | 94 | ||||
-rw-r--r-- | buildstream/source.py | 231 | ||||
-rw-r--r-- | buildstream/utils.py | 4 | ||||
-rw-r--r-- | doc/source/examples/git-mirror.rst | 144 | ||||
-rw-r--r-- | doc/source/examples/tar-mirror.rst | 103 | ||||
-rw-r--r-- | doc/source/format_project.rst | 37 | ||||
-rw-r--r-- | doc/source/main_install.rst | 7 | ||||
-rw-r--r-- | doc/source/using_config.rst | 21 | ||||
-rw-r--r-- | doc/source/using_examples.rst | 2 | ||||
-rw-r--r-- | tests/completions/completions.py | 1 | ||||
-rw-r--r-- | tests/frontend/mirror.py | 402 | ||||
-rw-r--r-- | tests/frontend/project/sources/fetch_source.py | 85 | ||||
-rw-r--r-- | tests/testutils/repo/repo.py | 21 |
23 files changed, 1188 insertions, 80 deletions
@@ -5,6 +5,10 @@ buildstream 1.3.1 o Add a `--tar` option to `bst checkout` which allows a tarball to be created from the artifact contents. + o Fetching and tracking will consult mirrors defined in project config, + and the preferred mirror to fetch from can be defined in the command + line or user config. + ================= buildstream 1.1.4 ================= diff --git a/README.rst b/README.rst index 6f64f47a6..c553da068 100644 --- a/README.rst +++ b/README.rst @@ -25,7 +25,7 @@ BuildStream offers the following advantages: * **Declarative build instructions/definitions** - BuildStream provides a a flexible and extensible framework for the modelling + BuildStream provides a flexible and extensible framework for the modelling of software build pipelines in a declarative YAML format, which allows you to manipulate filesystem data in a controlled, reproducible sandboxed environment. @@ -61,25 +61,29 @@ How does BuildStream work? ========================== BuildStream operates on a set of YAML files (.bst files), as follows: -* loads the YAML files which describe the target(s) and all dependencies -* evaluates the version information and build instructions to calculate a build +* Loads the YAML files which describe the target(s) and all dependencies. +* Evaluates the version information and build instructions to calculate a build graph for the target(s) and all dependencies and unique cache-keys for each - element -* retrieves elements from cache if they are already built, or builds them in a - sandboxed environment using the instructions declared in the .bst files -* transforms/configures and/or deploys the resulting target(s) based on the + element. +* Retrieves previously built elements (artifacts) from a local/remote cache, or + builds the elements in a sandboxed environment using the instructions declared + in the .bst files. +* Transforms/configures and/or deploys the resulting target(s) based on the instructions declared in the .bst files. How can I get started? ====================== -The easiest way to get started is to explore some existing .bst files, for example: +To start using BuildStream, first, +`install <https://buildstream.gitlab.io/buildstream/main_install.html>`_ +BuildStream onto your machine and then follow our +`tutorial <https://buildstream.gitlab.io/buildstream/using_tutorial.html>`_. + +We also recommend exploring some existing BuildStream projects: * https://gitlab.gnome.org/GNOME/gnome-build-meta/ * https://gitlab.com/freedesktop-sdk/freedesktop-sdk * https://gitlab.com/baserock/definitions -* https://gitlab.com/BuildStream/buildstream-examples/tree/master/build-x86image -* https://gitlab.com/BuildStream/buildstream-examples/tree/master/netsurf-flatpak If you have any questions please ask on our `#buildstream <irc://irc.gnome.org/buildstream>`_ channel in `irc.gnome.org <irc://irc.gnome.org>`_ diff --git a/buildstream/__init__.py b/buildstream/__init__.py index cf56ecfe1..895adc60f 100644 --- a/buildstream/__init__.py +++ b/buildstream/__init__.py @@ -29,7 +29,7 @@ if "_BST_COMPLETION" not in os.environ: from .utils import UtilError, ProgramNotFoundError from .sandbox import Sandbox, SandboxFlags from .plugin import Plugin - from .source import Source, SourceError, Consistency + from .source import Source, SourceError, Consistency, SourceFetcher from .element import Element, ElementError, Scope from .buildelement import BuildElement from .scriptelement import ScriptElement diff --git a/buildstream/_context.py b/buildstream/_context.py index fbb0a9686..7dc68af79 100644 --- a/buildstream/_context.py +++ b/buildstream/_context.py @@ -259,7 +259,7 @@ class Context(): # Shallow validation of overrides, parts of buildstream which rely # on the overrides are expected to validate elsewhere. for _, overrides in _yaml.node_items(self._project_overrides): - _yaml.node_validate(overrides, ['artifacts', 'options', 'strict']) + _yaml.node_validate(overrides, ['artifacts', 'options', 'strict', 'default-mirror']) profile_end(Topics.LOAD_CONTEXT, 'load') diff --git a/buildstream/_frontend/app.py b/buildstream/_frontend/app.py index 5fae0307b..04a90ea32 100644 --- a/buildstream/_frontend/app.py +++ b/buildstream/_frontend/app.py @@ -202,7 +202,8 @@ class App(): # Load the Project # try: - self.project = Project(directory, self.context, cli_options=self._main_options['option']) + self.project = Project(directory, self.context, cli_options=self._main_options['option'], + default_mirror=self._main_options.get('default_mirror')) except LoadError as e: # Let's automatically start a `bst init` session in this case diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py index bd2ce8a73..20624e2ac 100644 --- a/buildstream/_frontend/cli.py +++ b/buildstream/_frontend/cli.py @@ -217,6 +217,8 @@ def print_version(ctx, param, value): help="Elements must be rebuilt when their dependencies have changed") @click.option('--option', '-o', type=click.Tuple([str, str]), multiple=True, metavar='OPTION VALUE', help="Specify a project option") +@click.option('--default-mirror', default=None, + help="The mirror to fetch from first, before attempting other mirrors") @click.pass_context def cli(context, **kwargs): """Build and manipulate BuildStream projects diff --git a/buildstream/_loader/loader.py b/buildstream/_loader/loader.py index 07b0de996..e9b9d95f1 100644 --- a/buildstream/_loader/loader.py +++ b/buildstream/_loader/loader.py @@ -513,7 +513,7 @@ class Loader(): if self._fetch_subprojects: if ticker: ticker(filename, 'Fetching subproject from {} source'.format(source.get_kind())) - source.fetch() + source._fetch() else: detail = "Try fetching the project with `bst fetch {}`".format(filename) raise LoadError(LoadErrorReason.SUBPROJECT_FETCH_NEEDED, diff --git a/buildstream/_project.py b/buildstream/_project.py index 54ec9ee34..1c30fb9bb 100644 --- a/buildstream/_project.py +++ b/buildstream/_project.py @@ -19,7 +19,7 @@ import os import multiprocessing # for cpu_count() -from collections import Mapping +from collections import Mapping, OrderedDict from pluginbase import PluginBase from . import utils from . import _cachekey @@ -35,9 +35,6 @@ from ._projectrefs import ProjectRefs, ProjectRefStorage from ._versions import BST_FORMAT_VERSION -# The separator we use for user specified aliases -_ALIAS_SEPARATOR = ':' - # Project Configuration file _PROJECT_CONF_FILE = 'project.conf' @@ -70,7 +67,7 @@ class HostMount(): # class Project(): - def __init__(self, directory, context, *, junction=None, cli_options=None): + def __init__(self, directory, context, *, junction=None, cli_options=None, default_mirror=None): # The project name self.name = None @@ -94,6 +91,8 @@ class Project(): self.base_env_nocache = None # The base nocache mask (list) for the environment self.element_overrides = {} # Element specific configurations self.source_overrides = {} # Source specific configurations + self.mirrors = OrderedDict() # contains dicts of alias-mappings to URIs. + self.default_mirror = default_mirror # The name of the preferred mirror. # # Private Members @@ -133,8 +132,8 @@ class Project(): # fully qualified urls based on the shorthand which is allowed # to be specified in the YAML def translate_url(self, url): - if url and _ALIAS_SEPARATOR in url: - url_alias, url_body = url.split(_ALIAS_SEPARATOR, 1) + if url and utils._ALIAS_SEPARATOR in url: + url_alias, url_body = url.split(utils._ALIAS_SEPARATOR, 1) alias_url = self._aliases.get(url_alias) if alias_url: url = alias_url + url_body @@ -202,6 +201,36 @@ class Project(): self._assert_plugin_format(source, version) return source + # get_alias_uri() + # + # Returns the URI for a given alias, if it exists + # + # Args: + # alias (str): The alias. + # + # Returns: + # str: The URI for the given alias; or None: if there is no URI for + # that alias. + def get_alias_uri(self, alias): + return self._aliases.get(alias) + + # get_alias_uris() + # + # Returns a list of every URI to replace an alias with + def get_alias_uris(self, alias): + if not alias or alias not in self._aliases: + return [None] + + mirror_list = [] + for key, alias_mapping in self.mirrors.items(): + if alias in alias_mapping: + if key == self.default_mirror: + mirror_list = alias_mapping[alias] + mirror_list + else: + mirror_list += alias_mapping[alias] + mirror_list.append(self._aliases[alias]) + return mirror_list + # _load(): # # Loads the project configuration file in the project directory. @@ -249,7 +278,7 @@ class Project(): 'aliases', 'name', 'artifacts', 'options', 'fail-on-overlap', 'shell', - 'ref-storage', 'sandbox' + 'ref-storage', 'sandbox', 'mirrors', ]) # The project name, element path and option declarations @@ -290,6 +319,10 @@ class Project(): # self.options.process_node(config) + # Override default_mirror if not set by command-line + if not self.default_mirror: + self.default_mirror = _yaml.node_get(overrides, str, 'default-mirror', default_value=None) + # # Now all YAML composition is done, from here on we just load # the values from our loaded configuration dictionary. @@ -414,6 +447,21 @@ class Project(): self._shell_host_files.append(mount) + mirrors = _yaml.node_get(config, list, 'mirrors', default_value=[]) + for mirror in mirrors: + allowed_mirror_fields = [ + 'name', 'aliases' + ] + _yaml.node_validate(mirror, allowed_mirror_fields) + mirror_name = _yaml.node_get(mirror, str, 'name') + alias_mappings = {} + for alias_mapping, uris in _yaml.node_items(mirror['aliases']): + assert isinstance(uris, list) + alias_mappings[alias_mapping] = list(uris) + self.mirrors[mirror_name] = alias_mappings + if not self.default_mirror: + self.default_mirror = mirror_name + # _assert_plugin_format() # # Helper to raise a PluginError if the loaded plugin is of a lesser version then diff --git a/buildstream/_versions.py b/buildstream/_versions.py index 19699e125..4531d9a72 100644 --- a/buildstream/_versions.py +++ b/buildstream/_versions.py @@ -23,7 +23,7 @@ # This version is bumped whenever enhancements are made # to the `project.conf` format or the core element format. # -BST_FORMAT_VERSION = 10 +BST_FORMAT_VERSION = 11 # The base BuildStream artifact version diff --git a/buildstream/plugins/sources/bzr.py b/buildstream/plugins/sources/bzr.py index 9fd2eb60c..3afd0f04f 100644 --- a/buildstream/plugins/sources/bzr.py +++ b/buildstream/plugins/sources/bzr.py @@ -102,7 +102,7 @@ class BzrSource(Source): def track(self): with self.timed_activity("Tracking {}".format(self.url), silent_nested=True): - self._ensure_mirror() + self._ensure_mirror(skip_ref_check=True) ret, out = self.check_output([self.host_bzr, "version-info", "--custom", "--template={revno}", self._get_branch_dir()], @@ -214,7 +214,7 @@ class BzrSource(Source): yield repodir self._atomic_replace_mirrordir(repodir) - def _ensure_mirror(self): + def _ensure_mirror(self, skip_ref_check=False): with self._atomic_repodir() as repodir: # Initialize repo if no metadata bzr_metadata_dir = os.path.join(repodir, ".bzr") @@ -223,18 +223,21 @@ class BzrSource(Source): fail="Failed to initialize bzr repository") branch_dir = os.path.join(repodir, self.tracking) + branch_url = self.url + "/" + self.tracking if not os.path.exists(branch_dir): # `bzr branch` the branch if it doesn't exist # to get the upstream code - branch_url = self.url + "/" + self.tracking self.call([self.host_bzr, "branch", branch_url, branch_dir], fail="Failed to branch from {} to {}".format(branch_url, branch_dir)) else: # `bzr pull` the branch if it does exist # to get any changes to the upstream code - self.call([self.host_bzr, "pull", "--directory={}".format(branch_dir)], + self.call([self.host_bzr, "pull", "--directory={}".format(branch_dir), branch_url], fail="Failed to pull new changes for {}".format(branch_dir)) + if not skip_ref_check and not self._check_ref(): + raise SourceError("Failed to ensure ref '{}' was mirrored".format(self.ref), + reason="ref-not-mirrored") def setup(): diff --git a/buildstream/plugins/sources/git.py b/buildstream/plugins/sources/git.py index 6762f6025..bccf5d6b7 100644 --- a/buildstream/plugins/sources/git.py +++ b/buildstream/plugins/sources/git.py @@ -78,7 +78,7 @@ from io import StringIO from configparser import RawConfigParser -from buildstream import Source, SourceError, Consistency +from buildstream import Source, SourceError, Consistency, SourceFetcher from buildstream import utils GIT_MODULES = '.gitmodules' @@ -88,18 +88,20 @@ GIT_MODULES = '.gitmodules' # for the primary git source and also for each submodule it # might have at a given time # -class GitMirror(): +class GitMirror(SourceFetcher): def __init__(self, source, path, url, ref): + super().__init__() self.source = source self.path = path - self.url = source.translate_url(url) + self.url = url self.ref = ref - self.mirror = os.path.join(source.get_mirror_directory(), utils.url_directory_name(self.url)) + self.mirror = os.path.join(source.get_mirror_directory(), utils.url_directory_name(url)) + self.mark_download_url(url) # Ensures that the mirror exists - def ensure(self): + def ensure(self, alias_override=None): # Unfortunately, git does not know how to only clone just a specific ref, # so we have to download all of those gigs even if we only need a couple @@ -112,22 +114,47 @@ class GitMirror(): # system configured tmpdir is not on the same partition. # with self.source.tempdir() as tmpdir: - self.source.call([self.source.host_git, 'clone', '--mirror', '-n', self.url, tmpdir], - fail="Failed to clone git repository {}".format(self.url), + url = self.source.translate_url(self.url, alias_override=alias_override) + self.source.call([self.source.host_git, 'clone', '--mirror', '-n', url, tmpdir], + fail="Failed to clone git repository {}".format(url), fail_temporarily=True) try: shutil.move(tmpdir, self.mirror) except (shutil.Error, OSError) as e: raise SourceError("{}: Failed to move cloned git repository {} from '{}' to '{}'" - .format(self.source, self.url, tmpdir, self.mirror)) from e + .format(self.source, url, tmpdir, self.mirror)) from e + + def _fetch(self, alias_override=None): + url = self.source.translate_url(self.url, alias_override=alias_override) + + if alias_override: + remote_name = utils.url_directory_name(alias_override) + _, remotes = self.source.check_output( + [self.source.host_git, 'remote'], + fail="Failed to retrieve list of remotes in {}".format(self.mirror), + cwd=self.mirror + ) + if remote_name not in remotes: + self.source.call( + [self.source.host_git, 'remote', 'add', remote_name, url], + fail="Failed to add remote {} with url {}".format(remote_name, url), + cwd=self.mirror + ) + else: + remote_name = "origin" - def fetch(self): - self.source.call([self.source.host_git, 'fetch', 'origin', '--prune'], - fail="Failed to fetch from remote git repository: {}".format(self.url), + self.source.call([self.source.host_git, 'fetch', remote_name, '--prune'], + fail="Failed to fetch from remote git repository: {}".format(url), fail_temporarily=True, cwd=self.mirror) + def fetch(self, alias_override=None): + self.ensure(alias_override) + if not self.has_ref(): + self._fetch(alias_override) + self.assert_ref() + def has_ref(self): if not self.ref: return False @@ -171,13 +198,14 @@ class GitMirror(): def init_workspace(self, directory): fullpath = os.path.join(directory, self.path) + url = self.source.translate_url(self.url) self.source.call([self.source.host_git, 'clone', '--no-checkout', self.mirror, fullpath], fail="Failed to clone git mirror {} in directory: {}".format(self.mirror, fullpath), fail_temporarily=True) - self.source.call([self.source.host_git, 'remote', 'set-url', 'origin', self.url], - fail='Failed to add remote origin "{}"'.format(self.url), + self.source.call([self.source.host_git, 'remote', 'set-url', 'origin', url], + fail='Failed to add remote origin "{}"'.format(url), cwd=fullpath) self.source.call([self.source.host_git, 'checkout', '--force', self.ref], @@ -277,6 +305,8 @@ class GitSource(Source): checkout = self.node_get_member(submodule, bool, 'checkout') self.submodule_checkout_overrides[path] = checkout + self.mark_download_url(self.original_url) + def preflight(self): # Check if git is installed, get the binary at the same time self.host_git = utils.get_host_tool('git') @@ -328,31 +358,13 @@ class GitSource(Source): .format(self.tracking, self.mirror.url), silent_nested=True): self.mirror.ensure() - self.mirror.fetch() + self.mirror._fetch() # Update self.mirror.ref and node.ref from the self.tracking branch ret = self.mirror.latest_commit(self.tracking) return ret - def fetch(self): - - with self.timed_activity("Fetching {}".format(self.mirror.url), silent_nested=True): - - # Here we are only interested in ensuring that our mirror contains - # the self.mirror.ref commit. - self.mirror.ensure() - if not self.mirror.has_ref(): - self.mirror.fetch() - - self.mirror.assert_ref() - - # Here after performing any fetches, we need to also ensure that - # we've cached the desired refs in our mirrors of submodules. - # - self.refresh_submodules() - self.fetch_submodules() - def init_workspace(self, directory): # XXX: may wish to refactor this as some code dupe with stage() self.refresh_submodules() @@ -384,6 +396,10 @@ class GitSource(Source): if checkout: mirror.stage(directory) + def get_source_fetchers(self): + self.refresh_submodules() + return [self.mirror] + self.submodules + ########################################################### # Local Functions # ########################################################### @@ -405,6 +421,7 @@ class GitSource(Source): # Assumes that we have our mirror and we have the ref which we point to # def refresh_submodules(self): + self.mirror.ensure() submodules = [] # XXX Here we should issue a warning if either: @@ -426,19 +443,6 @@ class GitSource(Source): self.submodules = submodules - # Ensures that we have mirrored git repositories for all - # the submodules existing at the given commit of the main git source. - # - # Also ensure that these mirrors have the required commits - # referred to at the given commit of the main git source. - # - def fetch_submodules(self): - for mirror in self.submodules: - mirror.ensure() - if not mirror.has_ref(): - mirror.fetch() - mirror.assert_ref() - # Plugin entry point def setup(): diff --git a/buildstream/source.py b/buildstream/source.py index 2db4f8eea..9866c99e9 100644 --- a/buildstream/source.py +++ b/buildstream/source.py @@ -65,6 +65,33 @@ these methods are mandatory to implement. **Optional**: If left unimplemented, this will default to calling :func:`Source.stage() <buildstream.source.Source.stage>` + +* :func:`Source.get_source_fetchers() <buildstream.source.Source.get_source_fetchers>` + + Get the objects that are used for fetching. + + **Optional**: This only needs to be implemented for sources that need to + download from multiple URLs while fetching (e.g. a git repo and its + submodules). For details on how to define a SourceFetcher, see + :ref:`SourceFetcher <core_source_fetcher>`. + + +.. _core_source_fetcher: + +SourceFetcher - Object for fetching individual URLs +=================================================== + + +Abstract Methods +---------------- +SourceFetchers expose the following abstract methods. Unless explicitly +mentioned, these methods are mandatory to implement. + +* :func:`SourceFetcher.fetch() <buildstream.source.SourceFetcher.fetch>` + + Fetches the URL associated with this SourceFetcher, optionally taking an + alias override. + """ import os @@ -114,6 +141,63 @@ class SourceError(BstError): super().__init__(message, detail=detail, domain=ErrorDomain.SOURCE, reason=reason, temporary=temporary) +class SourceFetcher(): + """SourceFetcher() + + This interface exists so that a source that downloads from multiple + places (e.g. a git source with submodules) has a consistent interface for + fetching and substituting aliases. + + *Since: 1.4* + """ + def __init__(self): + self.__alias = None + + ############################################################# + # Abstract Methods # + ############################################################# + def fetch(self, alias_override=None): + """Fetch remote sources and mirror them locally, ensuring at least + that the specific reference is cached locally. + + Args: + alias_override (str): The alias to use instead of the default one + defined by the :ref:`aliases <project_source_aliases>` field + in the project's config. + + Raises: + :class:`.SourceError` + + Implementors should raise :class:`.SourceError` if the there is some + network error or if the source reference could not be matched. + """ + raise ImplError("Source fetcher '{}' does not implement fetch()".format(type(self))) + + ############################################################# + # Public Methods # + ############################################################# + def mark_download_url(self, url): + """Identifies the URL that this SourceFetcher uses to download + + This must be called during the fetcher's initialization + + Args: + url (str): The url used to download. + """ + # Not guaranteed to be a valid alias yet. + # Ensuring it's a valid alias currently happens in Project.get_alias_uris + alias, _ = url.split(utils._ALIAS_SEPARATOR, 1) + self.__alias = alias + + ############################################################# + # Private Methods used in BuildStream # + ############################################################# + + # Returns the alias used by this fetcher + def _get_alias(self): + return self.__alias + + class Source(Plugin): """Source() @@ -125,7 +209,7 @@ class Source(Plugin): __defaults = {} # The defaults from the project __defaults_set = False # Flag, in case there are not defaults at all - def __init__(self, context, project, meta): + def __init__(self, context, project, meta, *, alias_override=None): provenance = _yaml.node_get_provenance(meta.config) super().__init__("{}-{}".format(meta.element_name, meta.element_index), context, project, provenance, "source") @@ -135,6 +219,11 @@ class Source(Plugin): self.__element_kind = meta.element_kind # The kind of the element owning this source self.__directory = meta.directory # Staging relative directory self.__consistency = Consistency.INCONSISTENT # Cached consistency state + self.__alias_override = alias_override # Tuple of alias and its override to use instead + self.__expected_alias = None # A hacky way to store the first alias used + + # FIXME: Reconstruct a MetaSource from a Source instead of storing it. + self.__meta = meta # MetaSource stored so we can copy this source later. # Collect the composited element configuration and # ask the element to configure itself. @@ -284,6 +373,36 @@ class Source(Plugin): """ self.stage(directory) + def mark_download_url(self, url): + """Identifies the URL that this Source uses to download + + This must be called during :func:`~buildstream.plugin.Plugin.configure` if + :func:`~buildstream.source.Source.translate_url` is not called. + + Args: + url (str): The url used to download + + *Since: 1.4* + """ + alias, _ = url.split(utils._ALIAS_SEPARATOR, 1) + self.__expected_alias = alias + + def get_source_fetchers(self): + """Get the objects that are used for fetching + + If this source doesn't download from multiple URLs, + returning None and falling back on the default behaviour + is recommended. + + Returns: + list: A list of SourceFetchers. If SourceFetchers are not supported, + this will be an empty list. + + *Since: 1.4* + """ + + return [] + ############################################################# # Public Methods # ############################################################# @@ -300,18 +419,42 @@ class Source(Plugin): os.makedirs(directory, exist_ok=True) return directory - def translate_url(self, url): + def translate_url(self, url, *, alias_override=None): """Translates the given url which may be specified with an alias into a fully qualified url. Args: url (str): A url, which may be using an alias + alias_override (str): Optionally, an URI to override the alias with. (*Since: 1.4*) Returns: str: The fully qualified url, with aliases resolved """ - project = self._get_project() - return project.translate_url(url) + # Alias overriding can happen explicitly (by command-line) or + # implicitly (the Source being constructed with an __alias_override). + if alias_override or self.__alias_override: + url_alias, url_body = url.split(utils._ALIAS_SEPARATOR, 1) + if url_alias: + if alias_override: + url = alias_override + url_body + else: + # Implicit alias overrides may only be done for one + # specific alias, so that sources that fetch from multiple + # URLs and use different aliases default to only overriding + # one alias, rather than getting confused. + override_alias = self.__alias_override[0] + override_url = self.__alias_override[1] + if url_alias == override_alias: + url = override_url + url_body + return url + else: + # Sneakily store the alias if it hasn't already been stored + if not self.__expected_alias and url and utils._ALIAS_SEPARATOR in url: + url_alias, _ = url.split(utils._ALIAS_SEPARATOR, 1) + self.__expected_alias = url_alias + + project = self._get_project() + return project.translate_url(url) def get_project_directory(self): """Fetch the project base directory @@ -375,7 +518,45 @@ class Source(Plugin): # Wrapper function around plugin provided fetch method # def _fetch(self): - self.fetch() + project = self._get_project() + source_fetchers = self.get_source_fetchers() + if source_fetchers: + for fetcher in source_fetchers: + alias = fetcher._get_alias() + success = False + for uri in project.get_alias_uris(alias): + try: + fetcher.fetch(uri) + # FIXME: Need to consider temporary vs. permanent failures, + # and how this works with retries. + except BstError as e: + last_error = e + continue + success = True + break + if not success: + raise last_error + else: + alias = self._get_alias() + if not project.mirrors or not alias: + self.fetch() + return + + context = self._get_context() + source_kind = type(self) + for uri in project.get_alias_uris(alias): + new_source = source_kind(context, project, self.__meta, + alias_override=(alias, uri)) + new_source._preflight() + try: + new_source.fetch() + # FIXME: Need to consider temporary vs. permanent failures, + # and how this works with retries. + except BstError as e: + last_error = e + continue + return + raise last_error # Wrapper for stage() api which gives the source # plugin a fully constructed path considering the @@ -582,7 +763,7 @@ class Source(Plugin): # Wrapper for track() # def _track(self): - new_ref = self.track() + new_ref = self.__do_track() current_ref = self.get_ref() if new_ref is None: @@ -594,10 +775,48 @@ class Source(Plugin): return new_ref + # Returns the alias if it's defined in the project + def _get_alias(self): + alias = self.__expected_alias + project = self._get_project() + if project.get_alias_uri(alias): + # The alias must already be defined in the project's aliases + # otherwise http://foo gets treated like it contains an alias + return alias + else: + return None + ############################################################# # Local Private Methods # ############################################################# + # Tries to call track for every mirror, stopping once it succeeds + def __do_track(self): + project = self._get_project() + # If there are no mirrors, or no aliases to replace, there's nothing to do here. + alias = self._get_alias() + if not project.mirrors or not alias: + return self.track() + + context = self._get_context() + source_kind = type(self) + + # NOTE: We are assuming here that tracking only requires substituting the + # first alias used + for uri in reversed(project.get_alias_uris(alias)): + new_source = source_kind(context, project, self.__meta, + alias_override=(alias, uri)) + new_source._preflight() + try: + ref = new_source.track() + # FIXME: Need to consider temporary vs. permanent failures, + # and how this works with retries. + except BstError as e: + last_error = e + continue + return ref + raise last_error + # Ensures a fully constructed path and returns it def __ensure_directory(self, directory): diff --git a/buildstream/utils.py b/buildstream/utils.py index e8270d82f..bfb58c9ef 100644 --- a/buildstream/utils.py +++ b/buildstream/utils.py @@ -42,6 +42,10 @@ from . import _signals from ._exceptions import BstError, ErrorDomain +# The separator we use for user specified aliases +_ALIAS_SEPARATOR = ':' + + class UtilError(BstError): """Raised by utility functions when system calls fail. diff --git a/doc/source/examples/git-mirror.rst b/doc/source/examples/git-mirror.rst new file mode 100644 index 000000000..1b05bf607 --- /dev/null +++ b/doc/source/examples/git-mirror.rst @@ -0,0 +1,144 @@ + + +Creating and using a git mirror +''''''''''''''''''''''''''''''' +This is an example of how to create a git mirror using git's +`git-http-backend <https://git-scm.com/docs/git-http-backend>`_ and +`lighttpd <https://redmine.lighttpd.net/projects/1/wiki/TutorialConfiguration>`_. + + +Prerequisites +============= +You will need git installed, and git-http-backend must be present. It is assumed +that the git-http-backend binary exists at `/usr/lib/git-core/git-http-backend`. + +You will need `lighttpd` installed, and at the bare minimum has the modules +`mod_alias`, `mod_cgi`, and `mod_setenv`. + +I will be using gnome-modulesets as an example, which can be cloned from +`http://gnome7.codethink.co.uk/gnome-modulesets.git`. + + +Starting a git http server +========================== + + +1. Set up a directory containing mirrors +---------------------------------------- +Choose a suitable directory to hold your mirrors, e.g. `/var/www/git`. + +Place the git repositories you want to use as mirrors in the mirror dir, e.g. +``git clone --mirror http://git.gnome.org/browse/yelp-xsl /var/www/git/yelp-xsl.git``. + + +2. Configure lighttpd +--------------------- +Write out a lighttpd.conf as follows: + +:: + + server.document-root = "/var/www/git/" + server.port = 3000 + server.modules = ( + "mod_alias", + "mod_cgi", + "mod_setenv", + ) + + alias.url += ( "/git" => "/usr/lib/git-core/git-http-backend" ) + $HTTP["url"] =~ "^/git" { + cgi.assign = ("" => "") + setenv.add-environment = ( + "GIT_PROJECT_ROOT" => "/var/www/git", + "GIT_HTTP_EXPORT_ALL" => "" + ) + } + +.. note:: + + If you have your mirrors in another directory, replace /var/www/git/ with that directory. + + +3. Start lighttpd +----------------- +lighttpd can be invoked with the command-line ``lighttpd -D -f lighttpd.conf``. + + +4. Test that you can fetch from it +---------------------------------- +We can then clone the mirrored repo using git via http with +``git clone http://127.0.0.1:3000/git/yelp-xsl``. + +.. note:: + + If you have set server.port to something other than the default, you will + need to replace the '3000' in the command-line. + + +5. Configure the project to use the mirror +------------------------------------------ +To add this local http server as a mirror, add the following to the project.conf: + +.. code:: yaml + + mirrors: + - name: local-mirror + aliases: + git_gnome_org: + - http://127.0.0.1:3000/git/ + + +6. Test that the mirror works +----------------------------- +We can make buildstream use the mirror by setting the alias to an invalid URL, e.g. + +.. code:: yaml + + aliases: + git_gnome_org: https://www.example.com/invalid/url/ + +Now, if you build an element that uses the source you placed in the mirror +(e.g. ``bst build core-deps/yelp-xsl.bst``), you will see that it uses your mirror. + + +.. _lighttpd_git_tar_conf: + +Bonus: lighttpd conf for git and tar +==================================== +For those who have also used the :ref:`tar-mirror tutorial <using_tar_mirror>`, +a combined lighttpd.conf is below: + +:: + + server.document-root = "/var/www/" + server.port = 3000 + server.modules = ( + "mod_alias", + "mod_cgi", + "mod_setenv", + ) + + alias.url += ( "/git" => "/usr/lib/git-core/git-http-backend" ) + $HTTP["url"] =~ "^/git" { + cgi.assign = ("" => "") + setenv.add-environment = ( + "GIT_PROJECT_ROOT" => "/var/www/git", + "GIT_HTTP_EXPORT_ALL" => "" + ) + } else $HTTP["url"] =~ "^/tar" { + dir-listing.activate = "enable" + } + + +Further reading +=============== +If this mirror isn't being used exclusively in a secure network, it is strongly +recommended you `use SSL <https://redmine.lighttpd.net/projects/1/wiki/HowToSimpleSSL>`_. + +This is the bare minimum required to set up a git mirror. A large, public project +would prefer to set it up using the +`git protocol <https://git-scm.com/book/en/v1/Git-on-the-Server-Git-Daemon>`_, +and a security-conscious project would be configured to use +`git over SSH <https://git-scm.com/book/en/v1/Git-on-the-Server-Getting-Git-on-a-Server#Small-Setups>`_. + +Lighttpd is documented on `its wiki <https://redmine.lighttpd.net/projects/lighttpd/wiki>`_. diff --git a/doc/source/examples/tar-mirror.rst b/doc/source/examples/tar-mirror.rst new file mode 100644 index 000000000..7a4a2f51e --- /dev/null +++ b/doc/source/examples/tar-mirror.rst @@ -0,0 +1,103 @@ + + +.. _using_tar_mirror: + +Creating and using a tar mirror +''''''''''''''''''''''''''''''' +This is an example of how to create a tar mirror using +`lighttpd <https://redmine.lighttpd.net/projects/1/wiki/TutorialConfiguration>`_. + + +Prerequisites +============= +You will need `lighttpd` installed. + + +I will be using gnome-modulesets as an example, which can be cloned from +`http://gnome7.codethink.co.uk/gnome-modulesets.git`. + + +Starting a tar server +===================== + + +1. Set up a directory containing mirrors +---------------------------------------- +Choose a suitable directory to hold your mirrored tar files, e.g. `/var/www/tar`. + +Place the tar files you want to use as mirrors in your mirror dir, e.g. + +.. code:: + + mkdir -p /var/www/tar/gettext + wget -O /var/www/tar/gettext/gettext-0.19.8.1.tar.xz https://ftp.gnu.org/gnu/gettext/gettext-0.19.8.1.tar.xz + + +2. Configure lighttpd +--------------------- +Write out a lighttpd.conf as follows: + +:: + + server.document-root = "/var/www/tar/" + server.port = 3000 + + dir-listing.activate = "enable" + +.. note:: + + If you have your mirrors in another directory, replace /var/www/tar/ with that directory. + +.. note:: + + An example lighttpd.conf that works for both git and tar services is available + :ref:`here <lighttpd_git_tar_conf>` + + +3. Start lighttpd +----------------- +lighttpd can be invoked with the command-line ``lighttpd -D -f lighttpd.conf``. + + +4. Test that you can fetch from it +---------------------------------- +We can then download the mirrored file with ``wget 127.0.0.1:3000/tar/gettext/gettext-0.19.8.1.tar.xz``. + +.. note:: + + If you have set server.port to something other than the default, you will need + to replace the '3000' in the command-line. + + +5. Configure the project to use the mirror +------------------------------------------ +To add this local http server as a mirror, add the following to the project.conf: + +.. code:: yaml + + mirrors: + - name: local-mirror + aliases: + ftp_gnu_org: + - http://127.0.0.1:3000/tar/ + + +6. Test that the mirror works +----------------------------- +We can make buildstream use the mirror by setting the alias to an invalid URL, e.g. + +.. code:: yaml + + aliases: + ftp_gnu_org: https://www.example.com/invalid/url/ + +Now, if you build an element that uses the source you placed in the mirror +(e.g. ``bst build core-deps/gettext.bst``), you will see that it uses your mirror. + + +Further reading +=============== +If this mirror isn't being used exclusively in a secure network, it is strongly +recommended you `use SSL <https://redmine.lighttpd.net/projects/1/wiki/HowToSimpleSSL>`_. + +Lighttpd is documented on `its wiki <https://redmine.lighttpd.net/projects/lighttpd/wiki>`_. diff --git a/doc/source/format_project.rst b/doc/source/format_project.rst index 36de41a55..cf0970f77 100644 --- a/doc/source/format_project.rst +++ b/doc/source/format_project.rst @@ -198,6 +198,43 @@ You can also specify a list of caches here; earlier entries in the list will have higher priority than later ones. +.. _project_essentials_mirrors: + +Mirrors +~~~~~~~ +A list of mirrors can be defined that couple a location to a mapping of aliases to a +list of URIs, e.g. + +.. code:: yaml + + mirrors: + - name: middle-earth + aliases: + foo: + - http://www.middle-earth.com/foo/1 + - http://www.middle-earth.com/foo/2 + bar: + - http://www.middle-earth.com/bar/1 + - http://www.middle-earth.com/bar/2 + - name: oz + aliases: + foo: + - http://www.oz.com/foo + bar: + - http://www.oz.com/bar + +The order that the mirrors (and the URIs therein) are consulted is in the order +they are defined when fetching, and in reverse-order when tracking. + +A default mirror to consult first can be defined via +:ref:`user config <config_default_mirror>`, or the command-line argument +:ref:`--default-mirror <invoking_bst>`. + +.. note:: + + The ``mirrors`` field is available since :ref:`format version 11 <project_format_version>` + + .. _project_plugins: External plugins diff --git a/doc/source/main_install.rst b/doc/source/main_install.rst index ebd55a31e..1a4895903 100644 --- a/doc/source/main_install.rst +++ b/doc/source/main_install.rst @@ -1,7 +1,12 @@ Install ======= -This section covers how to install BuildStream onto your machine, how to run BuildStream inside a docker image and also how to configure an artifact server. +This section covers how to install BuildStream onto your machine, how to run +BuildStream inside a docker image and also how to configure an artifact server. +.. note:: + + BuildStream is not currently supported natively on macOS and Windows. Windows + and macOS users should refer to :ref:`docker`. .. toctree:: :maxdepth: 2 diff --git a/doc/source/using_config.rst b/doc/source/using_config.rst index d02aaf3d4..c707cd04b 100644 --- a/doc/source/using_config.rst +++ b/doc/source/using_config.rst @@ -89,6 +89,27 @@ modifying some low level component. the ``--strict`` and ``--no-strict`` command line options. +.. _config_default_mirror: + +Default Mirror +~~~~~~~~~~~~~~ +When using :ref:`mirrors <project_essentials_mirrors>`, a default mirror can +be defined to be fetched first. +The default mirror is defined by its name, e.g. + +.. code:: yaml + + projects: + project-name: + default-mirror: oz + + +.. note:: + + It is possible to override this at invocation time using the + ``--default-mirror`` command-line option. + + Default configuration --------------------- The default BuildStream configuration is specified here for reference: diff --git a/doc/source/using_examples.rst b/doc/source/using_examples.rst index aa100d007..622b09e32 100644 --- a/doc/source/using_examples.rst +++ b/doc/source/using_examples.rst @@ -10,3 +10,5 @@ maintained and work as expected. :maxdepth: 1 examples/flatpak-autotools + examples/tar-mirror + examples/git-mirror diff --git a/tests/completions/completions.py b/tests/completions/completions.py index 7c169c2d2..1ff026ea5 100644 --- a/tests/completions/completions.py +++ b/tests/completions/completions.py @@ -27,6 +27,7 @@ MAIN_OPTIONS = [ "--colors ", "--config ", "--debug ", + "--default-mirror ", "--directory ", "--error-lines ", "--fetchers ", diff --git a/tests/frontend/mirror.py b/tests/frontend/mirror.py new file mode 100644 index 000000000..62c796ab8 --- /dev/null +++ b/tests/frontend/mirror.py @@ -0,0 +1,402 @@ +import os +import pytest + +from tests.testutils import cli, create_repo, ALL_REPO_KINDS + +from buildstream import _yaml + + +# Project directory +TOP_DIR = os.path.dirname(os.path.realpath(__file__)) +DATA_DIR = os.path.join(TOP_DIR, 'project') + + +def generate_element(output_file): + element = { + 'kind': 'import', + 'sources': [ + { + 'kind': 'fetch_source', + "output-text": output_file, + "urls": ["foo:repo1", "bar:repo2"], + "fetch-succeeds": { + "FOO/repo1": True, + "BAR/repo2": False, + "OOF/repo1": False, + "RAB/repo2": True, + "OFO/repo1": False, + "RBA/repo2": False, + "ooF/repo1": False, + "raB/repo2": False, + } + } + ] + } + return element + + +def generate_project(): + project = { + 'name': 'test', + 'element-path': 'elements', + 'aliases': { + 'foo': 'FOO/', + 'bar': 'BAR/', + }, + 'mirrors': [ + { + 'name': 'middle-earth', + 'aliases': { + 'foo': ['OOF/'], + 'bar': ['RAB/'], + }, + }, + { + 'name': 'arrakis', + 'aliases': { + 'foo': ['OFO/'], + 'bar': ['RBA/'], + }, + }, + { + 'name': 'oz', + 'aliases': { + 'foo': ['ooF/'], + 'bar': ['raB/'], + } + }, + ], + 'plugins': [ + { + 'origin': 'local', + 'path': 'sources', + 'sources': { + 'fetch_source': 0 + } + } + ] + } + return project + + +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS]) +def test_mirror_fetch(cli, tmpdir, datafiles, kind): + bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr') + dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr') + upstream_repodir = os.path.join(str(tmpdir), 'upstream') + mirror_repodir = os.path.join(str(tmpdir), 'mirror') + project_dir = os.path.join(str(tmpdir), 'project') + os.makedirs(project_dir) + element_dir = os.path.join(project_dir, 'elements') + + # Create repo objects of the upstream and mirror + upstream_repo = create_repo(kind, upstream_repodir) + upstream_ref = upstream_repo.create(bin_files_path) + mirror_repo = upstream_repo.copy(mirror_repodir) + mirror_ref = upstream_ref + upstream_ref = upstream_repo.create(dev_files_path) + + element = { + 'kind': 'import', + 'sources': [ + upstream_repo.source_config(ref=upstream_ref) + ] + } + element_name = 'test.bst' + element_path = os.path.join(element_dir, element_name) + full_repo = element['sources'][0]['url'] + upstream_map, repo_name = os.path.split(full_repo) + alias = 'foo-' + kind + aliased_repo = alias + ':' + repo_name + element['sources'][0]['url'] = aliased_repo + full_mirror = mirror_repo.source_config()['url'] + mirror_map, _ = os.path.split(full_mirror) + os.makedirs(element_dir) + _yaml.dump(element, element_path) + + project = { + 'name': 'test', + 'element-path': 'elements', + 'aliases': { + alias: upstream_map + "/" + }, + 'mirrors': [ + { + 'name': 'middle-earth', + 'aliases': { + alias: [mirror_map + "/"], + }, + }, + ] + } + project_file = os.path.join(project_dir, 'project.conf') + _yaml.dump(project, project_file) + + # No obvious ways of checking that the mirror has been fetched + # But at least we can be sure it succeeds + result = cli.run(project=project_dir, args=['fetch', element_name]) + result.assert_success() + + +@pytest.mark.datafiles(DATA_DIR) +def test_mirror_fetch_multi(cli, tmpdir, datafiles): + output_file = os.path.join(str(tmpdir), "output.txt") + project_dir = str(tmpdir) + element_dir = os.path.join(project_dir, 'elements') + os.makedirs(element_dir, exist_ok=True) + element_name = "test.bst" + element_path = os.path.join(element_dir, element_name) + element = generate_element(output_file) + _yaml.dump(element, element_path) + + project_file = os.path.join(project_dir, 'project.conf') + project = generate_project() + _yaml.dump(project, project_file) + + result = cli.run(project=project_dir, args=['fetch', element_name]) + result.assert_success() + with open(output_file) as f: + contents = f.read() + assert "Fetch foo:repo1 succeeded from FOO/repo1" in contents + assert "Fetch bar:repo2 succeeded from RAB/repo2" in contents + + +@pytest.mark.datafiles(DATA_DIR) +def test_mirror_fetch_default_cmdline(cli, tmpdir, datafiles): + output_file = os.path.join(str(tmpdir), "output.txt") + project_dir = str(tmpdir) + element_dir = os.path.join(project_dir, 'elements') + os.makedirs(element_dir, exist_ok=True) + element_name = "test.bst" + element_path = os.path.join(element_dir, element_name) + element = generate_element(output_file) + _yaml.dump(element, element_path) + + project_file = os.path.join(project_dir, 'project.conf') + project = generate_project() + _yaml.dump(project, project_file) + + result = cli.run(project=project_dir, args=['--default-mirror', 'arrakis', 'fetch', element_name]) + result.assert_success() + with open(output_file) as f: + contents = f.read() + print(contents) + # Success if fetching from arrakis' mirror happened before middle-earth's + arrakis_str = "OFO/repo1" + arrakis_pos = contents.find(arrakis_str) + assert arrakis_pos != -1, "'{}' wasn't found".format(arrakis_str) + me_str = "OOF/repo1" + me_pos = contents.find(me_str) + assert me_pos != -1, "'{}' wasn't found".format(me_str) + assert arrakis_pos < me_pos, "'{}' wasn't found before '{}'".format(arrakis_str, me_str) + + +@pytest.mark.datafiles(DATA_DIR) +def test_mirror_fetch_default_userconfig(cli, tmpdir, datafiles): + output_file = os.path.join(str(tmpdir), "output.txt") + project_dir = str(tmpdir) + element_dir = os.path.join(project_dir, 'elements') + os.makedirs(element_dir, exist_ok=True) + element_name = "test.bst" + element_path = os.path.join(element_dir, element_name) + element = generate_element(output_file) + _yaml.dump(element, element_path) + + project_file = os.path.join(project_dir, 'project.conf') + project = generate_project() + _yaml.dump(project, project_file) + + userconfig = { + 'projects': { + 'test': { + 'default-mirror': 'oz' + } + } + } + cli.configure(userconfig) + + result = cli.run(project=project_dir, args=['fetch', element_name]) + result.assert_success() + with open(output_file) as f: + contents = f.read() + print(contents) + # Success if fetching from Oz' mirror happened before middle-earth's + oz_str = "ooF/repo1" + oz_pos = contents.find(oz_str) + assert oz_pos != -1, "'{}' wasn't found".format(oz_str) + me_str = "OOF/repo1" + me_pos = contents.find(me_str) + assert me_pos != -1, "'{}' wasn't found".format(me_str) + assert oz_pos < me_pos, "'{}' wasn't found before '{}'".format(oz_str, me_str) + + +@pytest.mark.datafiles(DATA_DIR) +def test_mirror_fetch_default_cmdline_overrides_config(cli, tmpdir, datafiles): + output_file = os.path.join(str(tmpdir), "output.txt") + project_dir = str(tmpdir) + element_dir = os.path.join(project_dir, 'elements') + os.makedirs(element_dir, exist_ok=True) + element_name = "test.bst" + element_path = os.path.join(element_dir, element_name) + element = generate_element(output_file) + _yaml.dump(element, element_path) + + project_file = os.path.join(project_dir, 'project.conf') + project = generate_project() + _yaml.dump(project, project_file) + + userconfig = { + 'projects': { + 'test': { + 'default-mirror': 'oz' + } + } + } + cli.configure(userconfig) + + result = cli.run(project=project_dir, args=['--default-mirror', 'arrakis', 'fetch', element_name]) + result.assert_success() + with open(output_file) as f: + contents = f.read() + print(contents) + # Success if fetching from arrakis' mirror happened before middle-earth's + arrakis_str = "OFO/repo1" + arrakis_pos = contents.find(arrakis_str) + assert arrakis_pos != -1, "'{}' wasn't found".format(arrakis_str) + me_str = "OOF/repo1" + me_pos = contents.find(me_str) + assert me_pos != -1, "'{}' wasn't found".format(me_str) + assert arrakis_pos < me_pos, "'{}' wasn't found before '{}'".format(arrakis_str, me_str) + + +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS]) +def test_mirror_track_upstream_present(cli, tmpdir, datafiles, kind): + bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr') + dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr') + upstream_repodir = os.path.join(str(tmpdir), 'upstream') + mirror_repodir = os.path.join(str(tmpdir), 'mirror') + project_dir = os.path.join(str(tmpdir), 'project') + os.makedirs(project_dir) + element_dir = os.path.join(project_dir, 'elements') + + # Create repo objects of the upstream and mirror + upstream_repo = create_repo(kind, upstream_repodir) + upstream_ref = upstream_repo.create(bin_files_path) + mirror_repo = upstream_repo.copy(mirror_repodir) + mirror_ref = upstream_ref + upstream_ref = upstream_repo.create(dev_files_path) + + element = { + 'kind': 'import', + 'sources': [ + upstream_repo.source_config(ref=upstream_ref) + ] + } + + element['sources'][0] + element_name = 'test.bst' + element_path = os.path.join(element_dir, element_name) + full_repo = element['sources'][0]['url'] + upstream_map, repo_name = os.path.split(full_repo) + alias = 'foo-' + kind + aliased_repo = alias + ':' + repo_name + element['sources'][0]['url'] = aliased_repo + full_mirror = mirror_repo.source_config()['url'] + mirror_map, _ = os.path.split(full_mirror) + os.makedirs(element_dir) + _yaml.dump(element, element_path) + + project = { + 'name': 'test', + 'element-path': 'elements', + 'aliases': { + alias: upstream_map + "/" + }, + 'mirrors': [ + { + 'name': 'middle-earth', + 'aliases': { + alias: [mirror_map + "/"], + }, + }, + ] + } + project_file = os.path.join(project_dir, 'project.conf') + _yaml.dump(project, project_file) + + result = cli.run(project=project_dir, args=['track', element_name]) + result.assert_success() + + # Tracking tries upstream first. Check the ref is from upstream. + new_element = _yaml.load(element_path) + source = new_element['sources'][0] + if 'ref' in source: + assert source['ref'] == upstream_ref + + +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS]) +def test_mirror_track_upstream_absent(cli, tmpdir, datafiles, kind): + bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr') + dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr') + upstream_repodir = os.path.join(str(tmpdir), 'upstream') + mirror_repodir = os.path.join(str(tmpdir), 'mirror') + project_dir = os.path.join(str(tmpdir), 'project') + os.makedirs(project_dir) + element_dir = os.path.join(project_dir, 'elements') + + # Create repo objects of the upstream and mirror + upstream_repo = create_repo(kind, upstream_repodir) + upstream_ref = upstream_repo.create(bin_files_path) + mirror_repo = upstream_repo.copy(mirror_repodir) + mirror_ref = upstream_ref + upstream_ref = upstream_repo.create(dev_files_path) + + element = { + 'kind': 'import', + 'sources': [ + upstream_repo.source_config(ref=upstream_ref) + ] + } + + element['sources'][0] + element_name = 'test.bst' + element_path = os.path.join(element_dir, element_name) + full_repo = element['sources'][0]['url'] + upstream_map, repo_name = os.path.split(full_repo) + alias = 'foo-' + kind + aliased_repo = alias + ':' + repo_name + element['sources'][0]['url'] = aliased_repo + full_mirror = mirror_repo.source_config()['url'] + mirror_map, _ = os.path.split(full_mirror) + os.makedirs(element_dir) + _yaml.dump(element, element_path) + + project = { + 'name': 'test', + 'element-path': 'elements', + 'aliases': { + alias: 'http://www.example.com/' + }, + 'mirrors': [ + { + 'name': 'middle-earth', + 'aliases': { + alias: [mirror_map + "/"], + }, + }, + ] + } + project_file = os.path.join(project_dir, 'project.conf') + _yaml.dump(project, project_file) + + result = cli.run(project=project_dir, args=['track', element_name]) + result.assert_success() + + # Check that tracking fell back to the mirror + new_element = _yaml.load(element_path) + source = new_element['sources'][0] + if 'ref' in source: + assert source['ref'] == mirror_ref diff --git a/tests/frontend/project/sources/fetch_source.py b/tests/frontend/project/sources/fetch_source.py new file mode 100644 index 000000000..ebd3fe757 --- /dev/null +++ b/tests/frontend/project/sources/fetch_source.py @@ -0,0 +1,85 @@ +import os +import sys + +from buildstream import Source, Consistency, SourceError, SourceFetcher + +# Expected config +# sources: +# - output-text: $FILE +# urls: +# - foo:bar +# - baz:quux +# fetch-succeeds: +# Foo/bar: true +# ooF/bar: false + + +class FetchFetcher(SourceFetcher): + def __init__(self, source, url): + super().__init__() + self.source = source + self.original_url = url + self.mark_download_url(url) + + def fetch(self, alias_override=None): + url = self.source.translate_url(self.original_url, alias_override=alias_override) + with open(self.source.output_file, "a") as f: + success = url in self.source.fetch_succeeds and self.source.fetch_succeeds[url] + message = "Fetch {} {} from {}\n".format(self.original_url, + "succeeded" if success else "failed", + url) + f.write(message) + if not success: + raise SourceError("Failed to fetch {}".format(url)) + + +class FetchSource(Source): + # Read config to know which URLs to fetch + def configure(self, node): + self.original_urls = self.node_get_member(node, list, 'urls') + self.fetchers = [FetchFetcher(self, url) for url in self.original_urls] + self.output_file = self.node_get_member(node, str, 'output-text') + self.fetch_succeeds = {} + if 'fetch-succeeds' in node: + self.fetch_succeeds = {x[0]: x[1] for x in self.node_items(node['fetch-succeeds'])} + + def get_source_fetchers(self): + return self.fetchers + + def preflight(self): + output_dir = os.path.dirname(self.output_file) + if not os.path.exists(output_dir): + raise SourceError("Directory '{}' does not exist".format(output_dir)) + + def fetch(self): + for fetcher in self.fetchers: + fetcher.fetch() + + def get_unique_key(self): + return {"urls": self.original_urls, "output_file": self.output_file} + + def get_consistency(self): + if not os.path.exists(self.output_file): + return Consistency.RESOLVED + + with open(self.output_file, "r") as f: + contents = f.read() + for url in self.original_urls: + if url not in contents: + return Consistency.RESOLVED + + return Consistency.CACHED + + # We dont have a ref, we're a local file... + def load_ref(self, node): + pass + + def get_ref(self): + return None # pragma: nocover + + def set_ref(self, ref, node): + pass # pragma: nocover + + +def setup(): + return FetchSource diff --git a/tests/testutils/repo/repo.py b/tests/testutils/repo/repo.py index 4c9ee59a9..234aa374c 100644 --- a/tests/testutils/repo/repo.py +++ b/tests/testutils/repo/repo.py @@ -22,7 +22,7 @@ class Repo(): # The directory the actual repo will be stored in self.repo = os.path.join(self.directory, subdir) - os.makedirs(self.repo) + os.makedirs(self.repo, exist_ok=True) # create(): # @@ -69,3 +69,22 @@ class Repo(): shutil.copytree(src_path, dest_path) else: shutil.copy2(src_path, dest_path) + + # copy(): + # + # Creates a copy of this repository in the specified + # destination. + # + # Args: + # dest (str): The destination directory + # + # Returns: + # (Repo): A Repo object for the new repository. + def copy(self, dest): + subdir = self.repo[len(self.directory):].lstrip(os.sep) + new_dir = os.path.join(dest, subdir) + os.makedirs(new_dir, exist_ok=True) + self.copy_directory(self.repo, new_dir) + repo_type = type(self) + new_repo = repo_type(dest, subdir) + return new_repo |