From 9d2442b2794d7a531f50b3f1f9a0c6e4236bd9be Mon Sep 17 00:00:00 2001 From: Chandan Singh Date: Wed, 25 Jul 2018 15:01:32 +0100 Subject: Allow source plugins to access previous sources Source plugin implementations can now specify that they need access to previously staged sources by specifying `BST_REQUIRES_PREVIOUS_SOURCES_TRACK` and/or `BST_REQUIRES_PREVIOUS_SOURCES_FETCH`, corresponding to access at `track` and `fetch` times respectively. Fixes #381. Replaces !505. For relevant discussion, see this discussion: https://gitlab.com/BuildStream/buildstream/merge_requests/505#note_83780747 --- buildstream/_loader/loader.py | 5 +- buildstream/_scheduler/queues/fetchqueue.py | 4 +- buildstream/element.py | 10 +- buildstream/source.py | 223 ++++++++++++++++----- tests/sources/previous_source_access.py | 47 +++++ .../previous_source_access/elements/target.bst | 6 + tests/sources/previous_source_access/files/file | 1 + .../plugins/sources/foo_transform.py | 87 ++++++++ tests/sources/previous_source_access/project.conf | 10 + 9 files changed, 337 insertions(+), 56 deletions(-) create mode 100644 tests/sources/previous_source_access.py create mode 100644 tests/sources/previous_source_access/elements/target.bst create mode 100644 tests/sources/previous_source_access/files/file create mode 100644 tests/sources/previous_source_access/plugins/sources/foo_transform.py create mode 100644 tests/sources/previous_source_access/project.conf diff --git a/buildstream/_loader/loader.py b/buildstream/_loader/loader.py index 6e46197ab..275bc20cf 100644 --- a/buildstream/_loader/loader.py +++ b/buildstream/_loader/loader.py @@ -522,14 +522,15 @@ class Loader(): element = Element._new_from_meta(meta_element, platform.artifactcache) element._preflight() - for source in element.sources(): + sources = list(element.sources()) + for idx, source in enumerate(sources): # Handle the case where a subproject needs to be fetched # if source.get_consistency() == Consistency.RESOLVED: if fetch_subprojects: if ticker: ticker(filename, 'Fetching subproject from {} source'.format(source.get_kind())) - source._fetch() + source._fetch(sources[0:idx]) else: detail = "Try fetching the project with `bst fetch {}`".format(filename) raise LoadError(LoadErrorReason.SUBPROJECT_FETCH_NEEDED, diff --git a/buildstream/_scheduler/queues/fetchqueue.py b/buildstream/_scheduler/queues/fetchqueue.py index 265890b7a..bd90a13b6 100644 --- a/buildstream/_scheduler/queues/fetchqueue.py +++ b/buildstream/_scheduler/queues/fetchqueue.py @@ -40,8 +40,10 @@ class FetchQueue(Queue): self._skip_cached = skip_cached def process(self, element): + previous_sources = [] for source in element.sources(): - source._fetch() + source._fetch(previous_sources) + previous_sources.append(source) def status(self, element): # state of dependencies may have changed, recalculate element state diff --git a/buildstream/element.py b/buildstream/element.py index 40cac47cd..a34b1ca36 100644 --- a/buildstream/element.py +++ b/buildstream/element.py @@ -1229,6 +1229,12 @@ class Element(Plugin): # Prepend provenance to the error raise ElementError("{}: {}".format(self, e), reason=e.reason) from e + # Ensure that the first source does not need access to previous soruces + if self.__sources and self.__sources[0]._requires_previous_sources(): + raise ElementError("{}: {} cannot be the first source of an element " + "as it requires access to previous sources" + .format(self, self.__sources[0])) + # Preflight the sources for source in self.sources(): source._preflight() @@ -1272,9 +1278,9 @@ class Element(Plugin): # def _track(self): refs = [] - for source in self.__sources: + for index, source in enumerate(self.__sources): old_ref = source.get_ref() - new_ref = source._track() + new_ref = source._track(self.__sources[0:index]) refs.append((source._get_unique_id(), new_ref)) # Complimentary warning that the new ref will be unused. diff --git a/buildstream/source.py b/buildstream/source.py index d58bfe2a3..7a0a0ec88 100644 --- a/buildstream/source.py +++ b/buildstream/source.py @@ -76,6 +76,39 @@ these methods are mandatory to implement. :ref:`SourceFetcher `. +Accessing previous sources +-------------------------- +*Since: 1.4* + +In the general case, all sources are fetched and tracked independently of one +another. In situations where a source needs to access previous source(s) in +order to perform its own track and/or fetch, following attributes can be set to +request access to previous sources: + +* :attr:`~buildstream.source.Source.BST_REQUIRES_PREVIOUS_SOURCES_TRACK` + + Indicate that access to previous sources is required during track + +* :attr:`~buildstream.source.Source.BST_REQUIRES_PREVIOUS_SOURCES_FETCH` + + Indicate that access to previous sources is required during fetch + +The intended use of such plugins is to fetch external dependencies of other +sources, typically using some kind of package manager, such that all the +dependencies of the original source(s) are available at build time. + +When implementing such a plugin, implementors should adhere to the following +guidelines: + +* Implementations must be able to store the obtained artifacts in a + subdirectory. + +* Implementations must be able to deterministically generate a unique ref, such + that two refs are different if and only if they produce different outputs. + +* Implementations must not introduce host contamination. + + .. _core_source_fetcher: SourceFetcher - Object for fetching individual URLs @@ -92,6 +125,8 @@ mentioned, these methods are mandatory to implement. Fetches the URL associated with this SourceFetcher, optionally taking an alias override. +Class Reference +--------------- """ import os @@ -156,7 +191,7 @@ class SourceFetcher(): ############################################################# # Abstract Methods # ############################################################# - def fetch(self, alias_override=None): + def fetch(self, alias_override=None, **kwargs): """Fetch remote sources and mirror them locally, ensuring at least that the specific reference is cached locally. @@ -209,6 +244,32 @@ class Source(Plugin): __defaults = {} # The defaults from the project __defaults_set = False # Flag, in case there are not defaults at all + BST_REQUIRES_PREVIOUS_SOURCES_TRACK = False + """Whether access to previous sources is required during track + + When set to True: + * all sources listed before this source in the given element will be + fetched before this source is tracked + * Source.track() will be called with an additional keyword argument + `previous_sources_dir` where previous sources will be staged + * this source can not be the first source for an element + + *Since: 1.4* + """ + + BST_REQUIRES_PREVIOUS_SOURCES_FETCH = False + """Whether access to previous sources is required during fetch + + When set to True: + * all sources listed before this source in the given element will be + fetched before this source is fetched + * Source.fetch() will be called with an additional keyword argument + `previous_sources_dir` where previous sources will be staged + * this source can not be the first source for an element + + *Since: 1.4* + """ + def __init__(self, context, project, meta, *, alias_override=None): provenance = _yaml.node_get_provenance(meta.config) super().__init__("{}-{}".format(meta.element_name, meta.element_index), @@ -305,9 +366,15 @@ class Source(Plugin): """ raise ImplError("Source plugin '{}' does not implement set_ref()".format(self.get_kind())) - def track(self): + def track(self, **kwargs): """Resolve a new ref from the plugin's track option + Args: + previous_sources_dir (str): directory where previous sources are staged. + Note that this keyword argument is available only when + :attr:`~buildstream.source.Source.BST_REQUIRES_PREVIOUS_SOURCES_TRACK` + is set to True. + Returns: (simple object): A new internal source reference, or None @@ -326,10 +393,16 @@ class Source(Plugin): # Allow a non implementation return None - def fetch(self): + def fetch(self, **kwargs): """Fetch remote sources and mirror them locally, ensuring at least that the specific reference is cached locally. + Args: + previous_sources_dir (str): directory where previous sources are staged. + Note that this keyword argument is available only when + :attr:`~buildstream.source.Source.BST_REQUIRES_PREVIOUS_SOURCES_FETCH` + is set to True. + Raises: :class:`.SourceError` @@ -519,50 +592,19 @@ class Source(Plugin): # Wrapper function around plugin provided fetch method # - def _fetch(self): - project = self._get_project() - source_fetchers = self.get_source_fetchers() - if source_fetchers: - for fetcher in source_fetchers: - alias = fetcher._get_alias() - success = False - for uri in project.get_alias_uris(alias, first_pass=self.__first_pass): - try: - fetcher.fetch(uri) - # FIXME: Need to consider temporary vs. permanent failures, - # and how this works with retries. - except BstError as e: - last_error = e - continue - success = True - break - if not success: - raise last_error + # Args: + # previous_sources (list): List of Sources listed prior to this source + # + def _fetch(self, previous_sources): + + if self.BST_REQUIRES_PREVIOUS_SOURCES_FETCH: + self.__ensure_previous_sources(previous_sources) + with self.tempdir() as staging_directory: + for src in previous_sources: + src._stage(staging_directory) + self.__do_fetch(previous_sources_dir=self.__ensure_directory(staging_directory)) else: - alias = self._get_alias() - if self.__first_pass: - mirrors = project.first_pass_config.mirrors - else: - mirrors = project.config.mirrors - if not mirrors or not alias: - self.fetch() - return - - context = self._get_context() - source_kind = type(self) - for uri in project.get_alias_uris(alias, first_pass=self.__first_pass): - new_source = source_kind(context, project, self.__meta, - alias_override=(alias, uri)) - new_source._preflight() - try: - new_source.fetch() - # FIXME: Need to consider temporary vs. permanent failures, - # and how this works with retries. - except BstError as e: - last_error = e - continue - return - raise last_error + self.__do_fetch() # Wrapper for stage() api which gives the source # plugin a fully constructed path considering the @@ -773,8 +815,19 @@ class Source(Plugin): # Wrapper for track() # - def _track(self): - new_ref = self.__do_track() + # Args: + # previous_sources (list): List of Sources listed prior to this source + # + def _track(self, previous_sources): + if self.BST_REQUIRES_PREVIOUS_SOURCES_TRACK: + self.__ensure_previous_sources(previous_sources) + with self.tempdir() as staging_directory: + for src in previous_sources: + src._stage(staging_directory) + new_ref = self.__do_track(previous_sources_dir=self.__ensure_directory(staging_directory)) + else: + new_ref = self.__do_track() + current_ref = self.get_ref() if new_ref is None: @@ -786,6 +839,17 @@ class Source(Plugin): return new_ref + # _requires_previous_sources() + # + # If a plugin requires access to previous sources at track or fetch time, + # then it cannot be the first source of an elemenet. + # + # Returns: + # (bool): Whether this source requires access to previous sources + # + def _requires_previous_sources(self): + return self.BST_REQUIRES_PREVIOUS_SOURCES_TRACK or self.BST_REQUIRES_PREVIOUS_SOURCES_FETCH + # Returns the alias if it's defined in the project def _get_alias(self): alias = self.__expected_alias @@ -801,8 +865,54 @@ class Source(Plugin): # Local Private Methods # ############################################################# + # Tries to call fetch for every mirror, stopping once it succeeds + def __do_fetch(self, **kwargs): + project = self._get_project() + source_fetchers = self.get_source_fetchers() + if source_fetchers: + for fetcher in source_fetchers: + alias = fetcher._get_alias() + success = False + for uri in project.get_alias_uris(alias, first_pass=self.__first_pass): + try: + fetcher.fetch(uri) + # FIXME: Need to consider temporary vs. permanent failures, + # and how this works with retries. + except BstError as e: + last_error = e + continue + success = True + break + if not success: + raise last_error + else: + alias = self._get_alias() + if self.__first_pass: + mirrors = project.first_pass_config.mirrors + else: + mirrors = project.config.mirrors + if not mirrors or not alias: + self.fetch(**kwargs) + return + + context = self._get_context() + source_kind = type(self) + for uri in project.get_alias_uris(alias, first_pass=self.__first_pass): + new_source = source_kind(context, project, self.__meta, + alias_override=(alias, uri)) + new_source._preflight() + try: + new_source.fetch(**kwargs) + # FIXME: Need to consider temporary vs. permanent failures, + # and how this works with retries. + except BstError as e: + last_error = e + continue + return + raise last_error + # Tries to call track for every mirror, stopping once it succeeds - def __do_track(self): + def __do_track(self, **kwargs): project = self._get_project() # If there are no mirrors, or no aliases to replace, there's nothing to do here. alias = self._get_alias() @@ -811,7 +921,7 @@ class Source(Plugin): else: mirrors = project.config.mirrors if not mirrors or not alias: - return self.track() + return self.track(**kwargs) context = self._get_context() source_kind = type(self) @@ -823,7 +933,7 @@ class Source(Plugin): alias_override=(alias, uri)) new_source._preflight() try: - ref = new_source.track() + ref = new_source.track(**kwargs) # FIXME: Need to consider temporary vs. permanent failures, # and how this works with retries. except BstError as e: @@ -867,3 +977,14 @@ class Source(Plugin): _yaml.node_final_assertions(config) return config + + # Ensures that previous sources have been tracked and fetched. + # + def __ensure_previous_sources(self, previous_sources): + for index, src in enumerate(previous_sources): + if src.get_consistency() == Consistency.RESOLVED: + src._fetch(previous_sources[0:index]) + elif src.get_consistency() == Consistency.INCONSISTENT: + new_ref = src._track(previous_sources[0:index]) + src._save_ref(new_ref) + src._fetch(previous_sources[0:index]) diff --git a/tests/sources/previous_source_access.py b/tests/sources/previous_source_access.py new file mode 100644 index 000000000..f7045383c --- /dev/null +++ b/tests/sources/previous_source_access.py @@ -0,0 +1,47 @@ +import os +import pytest + +from tests.testutils import cli + +DATA_DIR = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + 'previous_source_access' +) + + +################################################################## +# Tests # +################################################################## +# Test that plugins can access data from previous sources +@pytest.mark.datafiles(DATA_DIR) +def test_custom_transform_source(cli, tmpdir, datafiles): + project = os.path.join(datafiles.dirname, datafiles.basename) + + # Ensure we can track + result = cli.run(project=project, args=[ + 'track', 'target.bst' + ]) + result.assert_success() + + # Ensure we can fetch + result = cli.run(project=project, args=[ + 'fetch', 'target.bst' + ]) + result.assert_success() + + # Ensure we get correct output from foo_transform + result = cli.run(project=project, args=[ + 'build', 'target.bst' + ]) + destpath = os.path.join(cli.directory, 'checkout') + result = cli.run(project=project, args=[ + 'checkout', 'target.bst', destpath + ]) + result.assert_success() + # Assert that files from both sources exist, and that they have + # the same content + assert os.path.exists(os.path.join(destpath, 'file')) + assert os.path.exists(os.path.join(destpath, 'filetransform')) + with open(os.path.join(destpath, 'file')) as file1: + with open(os.path.join(destpath, 'filetransform')) as file2: + assert file1.read() == file2.read() diff --git a/tests/sources/previous_source_access/elements/target.bst b/tests/sources/previous_source_access/elements/target.bst new file mode 100644 index 000000000..c9d3b9bb6 --- /dev/null +++ b/tests/sources/previous_source_access/elements/target.bst @@ -0,0 +1,6 @@ +kind: import + +sources: +- kind: local + path: files/file +- kind: foo_transform diff --git a/tests/sources/previous_source_access/files/file b/tests/sources/previous_source_access/files/file new file mode 100644 index 000000000..980a0d5f1 --- /dev/null +++ b/tests/sources/previous_source_access/files/file @@ -0,0 +1 @@ +Hello World! diff --git a/tests/sources/previous_source_access/plugins/sources/foo_transform.py b/tests/sources/previous_source_access/plugins/sources/foo_transform.py new file mode 100644 index 000000000..7101bfd24 --- /dev/null +++ b/tests/sources/previous_source_access/plugins/sources/foo_transform.py @@ -0,0 +1,87 @@ +""" +foo_transform - transform "file" from previous sources into "filetransform" +=========================================================================== + +This is a test source plugin that looks for a file named "file" staged by +previous sources, and copies its contents to a file called "filetransform". + +""" + +import os +import hashlib + +from buildstream import Consistency, Source, SourceError, utils + + +class FooTransformSource(Source): + + # We need access to previous both at track time and fetch time + BST_REQUIRES_PREVIOUS_SOURCES_TRACK = True + BST_REQUIRES_PREVIOUS_SOURCES_FETCH = True + + @property + def mirror(self): + """Directory where this source should stage its files + + """ + path = os.path.join(self.get_mirror_directory(), self.name, + self.ref.strip()) + os.makedirs(path, exist_ok=True) + return path + + def configure(self, node): + self.node_validate(node, ['ref'] + Source.COMMON_CONFIG_KEYS) + self.ref = self.node_get_member(node, str, 'ref', None) + + def preflight(self): + pass + + def get_unique_key(self): + return (self.ref,) + + def get_consistency(self): + if self.ref is None: + return Consistency.INCONSISTENT + # If we have a file called "filetransform", verify that its checksum + # matches our ref. Otherwise, it resolved but not cached. + fpath = os.path.join(self.mirror, 'filetransform') + try: + with open(fpath, 'rb') as f: + if hashlib.sha256(f.read()).hexdigest() == self.ref.strip(): + return Consistency.CACHED + except Exception: + pass + return Consistency.RESOLVED + + def get_ref(self): + return self.ref + + def set_ref(self, ref, node): + self.ref = node['ref'] = ref + + def track(self, previous_sources_dir): + # Store the checksum of the file from previous source as our ref + fpath = os.path.join(previous_sources_dir, 'file') + with open(fpath, 'rb') as f: + return hashlib.sha256(f.read()).hexdigest() + + def fetch(self, previous_sources_dir): + fpath = os.path.join(previous_sources_dir, 'file') + # Verify that the checksum of the file from previous source matches + # our ref + with open(fpath, 'rb') as f: + if hashlib.sha256(f.read()).hexdigest() != self.ref.strip(): + raise SourceError("Element references do not match") + + # Copy "file" as "filetransform" + newfpath = os.path.join(self.mirror, 'filetransform') + utils.safe_copy(fpath, newfpath) + + def stage(self, directory): + # Simply stage the "filetransform" file + utils.safe_copy(os.path.join(self.mirror, 'filetransform'), + os.path.join(directory, 'filetransform')) + + +def setup(): + return FooTransformSource diff --git a/tests/sources/previous_source_access/project.conf b/tests/sources/previous_source_access/project.conf new file mode 100644 index 000000000..1749b3dba --- /dev/null +++ b/tests/sources/previous_source_access/project.conf @@ -0,0 +1,10 @@ +# Project with local source plugins +name: foo + +element-path: elements + +plugins: +- origin: local + path: plugins/sources + sources: + foo_transform: 0 -- cgit v1.2.1 From a7a8265090eaba5cb99fec7ad9acbadb84e998de Mon Sep 17 00:00:00 2001 From: Chandan Singh Date: Mon, 30 Jul 2018 19:41:57 +0100 Subject: Add pip source plugin `pip` source plugin can stage python packages that are either specified directly in the element definition or picked up from `requirements.txt` from previous sources. In order to support the latter use-case (which is also the primary motivation for this plugin), this plugin requires access to previous sources and hence is an example of a Source Transform source. Also, bump `BST_FORMAT_VERSION` as this patch adds a new core plugin. --- buildstream/_versions.py | 2 +- buildstream/plugins/sources/pip.py | 237 +++++++++++++++++++++ buildstream/source.py | 8 +- doc/source/core_plugins.rst | 1 + tests/cachekey/project/sources/pip1.bst | 12 ++ tests/cachekey/project/sources/pip1.expected | 1 + tests/cachekey/project/target.bst | 1 + tests/cachekey/project/target.expected | 2 +- tests/integration/pip.py | 66 ------ tests/integration/pip_element.py | 66 ++++++ tests/integration/pip_source.py | 99 +++++++++ tests/integration/project/files/pip-source/app1.py | 11 + .../project/files/pip-source/myreqs.txt | 1 + .../project/files/pypi-repo/app2/App2-0.1.tar.gz | Bin 0 -> 769 bytes .../project/files/pypi-repo/app2/index.html | 8 + .../files/pypi-repo/hellolib/HelloLib-0.1.tar.gz | Bin 0 -> 734 bytes .../project/files/pypi-repo/hellolib/index.html | 8 + tests/sources/pip.py | 47 ++++ tests/sources/pip/first-source-pip/target.bst | 6 + tests/sources/pip/no-packages/file | 1 + tests/sources/pip/no-packages/target.bst | 6 + tests/sources/pip/no-ref/file | 1 + tests/sources/pip/no-ref/target.bst | 8 + 23 files changed, 520 insertions(+), 72 deletions(-) create mode 100644 buildstream/plugins/sources/pip.py create mode 100644 tests/cachekey/project/sources/pip1.bst create mode 100644 tests/cachekey/project/sources/pip1.expected delete mode 100644 tests/integration/pip.py create mode 100644 tests/integration/pip_element.py create mode 100644 tests/integration/pip_source.py create mode 100644 tests/integration/project/files/pip-source/app1.py create mode 100644 tests/integration/project/files/pip-source/myreqs.txt create mode 100644 tests/integration/project/files/pypi-repo/app2/App2-0.1.tar.gz create mode 100644 tests/integration/project/files/pypi-repo/app2/index.html create mode 100644 tests/integration/project/files/pypi-repo/hellolib/HelloLib-0.1.tar.gz create mode 100644 tests/integration/project/files/pypi-repo/hellolib/index.html create mode 100644 tests/sources/pip.py create mode 100644 tests/sources/pip/first-source-pip/target.bst create mode 100644 tests/sources/pip/no-packages/file create mode 100644 tests/sources/pip/no-packages/target.bst create mode 100644 tests/sources/pip/no-ref/file create mode 100644 tests/sources/pip/no-ref/target.bst diff --git a/buildstream/_versions.py b/buildstream/_versions.py index a722e11a5..713cb9d67 100644 --- a/buildstream/_versions.py +++ b/buildstream/_versions.py @@ -23,7 +23,7 @@ # This version is bumped whenever enhancements are made # to the `project.conf` format or the core element format. # -BST_FORMAT_VERSION = 15 +BST_FORMAT_VERSION = 16 # The base BuildStream artifact version diff --git a/buildstream/plugins/sources/pip.py b/buildstream/plugins/sources/pip.py new file mode 100644 index 000000000..18e65c73d --- /dev/null +++ b/buildstream/plugins/sources/pip.py @@ -0,0 +1,237 @@ +# +# Copyright 2018 Bloomberg Finance LP +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . +# +# Authors: +# Chandan Singh + +""" +pip - stage python packages using pip +===================================== + +**Host depndencies:** + + * ``pip`` python module + +This plugin will download source distributions for specified packages using +``pip`` but will not install them. It is expected that the elements using this +source will install the downloaded packages. + +Downloaded tarballs will be stored in a directory called ".bst_pip_downloads". + +**Usage:** + +.. code:: yaml + + # Specify the pip source kind + kind: pip + + # Optionally specify index url, defaults to PyPi + # This url is used to discover new versions of packages and download them + # Projects intending to mirror their sources to a permanent location should + # use an aliased url, and declare the alias in the project configuration + url: https://mypypi.example.com/simple + + # Optionally specify the path to requirements files + # Note that either 'requirements-files' or 'packages' must be defined + requirements-files: + - requirements.txt + + # Optionally specify a list of additional packages + # Note that either 'requirements-files' or 'packages' must be defined + packages: + - flake8 + + # Optionally specify a relative staging directory + directory: path/to/stage + + # Specify the ref. It is a list of strings of format + # "==", separated by "\\n". + # Usually this will be contents of a requirements.txt file where all + # package versions have been frozen. + ref: "flake8==3.5.0\\nmccabe==0.6.1\\npkg-resources==0.0.0\\npycodestyle==2.3.1\\npyflakes==1.6.0" + +.. note:: + + The ``pip`` plugin is available since :ref:`format version 16 ` + +""" + +import errno +import hashlib +import os +import re + +from buildstream import Consistency, Source, SourceError, utils + +_OUTPUT_DIRNAME = '.bst_pip_downloads' +_PYPI_INDEX_URL = 'https://pypi.org/simple/' + +# Used only for finding pip command +_PYTHON_VERSIONS = [ + 'python2.7', + 'python3.0', + 'python3.1', + 'python3.2', + 'python3.3', + 'python3.4', + 'python3.5', + 'python3.6', + 'python3.7', +] + +# List of allowed extensions taken from +# https://docs.python.org/3/distutils/sourcedist.html. +# Names of source distribution archives must be of the form +# '%{package-name}-%{version}.%{extension}'. +_SDIST_RE = re.compile( + r'^([a-zA-Z0-9]+?)-(.+).(?:tar|tar.bz2|tar.gz|tar.xz|tar.Z|zip)$', + re.IGNORECASE) + + +class PipSource(Source): + # pylint: disable=attribute-defined-outside-init + + # We need access to previous sources at track time to use requirements.txt + # but not at fetch time as self.ref should contain sufficient information + # for this plugin + BST_REQUIRES_PREVIOUS_SOURCES_TRACK = True + + def configure(self, node): + self.node_validate(node, ['url', 'packages', 'ref', 'requirements-files'] + + Source.COMMON_CONFIG_KEYS) + self.ref = self.node_get_member(node, str, 'ref', None) + self.original_url = self.node_get_member(node, str, 'url', _PYPI_INDEX_URL) + self.index_url = self.translate_url(self.original_url) + self.packages = self.node_get_member(node, list, 'packages', []) + self.requirements_files = self.node_get_member(node, list, 'requirements-files', []) + + if not (self.packages or self.requirements_files): + raise SourceError("{}: Either 'packages' or 'requirements-files' must be specified". format(self)) + + def preflight(self): + # Try to find a pip version that supports download command + self.host_pip = None + for python in reversed(_PYTHON_VERSIONS): + try: + host_python = utils.get_host_tool(python) + rc = self.call([host_python, '-m', 'pip', 'download', '--help']) + if rc == 0: + self.host_pip = [host_python, '-m', 'pip'] + break + except utils.ProgramNotFoundError: + pass + + if self.host_pip is None: + raise SourceError("{}: Unable to find a suitable pip command".format(self)) + + def get_unique_key(self): + return [self.original_url, self.ref] + + def get_consistency(self): + if not self.ref: + return Consistency.INCONSISTENT + if os.path.exists(self._mirror) and os.listdir(self._mirror): + return Consistency.CACHED + return Consistency.RESOLVED + + def get_ref(self): + return self.ref + + def load_ref(self, node): + self.ref = self.node_get_member(node, str, 'ref', None) + + def set_ref(self, ref, node): + node['ref'] = self.ref = ref + + def track(self, previous_sources_dir): + # XXX pip does not offer any public API other than the CLI tool so it + # is not feasible to correctly parse the requirements file or to check + # which package versions pip is going to install. + # See https://pip.pypa.io/en/stable/user_guide/#using-pip-from-your-program + # for details. + # As a result, we have to wastefully install the packages during track. + with self.tempdir() as tmpdir: + install_args = self.host_pip + ['download', + '--no-binary', ':all:', + '--index-url', self.index_url, + '--dest', tmpdir] + for requirement_file in self.requirements_files: + fpath = os.path.join(previous_sources_dir, requirement_file) + install_args += ['-r', fpath] + install_args += self.packages + + self.call(install_args, fail="Failed to install python packages") + reqs = self._parse_sdist_names(tmpdir) + + return '\n'.join(["{}=={}".format(pkg, ver) for pkg, ver in reqs]) + + def fetch(self): + with self.tempdir() as tmpdir: + packages = self.ref.strip().split('\n') + package_dir = os.path.join(tmpdir, 'packages') + os.makedirs(package_dir) + self.call(self.host_pip + ['download', + '--no-binary', ':all:', + '--index-url', self.index_url, + '--dest', package_dir] + packages, + fail="Failed to install python packages: {}".format(packages)) + + # If the mirror directory already exists, assume that some other + # process has fetched the sources before us and ensure that we do + # not raise an error in that case. + try: + os.makedirs(self._mirror) + os.rename(package_dir, self._mirror) + except FileExistsError: + return + except OSError as e: + if e.errno != errno.ENOTEMPTY: + raise + + def stage(self, directory): + with self.timed_activity("Staging Python packages", silent_nested=True): + utils.copy_files(self._mirror, os.path.join(directory, _OUTPUT_DIRNAME)) + + # Directory where this source should stage its files + # + @property + def _mirror(self): + if not self.ref: + return None + return os.path.join(self.get_mirror_directory(), + utils.url_directory_name(self.original_url), + hashlib.sha256(self.ref.encode()).hexdigest()) + + # Parse names of downloaded source distributions + # + # Args: + # basedir (str): Directory containing source distribution archives + # + # Returns: + # (list): List of (package_name, version) tuples in sorted order + # + def _parse_sdist_names(self, basedir): + reqs = [] + for f in os.listdir(basedir): + pkg_match = _SDIST_RE.match(f) + if pkg_match: + reqs.append(pkg_match.groups()) + + return sorted(reqs) + + +def setup(): + return PipSource diff --git a/buildstream/source.py b/buildstream/source.py index 7a0a0ec88..9822beeec 100644 --- a/buildstream/source.py +++ b/buildstream/source.py @@ -982,9 +982,9 @@ class Source(Plugin): # def __ensure_previous_sources(self, previous_sources): for index, src in enumerate(previous_sources): + # BuildStream should track sources in the order they appear so + # previous sources should never be in an inconsistent state + assert src.get_consistency() != Consistency.INCONSISTENT + if src.get_consistency() == Consistency.RESOLVED: src._fetch(previous_sources[0:index]) - elif src.get_consistency() == Consistency.INCONSISTENT: - new_ref = src._track(previous_sources[0:index]) - src._save_ref(new_ref) - src._fetch(previous_sources[0:index]) diff --git a/doc/source/core_plugins.rst b/doc/source/core_plugins.rst index da7c607c8..c82ffe52f 100644 --- a/doc/source/core_plugins.rst +++ b/doc/source/core_plugins.rst @@ -58,6 +58,7 @@ Sources sources/ostree sources/patch sources/deb + sources/pip External plugins diff --git a/tests/cachekey/project/sources/pip1.bst b/tests/cachekey/project/sources/pip1.bst new file mode 100644 index 000000000..ee69efad6 --- /dev/null +++ b/tests/cachekey/project/sources/pip1.bst @@ -0,0 +1,12 @@ +kind: import + +sources: +- kind: git + url: https://example.com/foo/foobar.git + ref: b99955530263172ed1beae52aed7a33885ef781f +- kind: pip + url: https://pypi.example.com/simple + packages: + - horses + - ponies + ref: 'horses==0.0.1\nponies==0.0.2' diff --git a/tests/cachekey/project/sources/pip1.expected b/tests/cachekey/project/sources/pip1.expected new file mode 100644 index 000000000..11d7c5fae --- /dev/null +++ b/tests/cachekey/project/sources/pip1.expected @@ -0,0 +1 @@ +880d0dc27d6683725cfd68d60156058115a9a53793b14b727fc6d0588a473763 \ No newline at end of file diff --git a/tests/cachekey/project/target.bst b/tests/cachekey/project/target.bst index 174215a49..d96645da8 100644 --- a/tests/cachekey/project/target.bst +++ b/tests/cachekey/project/target.bst @@ -13,6 +13,7 @@ depends: - sources/patch1.bst - sources/patch2.bst - sources/patch3.bst +- sources/pip1.bst - sources/remote1.bst - sources/remote2.bst - sources/tar1.bst diff --git a/tests/cachekey/project/target.expected b/tests/cachekey/project/target.expected index d6d99895a..3c1cc261f 100644 --- a/tests/cachekey/project/target.expected +++ b/tests/cachekey/project/target.expected @@ -1 +1 @@ -e045df890262f7b7c663e64ad0bfba428d9d80ec514df3ddb13313d4ef669b73 \ No newline at end of file +09620aa58875d96611d22632b7585a0f22f88f5ecca6f5d1915d3e529d036bd8 \ No newline at end of file diff --git a/tests/integration/pip.py b/tests/integration/pip.py deleted file mode 100644 index 6c6de8bf8..000000000 --- a/tests/integration/pip.py +++ /dev/null @@ -1,66 +0,0 @@ -import os -import sys -import pytest - -from buildstream import _yaml - -from tests.testutils import cli_integration as cli -from tests.testutils.integration import assert_contains - - -pytestmark = pytest.mark.integration - - -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project" -) - - -@pytest.mark.datafiles(DATA_DIR) -def test_pip_build(cli, tmpdir, datafiles): - project = os.path.join(datafiles.dirname, datafiles.basename) - checkout = os.path.join(cli.directory, 'checkout') - element_path = os.path.join(project, 'elements') - element_name = 'pip/hello.bst' - - element = { - 'kind': 'pip', - 'variables': { - 'pip': 'pip3' - }, - 'depends': [{ - 'filename': 'base.bst' - }], - 'sources': [{ - 'kind': 'tar', - 'url': 'file://{}/files/hello.tar.xz'.format(project), - 'ref': 'ad96570b552498807abec33c06210bf68378d854ced6753b77916c5ed517610d' - - }] - } - os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True) - _yaml.dump(element, os.path.join(element_path, element_name)) - - result = cli.run(project=project, args=['build', element_name]) - assert result.exit_code == 0 - - result = cli.run(project=project, args=['checkout', element_name, checkout]) - assert result.exit_code == 0 - - assert_contains(checkout, ['/usr', '/usr/lib', '/usr/bin', - '/usr/bin/hello', '/usr/lib/python3.6']) - - -# Test running an executable built with pip -@pytest.mark.datafiles(DATA_DIR) -def test_pip_run(cli, tmpdir, datafiles): - # Create and build our test element - test_pip_build(cli, tmpdir, datafiles) - - project = os.path.join(datafiles.dirname, datafiles.basename) - element_name = 'pip/hello.bst' - - result = cli.run(project=project, args=['shell', element_name, '/usr/bin/hello']) - assert result.exit_code == 0 - assert result.output == 'Hello, world!\n' diff --git a/tests/integration/pip_element.py b/tests/integration/pip_element.py new file mode 100644 index 000000000..6c6de8bf8 --- /dev/null +++ b/tests/integration/pip_element.py @@ -0,0 +1,66 @@ +import os +import sys +import pytest + +from buildstream import _yaml + +from tests.testutils import cli_integration as cli +from tests.testutils.integration import assert_contains + + +pytestmark = pytest.mark.integration + + +DATA_DIR = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + "project" +) + + +@pytest.mark.datafiles(DATA_DIR) +def test_pip_build(cli, tmpdir, datafiles): + project = os.path.join(datafiles.dirname, datafiles.basename) + checkout = os.path.join(cli.directory, 'checkout') + element_path = os.path.join(project, 'elements') + element_name = 'pip/hello.bst' + + element = { + 'kind': 'pip', + 'variables': { + 'pip': 'pip3' + }, + 'depends': [{ + 'filename': 'base.bst' + }], + 'sources': [{ + 'kind': 'tar', + 'url': 'file://{}/files/hello.tar.xz'.format(project), + 'ref': 'ad96570b552498807abec33c06210bf68378d854ced6753b77916c5ed517610d' + + }] + } + os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True) + _yaml.dump(element, os.path.join(element_path, element_name)) + + result = cli.run(project=project, args=['build', element_name]) + assert result.exit_code == 0 + + result = cli.run(project=project, args=['checkout', element_name, checkout]) + assert result.exit_code == 0 + + assert_contains(checkout, ['/usr', '/usr/lib', '/usr/bin', + '/usr/bin/hello', '/usr/lib/python3.6']) + + +# Test running an executable built with pip +@pytest.mark.datafiles(DATA_DIR) +def test_pip_run(cli, tmpdir, datafiles): + # Create and build our test element + test_pip_build(cli, tmpdir, datafiles) + + project = os.path.join(datafiles.dirname, datafiles.basename) + element_name = 'pip/hello.bst' + + result = cli.run(project=project, args=['shell', element_name, '/usr/bin/hello']) + assert result.exit_code == 0 + assert result.output == 'Hello, world!\n' diff --git a/tests/integration/pip_source.py b/tests/integration/pip_source.py new file mode 100644 index 000000000..fc5b56a7c --- /dev/null +++ b/tests/integration/pip_source.py @@ -0,0 +1,99 @@ +import os +import pytest + +from buildstream import _yaml + +from tests.testutils import cli_integration as cli +from tests.testutils.integration import assert_contains + + +pytestmark = pytest.mark.integration + + +DATA_DIR = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + "project" +) + + +@pytest.mark.datafiles(DATA_DIR) +def test_pip_source_import(cli, tmpdir, datafiles): + project = os.path.join(datafiles.dirname, datafiles.basename) + checkout = os.path.join(cli.directory, 'checkout') + element_path = os.path.join(project, 'elements') + element_name = 'pip/hello.bst' + + element = { + 'kind': 'import', + 'sources': [ + { + 'kind': 'local', + 'path': 'files/pip-source' + }, + { + 'kind': 'pip', + 'url': 'file://{}'.format(os.path.realpath(os.path.join(project, 'files', 'pypi-repo'))), + 'requirements-files': ['myreqs.txt'], + 'packages': ['app2'] + } + ] + } + os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True) + _yaml.dump(element, os.path.join(element_path, element_name)) + + result = cli.run(project=project, args=['track', element_name]) + assert result.exit_code == 0 + + result = cli.run(project=project, args=['build', element_name]) + assert result.exit_code == 0 + + result = cli.run(project=project, args=['checkout', element_name, checkout]) + assert result.exit_code == 0 + + assert_contains(checkout, ['/.bst_pip_downloads', + '/.bst_pip_downloads/HelloLib-0.1.tar.gz', + '/.bst_pip_downloads/App2-0.1.tar.gz']) + + +@pytest.mark.datafiles(DATA_DIR) +def test_pip_source_build(cli, tmpdir, datafiles): + project = os.path.join(datafiles.dirname, datafiles.basename) + element_path = os.path.join(project, 'elements') + element_name = 'pip/hello.bst' + + element = { + 'kind': 'manual', + 'depends': ['base.bst'], + 'sources': [ + { + 'kind': 'local', + 'path': 'files/pip-source' + }, + { + 'kind': 'pip', + 'url': 'file://{}'.format(os.path.realpath(os.path.join(project, 'files', 'pypi-repo'))), + 'requirements-files': ['myreqs.txt'], + 'packages': ['app2'] + } + ], + 'config': { + 'install-commands': [ + 'pip3 install --no-index --prefix %{install-root}/usr .bst_pip_downloads/*.tar.gz', + 'chmod +x app1.py', + 'install app1.py %{install-root}/usr/bin/' + ] + } + } + os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True) + _yaml.dump(element, os.path.join(element_path, element_name)) + + result = cli.run(project=project, args=['track', element_name]) + assert result.exit_code == 0 + + result = cli.run(project=project, args=['build', element_name]) + assert result.exit_code == 0 + + result = cli.run(project=project, args=['shell', element_name, '/usr/bin/app1.py']) + assert result.exit_code == 0 + assert result.output == """Hello App1! +""" diff --git a/tests/integration/project/files/pip-source/app1.py b/tests/integration/project/files/pip-source/app1.py new file mode 100644 index 000000000..ab1005ba4 --- /dev/null +++ b/tests/integration/project/files/pip-source/app1.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python3 + +from hellolib import hello + + +def main(): + hello('App1') + + +if __name__ == '__main__': + main() diff --git a/tests/integration/project/files/pip-source/myreqs.txt b/tests/integration/project/files/pip-source/myreqs.txt new file mode 100644 index 000000000..c805aae53 --- /dev/null +++ b/tests/integration/project/files/pip-source/myreqs.txt @@ -0,0 +1 @@ +hellolib diff --git a/tests/integration/project/files/pypi-repo/app2/App2-0.1.tar.gz b/tests/integration/project/files/pypi-repo/app2/App2-0.1.tar.gz new file mode 100644 index 000000000..86cb43cfe Binary files /dev/null and b/tests/integration/project/files/pypi-repo/app2/App2-0.1.tar.gz differ diff --git a/tests/integration/project/files/pypi-repo/app2/index.html b/tests/integration/project/files/pypi-repo/app2/index.html new file mode 100644 index 000000000..5bc72e47c --- /dev/null +++ b/tests/integration/project/files/pypi-repo/app2/index.html @@ -0,0 +1,8 @@ + + + Links for app1 + + + App2-0.1.tar.gz
+ + diff --git a/tests/integration/project/files/pypi-repo/hellolib/HelloLib-0.1.tar.gz b/tests/integration/project/files/pypi-repo/hellolib/HelloLib-0.1.tar.gz new file mode 100644 index 000000000..3b0884c66 Binary files /dev/null and b/tests/integration/project/files/pypi-repo/hellolib/HelloLib-0.1.tar.gz differ diff --git a/tests/integration/project/files/pypi-repo/hellolib/index.html b/tests/integration/project/files/pypi-repo/hellolib/index.html new file mode 100644 index 000000000..eb9935c7f --- /dev/null +++ b/tests/integration/project/files/pypi-repo/hellolib/index.html @@ -0,0 +1,8 @@ + + + Links for app1 + + + HelloLib-0.1.tar.gz
+ + diff --git a/tests/sources/pip.py b/tests/sources/pip.py new file mode 100644 index 000000000..8b4c213dc --- /dev/null +++ b/tests/sources/pip.py @@ -0,0 +1,47 @@ +import os +import pytest + +from buildstream._exceptions import ErrorDomain +from buildstream import _yaml +from tests.testutils import cli + +DATA_DIR = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + 'pip', +) + + +def generate_project(project_dir, tmpdir): + project_file = os.path.join(project_dir, "project.conf") + _yaml.dump({'name': 'foo'}, project_file) + + +# Test that without ref, consistency is set appropriately. +@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-ref')) +def test_no_ref(cli, tmpdir, datafiles): + project = os.path.join(datafiles.dirname, datafiles.basename) + generate_project(project, tmpdir) + assert cli.get_element_state(project, 'target.bst') == 'no reference' + + +# Test that pip is not allowed to be the first source +@pytest.mark.datafiles(os.path.join(DATA_DIR, 'first-source-pip')) +def test_first_source(cli, tmpdir, datafiles): + project = os.path.join(datafiles.dirname, datafiles.basename) + generate_project(project, tmpdir) + result = cli.run(project=project, args=[ + 'show', 'target.bst' + ]) + result.assert_main_error(ErrorDomain.ELEMENT, None) + + +# Test that error is raised when neither packges nor requirements files +# have been specified +@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-packages')) +def test_no_packages(cli, tmpdir, datafiles): + project = os.path.join(datafiles.dirname, datafiles.basename) + generate_project(project, tmpdir) + result = cli.run(project=project, args=[ + 'show', 'target.bst' + ]) + result.assert_main_error(ErrorDomain.SOURCE, None) diff --git a/tests/sources/pip/first-source-pip/target.bst b/tests/sources/pip/first-source-pip/target.bst new file mode 100644 index 000000000..e5f20ab0b --- /dev/null +++ b/tests/sources/pip/first-source-pip/target.bst @@ -0,0 +1,6 @@ +kind: import +description: pip should not be allowed to be the first source +sources: +- kind: pip + packages: + - flake8 diff --git a/tests/sources/pip/no-packages/file b/tests/sources/pip/no-packages/file new file mode 100644 index 000000000..980a0d5f1 --- /dev/null +++ b/tests/sources/pip/no-packages/file @@ -0,0 +1 @@ +Hello World! diff --git a/tests/sources/pip/no-packages/target.bst b/tests/sources/pip/no-packages/target.bst new file mode 100644 index 000000000..0d8b948c4 --- /dev/null +++ b/tests/sources/pip/no-packages/target.bst @@ -0,0 +1,6 @@ +kind: import +description: The kind of this element is irrelevant. +sources: +- kind: local + path: file +- kind: pip diff --git a/tests/sources/pip/no-ref/file b/tests/sources/pip/no-ref/file new file mode 100644 index 000000000..980a0d5f1 --- /dev/null +++ b/tests/sources/pip/no-ref/file @@ -0,0 +1 @@ +Hello World! diff --git a/tests/sources/pip/no-ref/target.bst b/tests/sources/pip/no-ref/target.bst new file mode 100644 index 000000000..ec450b7ef --- /dev/null +++ b/tests/sources/pip/no-ref/target.bst @@ -0,0 +1,8 @@ +kind: import +description: The kind of this element is irrelevant. +sources: +- kind: local + path: file +- kind: pip + packages: + - flake8 -- cgit v1.2.1 From 4a2dd6afe5e04c8f12a2680a09a1225831565731 Mon Sep 17 00:00:00 2001 From: Chandan Singh Date: Wed, 15 Aug 2018 11:21:30 +0100 Subject: Add NEWS entry for Source Transform and pip source --- NEWS | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/NEWS b/NEWS index e1f5b9f0f..bc35ff79a 100644 --- a/NEWS +++ b/NEWS @@ -17,6 +17,13 @@ buildstream 1.3.1 to avoid having to specify the dependency type for every entry in 'depends'. + o Source plugins may now request access access to previous during track and + fetch by setting `BST_REQUIRES_PREVIOUS_SOURCES_TRACK` and/or + `BST_REQUIRES_PREVIOUS_SOURCES_FETCH` attributes. + + o Add new `pip` source plugin for downloading python packages using pip, + based on requirements files from previous sources. + ================= buildstream 1.1.5 -- cgit v1.2.1