diff options
22 files changed, 454 insertions, 6 deletions
diff --git a/buildstream/_versions.py b/buildstream/_versions.py index eddb34fc6..bbb43000e 100644 --- a/buildstream/_versions.py +++ b/buildstream/_versions.py @@ -23,7 +23,7 @@ # This version is bumped whenever enhancements are made # to the `project.conf` format or the core element format. # -BST_FORMAT_VERSION = 12 +BST_FORMAT_VERSION = 13 # The base BuildStream artifact version diff --git a/buildstream/plugins/sources/pip.py b/buildstream/plugins/sources/pip.py new file mode 100644 index 000000000..18e65c73d --- /dev/null +++ b/buildstream/plugins/sources/pip.py @@ -0,0 +1,237 @@ +# +# Copyright 2018 Bloomberg Finance LP +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Chandan Singh <csingh43@bloomberg.net> + +""" +pip - stage python packages using pip +===================================== + +**Host depndencies:** + + * ``pip`` python module + +This plugin will download source distributions for specified packages using +``pip`` but will not install them. It is expected that the elements using this +source will install the downloaded packages. + +Downloaded tarballs will be stored in a directory called ".bst_pip_downloads". + +**Usage:** + +.. code:: yaml + + # Specify the pip source kind + kind: pip + + # Optionally specify index url, defaults to PyPi + # This url is used to discover new versions of packages and download them + # Projects intending to mirror their sources to a permanent location should + # use an aliased url, and declare the alias in the project configuration + url: https://mypypi.example.com/simple + + # Optionally specify the path to requirements files + # Note that either 'requirements-files' or 'packages' must be defined + requirements-files: + - requirements.txt + + # Optionally specify a list of additional packages + # Note that either 'requirements-files' or 'packages' must be defined + packages: + - flake8 + + # Optionally specify a relative staging directory + directory: path/to/stage + + # Specify the ref. It is a list of strings of format + # "<package-name>==<version>", separated by "\\n". + # Usually this will be contents of a requirements.txt file where all + # package versions have been frozen. + ref: "flake8==3.5.0\\nmccabe==0.6.1\\npkg-resources==0.0.0\\npycodestyle==2.3.1\\npyflakes==1.6.0" + +.. note:: + + The ``pip`` plugin is available since :ref:`format version 16 <project_format_version>` + +""" + +import errno +import hashlib +import os +import re + +from buildstream import Consistency, Source, SourceError, utils + +_OUTPUT_DIRNAME = '.bst_pip_downloads' +_PYPI_INDEX_URL = 'https://pypi.org/simple/' + +# Used only for finding pip command +_PYTHON_VERSIONS = [ + 'python2.7', + 'python3.0', + 'python3.1', + 'python3.2', + 'python3.3', + 'python3.4', + 'python3.5', + 'python3.6', + 'python3.7', +] + +# List of allowed extensions taken from +# https://docs.python.org/3/distutils/sourcedist.html. +# Names of source distribution archives must be of the form +# '%{package-name}-%{version}.%{extension}'. +_SDIST_RE = re.compile( + r'^([a-zA-Z0-9]+?)-(.+).(?:tar|tar.bz2|tar.gz|tar.xz|tar.Z|zip)$', + re.IGNORECASE) + + +class PipSource(Source): + # pylint: disable=attribute-defined-outside-init + + # We need access to previous sources at track time to use requirements.txt + # but not at fetch time as self.ref should contain sufficient information + # for this plugin + BST_REQUIRES_PREVIOUS_SOURCES_TRACK = True + + def configure(self, node): + self.node_validate(node, ['url', 'packages', 'ref', 'requirements-files'] + + Source.COMMON_CONFIG_KEYS) + self.ref = self.node_get_member(node, str, 'ref', None) + self.original_url = self.node_get_member(node, str, 'url', _PYPI_INDEX_URL) + self.index_url = self.translate_url(self.original_url) + self.packages = self.node_get_member(node, list, 'packages', []) + self.requirements_files = self.node_get_member(node, list, 'requirements-files', []) + + if not (self.packages or self.requirements_files): + raise SourceError("{}: Either 'packages' or 'requirements-files' must be specified". format(self)) + + def preflight(self): + # Try to find a pip version that supports download command + self.host_pip = None + for python in reversed(_PYTHON_VERSIONS): + try: + host_python = utils.get_host_tool(python) + rc = self.call([host_python, '-m', 'pip', 'download', '--help']) + if rc == 0: + self.host_pip = [host_python, '-m', 'pip'] + break + except utils.ProgramNotFoundError: + pass + + if self.host_pip is None: + raise SourceError("{}: Unable to find a suitable pip command".format(self)) + + def get_unique_key(self): + return [self.original_url, self.ref] + + def get_consistency(self): + if not self.ref: + return Consistency.INCONSISTENT + if os.path.exists(self._mirror) and os.listdir(self._mirror): + return Consistency.CACHED + return Consistency.RESOLVED + + def get_ref(self): + return self.ref + + def load_ref(self, node): + self.ref = self.node_get_member(node, str, 'ref', None) + + def set_ref(self, ref, node): + node['ref'] = self.ref = ref + + def track(self, previous_sources_dir): + # XXX pip does not offer any public API other than the CLI tool so it + # is not feasible to correctly parse the requirements file or to check + # which package versions pip is going to install. + # See https://pip.pypa.io/en/stable/user_guide/#using-pip-from-your-program + # for details. + # As a result, we have to wastefully install the packages during track. + with self.tempdir() as tmpdir: + install_args = self.host_pip + ['download', + '--no-binary', ':all:', + '--index-url', self.index_url, + '--dest', tmpdir] + for requirement_file in self.requirements_files: + fpath = os.path.join(previous_sources_dir, requirement_file) + install_args += ['-r', fpath] + install_args += self.packages + + self.call(install_args, fail="Failed to install python packages") + reqs = self._parse_sdist_names(tmpdir) + + return '\n'.join(["{}=={}".format(pkg, ver) for pkg, ver in reqs]) + + def fetch(self): + with self.tempdir() as tmpdir: + packages = self.ref.strip().split('\n') + package_dir = os.path.join(tmpdir, 'packages') + os.makedirs(package_dir) + self.call(self.host_pip + ['download', + '--no-binary', ':all:', + '--index-url', self.index_url, + '--dest', package_dir] + packages, + fail="Failed to install python packages: {}".format(packages)) + + # If the mirror directory already exists, assume that some other + # process has fetched the sources before us and ensure that we do + # not raise an error in that case. + try: + os.makedirs(self._mirror) + os.rename(package_dir, self._mirror) + except FileExistsError: + return + except OSError as e: + if e.errno != errno.ENOTEMPTY: + raise + + def stage(self, directory): + with self.timed_activity("Staging Python packages", silent_nested=True): + utils.copy_files(self._mirror, os.path.join(directory, _OUTPUT_DIRNAME)) + + # Directory where this source should stage its files + # + @property + def _mirror(self): + if not self.ref: + return None + return os.path.join(self.get_mirror_directory(), + utils.url_directory_name(self.original_url), + hashlib.sha256(self.ref.encode()).hexdigest()) + + # Parse names of downloaded source distributions + # + # Args: + # basedir (str): Directory containing source distribution archives + # + # Returns: + # (list): List of (package_name, version) tuples in sorted order + # + def _parse_sdist_names(self, basedir): + reqs = [] + for f in os.listdir(basedir): + pkg_match = _SDIST_RE.match(f) + if pkg_match: + reqs.append(pkg_match.groups()) + + return sorted(reqs) + + +def setup(): + return PipSource diff --git a/buildstream/source.py b/buildstream/source.py index 6b6d3126b..2a3218cfc 100644 --- a/buildstream/source.py +++ b/buildstream/source.py @@ -1074,12 +1074,12 @@ class Source(Plugin): # def __ensure_previous_sources(self, previous_sources): for index, src in enumerate(previous_sources): + # BuildStream should track sources in the order they appear so + # previous sources should never be in an inconsistent state + assert src.get_consistency() != Consistency.INCONSISTENT + if src.get_consistency() == Consistency.RESOLVED: src._fetch(previous_sources[0:index]) - elif src.get_consistency() == Consistency.INCONSISTENT: - new_ref = src._track(previous_sources[0:index]) - src._save_ref(new_ref) - src._fetch(previous_sources[0:index]) def _extract_alias(url): diff --git a/doc/source/core_plugins.rst b/doc/source/core_plugins.rst index da7c607c8..c82ffe52f 100644 --- a/doc/source/core_plugins.rst +++ b/doc/source/core_plugins.rst @@ -58,6 +58,7 @@ Sources sources/ostree sources/patch sources/deb + sources/pip External plugins diff --git a/tests/cachekey/project/sources/pip1.bst b/tests/cachekey/project/sources/pip1.bst new file mode 100644 index 000000000..ee69efad6 --- /dev/null +++ b/tests/cachekey/project/sources/pip1.bst @@ -0,0 +1,12 @@ +kind: import + +sources: +- kind: git + url: https://example.com/foo/foobar.git + ref: b99955530263172ed1beae52aed7a33885ef781f +- kind: pip + url: https://pypi.example.com/simple + packages: + - horses + - ponies + ref: 'horses==0.0.1\nponies==0.0.2' diff --git a/tests/cachekey/project/sources/pip1.expected b/tests/cachekey/project/sources/pip1.expected new file mode 100644 index 000000000..7ab6fd13f --- /dev/null +++ b/tests/cachekey/project/sources/pip1.expected @@ -0,0 +1 @@ +a36bfabe4365076681469b8b6f580d1adb7da5d88a69c168a7bb831fa15651a7
\ No newline at end of file diff --git a/tests/cachekey/project/target.bst b/tests/cachekey/project/target.bst index 325cd235e..7aedfcc7a 100644 --- a/tests/cachekey/project/target.bst +++ b/tests/cachekey/project/target.bst @@ -13,6 +13,7 @@ depends: - sources/patch1.bst - sources/patch2.bst - sources/patch3.bst +- sources/pip1.bst - sources/tar1.bst - sources/tar2.bst - sources/zip1.bst diff --git a/tests/cachekey/project/target.expected b/tests/cachekey/project/target.expected index 5fcd89438..36b583e1d 100644 --- a/tests/cachekey/project/target.expected +++ b/tests/cachekey/project/target.expected @@ -1 +1 @@ -46f48e5c0ff52370ff0cf2bb23bd2c79da23141e6c17b9aa720f7d97b7194340
\ No newline at end of file +7534baaacec89d9583a09aa016979c182b5c22f946100050ee5fb44a07ab554d diff --git a/tests/integration/pip.py b/tests/integration/pip_element.py index 6c6de8bf8..6c6de8bf8 100644 --- a/tests/integration/pip.py +++ b/tests/integration/pip_element.py diff --git a/tests/integration/pip_source.py b/tests/integration/pip_source.py new file mode 100644 index 000000000..fc5b56a7c --- /dev/null +++ b/tests/integration/pip_source.py @@ -0,0 +1,99 @@ +import os +import pytest + +from buildstream import _yaml + +from tests.testutils import cli_integration as cli +from tests.testutils.integration import assert_contains + + +pytestmark = pytest.mark.integration + + +DATA_DIR = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + "project" +) + + +@pytest.mark.datafiles(DATA_DIR) +def test_pip_source_import(cli, tmpdir, datafiles): + project = os.path.join(datafiles.dirname, datafiles.basename) + checkout = os.path.join(cli.directory, 'checkout') + element_path = os.path.join(project, 'elements') + element_name = 'pip/hello.bst' + + element = { + 'kind': 'import', + 'sources': [ + { + 'kind': 'local', + 'path': 'files/pip-source' + }, + { + 'kind': 'pip', + 'url': 'file://{}'.format(os.path.realpath(os.path.join(project, 'files', 'pypi-repo'))), + 'requirements-files': ['myreqs.txt'], + 'packages': ['app2'] + } + ] + } + os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True) + _yaml.dump(element, os.path.join(element_path, element_name)) + + result = cli.run(project=project, args=['track', element_name]) + assert result.exit_code == 0 + + result = cli.run(project=project, args=['build', element_name]) + assert result.exit_code == 0 + + result = cli.run(project=project, args=['checkout', element_name, checkout]) + assert result.exit_code == 0 + + assert_contains(checkout, ['/.bst_pip_downloads', + '/.bst_pip_downloads/HelloLib-0.1.tar.gz', + '/.bst_pip_downloads/App2-0.1.tar.gz']) + + +@pytest.mark.datafiles(DATA_DIR) +def test_pip_source_build(cli, tmpdir, datafiles): + project = os.path.join(datafiles.dirname, datafiles.basename) + element_path = os.path.join(project, 'elements') + element_name = 'pip/hello.bst' + + element = { + 'kind': 'manual', + 'depends': ['base.bst'], + 'sources': [ + { + 'kind': 'local', + 'path': 'files/pip-source' + }, + { + 'kind': 'pip', + 'url': 'file://{}'.format(os.path.realpath(os.path.join(project, 'files', 'pypi-repo'))), + 'requirements-files': ['myreqs.txt'], + 'packages': ['app2'] + } + ], + 'config': { + 'install-commands': [ + 'pip3 install --no-index --prefix %{install-root}/usr .bst_pip_downloads/*.tar.gz', + 'chmod +x app1.py', + 'install app1.py %{install-root}/usr/bin/' + ] + } + } + os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True) + _yaml.dump(element, os.path.join(element_path, element_name)) + + result = cli.run(project=project, args=['track', element_name]) + assert result.exit_code == 0 + + result = cli.run(project=project, args=['build', element_name]) + assert result.exit_code == 0 + + result = cli.run(project=project, args=['shell', element_name, '/usr/bin/app1.py']) + assert result.exit_code == 0 + assert result.output == """Hello App1! +""" diff --git a/tests/integration/project/files/pip-source/app1.py b/tests/integration/project/files/pip-source/app1.py new file mode 100644 index 000000000..ab1005ba4 --- /dev/null +++ b/tests/integration/project/files/pip-source/app1.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python3 + +from hellolib import hello + + +def main(): + hello('App1') + + +if __name__ == '__main__': + main() diff --git a/tests/integration/project/files/pip-source/myreqs.txt b/tests/integration/project/files/pip-source/myreqs.txt new file mode 100644 index 000000000..c805aae53 --- /dev/null +++ b/tests/integration/project/files/pip-source/myreqs.txt @@ -0,0 +1 @@ +hellolib diff --git a/tests/integration/project/files/pypi-repo/app2/App2-0.1.tar.gz b/tests/integration/project/files/pypi-repo/app2/App2-0.1.tar.gz Binary files differnew file mode 100644 index 000000000..86cb43cfe --- /dev/null +++ b/tests/integration/project/files/pypi-repo/app2/App2-0.1.tar.gz diff --git a/tests/integration/project/files/pypi-repo/app2/index.html b/tests/integration/project/files/pypi-repo/app2/index.html new file mode 100644 index 000000000..5bc72e47c --- /dev/null +++ b/tests/integration/project/files/pypi-repo/app2/index.html @@ -0,0 +1,8 @@ +<html> + <head> + <title>Links for app1</title> + </head> + <body> + <a href='App2-0.1.tar.gz'>App2-0.1.tar.gz</a><br /> + </body> +</html> diff --git a/tests/integration/project/files/pypi-repo/hellolib/HelloLib-0.1.tar.gz b/tests/integration/project/files/pypi-repo/hellolib/HelloLib-0.1.tar.gz Binary files differnew file mode 100644 index 000000000..3b0884c66 --- /dev/null +++ b/tests/integration/project/files/pypi-repo/hellolib/HelloLib-0.1.tar.gz diff --git a/tests/integration/project/files/pypi-repo/hellolib/index.html b/tests/integration/project/files/pypi-repo/hellolib/index.html new file mode 100644 index 000000000..eb9935c7f --- /dev/null +++ b/tests/integration/project/files/pypi-repo/hellolib/index.html @@ -0,0 +1,8 @@ +<html> + <head> + <title>Links for app1</title> + </head> + <body> + <a href='HelloLib-0.1.tar.gz'>HelloLib-0.1.tar.gz</a><br /> + </body> +</html> diff --git a/tests/sources/pip.py b/tests/sources/pip.py new file mode 100644 index 000000000..8b4c213dc --- /dev/null +++ b/tests/sources/pip.py @@ -0,0 +1,47 @@ +import os +import pytest + +from buildstream._exceptions import ErrorDomain +from buildstream import _yaml +from tests.testutils import cli + +DATA_DIR = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + 'pip', +) + + +def generate_project(project_dir, tmpdir): + project_file = os.path.join(project_dir, "project.conf") + _yaml.dump({'name': 'foo'}, project_file) + + +# Test that without ref, consistency is set appropriately. +@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-ref')) +def test_no_ref(cli, tmpdir, datafiles): + project = os.path.join(datafiles.dirname, datafiles.basename) + generate_project(project, tmpdir) + assert cli.get_element_state(project, 'target.bst') == 'no reference' + + +# Test that pip is not allowed to be the first source +@pytest.mark.datafiles(os.path.join(DATA_DIR, 'first-source-pip')) +def test_first_source(cli, tmpdir, datafiles): + project = os.path.join(datafiles.dirname, datafiles.basename) + generate_project(project, tmpdir) + result = cli.run(project=project, args=[ + 'show', 'target.bst' + ]) + result.assert_main_error(ErrorDomain.ELEMENT, None) + + +# Test that error is raised when neither packges nor requirements files +# have been specified +@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-packages')) +def test_no_packages(cli, tmpdir, datafiles): + project = os.path.join(datafiles.dirname, datafiles.basename) + generate_project(project, tmpdir) + result = cli.run(project=project, args=[ + 'show', 'target.bst' + ]) + result.assert_main_error(ErrorDomain.SOURCE, None) diff --git a/tests/sources/pip/first-source-pip/target.bst b/tests/sources/pip/first-source-pip/target.bst new file mode 100644 index 000000000..e5f20ab0b --- /dev/null +++ b/tests/sources/pip/first-source-pip/target.bst @@ -0,0 +1,6 @@ +kind: import +description: pip should not be allowed to be the first source +sources: +- kind: pip + packages: + - flake8 diff --git a/tests/sources/pip/no-packages/file b/tests/sources/pip/no-packages/file new file mode 100644 index 000000000..980a0d5f1 --- /dev/null +++ b/tests/sources/pip/no-packages/file @@ -0,0 +1 @@ +Hello World! diff --git a/tests/sources/pip/no-packages/target.bst b/tests/sources/pip/no-packages/target.bst new file mode 100644 index 000000000..0d8b948c4 --- /dev/null +++ b/tests/sources/pip/no-packages/target.bst @@ -0,0 +1,6 @@ +kind: import +description: The kind of this element is irrelevant. +sources: +- kind: local + path: file +- kind: pip diff --git a/tests/sources/pip/no-ref/file b/tests/sources/pip/no-ref/file new file mode 100644 index 000000000..980a0d5f1 --- /dev/null +++ b/tests/sources/pip/no-ref/file @@ -0,0 +1 @@ +Hello World! diff --git a/tests/sources/pip/no-ref/target.bst b/tests/sources/pip/no-ref/target.bst new file mode 100644 index 000000000..ec450b7ef --- /dev/null +++ b/tests/sources/pip/no-ref/target.bst @@ -0,0 +1,8 @@ +kind: import +description: The kind of this element is irrelevant. +sources: +- kind: local + path: file +- kind: pip + packages: + - flake8 |