pip.py: Remove the 'pip' source plugin, it's in bst-plugins-experimentalbschubert/remove-pip-source

14 files changed, 2 insertions, 536 deletions

diff --git a/NEWS b/NEWS
index af362eb2f..19ad35eee 100644
--- a/NEWS
+++ b/NEWS
@@ -24,6 +24,7 @@ Plugins
     variable was not included in the cache key correctly.
 
   o The `pip` element has been removed. Please use the one from bst-plugins-experimental
+  o The `pip` source has been removed. Please use the one from bst-plugins-experimental
 
 
 API
diff --git a/doc/source/core_plugins.rst b/doc/source/core_plugins.rst
index b40728669..900c7e4b6 100644
--- a/doc/source/core_plugins.rst
+++ b/doc/source/core_plugins.rst
@@ -54,7 +54,6 @@ information.
    sources/git
    sources/bzr
    sources/patch
-   sources/pip
 
 
 .. _plugins_external:
diff --git a/src/buildstream/plugins/sources/pip.py b/src/buildstream/plugins/sources/pip.py
deleted file mode 100644
index c0885ce5a..000000000
--- a/src/buildstream/plugins/sources/pip.py
+++ /dev/null
@@ -1,263 +0,0 @@
-#
-#  Copyright 2018 Bloomberg Finance LP
-#
-#  This program is free software; you can redistribute it and/or
-#  modify it under the terms of the GNU Lesser General Public
-#  License as published by the Free Software Foundation; either
-#  version 2 of the License, or (at your option) any later version.
-#
-#  This library is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#  Lesser General Public License for more details.
-#
-#  You should have received a copy of the GNU Lesser General Public
-#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
-#
-#  Authors:
-#        Chandan Singh <csingh43@bloomberg.net>
-
-"""
-pip - stage python packages using pip
-=====================================
-
-**Host depndencies:**
-
-  * ``pip`` python module
-
-This plugin will download source distributions for specified packages using
-``pip`` but will not install them. It is expected that the elements using this
-source will install the downloaded packages.
-
-Downloaded tarballs will be stored in a directory called ".bst_pip_downloads".
-
-**Usage:**
-
-.. code:: yaml
-
-   # Specify the pip source kind
-   kind: pip
-
-   # Optionally specify index url, defaults to PyPi
-   # This url is used to discover new versions of packages and download them
-   # Projects intending to mirror their sources to a permanent location should
-   # use an aliased url, and declare the alias in the project configuration
-   url: https://mypypi.example.com/simple
-
-   # Optionally specify the path to requirements files
-   # Note that either 'requirements-files' or 'packages' must be defined
-   requirements-files:
-   - requirements.txt
-
-   # Optionally specify a list of additional packages
-   # Note that either 'requirements-files' or 'packages' must be defined
-   packages:
-   - flake8
-
-   # Specify the ref. It is a list of strings of format
-   # "<package-name>==<version>", separated by "\\n".
-   # Usually this will be contents of a requirements.txt file where all
-   # package versions have been frozen.
-   ref: "flake8==3.5.0\\nmccabe==0.6.1\\npkg-resources==0.0.0\\npycodestyle==2.3.1\\npyflakes==1.6.0"
-
-See :ref:`built-in functionality doumentation <core_source_builtins>` for
-details on common configuration options for sources.
-"""
-
-import hashlib
-import os
-import re
-
-from buildstream import Source, SourceError, utils
-
-_OUTPUT_DIRNAME = ".bst_pip_downloads"
-_PYPI_INDEX_URL = "https://pypi.org/simple/"
-
-# Used only for finding pip command
-_PYTHON_VERSIONS = [
-    "python",  # when running in a venv, we might not have the exact version
-    "python2.7",
-    "python3.0",
-    "python3.1",
-    "python3.2",
-    "python3.3",
-    "python3.4",
-    "python3.5",
-    "python3.6",
-    "python3.7",
-    "python3.8",
-]
-
-# List of allowed extensions taken from
-# https://docs.python.org/3/distutils/sourcedist.html.
-# Names of source distribution archives must be of the form
-# '%{package-name}-%{version}.%{extension}'.
-_SDIST_RE = re.compile(r"^([\w.-]+?)-((?:[\d.]+){2,})\.(?:tar|tar.bz2|tar.gz|tar.xz|tar.Z|zip)$", re.IGNORECASE)
-
-
-class PipSource(Source):
-    # pylint: disable=attribute-defined-outside-init
-
-    BST_MIN_VERSION = "2.0"
-
-    # We need access to previous sources at track time to use requirements.txt
-    # but not at fetch time as self.ref should contain sufficient information
-    # for this plugin
-    BST_REQUIRES_PREVIOUS_SOURCES_TRACK = True
-
-    def configure(self, node):
-        node.validate_keys(["url", "packages", "ref", "requirements-files"] + Source.COMMON_CONFIG_KEYS)
-        self.ref = node.get_str("ref", None)
-        self.original_url = node.get_str("url", _PYPI_INDEX_URL)
-        self.index_url = self.translate_url(self.original_url)
-        self.packages = node.get_str_list("packages", [])
-        self.requirements_files = node.get_str_list("requirements-files", [])
-
-        if not (self.packages or self.requirements_files):
-            raise SourceError("{}: Either 'packages' or 'requirements-files' must be specified".format(self))
-
-    def preflight(self):
-        # Try to find a pip version that spports download command
-        self.host_pip = None
-        for python in reversed(_PYTHON_VERSIONS):
-            try:
-                host_python = utils.get_host_tool(python)
-                rc = self.call([host_python, "-m", "pip", "download", "--help"])
-                if rc == 0:
-                    self.host_pip = [host_python, "-m", "pip"]
-                    break
-            except utils.ProgramNotFoundError:
-                pass
-
-        if self.host_pip is None:
-            raise SourceError("{}: Unable to find a suitable pip command".format(self))
-
-    def get_unique_key(self):
-        return [self.original_url, self.ref]
-
-    def is_cached(self):
-        return os.path.exists(self._mirror) and os.listdir(self._mirror)
-
-    def get_ref(self):
-        return self.ref
-
-    def load_ref(self, node):
-        self.ref = node.get_str("ref", None)
-
-    def set_ref(self, ref, node):
-        node["ref"] = self.ref = ref
-
-    def track(self, previous_sources_dir):  # pylint: disable=arguments-differ
-        # XXX pip does not offer any public API other than the CLI tool so it
-        # is not feasible to correctly parse the requirements file or to check
-        # which package versions pip is going to install.
-        # See https://pip.pypa.io/en/stable/user_guide/#using-pip-from-your-program
-        # for details.
-        # As a result, we have to wastefully install the packages during track.
-        with self.tempdir() as tmpdir:
-            install_args = self.host_pip + [
-                "download",
-                "--no-binary",
-                ":all:",
-                "--index-url",
-                self.index_url,
-                "--dest",
-                tmpdir,
-            ]
-            for requirement_file in self.requirements_files:
-                fpath = os.path.join(previous_sources_dir, requirement_file)
-                install_args += ["-r", fpath]
-            install_args += self.packages
-
-            self.call(install_args, fail="Failed to install python packages")
-            reqs = self._parse_sdist_names(tmpdir)
-
-        return "\n".join(["{}=={}".format(pkg, ver) for pkg, ver in reqs])
-
-    def fetch(self):  # pylint: disable=arguments-differ
-        with self.tempdir() as tmpdir:
-            packages = self.ref.strip().split("\n")
-            package_dir = os.path.join(tmpdir, "packages")
-            os.makedirs(package_dir)
-            self.call(
-                [
-                    *self.host_pip,
-                    "download",
-                    "--no-binary",
-                    ":all:",
-                    "--index-url",
-                    self.index_url,
-                    "--dest",
-                    package_dir,
-                    *packages,
-                ],
-                fail="Failed to install python packages: {}".format(packages),
-            )
-
-            # If the mirror directory already exists, assume that some other
-            # process has fetched the sources before us and ensure that we do
-            # not raise an error in that case.
-            try:
-                utils.move_atomic(package_dir, self._mirror)
-            except utils.DirectoryExistsError:
-                # Another process has beaten us and has fetched the sources
-                # before us.
-                pass
-            except OSError as e:
-                raise SourceError(
-                    "{}: Failed to move downloaded pip packages from '{}' to '{}': {}".format(
-                        self, package_dir, self._mirror, e
-                    )
-                ) from e
-
-    def stage(self, directory):
-        with self.timed_activity("Staging Python packages", silent_nested=True):
-            utils.copy_files(self._mirror, os.path.join(directory, _OUTPUT_DIRNAME))
-
-    # Directory where this source should stage its files
-    #
-    @property
-    def _mirror(self):
-        if not self.ref:
-            return None
-        return os.path.join(
-            self.get_mirror_directory(),
-            utils.url_directory_name(self.original_url),
-            hashlib.sha256(self.ref.encode()).hexdigest(),
-        )
-
-    # Parse names of downloaded source distributions
-    #
-    # Args:
-    #    basedir (str): Directory containing source distribution archives
-    #
-    # Returns:
-    #    (list): List of (package_name, version) tuples in sorted order
-    #
-    def _parse_sdist_names(self, basedir):
-        reqs = []
-        for f in os.listdir(basedir):
-            pkg = _match_package_name(f)
-            if pkg is not None:
-                reqs.append(pkg)
-
-        return sorted(reqs)
-
-
-# Extract the package name and version of a source distribution
-#
-# Args:
-#    filename (str): Filename of the source distribution
-#
-# Returns:
-#    (tuple): A tuple of (package_name, version)
-#
-def _match_package_name(filename):
-    pkg_match = _SDIST_RE.match(filename)
-    if pkg_match is None:
-        return None
-    return pkg_match.groups()
-
-
-def setup():
-    return PipSource
diff --git a/tests/cachekey/project/sources/pip1.bst b/tests/cachekey/project/sources/pip1.bst
deleted file mode 100644
index ee69efad6..000000000
--- a/tests/cachekey/project/sources/pip1.bst
+++ /dev/null
@@ -1,12 +0,0 @@
-kind: import
-
-sources:
-- kind: git
-  url: https://example.com/foo/foobar.git
-  ref: b99955530263172ed1beae52aed7a33885ef781f
-- kind: pip
-  url: https://pypi.example.com/simple
-  packages:
-  - horses
-  - ponies
-  ref: 'horses==0.0.1\nponies==0.0.2'
diff --git a/tests/cachekey/project/sources/pip1.expected b/tests/cachekey/project/sources/pip1.expected
deleted file mode 100644
index 844f4c54c..000000000
--- a/tests/cachekey/project/sources/pip1.expected
+++ /dev/null
@@ -1 +0,0 @@
-70870ffa81e4527a2d812c809b1e99025c72d1ae289be38e45a5b22dc0262eac
\ No newline at end of file
diff --git a/tests/cachekey/project/target.bst b/tests/cachekey/project/target.bst
index cabf3f7bf..1c6331754 100644
--- a/tests/cachekey/project/target.bst
+++ b/tests/cachekey/project/target.bst
@@ -13,7 +13,6 @@ depends:
 - sources/patch1.bst
 - sources/patch2.bst
 - sources/patch3.bst
-- sources/pip1.bst
 - sources/remote1.bst
 - sources/remote2.bst
 - sources/tar1.bst
diff --git a/tests/cachekey/project/target.expected b/tests/cachekey/project/target.expected
index cf38ddcfe..7f861c605 100644
--- a/tests/cachekey/project/target.expected
+++ b/tests/cachekey/project/target.expected
@@ -1 +1 @@
-a57f84353d1528b3ecbd9993ac02a20d0355f2451e19700172c95575af9a5b48
\ No newline at end of file
+fbf0b4dfc7d28cc4b534ea1c0d8af38a72a51d540cf903bca2c04389aadbe135
diff --git a/tests/integration/pip_source.py b/tests/integration/pip_source.py
deleted file mode 100644
index 5d314974d..000000000
--- a/tests/integration/pip_source.py
+++ /dev/null
@@ -1,177 +0,0 @@
-# Pylint doesn't play well with fixtures and dependency injection from pytest
-# pylint: disable=redefined-outer-name
-
-import os
-import pytest
-
-from buildstream import _yaml
-
-from buildstream.testing import cli_integration as cli  # pylint: disable=unused-import
-from buildstream.testing.integration import assert_contains
-from buildstream.testing._utils.site import HAVE_SANDBOX
-
-from tests.testutils.python_repo import setup_pypi_repo  # pylint: disable=unused-import
-
-
-pytestmark = pytest.mark.integration
-
-
-DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
-
-
-@pytest.mark.datafiles(DATA_DIR)
-def test_pip_source_import_packages(cli, datafiles, setup_pypi_repo):
-    project = str(datafiles)
-    checkout = os.path.join(cli.directory, "checkout")
-    element_path = os.path.join(project, "elements")
-    element_name = "pip/hello.bst"
-
-    # check that exotically named packages are imported correctly
-    myreqs_packages = "hellolib"
-    dependencies = ["app2", "app.3", "app-4", "app_5", "app.no.6", "app-no-7", "app_no_8"]
-    mock_packages = {myreqs_packages: {package: {} for package in dependencies}}
-
-    # create mock pypi repository
-    pypi_repo = os.path.join(project, "files", "pypi-repo")
-    os.makedirs(pypi_repo, exist_ok=True)
-    setup_pypi_repo(mock_packages, pypi_repo)
-
-    element = {
-        "kind": "import",
-        "sources": [
-            {"kind": "local", "path": "files/pip-source"},
-            {"kind": "pip", "url": "file://{}".format(os.path.realpath(pypi_repo)), "packages": [myreqs_packages]},
-        ],
-    }
-    os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True)
-    _yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
-
-    result = cli.run(project=project, args=["source", "track", element_name])
-    assert result.exit_code == 0
-
-    result = cli.run(project=project, args=["build", element_name])
-    assert result.exit_code == 0
-
-    result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
-    assert result.exit_code == 0
-
-    assert_contains(
-        checkout,
-        [
-            "/.bst_pip_downloads",
-            "/.bst_pip_downloads/hellolib-0.1.tar.gz",
-            "/.bst_pip_downloads/app2-0.1.tar.gz",
-            "/.bst_pip_downloads/app.3-0.1.tar.gz",
-            "/.bst_pip_downloads/app-4-0.1.tar.gz",
-            "/.bst_pip_downloads/app_5-0.1.tar.gz",
-            "/.bst_pip_downloads/app.no.6-0.1.tar.gz",
-            "/.bst_pip_downloads/app-no-7-0.1.tar.gz",
-            "/.bst_pip_downloads/app_no_8-0.1.tar.gz",
-        ],
-    )
-
-
-@pytest.mark.datafiles(DATA_DIR)
-def test_pip_source_import_requirements_files(cli, datafiles, setup_pypi_repo):
-    project = str(datafiles)
-    checkout = os.path.join(cli.directory, "checkout")
-    element_path = os.path.join(project, "elements")
-    element_name = "pip/hello.bst"
-
-    # check that exotically named packages are imported correctly
-    myreqs_packages = "hellolib"
-    dependencies = ["app2", "app.3", "app-4", "app_5", "app.no.6", "app-no-7", "app_no_8"]
-    mock_packages = {myreqs_packages: {package: {} for package in dependencies}}
-
-    # create mock pypi repository
-    pypi_repo = os.path.join(project, "files", "pypi-repo")
-    os.makedirs(pypi_repo, exist_ok=True)
-    setup_pypi_repo(mock_packages, pypi_repo)
-
-    element = {
-        "kind": "import",
-        "sources": [
-            {"kind": "local", "path": "files/pip-source"},
-            {
-                "kind": "pip",
-                "url": "file://{}".format(os.path.realpath(pypi_repo)),
-                "requirements-files": ["myreqs.txt"],
-            },
-        ],
-    }
-    os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True)
-    _yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
-
-    result = cli.run(project=project, args=["source", "track", element_name])
-    assert result.exit_code == 0
-
-    result = cli.run(project=project, args=["build", element_name])
-    assert result.exit_code == 0
-
-    result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
-    assert result.exit_code == 0
-
-    assert_contains(
-        checkout,
-        [
-            "/.bst_pip_downloads",
-            "/.bst_pip_downloads/hellolib-0.1.tar.gz",
-            "/.bst_pip_downloads/app2-0.1.tar.gz",
-            "/.bst_pip_downloads/app.3-0.1.tar.gz",
-            "/.bst_pip_downloads/app-4-0.1.tar.gz",
-            "/.bst_pip_downloads/app_5-0.1.tar.gz",
-            "/.bst_pip_downloads/app.no.6-0.1.tar.gz",
-            "/.bst_pip_downloads/app-no-7-0.1.tar.gz",
-            "/.bst_pip_downloads/app_no_8-0.1.tar.gz",
-        ],
-    )
-
-
-@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
-def test_pip_source_build(cli, datafiles, setup_pypi_repo):
-    project = str(datafiles)
-    element_path = os.path.join(project, "elements")
-    element_name = "pip/hello.bst"
-
-    # check that exotically named packages are imported correctly
-    myreqs_packages = "hellolib"
-    dependencies = ["app2", "app.3", "app-4", "app_5", "app.no.6", "app-no-7", "app_no_8"]
-    mock_packages = {myreqs_packages: {package: {} for package in dependencies}}
-
-    # create mock pypi repository
-    pypi_repo = os.path.join(project, "files", "pypi-repo")
-    os.makedirs(pypi_repo, exist_ok=True)
-    setup_pypi_repo(mock_packages, pypi_repo)
-
-    element = {
-        "kind": "manual",
-        "depends": ["base.bst"],
-        "sources": [
-            {"kind": "local", "path": "files/pip-source"},
-            {
-                "kind": "pip",
-                "url": "file://{}".format(os.path.realpath(pypi_repo)),
-                "requirements-files": ["myreqs.txt"],
-                "packages": dependencies,
-            },
-        ],
-        "config": {
-            "install-commands": [
-                "pip3 install --no-index --prefix %{install-root}/usr .bst_pip_downloads/*.tar.gz",
-                "install app1.py %{install-root}/usr/bin/",
-            ]
-        },
-    }
-    os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True)
-    _yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
-
-    result = cli.run(project=project, args=["source", "track", element_name])
-    assert result.exit_code == 0
-
-    result = cli.run(project=project, args=["build", element_name])
-    assert result.exit_code == 0
-
-    result = cli.run(project=project, args=["shell", element_name, "/usr/bin/app1.py"])
-    assert result.exit_code == 0
-    assert result.output == "Hello App1! This is hellolib\n"
diff --git a/tests/sources/pip.py b/tests/sources/pip.py
deleted file mode 100644
index 1eacb4462..000000000
--- a/tests/sources/pip.py
+++ /dev/null
@@ -1,58 +0,0 @@
-# Pylint doesn't play well with fixtures and dependency injection from pytest
-# pylint: disable=redefined-outer-name
-
-import os
-import pytest
-
-from buildstream.exceptions import ErrorDomain
-from buildstream.plugins.sources.pip import _match_package_name
-from buildstream.testing import cli, generate_project  # pylint: disable=unused-import
-
-DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "pip",)
-
-
-# Test that without ref, consistency is set appropriately.
-@pytest.mark.datafiles(os.path.join(DATA_DIR, "no-ref"))
-def test_no_ref(cli, datafiles):
-    project = str(datafiles)
-    generate_project(project)
-    assert cli.get_element_state(project, "target.bst") == "no reference"
-
-
-# Test that pip is not allowed to be the first source
-@pytest.mark.datafiles(os.path.join(DATA_DIR, "first-source-pip"))
-def test_first_source(cli, datafiles):
-    project = str(datafiles)
-    generate_project(project)
-    result = cli.run(project=project, args=["show", "target.bst"])
-    result.assert_main_error(ErrorDomain.ELEMENT, None)
-
-
-# Test that error is raised when neither packges nor requirements files
-# have been specified
-@pytest.mark.datafiles(os.path.join(DATA_DIR, "no-packages"))
-def test_no_packages(cli, datafiles):
-    project = str(datafiles)
-    generate_project(project)
-    result = cli.run(project=project, args=["show", "target.bst"])
-    result.assert_main_error(ErrorDomain.SOURCE, None)
-
-
-# Test that pip source parses tar ball names correctly for the ref
-@pytest.mark.parametrize(
-    "tarball, expected_name, expected_version",
-    [
-        ("dotted.package-0.9.8.tar.gz", "dotted.package", "0.9.8"),
-        ("hyphenated-package-2.6.0.tar.gz", "hyphenated-package", "2.6.0"),
-        ("underscore_pkg-3.1.0.tar.gz", "underscore_pkg", "3.1.0"),
-        ("numbers2and5-1.0.1.tar.gz", "numbers2and5", "1.0.1"),
-        ("multiple.dots.package-5.6.7.tar.gz", "multiple.dots.package", "5.6.7"),
-        ("multiple-hyphens-package-1.2.3.tar.gz", "multiple-hyphens-package", "1.2.3"),
-        ("multiple_underscore_pkg-3.4.5.tar.gz", "multiple_underscore_pkg", "3.4.5"),
-        ("shortversion-1.0.tar.gz", "shortversion", "1.0"),
-        ("longversion-1.2.3.4.tar.gz", "longversion", "1.2.3.4"),
-    ],
-)
-def test_match_package_name(tarball, expected_name, expected_version):
-    name, version = _match_package_name(tarball)
-    assert (expected_name, expected_version) == (name, version)
diff --git a/tests/sources/pip/first-source-pip/target.bst b/tests/sources/pip/first-source-pip/target.bst
deleted file mode 100644
index e5f20ab0b..000000000
--- a/tests/sources/pip/first-source-pip/target.bst
+++ /dev/null
@@ -1,6 +0,0 @@
-kind: import
-description: pip should not be allowed to be the first source
-sources:
-- kind: pip
-  packages:
-  - flake8
diff --git a/tests/sources/pip/no-packages/file b/tests/sources/pip/no-packages/file
deleted file mode 100644
index 980a0d5f1..000000000
--- a/tests/sources/pip/no-packages/file
+++ /dev/null
@@ -1 +0,0 @@
-Hello World!
diff --git a/tests/sources/pip/no-packages/target.bst b/tests/sources/pip/no-packages/target.bst
deleted file mode 100644
index 0d8b948c4..000000000
--- a/tests/sources/pip/no-packages/target.bst
+++ /dev/null
@@ -1,6 +0,0 @@
-kind: import
-description: The kind of this element is irrelevant.
-sources:
-- kind: local
-  path: file
-- kind: pip
diff --git a/tests/sources/pip/no-ref/file b/tests/sources/pip/no-ref/file
deleted file mode 100644
index 980a0d5f1..000000000
--- a/tests/sources/pip/no-ref/file
+++ /dev/null
@@ -1 +0,0 @@
-Hello World!
diff --git a/tests/sources/pip/no-ref/target.bst b/tests/sources/pip/no-ref/target.bst
deleted file mode 100644
index ec450b7ef..000000000
--- a/tests/sources/pip/no-ref/target.bst
+++ /dev/null
@@ -1,8 +0,0 @@
-kind: import
-description: The kind of this element is irrelevant.
-sources:
-- kind: local
-  path: file
-- kind: pip
-  packages:
-  - flake8