Add pip source plugin

`pip` source plugin can stage python packages that are either specified
directly in the element definition or picked up from `requirements.txt`
from previous sources. In order to support the latter use-case
(which is also the primary motivation for this plugin), this plugin
requires access to previous sources and hence is an example of a
Source Transform source.

Also, bump `BST_FORMAT_VERSION` as this patch adds a new core plugin.

18 files changed, 211 insertions, 1 deletions

diff --git a/tests/cachekey/project/sources/pip1.bst b/tests/cachekey/project/sources/pip1.bst
new file mode 100644
index 000000000..ee69efad6
--- /dev/null
+++ b/tests/cachekey/project/sources/pip1.bst
@@ -0,0 +1,12 @@
+kind: import
+
+sources:
+- kind: git
+  url: https://example.com/foo/foobar.git
+  ref: b99955530263172ed1beae52aed7a33885ef781f
+- kind: pip
+  url: https://pypi.example.com/simple
+  packages:
+  - horses
+  - ponies
+  ref: 'horses==0.0.1\nponies==0.0.2'
diff --git a/tests/cachekey/project/sources/pip1.expected b/tests/cachekey/project/sources/pip1.expected
new file mode 100644
index 000000000..11d7c5fae
--- /dev/null
+++ b/tests/cachekey/project/sources/pip1.expected
@@ -0,0 +1 @@
+880d0dc27d6683725cfd68d60156058115a9a53793b14b727fc6d0588a473763
\ No newline at end of file
diff --git a/tests/cachekey/project/target.bst b/tests/cachekey/project/target.bst
index 174215a49..d96645da8 100644
--- a/tests/cachekey/project/target.bst
+++ b/tests/cachekey/project/target.bst
@@ -13,6 +13,7 @@ depends:
 - sources/patch1.bst
 - sources/patch2.bst
 - sources/patch3.bst
+- sources/pip1.bst
 - sources/remote1.bst
 - sources/remote2.bst
 - sources/tar1.bst
diff --git a/tests/cachekey/project/target.expected b/tests/cachekey/project/target.expected
index d6d99895a..3c1cc261f 100644
--- a/tests/cachekey/project/target.expected
+++ b/tests/cachekey/project/target.expected
@@ -1 +1 @@
-e045df890262f7b7c663e64ad0bfba428d9d80ec514df3ddb13313d4ef669b73
\ No newline at end of file
+09620aa58875d96611d22632b7585a0f22f88f5ecca6f5d1915d3e529d036bd8
\ No newline at end of file
diff --git a/tests/integration/pip.py b/tests/integration/pip_element.py
index 6c6de8bf8..6c6de8bf8 100644
--- a/tests/integration/pip.py
+++ b/tests/integration/pip_element.py
diff --git a/tests/integration/pip_source.py b/tests/integration/pip_source.py
new file mode 100644
index 000000000..fc5b56a7c
--- /dev/null
+++ b/tests/integration/pip_source.py
@@ -0,0 +1,99 @@
+import os
+import pytest
+
+from buildstream import _yaml
+
+from tests.testutils import cli_integration as cli
+from tests.testutils.integration import assert_contains
+
+
+pytestmark = pytest.mark.integration
+
+
+DATA_DIR = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)),
+    "project"
+)
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_pip_source_import(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    checkout = os.path.join(cli.directory, 'checkout')
+    element_path = os.path.join(project, 'elements')
+    element_name = 'pip/hello.bst'
+
+    element = {
+        'kind': 'import',
+        'sources': [
+            {
+                'kind': 'local',
+                'path': 'files/pip-source'
+            },
+            {
+                'kind': 'pip',
+                'url': 'file://{}'.format(os.path.realpath(os.path.join(project, 'files', 'pypi-repo'))),
+                'requirements-files': ['myreqs.txt'],
+                'packages': ['app2']
+            }
+        ]
+    }
+    os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True)
+    _yaml.dump(element, os.path.join(element_path, element_name))
+
+    result = cli.run(project=project, args=['track', element_name])
+    assert result.exit_code == 0
+
+    result = cli.run(project=project, args=['build', element_name])
+    assert result.exit_code == 0
+
+    result = cli.run(project=project, args=['checkout', element_name, checkout])
+    assert result.exit_code == 0
+
+    assert_contains(checkout, ['/.bst_pip_downloads',
+                               '/.bst_pip_downloads/HelloLib-0.1.tar.gz',
+                               '/.bst_pip_downloads/App2-0.1.tar.gz'])
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_pip_source_build(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    element_path = os.path.join(project, 'elements')
+    element_name = 'pip/hello.bst'
+
+    element = {
+        'kind': 'manual',
+        'depends': ['base.bst'],
+        'sources': [
+            {
+                'kind': 'local',
+                'path': 'files/pip-source'
+            },
+            {
+                'kind': 'pip',
+                'url': 'file://{}'.format(os.path.realpath(os.path.join(project, 'files', 'pypi-repo'))),
+                'requirements-files': ['myreqs.txt'],
+                'packages': ['app2']
+            }
+        ],
+        'config': {
+            'install-commands': [
+                'pip3 install --no-index --prefix %{install-root}/usr .bst_pip_downloads/*.tar.gz',
+                'chmod +x app1.py',
+                'install app1.py  %{install-root}/usr/bin/'
+            ]
+        }
+    }
+    os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True)
+    _yaml.dump(element, os.path.join(element_path, element_name))
+
+    result = cli.run(project=project, args=['track', element_name])
+    assert result.exit_code == 0
+
+    result = cli.run(project=project, args=['build', element_name])
+    assert result.exit_code == 0
+
+    result = cli.run(project=project, args=['shell', element_name, '/usr/bin/app1.py'])
+    assert result.exit_code == 0
+    assert result.output == """Hello App1!
+"""
diff --git a/tests/integration/project/files/pip-source/app1.py b/tests/integration/project/files/pip-source/app1.py
new file mode 100644
index 000000000..ab1005ba4
--- /dev/null
+++ b/tests/integration/project/files/pip-source/app1.py
@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+
+from hellolib import hello
+
+
+def main():
+    hello('App1')
+
+
+if __name__ == '__main__':
+    main()
diff --git a/tests/integration/project/files/pip-source/myreqs.txt b/tests/integration/project/files/pip-source/myreqs.txt
new file mode 100644
index 000000000..c805aae53
--- /dev/null
+++ b/tests/integration/project/files/pip-source/myreqs.txt
@@ -0,0 +1 @@
+hellolib
diff --git a/tests/integration/project/files/pypi-repo/app2/App2-0.1.tar.gz b/tests/integration/project/files/pypi-repo/app2/App2-0.1.tar.gz
new file mode 100644
index 000000000..86cb43cfe
--- /dev/null
+++ b/tests/integration/project/files/pypi-repo/app2/App2-0.1.tar.gz
diff --git a/tests/integration/project/files/pypi-repo/app2/index.html b/tests/integration/project/files/pypi-repo/app2/index.html
new file mode 100644
index 000000000..5bc72e47c
--- /dev/null
+++ b/tests/integration/project/files/pypi-repo/app2/index.html
@@ -0,0 +1,8 @@
+<html>
+  <head>
+    <title>Links for app1</title>
+  </head>
+  <body>
+    <a href='App2-0.1.tar.gz'>App2-0.1.tar.gz</a><br />
+  </body>
+</html>
diff --git a/tests/integration/project/files/pypi-repo/hellolib/HelloLib-0.1.tar.gz b/tests/integration/project/files/pypi-repo/hellolib/HelloLib-0.1.tar.gz
new file mode 100644
index 000000000..3b0884c66
--- /dev/null
+++ b/tests/integration/project/files/pypi-repo/hellolib/HelloLib-0.1.tar.gz
diff --git a/tests/integration/project/files/pypi-repo/hellolib/index.html b/tests/integration/project/files/pypi-repo/hellolib/index.html
new file mode 100644
index 000000000..eb9935c7f
--- /dev/null
+++ b/tests/integration/project/files/pypi-repo/hellolib/index.html
@@ -0,0 +1,8 @@
+<html>
+  <head>
+    <title>Links for app1</title>
+  </head>
+  <body>
+    <a href='HelloLib-0.1.tar.gz'>HelloLib-0.1.tar.gz</a><br />
+  </body>
+</html>
diff --git a/tests/sources/pip.py b/tests/sources/pip.py
new file mode 100644
index 000000000..8b4c213dc
--- /dev/null
+++ b/tests/sources/pip.py
@@ -0,0 +1,47 @@
+import os
+import pytest
+
+from buildstream._exceptions import ErrorDomain
+from buildstream import _yaml
+from tests.testutils import cli
+
+DATA_DIR = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)),
+    'pip',
+)
+
+
+def generate_project(project_dir, tmpdir):
+    project_file = os.path.join(project_dir, "project.conf")
+    _yaml.dump({'name': 'foo'}, project_file)
+
+
+# Test that without ref, consistency is set appropriately.
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-ref'))
+def test_no_ref(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    generate_project(project, tmpdir)
+    assert cli.get_element_state(project, 'target.bst') == 'no reference'
+
+
+# Test that pip is not allowed to be the first source
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'first-source-pip'))
+def test_first_source(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    generate_project(project, tmpdir)
+    result = cli.run(project=project, args=[
+        'show', 'target.bst'
+    ])
+    result.assert_main_error(ErrorDomain.ELEMENT, None)
+
+
+# Test that error is raised when neither packges nor requirements files
+# have been specified
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-packages'))
+def test_no_packages(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    generate_project(project, tmpdir)
+    result = cli.run(project=project, args=[
+        'show', 'target.bst'
+    ])
+    result.assert_main_error(ErrorDomain.SOURCE, None)
diff --git a/tests/sources/pip/first-source-pip/target.bst b/tests/sources/pip/first-source-pip/target.bst
new file mode 100644
index 000000000..e5f20ab0b
--- /dev/null
+++ b/tests/sources/pip/first-source-pip/target.bst
@@ -0,0 +1,6 @@
+kind: import
+description: pip should not be allowed to be the first source
+sources:
+- kind: pip
+  packages:
+  - flake8
diff --git a/tests/sources/pip/no-packages/file b/tests/sources/pip/no-packages/file
new file mode 100644
index 000000000..980a0d5f1
--- /dev/null
+++ b/tests/sources/pip/no-packages/file
@@ -0,0 +1 @@
+Hello World!
diff --git a/tests/sources/pip/no-packages/target.bst b/tests/sources/pip/no-packages/target.bst
new file mode 100644
index 000000000..0d8b948c4
--- /dev/null
+++ b/tests/sources/pip/no-packages/target.bst
@@ -0,0 +1,6 @@
+kind: import
+description: The kind of this element is irrelevant.
+sources:
+- kind: local
+  path: file
+- kind: pip
diff --git a/tests/sources/pip/no-ref/file b/tests/sources/pip/no-ref/file
new file mode 100644
index 000000000..980a0d5f1
--- /dev/null
+++ b/tests/sources/pip/no-ref/file
@@ -0,0 +1 @@
+Hello World!
diff --git a/tests/sources/pip/no-ref/target.bst b/tests/sources/pip/no-ref/target.bst
new file mode 100644
index 000000000..ec450b7ef
--- /dev/null
+++ b/tests/sources/pip/no-ref/target.bst
@@ -0,0 +1,8 @@
+kind: import
+description: The kind of this element is irrelevant.
+sources:
+- kind: local
+  path: file
+- kind: pip
+  packages:
+  - flake8