Add `bst source-checkout` command

As discussed in
https://mail.gnome.org/archives/buildstream-list/2018-September/msg00064.html,
add `bst source-checkout` command. This will allow users to checkout
sources for a given target.

* _frontend/cli.py: Add source-checkout command
* _pipeline.py: Add assert_sources_cached() method
* _stream.py: Add source_checkout method, abstract out
  __check_location_writable() method that used to part of checkout()

3 files changed, 132 insertions, 0 deletions

diff --git a/tests/frontend/project/elements/checkout-deps.bst b/tests/frontend/project/elements/checkout-deps.bst
new file mode 100644
index 000000000..a2c1d93cc
--- /dev/null
+++ b/tests/frontend/project/elements/checkout-deps.bst
@@ -0,0 +1,10 @@
+kind: import
+description: It is important for this element to have both build and runtime dependencies
+sources:
+- kind: local
+  path: files/etc-files
+depends:
+- filename: import-dev.bst
+  type: build
+- filename: import-bin.bst
+  type: runtime
diff --git a/tests/frontend/project/files/etc-files/etc/buildstream/config b/tests/frontend/project/files/etc-files/etc/buildstream/config
new file mode 100644
index 000000000..04204c7c9
--- /dev/null
+++ b/tests/frontend/project/files/etc-files/etc/buildstream/config
@@ -0,0 +1 @@
+config
diff --git a/tests/frontend/source_checkout.py b/tests/frontend/source_checkout.py
new file mode 100644
index 000000000..58dfdbd42
--- /dev/null
+++ b/tests/frontend/source_checkout.py
@@ -0,0 +1,121 @@
+import os
+import pytest
+
+from tests.testutils import cli
+
+from buildstream import utils, _yaml
+from buildstream._exceptions import ErrorDomain, LoadErrorReason
+
+# Project directory
+DATA_DIR = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)),
+    'project',
+)
+
+
+def generate_remote_import_element(input_path, output_path):
+    return {
+        'kind': 'import',
+        'sources': [
+            {
+                'kind': 'remote',
+                'url': 'file://{}'.format(input_path),
+                'filename': output_path,
+                'ref': utils.sha256sum(input_path),
+            }
+        ]
+    }
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_source_checkout(datafiles, cli):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    checkout = os.path.join(cli.directory, 'source-checkout')
+    target = 'checkout-deps.bst'
+
+    result = cli.run(project=project, args=['source-checkout', target, '--deps', 'none', checkout])
+    result.assert_success()
+
+    assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize('deps', [('build'), ('none'), ('run'), ('all')])
+def test_source_checkout_deps(datafiles, cli, deps):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    checkout = os.path.join(cli.directory, 'source-checkout')
+    target = 'checkout-deps.bst'
+
+    result = cli.run(project=project, args=['source-checkout', target, '--deps', deps, checkout])
+    result.assert_success()
+
+    # Sources of the target
+    if deps == 'build':
+        assert not os.path.exists(os.path.join(checkout, 'checkout-deps'))
+    else:
+        assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
+
+    # Sources of the target's build dependencies
+    if deps in ('build', 'all'):
+        assert os.path.exists(os.path.join(checkout, 'import-dev', 'usr', 'include', 'pony.h'))
+    else:
+        assert not os.path.exists(os.path.join(checkout, 'import-dev'))
+
+    # Sources of the target's runtime dependencies
+    if deps in ('run', 'all'):
+        assert os.path.exists(os.path.join(checkout, 'import-bin', 'usr', 'bin', 'hello'))
+    else:
+        assert not os.path.exists(os.path.join(checkout, 'import-bin'))
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_source_checkout_except(datafiles, cli):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    checkout = os.path.join(cli.directory, 'source-checkout')
+    target = 'checkout-deps.bst'
+
+    result = cli.run(project=project, args=['source-checkout', target,
+                                            '--deps', 'all',
+                                            '--except', 'import-bin.bst',
+                                            checkout])
+    result.assert_success()
+
+    # Sources for the target should be present
+    assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
+
+    # Sources for import-bin.bst should not be present
+    assert not os.path.exists(os.path.join(checkout, 'import-bin'))
+
+    # Sources for other dependencies should be present
+    assert os.path.exists(os.path.join(checkout, 'import-dev', 'usr', 'include', 'pony.h'))
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize('fetch', [(False), (True)])
+def test_source_checkout_fetch(datafiles, cli, fetch):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    checkout = os.path.join(cli.directory, 'source-checkout')
+    target = 'remote-import-dev.bst'
+    target_path = os.path.join(project, 'elements', target)
+
+    # Create an element with remote source
+    element = generate_remote_import_element(
+        os.path.join(project, 'files', 'dev-files', 'usr', 'include', 'pony.h'),
+        'pony.h')
+    _yaml.dump(element, target_path)
+
+    # Testing --fetch option requires that we do not have the sources
+    # cached already
+    assert cli.get_element_state(project, target) == 'fetch needed'
+
+    args = ['source-checkout']
+    if fetch:
+        args += ['--fetch']
+    args += [target, checkout]
+    result = cli.run(project=project, args=args)
+
+    if fetch:
+        result.assert_success()
+        assert os.path.exists(os.path.join(checkout, 'remote-import-dev', 'pony.h'))
+    else:
+        result.assert_main_error(ErrorDomain.PIPELINE, 'uncached-sources')