Merge branch 'juerg/source-checkout' into 'master'

Fetch sources as needed for bst source checkout

See merge request BuildStream/buildstream!1427

26 files changed, 74 insertions, 60 deletions

diff --git a/man/bst-artifact-checkout.1 b/man/bst-artifact-checkout.1
index 27b9954f4..2bf023c8a 100644
--- a/man/bst-artifact-checkout.1
+++ b/man/bst-artifact-checkout.1
@@ -1,4 +1,4 @@
-.TH "BST ARTIFACT CHECKOUT" "1" "13-Mar-2019" "" "bst artifact checkout Manual"
+.TH "BST ARTIFACT CHECKOUT" "1" "27-Jun-2019" "" "bst artifact checkout Manual"
 .SH NAME
 bst\-artifact\-checkout \- Checkout contents of an artifact
 .SH SYNOPSIS
@@ -26,5 +26,8 @@ Checkout hardlinks instead of copying if possible
 \fB\-\-tar\fP LOCATION
 Create a tarball from the artifact contents instead of a file tree. If LOCATION is '-', the tarball will be dumped to the standard output.
 .TP
+\fB\-\-pull\fP
+Whether to pull the artifact if it's missing or incomplete.
+.TP
 \fB\-\-directory\fP DIRECTORY
 The directory to checkout the artifact to
diff --git a/man/bst-artifact-delete.1 b/man/bst-artifact-delete.1
index 43ef9bb47..4cc737d0e 100644
--- a/man/bst-artifact-delete.1
+++ b/man/bst-artifact-delete.1
@@ -1,11 +1,11 @@
-.TH "BST ARTIFACT DELETE" "1" "13-Mar-2019" "" "bst artifact delete Manual"
+.TH "BST ARTIFACT DELETE" "1" "27-Jun-2019" "" "bst artifact delete Manual"
 .SH NAME
 bst\-artifact\-delete \- Remove artifacts from the local cache
 .SH SYNOPSIS
 .B bst artifact delete
 [OPTIONS] [ARTIFACTS]...
 .SH DESCRIPTION
-Show logs of all artifacts
+Remove artifacts from the local cache
 .SH OPTIONS
 .TP
 \fB\-\-no\-prune\fP
diff --git a/man/bst-artifact-log.1 b/man/bst-artifact-log.1
index 2be77021f..88c797a78 100644
--- a/man/bst-artifact-log.1
+++ b/man/bst-artifact-log.1
@@ -1,8 +1,36 @@
-.TH "BST ARTIFACT LOG" "1" "13-Mar-2019" "" "bst artifact log Manual"
+.TH "BST ARTIFACT LOG" "1" "27-Jun-2019" "" "bst artifact log Manual"
 .SH NAME
-bst\-artifact\-log \- Show logs of an artifact
+bst\-artifact\-log \- Show logs of artifacts
 .SH SYNOPSIS
 .B bst artifact log
 [OPTIONS] [ARTIFACTS]...
 .SH DESCRIPTION
-Show logs of all artifacts
+Show logs of artifacts.
+.PP
+Note that 'artifacts' can be element references like "hello.bst", and they
+can also be artifact references. You may use shell-style wildcards for
+either.
+.PP
+Here are some examples of element references:
+.PP
+
+- `hello.bst`
+- `*.bst`
+.PP
+Note that element references must end with '.bst' to distinguish them from
+artifact references. Anything that does not end in '.bst' is an artifact
+ref.
+.PP
+Artifact references follow the format `<project_name>/<element>/<key>`.
+Note that 'element' is without the `.bst` extension.
+.PP
+Here are some examples of artifact references:
+.PP
+
+- `myproject/hello/*`
+- `myproject/*`
+- `*`
+- `myproject/hello/827637*`
+- `myproject/he*/827637*`
+- `myproject/he??o/827637*`
+- `m*/h*/8276376b077eda104c812e6ec2f488c7c9eea211ce572c83d734c10bf241209f`
diff --git a/man/bst-artifact-pull.1 b/man/bst-artifact-pull.1
index 890090a44..1369b97cd 100644
--- a/man/bst-artifact-pull.1
+++ b/man/bst-artifact-pull.1
@@ -1,4 +1,4 @@
-.TH "BST ARTIFACT PULL" "1" "13-Mar-2019" "" "bst artifact pull Manual"
+.TH "BST ARTIFACT PULL" "1" "27-Jun-2019" "" "bst artifact pull Manual"
 .SH NAME
 bst\-artifact\-pull \- Pull a built artifact
 .SH SYNOPSIS
diff --git a/man/bst-artifact-push.1 b/man/bst-artifact-push.1
index 09b068449..0e8bb424c 100644
--- a/man/bst-artifact-push.1
+++ b/man/bst-artifact-push.1
@@ -1,4 +1,4 @@
-.TH "BST ARTIFACT PUSH" "1" "13-Mar-2019" "" "bst artifact push Manual"
+.TH "BST ARTIFACT PUSH" "1" "27-Jun-2019" "" "bst artifact push Manual"
 .SH NAME
 bst\-artifact\-push \- Push a built artifact
 .SH SYNOPSIS
diff --git a/man/bst-artifact-server.1 b/man/bst-artifact-server.1
index d5d9b8cde..bf230c73e 100644
--- a/man/bst-artifact-server.1
+++ b/man/bst-artifact-server.1
@@ -1,4 +1,4 @@
-.TH "BST-ARTIFACT-SERVER" "1" "13-Mar-2019" "" "bst-artifact-server Manual"
+.TH "BST-ARTIFACT-SERVER" "1" "27-Jun-2019" "" "bst-artifact-server Manual"
 .SH NAME
 bst-artifact-server \- CAS Artifact Server
 .SH SYNOPSIS
diff --git a/man/bst-artifact.1 b/man/bst-artifact.1
index aa46e2ef0..dbb359bc6 100644
--- a/man/bst-artifact.1
+++ b/man/bst-artifact.1
@@ -1,4 +1,4 @@
-.TH "BST ARTIFACT" "1" "13-Mar-2019" "" "bst artifact Manual"
+.TH "BST ARTIFACT" "1" "27-Jun-2019" "" "bst artifact Manual"
 .SH NAME
 bst\-artifact \- Manipulate cached artifacts
 .SH SYNOPSIS
@@ -21,7 +21,7 @@ Manipulate cached artifacts
   See \fBbst artifact-push(1)\fP for full documentation on the \fBpush\fP command.
 .PP
 \fBlog\fP
-  Show logs of an artifact
+  Show logs of artifacts
   See \fBbst artifact-log(1)\fP for full documentation on the \fBlog\fP command.
 .PP
 \fBdelete\fP
diff --git a/man/bst-build.1 b/man/bst-build.1
index edb67a499..04b57455d 100644
--- a/man/bst-build.1
+++ b/man/bst-build.1
@@ -1,4 +1,4 @@
-.TH "BST BUILD" "1" "13-Mar-2019" "" "bst build Manual"
+.TH "BST BUILD" "1" "27-Jun-2019" "" "bst build Manual"
 .SH NAME
 bst\-build \- Build elements in a pipeline
 .SH SYNOPSIS
diff --git a/man/bst-help.1 b/man/bst-help.1
index feebf2463..69de66469 100644
--- a/man/bst-help.1
+++ b/man/bst-help.1
@@ -1,4 +1,4 @@
-.TH "BST HELP" "1" "13-Mar-2019" "" "bst help Manual"
+.TH "BST HELP" "1" "27-Jun-2019" "" "bst help Manual"
 .SH NAME
 bst\-help \- Print usage information
 .SH SYNOPSIS
diff --git a/man/bst-init.1 b/man/bst-init.1
index 6f41d97d9..351551c0e 100644
--- a/man/bst-init.1
+++ b/man/bst-init.1
@@ -1,4 +1,4 @@
-.TH "BST INIT" "1" "13-Mar-2019" "" "bst init Manual"
+.TH "BST INIT" "1" "27-Jun-2019" "" "bst init Manual"
 .SH NAME
 bst\-init \- Initialize a new BuildStream project
 .SH SYNOPSIS
@@ -18,7 +18,7 @@ interactive session.
 The project name to use
 .TP
 \fB\-\-format\-version\fP INTEGER
-The required format version (default: 23)
+The required format version (default: 24)
 .TP
 \fB\-\-element\-path\fP PATH
 The subdirectory to store elements in (default: elements)
diff --git a/man/bst-shell.1 b/man/bst-shell.1
index 376cddb4b..4a068f7a9 100644
--- a/man/bst-shell.1
+++ b/man/bst-shell.1
@@ -1,4 +1,4 @@
-.TH "BST SHELL" "1" "13-Mar-2019" "" "bst shell Manual"
+.TH "BST SHELL" "1" "27-Jun-2019" "" "bst shell Manual"
 .SH NAME
 bst\-shell \- Shell into an element's sandbox environment
 .SH SYNOPSIS
@@ -45,3 +45,6 @@ Create an isolated build sandbox
 .TP
 \fB\-t,\fP \-\-use\-buildtree [ask|try|always|never]
 Defaults to ask but if set to always the function will fail if a build tree is not available
+.TP
+\fB\-\-pull\fP
+Attempt to pull missing or incomplete artifacts
diff --git a/man/bst-show.1 b/man/bst-show.1
index 54d02c3b6..9c4958e61 100644
--- a/man/bst-show.1
+++ b/man/bst-show.1
@@ -1,4 +1,4 @@
-.TH "BST SHOW" "1" "13-Mar-2019" "" "bst show Manual"
+.TH "BST SHOW" "1" "27-Jun-2019" "" "bst show Manual"
 .SH NAME
 bst\-show \- Show elements in the pipeline
 .SH SYNOPSIS
diff --git a/man/bst-source-checkout.1 b/man/bst-source-checkout.1
index aed31db7a..57c43e926 100644
--- a/man/bst-source-checkout.1
+++ b/man/bst-source-checkout.1
@@ -1,4 +1,4 @@
-.TH "BST SOURCE CHECKOUT" "1" "13-Mar-2019" "" "bst source checkout Manual"
+.TH "BST SOURCE CHECKOUT" "1" "27-Jun-2019" "" "bst source checkout Manual"
 .SH NAME
 bst\-source\-checkout \- Checkout sources for an element
 .SH SYNOPSIS
@@ -20,9 +20,6 @@ Except certain dependencies
 \fB\-d,\fP \-\-deps [build|none|run|all]
 The dependencies whose sources to checkout (default: none)
 .TP
-\fB\-\-fetch\fP
-Fetch elements if they are not fetched
-.TP
 \fB\-\-tar\fP
 Create a tarball from the element's sources instead of a file tree.
 .TP
diff --git a/man/bst-source-fetch.1 b/man/bst-source-fetch.1
index ddcd557e4..cdd2bf57a 100644
--- a/man/bst-source-fetch.1
+++ b/man/bst-source-fetch.1
@@ -1,4 +1,4 @@
-.TH "BST SOURCE FETCH" "1" "13-Mar-2019" "" "bst source fetch Manual"
+.TH "BST SOURCE FETCH" "1" "27-Jun-2019" "" "bst source fetch Manual"
 .SH NAME
 bst\-source\-fetch \- Fetch sources in a pipeline
 .SH SYNOPSIS
@@ -38,3 +38,6 @@ Track new source references before fetching
 .TP
 \fB\-J,\fP \-\-track\-cross\-junctions
 Allow tracking to cross junction boundaries
+.TP
+\fB\-r,\fP \-\-remote TEXT
+The URL of the remote source cache (defaults to the first configured cache)
diff --git a/man/bst-source-track.1 b/man/bst-source-track.1
index 9a24619a1..92a415d03 100644
--- a/man/bst-source-track.1
+++ b/man/bst-source-track.1
@@ -1,4 +1,4 @@
-.TH "BST SOURCE TRACK" "1" "13-Mar-2019" "" "bst source track Manual"
+.TH "BST SOURCE TRACK" "1" "27-Jun-2019" "" "bst source track Manual"
 .SH NAME
 bst\-source\-track \- Track new source references
 .SH SYNOPSIS
diff --git a/man/bst-source.1 b/man/bst-source.1
index 4d396b3d0..cc28519e2 100644
--- a/man/bst-source.1
+++ b/man/bst-source.1
@@ -1,4 +1,4 @@
-.TH "BST SOURCE" "1" "13-Mar-2019" "" "bst source Manual"
+.TH "BST SOURCE" "1" "27-Jun-2019" "" "bst source Manual"
 .SH NAME
 bst\-source \- Manipulate sources for an element
 .SH SYNOPSIS
diff --git a/man/bst-workspace-close.1 b/man/bst-workspace-close.1
index 0b43cdf22..6dbd7be74 100644
--- a/man/bst-workspace-close.1
+++ b/man/bst-workspace-close.1
@@ -1,4 +1,4 @@
-.TH "BST WORKSPACE CLOSE" "1" "13-Mar-2019" "" "bst workspace close Manual"
+.TH "BST WORKSPACE CLOSE" "1" "27-Jun-2019" "" "bst workspace close Manual"
 .SH NAME
 bst\-workspace\-close \- Close workspaces
 .SH SYNOPSIS
diff --git a/man/bst-workspace-list.1 b/man/bst-workspace-list.1
index f5252670d..f318967b9 100644
--- a/man/bst-workspace-list.1
+++ b/man/bst-workspace-list.1
@@ -1,4 +1,4 @@
-.TH "BST WORKSPACE LIST" "1" "13-Mar-2019" "" "bst workspace list Manual"
+.TH "BST WORKSPACE LIST" "1" "27-Jun-2019" "" "bst workspace list Manual"
 .SH NAME
 bst\-workspace\-list \- List open workspaces
 .SH SYNOPSIS
diff --git a/man/bst-workspace-open.1 b/man/bst-workspace-open.1
index 4a1065d69..047330193 100644
--- a/man/bst-workspace-open.1
+++ b/man/bst-workspace-open.1
@@ -1,4 +1,4 @@
-.TH "BST WORKSPACE OPEN" "1" "13-Mar-2019" "" "bst workspace open Manual"
+.TH "BST WORKSPACE OPEN" "1" "27-Jun-2019" "" "bst workspace open Manual"
 .SH NAME
 bst\-workspace\-open \- Open a new workspace
 .SH SYNOPSIS
diff --git a/man/bst-workspace-reset.1 b/man/bst-workspace-reset.1
index d37a7f1c2..7f3e2c8ae 100644
--- a/man/bst-workspace-reset.1
+++ b/man/bst-workspace-reset.1
@@ -1,4 +1,4 @@
-.TH "BST WORKSPACE RESET" "1" "13-Mar-2019" "" "bst workspace reset Manual"
+.TH "BST WORKSPACE RESET" "1" "27-Jun-2019" "" "bst workspace reset Manual"
 .SH NAME
 bst\-workspace\-reset \- Reset a workspace to its original state
 .SH SYNOPSIS
diff --git a/man/bst-workspace.1 b/man/bst-workspace.1
index 337620584..4d2ee00ff 100644
--- a/man/bst-workspace.1
+++ b/man/bst-workspace.1
@@ -1,4 +1,4 @@
-.TH "BST WORKSPACE" "1" "13-Mar-2019" "" "bst workspace Manual"
+.TH "BST WORKSPACE" "1" "27-Jun-2019" "" "bst workspace Manual"
 .SH NAME
 bst\-workspace \- Manipulate developer workspaces
 .SH SYNOPSIS
diff --git a/man/bst.1 b/man/bst.1
index ff5e79b6f..3ca3ca3be 100644
--- a/man/bst.1
+++ b/man/bst.1
@@ -1,4 +1,4 @@
-.TH "BST" "1" "13-Mar-2019" "" "bst Manual"
+.TH "BST" "1" "27-Jun-2019" "" "bst Manual"
 .SH NAME
 bst \- Build and manipulate BuildStream projects...
 .SH SYNOPSIS
@@ -68,7 +68,7 @@ The mirror to fetch from first, before attempting other mirrors
 \fB\-\-pull\-buildtrees\fP
 Include an element's build tree when pulling remote element artifacts
 .TP
-\fB\-\-cache\-buildtrees\fP [always|failure|never]
+\fB\-\-cache\-buildtrees\fP [always|auto|never]
 Cache artifact build tree content on creation
 .SH COMMANDS
 .PP
diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py
index 53adb188c..bbb9240f4 100644
--- a/src/buildstream/_frontend/cli.py
+++ b/src/buildstream/_frontend/cli.py
@@ -751,8 +751,6 @@ def source_track(app, elements, deps, except_, cross_junctions):
 @click.option('--deps', '-d', default='none',
               type=click.Choice(['build', 'none', 'run', 'all']),
               help='The dependencies whose sources to checkout (default: none)')
-@click.option('--fetch', 'fetch_', default=False, is_flag=True,
-              help='Fetch elements if they are not fetched')
 @click.option('--tar', 'tar', default=False, is_flag=True,
               help='Create a tarball from the element\'s sources instead of a '
                    'file tree.')
@@ -760,7 +758,7 @@ def source_track(app, elements, deps, except_, cross_junctions):
 @click.argument('element', required=False, type=click.Path(readable=False))
 @click.argument('location', type=click.Path(), required=False)
 @click.pass_obj
-def source_checkout(app, element, location, force, deps, fetch_, except_,
+def source_checkout(app, element, location, force, deps, except_,
                     tar, build_scripts):
     """Checkout sources of an element to the specified location
 
@@ -786,7 +784,6 @@ def source_checkout(app, element, location, force, deps, fetch_, except_,
                                    location=location,
                                    force=force,
                                    deps=deps,
-                                   fetch=fetch_,
                                    except_targets=except_,
                                    tar=tar,
                                    include_build_scripts=build_scripts)
diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py
index a7db33bb9..195be55ba 100644
--- a/src/buildstream/_stream.py
+++ b/src/buildstream/_stream.py
@@ -631,14 +631,12 @@ class Stream():
     #    target (str): The target element whose sources to checkout
     #    location (str): Location to checkout the sources to
     #    deps (str): The dependencies to checkout
-    #    fetch (bool): Whether to fetch missing sources
     #    except_targets (list): List of targets to except from staging
     #
     def source_checkout(self, target, *,
                         location=None,
                         force=False,
                         deps='none',
-                        fetch=False,
                         except_targets=(),
                         tar=False,
                         include_build_scripts=False):
@@ -650,14 +648,13 @@ class Stream():
                                  except_targets=except_targets)
 
         # Assert all sources are cached in the source dir
-        if fetch:
-            self._fetch(elements, fetch_original=True)
+        self._fetch(elements)
         self._pipeline.assert_sources_cached(elements)
 
         # Stage all sources determined by scope
         try:
             self._source_checkout(elements, location, force, deps,
-                                  fetch, tar, include_build_scripts)
+                                  tar, include_build_scripts)
         except BstError as e:
             raise StreamError("Error while writing sources"
                               ": '{}'".format(e), detail=e.detail, reason=e.reason) from e
@@ -1340,7 +1337,6 @@ class Stream():
                          location=None,
                          force=False,
                          deps='none',
-                         fetch=False,
                          tar=False,
                          include_build_scripts=False):
         location = os.path.abspath(location)
diff --git a/tests/frontend/source_checkout.py b/tests/frontend/source_checkout.py
index 0bedfe1e2..38041c45d 100644
--- a/tests/frontend/source_checkout.py
+++ b/tests/frontend/source_checkout.py
@@ -9,7 +9,6 @@ import pytest
 from buildstream.testing import cli  # pylint: disable=unused-import
 
 from buildstream import utils, _yaml
-from buildstream._exceptions import ErrorDomain
 
 # Project directory
 DATA_DIR = os.path.join(
@@ -145,8 +144,7 @@ def test_source_checkout_except(datafiles, cli):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize('fetch', [(False), (True)])
-def test_source_checkout_fetch(datafiles, cli, fetch):
+def test_source_checkout_fetch(datafiles, cli):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'source-checkout')
     target = 'remote-import-dev.bst'
@@ -158,21 +156,16 @@ def test_source_checkout_fetch(datafiles, cli, fetch):
         'pony.h')
     _yaml.dump(element, target_path)
 
-    # Testing --fetch option requires that we do not have the sources
+    # Testing implicit fetching requires that we do not have the sources
     # cached already
     assert cli.get_element_state(project, target) == 'fetch needed'
 
     args = ['source', 'checkout']
-    if fetch:
-        args += ['--fetch']
     args += [target, checkout]
     result = cli.run(project=project, args=args)
 
-    if fetch:
-        result.assert_success()
-        assert os.path.exists(os.path.join(checkout, 'remote-import-dev', 'pony.h'))
-    else:
-        result.assert_main_error(ErrorDomain.PIPELINE, 'uncached-sources')
+    result.assert_success()
+    assert os.path.exists(os.path.join(checkout, 'remote-import-dev', 'pony.h'))
 
 
 @pytest.mark.datafiles(DATA_DIR)
diff --git a/tests/sourcecache/source-checkout.py b/tests/sourcecache/source-checkout.py
index 15c80fde9..c2c7fe3cd 100644
--- a/tests/sourcecache/source-checkout.py
+++ b/tests/sourcecache/source-checkout.py
@@ -48,14 +48,8 @@ def test_source_checkout(tmpdir, datafiles, cli):
 
     repo = create_element_size('target.bst', project_dir, element_path, [], 100000)
 
-    # without fetch it should fail
+    # check implicit fetching
     res = cli.run(project=project_dir, args=['source', 'checkout', 'target.bst', target_dir])
-    res.assert_main_error(ErrorDomain.PIPELINE, "uncached-sources")
-
-    # fetch and check it works
-    res = cli.run(project=project_dir,
-                  args=['source', 'checkout', '--fetch', 'target.bst',
-                        target_dir])
     res.assert_success()
     assert "Fetching from" in res.stderr
 
@@ -75,5 +69,5 @@ def test_source_checkout(tmpdir, datafiles, cli):
     shutil.rmtree(os.path.join(cache_dir, 'cas'))
 
     res = cli.run(project=project_dir,
-                  args=['source', 'checkout', '--fetch', 'target.bst', target_dir])
+                  args=['source', 'checkout', 'target.bst', target_dir])
     res.assert_task_error(ErrorDomain.PLUGIN, None)