Add ability to build without interacting with available artifacttpollard/752

servers.

By adding a new user context and bst main option, we can provide
more granular control of what remotes are utilised during relevant
bst pull and push actions.

_artifactcache/artifactcache.py: Expand setup_remotes() to consider
the user context value of 'use_remotes'. Allowing the user to omit
interacting with all, or all non user defined remotes.

tests/frontend: pull.py & push.py addition of tests for checking
expected behaviour. Ensures interaction with user, project or no
remotes.

4 files changed, 157 insertions, 6 deletions

diff --git a/NEWS b/NEWS
index 449a23ff1..cf5a22dbd 100644
--- a/NEWS
+++ b/NEWS
@@ -122,6 +122,15 @@ buildstream 1.3.1
     'shell', 'show', 'source-checkout', 'track', 'workspace close' and 'workspace reset'
     commands are affected.
 
+  o bst interaction with defined artifact servers can be controlled more granularly.
+    This can be done via the user configuration option `useremotes` or via the bst cli
+    main option '--use-remotes'. This can be set as 'none', 'user' or the default value
+    'all'. Unless specifically overriden, when considering wether to pull or push to
+    available artifact servers (be it user or project defined) this optional config option
+    will be used. Setting this value to 'user' for example and performing a build would
+    lead to any project or junction defined artifact server to be ignored, whilst still
+    attempting to any user defined remotes.
+
 
 =================
 buildstream 1.1.5
diff --git a/buildstream/_artifactcache.py b/buildstream/_artifactcache.py
index 48ab278c4..c162c3974 100644
--- a/buildstream/_artifactcache.py
+++ b/buildstream/_artifactcache.py
@@ -148,7 +148,7 @@ class ArtifactCache():
     # Sets up which remotes to use
     #
     # Args:
-    #    use_config (bool): Whether to use project configuration
+    #    use_config (bool): Whether to use configuration
     #    remote_url (str): Remote artifact cache URL
     #
     # This requires that all of the projects which are to be processed in the session
@@ -167,11 +167,16 @@ class ArtifactCache():
             self._set_remotes([ArtifactCacheSpec(remote_url, push=True)])
             has_remote_caches = True
         if use_config:
-            for project in self.context.get_projects():
-                artifact_caches = _configured_remote_artifact_cache_specs(self.context, project)
-                if artifact_caches:  # artifact_caches is a list of ArtifactCacheSpec instances
-                    self._set_remotes(artifact_caches, project=project)
-                    has_remote_caches = True
+            if self.context.use_remotes == 'all':
+                for project in self.context.get_projects():
+                    artifact_caches = _configured_remote_artifact_cache_specs(self.context, project)
+                    if artifact_caches:  # artifact_caches is a list of ArtifactCacheSpec instances
+                        self._set_remotes(artifact_caches, project=project)
+                        has_remote_caches = True
+            # If configured to only use user configured remotes, pass existing user cache spec
+            elif self.context.use_remotes == 'user' and self.context.artifact_cache_specs:
+                self._set_remotes(self.context.artifact_cache_specs)
+                has_remote_caches = True
         if has_remote_caches:
             self._initialize_remotes()
 
diff --git a/tests/frontend/pull.py b/tests/frontend/pull.py
index 20b740948..57e1bfdb4 100644
--- a/tests/frontend/pull.py
+++ b/tests/frontend/pull.py
@@ -408,3 +408,75 @@ def test_pull_missing_notifies_user(caplog, cli, tmpdir, datafiles):
 
         assert "INFO    Remote ({}) does not have".format(share.repo) in result.stderr
         assert "SKIPPED Pull" in result.stderr
+
+
+# Tests that:
+#
+#  * The bst main option --use-remotes limits remote action
+#    as expected for pull jobs
+#
+@pytest.mark.datafiles(DATA_DIR)
+def test_useremotes_cli_options(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+
+    with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare1')) as shareuser,\
+        create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) as shareproject:
+
+        # Add shareproject repo url to project.conf
+        with open(os.path.join(project, "project.conf"), "a") as projconf:
+            projconf.write("artifacts:\n  url: {}\n  push: True".format(shareproject.repo))
+
+        # First build the target element and push to the remotes.
+        # We need the artifact available in the remotes to test against.
+        cli.configure({
+            'artifacts': {'url': shareuser.repo, 'push': True}
+        })
+        result = cli.run(project=project, args=['build', 'target.bst'])
+        result.assert_success()
+        assert cli.get_element_state(project, 'target.bst') == 'cached'
+
+        # Assert that everything is now cached in the remotes.
+        all_elements = ['target.bst', 'import-bin.bst', 'compose-all.bst']
+        for element_name in all_elements:
+            assert_shared(cli, shareuser, project, element_name)
+            assert_shared(cli, shareproject, project, element_name)
+
+        # Now we've pushed, delete the user's local artifact cache
+        artifacts = os.path.join(cli.directory, 'artifacts')
+        shutil.rmtree(artifacts)
+
+        # Assert that nothing is cached locally anymore
+        for element_name in all_elements:
+            assert cli.get_element_state(project, element_name) != 'cached'
+
+        # Attempt bst build with --use-remotes set as none, this should lead to
+        # a complete rebuild without pulling from either artifact remote cache
+        result = cli.run(project=project, args=['--use-remotes', 'none', 'build', 'target.bst'])
+        result.assert_success()
+        for element_name in all_elements:
+            assert element_name not in result.get_pulled_elements()
+
+        # Delete local cache again
+        artifacts = os.path.join(cli.directory, 'artifacts')
+        shutil.rmtree(artifacts)
+
+        # Attempt bst build with --use-remotes set as user, as the shareuser is
+        # passed in as user config and not via a project, assert project remote
+        # was not attempted by it not being in the output
+        result = cli.run(project=project, args=['--use-remotes', 'user', 'build', 'target.bst'])
+        result.assert_success()
+        for element_name in all_elements:
+            assert element_name in result.get_pulled_elements()
+        assert shareproject.repo not in result.stderr
+
+        # Delete local cache again
+        artifacts = os.path.join(cli.directory, 'artifacts')
+        shutil.rmtree(artifacts)
+
+        # Attempt bst build with --use-remotes set as all, this time
+        # assert that project remote is attempted and in the output
+        result = cli.run(project=project, args=['--use-remotes', 'all', 'build', 'target.bst'])
+        result.assert_success()
+        for element_name in all_elements:
+            assert element_name in result.get_pulled_elements()
+        assert shareproject.repo in result.stderr
diff --git a/tests/frontend/push.py b/tests/frontend/push.py
index f3f9b52f1..b728a2f58 100644
--- a/tests/frontend/push.py
+++ b/tests/frontend/push.py
@@ -416,3 +416,68 @@ def test_push_already_cached(caplog, cli, tmpdir, datafiles):
         assert not result.get_pushed_elements(), "No elements should have been pushed since the cache was populated"
         assert "INFO    Remote ({}) already has ".format(share.repo) in result.stderr
         assert "SKIPPED Push" in result.stderr
+
+
+# Tests that:
+#
+#  * The bst main option --use-remotes limits remote action
+#    as expected for push jobs
+#
+@pytest.mark.datafiles(DATA_DIR)
+def test_useremotes_cli_options(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+
+    with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare1')) as shareuser,\
+        create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) as shareproject:
+
+        # Add shareproject repo url to project.conf
+        with open(os.path.join(project, "project.conf"), "a") as projconf:
+            projconf.write("artifacts:\n  url: {}\n  push: True".format(shareproject.repo))
+
+        # Configure shareuser remote in user conf
+        cli.configure({
+            'artifacts': {'url': shareuser.repo, 'push': True}
+        })
+
+        # First build the target element with --use-remotes set as none.
+        # This should lead to a complete build without pushing to either artifact
+        # remote cache
+        result = cli.run(project=project, args=['--use-remotes', 'none', 'build', 'target.bst'])
+        result.assert_success()
+        assert not result.get_pushed_elements()
+        assert cli.get_element_state(project, 'target.bst') == 'cached'
+
+        # Delete the artifacts from the local artifact cache
+        all_elements = ['target.bst', 'import-bin.bst', 'compose-all.bst']
+        for element_name in all_elements:
+            cli.remove_artifact_from_cache(project, element_name)
+
+        # Assert that nothing is cached locally anymore
+        for element_name in all_elements:
+            assert cli.get_element_state(project, element_name) != 'cached'
+
+        # Attempt bst build with --use-remotes set as user, this should lead to
+        # a complete rebuild, with artifacts pushed to the shareuser remote artifact cache
+        # only. Assert project remote was not attempted by it not being in the output
+        result = cli.run(project=project, args=['--use-remotes', 'user', 'build', 'target.bst'])
+        result.assert_success()
+        for element_name in all_elements:
+            assert element_name in result.get_pushed_elements()
+        for element_name in all_elements:
+            assert_shared(cli, shareuser, project, element_name)
+        assert shareproject.repo not in result.stderr
+
+        # Delete the artifacts from the local artifact cache
+        all_elements = ['target.bst', 'import-bin.bst', 'compose-all.bst']
+        for element_name in all_elements:
+            cli.remove_artifact_from_cache(project, element_name)
+
+        # Attempt bst build with --use-remotes set as all, this should lead to
+        # a complete rebuild, with artifacts pushed to both the shareuser and
+        # shareproject remote artifacts caches
+        result = cli.run(project=project, args=['--use-remotes', 'all', 'build', 'target.bst'])
+        result.assert_success()
+        for element_name in all_elements:
+            assert element_name in result.get_pushed_elements()
+        for element_name in all_elements:
+            assert_shared(cli, shareproject, project, element_name)