WIP: Make uploading of build trees configurabletpollard/566

8 files changed, 403 insertions, 54 deletions

diff --git a/buildstream/_artifactcache.py b/buildstream/_artifactcache.py
index 5404dc12e..ee2cf5379 100644
--- a/buildstream/_artifactcache.py
+++ b/buildstream/_artifactcache.py
@@ -107,6 +107,7 @@ class ArtifactCache():
 
         self._has_fetch_remotes = False
         self._has_push_remotes = False
+        self._has_partial_push_remotes = False
 
         os.makedirs(self.extractdir, exist_ok=True)
 
@@ -488,6 +489,9 @@ class ArtifactCache():
                 self._has_fetch_remotes = True
                 if remote_spec.push:
                     self._has_push_remotes = True
+                    # Partial push requires generic push option to also be set
+                    if remote_spec.partial_push:
+                        self._has_partial_push_remotes = True
 
                 remotes[remote_spec.url] = CASRemote(remote_spec)
 
@@ -685,6 +689,32 @@ class ArtifactCache():
             remotes_for_project = self._remotes[element._get_project()]
             return any(remote.spec.push for remote in remotes_for_project)
 
+    # has_partial_push_remotes():
+    #
+    # Check whether any remote repositories are available for pushing
+    # non-complete artifacts. This option requires the generic push value
+    # to also be set.
+    #
+    # Args:
+    #     element (Element): The Element to check
+    #
+    # Returns:
+    #   (bool): True if any remote repository is configured for optional
+    #            partial pushes, False otherwise
+    #
+    def has_partial_push_remotes(self, *, element=None):
+        # If there's no partial push remotes available, we can't partial push at all
+        if not self._has_partial_push_remotes:
+            return False
+        elif element is None:
+            # At least one remote is set to allow partial pushes
+            return True
+        else:
+            # Check whether the specified element's project has push remotes configured
+            # to not accept partial artifact pushes
+            remotes_for_project = self._remotes[element._get_project()]
+            return any(remote.spec.partial_push for remote in remotes_for_project)
+
     # push():
     #
     # Push committed artifact to remote repository.
@@ -692,6 +722,8 @@ class ArtifactCache():
     # Args:
     #     element (Element): The Element whose artifact is to be pushed
     #     keys (list): The cache keys to use
+    #     partial(bool): If the artifact is cached in a partial state
+    #     subdir(string): Optional subdir to not push
     #
     # Returns:
     #   (bool): True if any remote was updated, False if no pushes were required
@@ -699,12 +731,25 @@ class ArtifactCache():
     # Raises:
     #   (ArtifactError): if there was an error
     #
-    def push(self, element, keys):
+    def push(self, element, keys, partial=False, subdir=None):
         refs = [self.get_artifact_fullname(element, key) for key in list(keys)]
 
         project = element._get_project()
 
-        push_remotes = [r for r in self._remotes[project] if r.spec.push]
+        push_remotes = []
+        partial_remotes = []
+
+        # Create list of remotes to push to, given current element and partial push config
+        if not partial:
+            push_remotes = [r for r in self._remotes[project] if (r.spec.push and not r.spec.partial_push)]
+
+        if self._has_partial_push_remotes:
+            # Create a specific list of the remotes expecting the artifact to be push in a partial
+            # state. This list needs to be pushed in a partial state, without the optional subdir if
+            # exists locally. No need to attempt pushing a partial artifact to a remote that is queued to
+            # to also recieve a full artifact
+            partial_remotes = [r for r in self._remotes[project] if (r.spec.partial_push and r.spec.push) and
+                               r not in push_remotes]
 
         pushed = False
 
@@ -713,7 +758,9 @@ class ArtifactCache():
             display_key = element._get_brief_display_key()
             element.status("Pushing artifact {} -> {}".format(display_key, remote.spec.url))
 
-            if self.cas.push(refs, remote):
+            # Passing the optional subdir allows for remote artifacts that are cached in a 'partial'
+            # state to be completed
+            if self.cas.push(refs, remote, subdir=subdir):
                 element.info("Pushed artifact {} -> {}".format(display_key, remote.spec.url))
                 pushed = True
             else:
@@ -721,6 +768,19 @@ class ArtifactCache():
                     remote.spec.url, element._get_brief_display_key()
                 ))
 
+        for remote in partial_remotes:
+            remote.init()
+            display_key = element._get_brief_display_key()
+            element.status("Pushing partial artifact {} -> {}".format(display_key, remote.spec.url))
+
+            if self.cas.push(refs, remote, excluded_subdirs=subdir):
+                element.info("Pushed partial artifact {} -> {}".format(display_key, remote.spec.url))
+                pushed = True
+            else:
+                element.info("Remote ({}) already has {} partial cached".format(
+                    remote.spec.url, element._get_brief_display_key()
+                ))
+
         return pushed
 
     # pull():
@@ -748,14 +808,23 @@ class ArtifactCache():
                 element.status("Pulling artifact {} <- {}".format(display_key, remote.spec.url))
 
                 if self.cas.pull(ref, remote, progress=progress, subdir=subdir, excluded_subdirs=excluded_subdirs):
-                    element.info("Pulled artifact {} <- {}".format(display_key, remote.spec.url))
                     if subdir:
-                        # Attempt to extract subdir into artifact extract dir if it already exists
-                        # without containing the subdir. If the respective artifact extract dir does not
-                        # exist a complete extraction will complete.
-                        self.extract(element, key, subdir)
-                    # no need to pull from additional remotes
-                    return True
+                        if not self.contains_subdir_artifact(element, key, subdir):
+                            # The pull was expecting the specific subdir to be present in the remote, attempt
+                            # to find it in other available remotes
+                            element.info("Pulled partial artifact {} <- {}. Attempting to retrieve {} from remotes"
+                                         .format(display_key, remote.spec.url, subdir))
+                        else:
+                            element.info("Pulled artifact {} <- {}".format(display_key, remote.spec.url))
+                            # Attempt to extract subdir into artifact extract dir if it already exists
+                            # without containing the subdir. If the respective artifact extract dir does not
+                            # exist a complete extraction will complete.
+                            self.extract(element, key, subdir)
+                            # no need to pull from additional remotes
+                            return True
+                    else:
+                        element.info("Pulled artifact {} <- {}".format(display_key, remote.spec.url))
+                        return True
                 else:
                     element.info("Remote ({}) does not have {} cached".format(
                         remote.spec.url, element._get_brief_display_key()
diff --git a/buildstream/_cas/cascache.py b/buildstream/_cas/cascache.py
index 9d7a121f4..e284d83b3 100644
--- a/buildstream/_cas/cascache.py
+++ b/buildstream/_cas/cascache.py
@@ -200,34 +200,47 @@ class CASCache():
     #   (bool): True if pull was successful, False if ref was not available
     #
     def pull(self, ref, remote, *, progress=None, subdir=None, excluded_subdirs=None):
-        try:
-            remote.init()
 
-            request = buildstream_pb2.GetReferenceRequest(instance_name=remote.spec.instance_name)
-            request.key = ref
-            response = remote.ref_storage.GetReference(request)
+        tree_found = False
 
-            tree = remote_execution_pb2.Digest()
-            tree.hash = response.digest.hash
-            tree.size_bytes = response.digest.size_bytes
+        while True:
+            try:
+                if not tree_found:
+                    remote.init()
 
-            # Check if the element artifact is present, if so just fetch the subdir.
-            if subdir and os.path.exists(self.objpath(tree)):
-                self._fetch_subdir(remote, tree, subdir)
-            else:
-                # Fetch artifact, excluded_subdirs determined in pullqueue
-                self._fetch_directory(remote, tree, excluded_subdirs=excluded_subdirs)
+                    request = buildstream_pb2.GetReferenceRequest(instance_name=remote.spec.instance_name)
+                    request.key = ref
+                    response = remote.ref_storage.GetReference(request)
 
-            self.set_ref(ref, tree)
+                    tree = remote_execution_pb2.Digest()
+                    tree.hash = response.digest.hash
+                    tree.size_bytes = response.digest.size_bytes
 
-            return True
-        except grpc.RpcError as e:
-            if e.code() != grpc.StatusCode.NOT_FOUND:
-                raise CASCacheError("Failed to pull ref {}: {}".format(ref, e)) from e
-            else:
-                return False
-        except BlobNotFound as e:
-            return False
+                # Check if the element artifact is present, if so just fetch the subdir.
+                if subdir and os.path.exists(self.objpath(tree)):
+                    self._fetch_subdir(remote, tree, subdir)
+                else:
+                    # Fetch artifact, excluded_subdirs determined in pullqueue
+                    self._fetch_directory(remote, tree, excluded_subdirs=excluded_subdirs)
+
+                self.set_ref(ref, tree)
+
+                return True
+            except grpc.RpcError as e:
+                if e.code() != grpc.StatusCode.NOT_FOUND:
+                    raise CASCacheError("Failed to pull ref {}: {}".format(ref, e)) from e
+                else:
+                    return False
+            except BlobNotFound as e:
+                if not excluded_subdirs and subdir:
+                    # The remote has the top level digest but could not complete a full pull,
+                    # attempt partial without the need to initialise and check for the artifact
+                    # digest. This default behaviour of dropping back to partial pulls could
+                    # be made a configurable warning given at artfictcache level.
+                    tree_found = True
+                    excluded_subdirs, subdir = subdir, excluded_subdirs
+                else:
+                    return False
 
     # pull_tree():
     #
@@ -272,6 +285,8 @@ class CASCache():
     # Args:
     #     refs (list): The refs to push
     #     remote (CASRemote): The remote to push to
+    #     subdir (string): Optional specific subdir to include in the push
+    #     excluded_subdirs (list): The optional list of subdirs to not push
     #
     # Returns:
     #   (bool): True if any remote was updated, False if no pushes were required
@@ -279,7 +294,7 @@ class CASCache():
     # Raises:
     #   (CASCacheError): if there was an error
     #
-    def push(self, refs, remote):
+    def push(self, refs, remote, *, subdir=None, excluded_subdirs=None):
         skipped_remote = True
         try:
             for ref in refs:
@@ -293,15 +308,18 @@ class CASCache():
                     response = remote.ref_storage.GetReference(request)
 
                     if response.digest.hash == tree.hash and response.digest.size_bytes == tree.size_bytes:
-                        # ref is already on the server with the same tree
-                        continue
+                        # ref is already on the server with the same tree, however it might be partially cached.
+                        # If artifact is not set to be pushed partially attempt to 'complete' the remote artifact if
+                        # needed, else continue.
+                        if excluded_subdirs or remote.verify_digest_on_remote(self._get_subdir(tree, subdir)):
+                            continue
 
                 except grpc.RpcError as e:
                     if e.code() != grpc.StatusCode.NOT_FOUND:
                         # Intentionally re-raise RpcError for outer except block.
                         raise
 
-                self._send_directory(remote, tree)
+                self._send_directory(remote, tree, excluded_dir=excluded_subdirs)
 
                 request = buildstream_pb2.UpdateReferenceRequest(instance_name=remote.spec.instance_name)
                 request.keys.append(ref)
@@ -784,10 +802,17 @@ class CASCache():
                 a += 1
                 b += 1
 
-    def _reachable_refs_dir(self, reachable, tree, update_mtime=False):
+    def _reachable_refs_dir(self, reachable, tree, update_mtime=False, subdir=False):
         if tree.hash in reachable:
             return
 
+        # If looping through subdir digests, skip processing if
+        # ref path does not exist, allowing for partial objects
+        if subdir and not os.path.exists(self.objpath(tree)):
+            return
+
+        # Raises FileNotFound exception is path does not exist,
+        # which should only be thrown on the top level digest
         if update_mtime:
             os.utime(self.objpath(tree))
 
@@ -804,9 +829,9 @@ class CASCache():
             reachable.add(filenode.digest.hash)
 
         for dirnode in directory.directories:
-            self._reachable_refs_dir(reachable, dirnode.digest, update_mtime=update_mtime)
+            self._reachable_refs_dir(reachable, dirnode.digest, update_mtime=update_mtime, subdir=True)
 
-    def _required_blobs(self, directory_digest):
+    def _required_blobs(self, directory_digest, excluded_dir=None):
         # parse directory, and recursively add blobs
         d = remote_execution_pb2.Digest()
         d.hash = directory_digest.hash
@@ -825,7 +850,8 @@ class CASCache():
             yield d
 
         for dirnode in directory.directories:
-            yield from self._required_blobs(dirnode.digest)
+            if dirnode.name != excluded_dir:
+                yield from self._required_blobs(dirnode.digest)
 
     # _ensure_blob():
     #
@@ -930,6 +956,7 @@ class CASCache():
             objpath = self._ensure_blob(remote, dir_digest)
 
             directory = remote_execution_pb2.Directory()
+
             with open(objpath, 'rb') as f:
                 directory.ParseFromString(f.read())
 
@@ -972,8 +999,8 @@ class CASCache():
 
         return dirdigest
 
-    def _send_directory(self, remote, digest, u_uid=uuid.uuid4()):
-        required_blobs = self._required_blobs(digest)
+    def _send_directory(self, remote, digest, u_uid=uuid.uuid4(), excluded_dir=None):
+        required_blobs = self._required_blobs(digest, excluded_dir=excluded_dir)
 
         missing_blobs = dict()
         # Limit size of FindMissingBlobs request
diff --git a/buildstream/_cas/casremote.py b/buildstream/_cas/casremote.py
index 56ba4c5d8..a953165f4 100644
--- a/buildstream/_cas/casremote.py
+++ b/buildstream/_cas/casremote.py
@@ -23,7 +23,8 @@ from .. import utils
 _MAX_PAYLOAD_BYTES = 1024 * 1024
 
 
-class CASRemoteSpec(namedtuple('CASRemoteSpec', 'url push server_cert client_key client_cert instance_name')):
+class CASRemoteSpec(namedtuple('CASRemoteSpec',
+                               'url push partial_push server_cert client_key client_cert instance_name')):
 
     # _new_from_config_node
     #
@@ -31,9 +32,18 @@ class CASRemoteSpec(namedtuple('CASRemoteSpec', 'url push server_cert client_key
     #
     @staticmethod
     def _new_from_config_node(spec_node, basedir=None):
-        _yaml.node_validate(spec_node, ['url', 'push', 'server-cert', 'client-key', 'client-cert', 'instance_name'])
+        _yaml.node_validate(spec_node, ['url', 'push', 'allow-partial-push', 'server-cert', 'client-key',
+                                        'client-cert', 'instance_name'])
         url = _yaml.node_get(spec_node, str, 'url')
         push = _yaml.node_get(spec_node, bool, 'push', default_value=False)
+        partial_push = _yaml.node_get(spec_node, bool, 'allow-partial-push', default_value=False)
+
+        # partial_push depends on push, raise error if not configured correctly
+        if partial_push and not push:
+            provenance = _yaml.node_get_provenance(spec_node, 'allow-partial-push')
+            raise LoadError(LoadErrorReason.INVALID_DATA,
+                            "{}: allow-partial-push also requires push to be set".format(provenance))
+
         if not url:
             provenance = _yaml.node_get_provenance(spec_node, 'url')
             raise LoadError(LoadErrorReason.INVALID_DATA,
@@ -63,10 +73,10 @@ class CASRemoteSpec(namedtuple('CASRemoteSpec', 'url push server_cert client_key
             raise LoadError(LoadErrorReason.INVALID_DATA,
                             "{}: 'client-cert' was specified without 'client-key'".format(provenance))
 
-        return CASRemoteSpec(url, push, server_cert, client_key, client_cert, instance_name)
+        return CASRemoteSpec(url, push, partial_push, server_cert, client_key, client_cert, instance_name)
 
 
-CASRemoteSpec.__new__.__defaults__ = (None, None, None, None)
+CASRemoteSpec.__new__.__defaults__ = (False, None, None, None, None)
 
 
 class BlobNotFound(CASRemoteError):
diff --git a/buildstream/element.py b/buildstream/element.py
index a243826ed..705c19e7e 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -1797,13 +1797,19 @@ class Element(Plugin):
     #   (bool): True if this element does not need a push job to be created
     #
     def _skip_push(self):
+
         if not self.__artifacts.has_push_remotes(element=self):
             # No push remotes for this element's project
             return True
 
         # Do not push elements that aren't cached, or that are cached with a dangling buildtree
-        # artifact unless element type is expected to have an an empty buildtree directory
-        if not self._cached_buildtree():
+        # artifact unless element type is expected to have an an empty buildtree directory. Check
+        # that this default behaviour is not overriden via a remote configured to allow pushing
+        # artifacts without their corresponding buildtree.
+        if not self._cached():
+            return True
+
+        if not self._cached_buildtree() and not self.__artifacts.has_partial_push_remotes(element=self):
             return True
 
         # Do not push tainted artifact
@@ -1814,11 +1820,14 @@ class Element(Plugin):
 
     # _push():
     #
-    # Push locally cached artifact to remote artifact repository.
+    # Push locally cached artifact to remote artifact repository. An attempt
+    # will be made to push partial artifacts if given current config dictates.
+    # If a remote set for 'full' artifact pushes is found to be cached partially
+    # in the remote, an attempt will be made to 'complete' it.
     #
     # Returns:
     #   (bool): True if the remote was updated, False if it already existed
-    #           and no updated was required
+    #           and no update was required
     #
     def _push(self):
         self.__assert_cached()
@@ -1827,8 +1836,17 @@ class Element(Plugin):
             self.warn("Not pushing tainted artifact.")
             return False
 
-        # Push all keys used for local commit
-        pushed = self.__artifacts.push(self, self.__get_cache_keys_for_commit())
+        # Push all keys used for local commit, this could be full or partial,
+        # given previous _skip_push() logic. If buildtree isn't cached, then
+        # set partial push
+
+        partial = False
+        subdir = 'buildtree'
+        if not self._cached_buildtree():
+            partial = True
+
+        pushed = self.__artifacts.push(self, self.__get_cache_keys_for_commit(), partial=partial, subdir=subdir)
+
         if not pushed:
             return False
 
diff --git a/buildstream/plugintestutils/runcli.py b/buildstream/plugintestutils/runcli.py
index fb7c23c6c..bd2f1e43e 100644
--- a/buildstream/plugintestutils/runcli.py
+++ b/buildstream/plugintestutils/runcli.py
@@ -238,6 +238,13 @@ class Result():
 
         return list(pushed)
 
+    def get_partial_pushed_elements(self):
+        pushed = re.findall(r'\[\s*push:(\S+)\s*\]\s*INFO\s*Pushed partial artifact', self.stderr)
+        if pushed is None:
+            return []
+
+        return list(pushed)
+
     def get_pulled_elements(self):
         pulled = re.findall(r'\[\s*pull:(\S+)\s*\]\s*INFO\s*Pulled artifact', self.stderr)
         if pulled is None:
@@ -245,6 +252,13 @@ class Result():
 
         return list(pulled)
 
+    def get_partial_pulled_elements(self):
+        pulled = re.findall(r'\[\s*pull:(\S+)\s*\]\s*INFO\s*Pulled partial artifact', self.stderr)
+        if pulled is None:
+            return []
+
+        return list(pulled)
+
 
 class Cli():
 
@@ -265,11 +279,15 @@ class Cli():
     #
     # Args:
     #    config (dict): The user configuration to use
+    #    reset (bool): Optional reset of stored config
     #
-    def configure(self, config):
+    def configure(self, config, reset=False):
         if self.config is None:
             self.config = {}
 
+        if reset:
+            self.config.clear()
+
         for key, val in config.items():
             self.config[key] = val
 
diff --git a/doc/source/using_config.rst b/doc/source/using_config.rst
index 71ca64a4a..e696a9d9d 100644
--- a/doc/source/using_config.rst
+++ b/doc/source/using_config.rst
@@ -59,6 +59,15 @@ configuration:
      # Add another cache to pull from
      - url: https://anothercache.com/artifacts:8080
        server-cert: another_server.crt
+     # Add a cache to push/pull to/from, specifying
+       that you wish to push artifacts in a 'partial'
+       state (this being without the respective buildtree).
+       Note that allow-partial-push requires push to also
+       be set.
+     - url: https://anothercache.com/artifacts:11003
+       push: true
+       allow-partial-push: true
+
 
 .. note::
 
@@ -86,6 +95,14 @@ configuration:
          # Add another cache to pull from
          - url: https://ourprojectcache.com/artifacts:8080
            server-cert: project_server.crt
+         # Add a cache to push/pull to/from, specifying
+           that you wish to push artifacts in a 'partial'
+           state (this being without the respective buildtree).
+           Note that allow-partial-push requires push to also
+           be set.
+         - url: https://anothercache.com/artifacts:11003
+           push: true
+           allow-partial-push: true
 
 
 .. note::
diff --git a/tests/artifactcache/config.py b/tests/artifactcache/config.py
index fda309725..d89e7f9b2 100644
--- a/tests/artifactcache/config.py
+++ b/tests/artifactcache/config.py
@@ -139,3 +139,28 @@ def test_missing_certs(cli, datafiles, config_key, config_value):
     # This does not happen for a simple `bst show`.
     result = cli.run(project=project, args=['artifact', 'pull', 'element.bst'])
     result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
+
+
+# Assert that if allow-partial-push is specified as true without push also being
+# set likewise, we get a comprehensive LoadError instead of an unhandled exception.
+@pytest.mark.datafiles(DATA_DIR)
+def test_partial_push_error(cli, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename, 'project', 'elements')
+
+    project_conf = {
+        'name': 'test',
+
+        'artifacts': {
+            'url': 'https://cache.example.com:12345',
+            'allow-partial-push': 'True'
+        }
+    }
+    project_conf_file = os.path.join(project, 'project.conf')
+    _yaml.dump(project_conf, project_conf_file)
+
+    # Use `pull` here to ensure we try to initialize the remotes, triggering the error
+    #
+    # This does not happen for a simple `bst show`.
+    result = cli.run(project=project, args=['artifact', 'pull', 'target.bst'])
+    result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
+    assert "allow-partial-push also requires push to be set" in result.stderr
diff --git a/tests/integration/pushbuildtrees.py b/tests/integration/pushbuildtrees.py
new file mode 100644
index 000000000..194d20602
--- /dev/null
+++ b/tests/integration/pushbuildtrees.py
@@ -0,0 +1,165 @@
+import os
+import shutil
+import pytest
+import subprocess
+
+from buildstream import _yaml
+from tests.testutils import create_artifact_share
+from tests.testutils.site import HAVE_SANDBOX
+from buildstream.plugintestutils import cli, cli_integration as cli2
+from buildstream.plugintestutils.integration import assert_contains
+from buildstream._exceptions import ErrorDomain, LoadErrorReason
+
+
+DATA_DIR = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)),
+    "project"
+)
+
+
+# Remove artifact cache & set cli2.config value of pull-buildtrees
+# to false, which is the default user context. The cache has to be
+# cleared as just forcefully removing the refpath leaves dangling objects.
+def default_state(cli2, tmpdir, share):
+    shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
+    cli2.configure({
+        'artifacts': {'url': share.repo, 'push': False},
+        'artifactdir': os.path.join(str(tmpdir), 'artifacts'),
+        'cache': {'pull-buildtrees': False},
+    })
+
+
+# Tests to capture the integration of the optionl push of buildtrees.
+# The behaviour should encompass pushing artifacts that are already cached
+# without a buildtree as well as artifacts that are cached with their buildtree.
+# This option is handled via 'allow-partial-push' on a per artifact remote config
+# node basis. Multiple remote config nodes can point to the same url and as such can
+# have different 'allow-partial-push' options, tests need to cover this using project
+# confs.
+@pytest.mark.integration
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+def test_pushbuildtrees(cli2, tmpdir, datafiles, integration_cache):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    element_name = 'autotools/amhello.bst'
+
+    # Create artifact shares for pull & push testing
+    with create_artifact_share(os.path.join(str(tmpdir), 'share1')) as share1,\
+        create_artifact_share(os.path.join(str(tmpdir), 'share2')) as share2,\
+        create_artifact_share(os.path.join(str(tmpdir), 'share3')) as share3,\
+        create_artifact_share(os.path.join(str(tmpdir), 'share4')) as share4:
+
+        cli2.configure({
+            'artifacts': {'url': share1.repo, 'push': True},
+            'artifactdir': os.path.join(str(tmpdir), 'artifacts')
+        })
+
+        cli2.configure({'artifacts': [{'url': share1.repo, 'push': True},
+                                     {'url': share2.repo, 'push': True, 'allow-partial-push': True}]})
+
+        # Build autotools element, checked pushed, delete local.
+        # As share 2 has push & allow-partial-push set a true, it
+        # should have pushed the artifacts, without the cached buildtrees,
+        # to it.
+        result = cli2.run(project=project, args=['build', element_name])
+        assert result.exit_code == 0
+        assert cli2.get_element_state(project, element_name) == 'cached'
+        elementdigest = share1.has_artifact('test', element_name, cli2.get_element_key(project, element_name))
+        buildtreedir = os.path.join(str(tmpdir), 'artifacts', 'extract', 'test', 'autotools-amhello',
+                                    elementdigest.hash, 'buildtree')
+        assert os.path.isdir(buildtreedir)
+        assert element_name in result.get_partial_pushed_elements()
+        assert element_name in result.get_pushed_elements()
+        assert share1.has_artifact('test', element_name, cli2.get_element_key(project, element_name))
+        assert share2.has_artifact('test', element_name, cli2.get_element_key(project, element_name))
+        default_state(cli2, tmpdir, share1)
+
+        # Check that after explictly pulling an artifact without it's buildtree,
+        # we can push it to another remote that is configured to accept the partial
+        # artifact
+        result = cli2.run(project=project, args=['artifact', 'pull', element_name])
+        assert element_name in result.get_pulled_elements()
+        cli2.configure({'artifacts': {'url': share3.repo, 'push': True, 'allow-partial-push': True}})
+        assert cli2.get_element_state(project, element_name) == 'cached'
+        assert not os.path.isdir(buildtreedir)
+        result = cli2.run(project=project, args=['artifact', 'push', element_name])
+        assert result.exit_code == 0
+        assert element_name in result.get_partial_pushed_elements()
+        assert element_name not in result.get_pushed_elements()
+        assert share3.has_artifact('test', element_name, cli2.get_element_key(project, element_name))
+        default_state(cli2, tmpdir, share3)
+
+        # Delete the local cache and pull the partial artifact from share 3,
+        # this should not include the buildtree when extracted locally, even when
+        # pull-buildtrees is given as a cli2 parameter as no available remotes will
+        # contain the buildtree
+        assert not os.path.isdir(buildtreedir)
+        assert cli2.get_element_state(project, element_name) != 'cached'
+        result = cli2.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])
+        assert element_name in result.get_partial_pulled_elements()
+        assert not os.path.isdir(buildtreedir)
+        default_state(cli2, tmpdir, share3)
+
+        # Delete the local cache and attempt to pull a 'full' artifact, including its
+        # buildtree. As with before share3 being the first listed remote will not have
+        # the buildtree available and should spawn a partial pull. Having share1 as the
+        # second available remote should allow the buildtree to be pulled thus 'completing'
+        # the artifact
+        cli2.configure({'artifacts': [{'url': share3.repo, 'push': True, 'allow-partial-push': True},
+                                     {'url': share1.repo, 'push': True}]})
+        assert cli2.get_element_state(project, element_name) != 'cached'
+        result = cli2.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])
+        assert element_name in result.get_partial_pulled_elements()
+        assert element_name in result.get_pulled_elements()
+        assert "Attempting to retrieve buildtree from remotes" in result.stderr
+        assert os.path.isdir(buildtreedir)
+        assert cli2.get_element_state(project, element_name) == 'cached'
+
+        # Test that we are able to 'complete' an artifact on a server which is cached partially,
+        # but has now been configured for full artifact pushing. This should require only pushing
+        # the missing blobs, which should be those of just the buildtree. In this case changing
+        # share3 to full pushes should exercise this
+        cli2.configure({'artifacts': {'url': share3.repo, 'push': True}})
+        result = cli2.run(project=project, args=['artifact', 'push', element_name])
+        assert element_name in result.get_pushed_elements()
+
+        # Ensure that the same remote url can be defined multiple times with differing push
+        # config. Buildstream supports the same remote having different configurations which
+        # partial pushing could be different for elements defined at a top level project.conf to
+        # those from a junctioned project. Assert that elements are pushed to the same remote in
+        # a state defined via their respective project.confs
+        default_state(cli2, tmpdir, share1)
+        cli2.configure({'artifactdir': os.path.join(str(tmpdir), 'artifacts')}, reset=True)
+        junction = os.path.join(project, 'elements', 'junction')
+        os.mkdir(junction)
+        shutil.copy2(os.path.join(project, 'elements', element_name), junction)
+
+        junction_conf = {}
+        project_conf = {}
+        junction_conf['name'] = 'amhello'
+        junction_conf['artifacts'] = {'url': share4.repo, 'push': True, 'allow-partial-push': True}
+        _yaml.dump(junction_conf, os.path.join(junction, 'project.conf'))
+        project_conf['artifacts'] = {'url': share4.repo, 'push': True}
+
+        # Read project.conf, the junction project.conf and buildstream.conf
+        # before running bst
+        with open(os.path.join(project, 'project.conf'), 'r') as f:
+            print(f.read())
+        with open(os.path.join(junction, 'project.conf'), 'r') as f:
+            print(f.read())
+        with open(os.path.join(project, 'cache', 'buildstream.conf'), 'r') as f:
+            print(f.read())
+
+        result = cli2.run(project=project, args=['build', 'junction/amhello.bst'], project_config=project_conf)
+
+        # Read project.conf, the junction project.conf and buildstream.conf
+        # after running bst
+        with open(os.path.join(project, 'project.conf'), 'r') as f:
+            print(f.read())
+        with open(os.path.join(junction, 'project.conf'), 'r') as f:
+            print(f.read())
+        with open(os.path.join(project, 'cache', 'buildstream.conf'), 'r') as f:
+            print(f.read())
+
+        assert 'junction/amhello.bst' in result.get_partial_pushed_elements()
+        assert 'base/base-alpine.bst' in result.get_pushed_elements()