Merge branch 'juerg/capture-tree' into 'master'

Use buildbox-casd CaptureTree() to import directories

Closes #1132

See merge request BuildStream/buildstream!1611

3 files changed, 65 insertions, 73 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 34067613d..cfbad44b3 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,7 +1,7 @@
 include:
   - template: Code-Quality.gitlab-ci.yml
 
-image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-debian:9-master-83237142
+image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-debian:9-master-85428307
 
 cache:
   key: "$CI_JOB_NAME-"
@@ -54,24 +54,24 @@ variables:
     - .coverage-reports
 
 tests-debian-9:
-  image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-debian:9-master-83237142
+  image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-debian:9-master-85428307
   <<: *tests
 
 tests-fedora-29:
-  image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:29-master-83237142
+  image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:29-master-85428307
   <<: *tests
 
 tests-fedora-30:
-  image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:30-master-83237142
+  image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:30-master-85428307
   <<: *tests
 
 tests-ubuntu-18.04:
-  image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-ubuntu:18.04-master-83237142
+  image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-ubuntu:18.04-master-85428307
   <<: *tests
 
 tests-centos-7.6:
   <<: *tests
-  image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-centos:7.6.1810-master-83237142
+  image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-centos:7.6.1810-master-85428307
 
 overnight-fedora-30-aarch64:
   image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:aarch64-30-master-59168197
@@ -87,7 +87,7 @@ overnight-fedora-30-aarch64:
 tests-unix:
   # Use fedora here, to a) run a test on fedora and b) ensure that we
   # can get rid of ostree - this is not possible with debian-8
-  image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:29-master-83237142
+  image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:29-master-85428307
   <<: *tests
   variables:
     BST_FORCE_SANDBOX: "chroot"
@@ -104,7 +104,7 @@ tests-unix:
     - ${TEST_COMMAND}
 
 tests-buildbox:
-  image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:29-master-83237142
+  image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:29-master-85428307
   <<: *tests
   variables:
     BST_FORCE_SANDBOX: "buildbox"
@@ -134,7 +134,7 @@ tests-buildbox:
 
 tests-fedora-missing-deps:
   # Ensure that tests behave nicely while missing bwrap and ostree
-  image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:29-master-83237142
+  image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:29-master-85428307
   <<: *tests
 
   script:
@@ -153,7 +153,7 @@ tests-fedora-update-deps:
   # Check if the tests pass after updating requirements to their latest
   # allowed version.
   allow_failure: true
-  image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:29-master-83237142
+  image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:29-master-85428307
   <<: *tests
 
   script:
@@ -167,7 +167,7 @@ tests-fedora-update-deps:
 
 tests-remote-execution:
   allow_failure: true
-  image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:29-master-83237142
+  image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:29-master-85428307
   <<: *tests
   before_script:
     - dnf install -y docker docker-compose
@@ -190,7 +190,7 @@ tests-remote-execution:
     PYTEST_ARGS: "--color=yes --remote-execution"
 
 tests-spawn-multiprocessing-start-method:
-  image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:29-master-83237142
+  image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:29-master-85428307
   <<: *tests
   variables:
     BST_FORCE_START_METHOD: "spawn"
@@ -230,8 +230,8 @@ tests-wsl-master:
   - PATH=/root/.local/bin:$PATH tox --version
   script:
   # Install static buildbox-casd binary
-  - wget https://buildbox-casd-binaries.nyc3.cdn.digitaloceanspaces.com/buildbox-casd-x86_64-linux-20190919-bc3eeb4b.tar.xz
-  - tar -C /root/.local/bin -xf buildbox-casd-x86_64-linux-20190919-bc3eeb4b.tar.xz
+  - wget https://buildbox-casd-binaries.nyc3.cdn.digitaloceanspaces.com/buildbox-casd-x86_64-linux-20190930-f9862db1.tar.xz
+  - tar -C /root/.local/bin -xf buildbox-casd-x86_64-linux-20190930-f9862db1.tar.xz
 
   - PATH=/root/.local/bin:$PATH ${TEST_COMMAND}
   only:
@@ -250,8 +250,8 @@ tests-wsl-non-master:
   - PATH=/root/.local/bin:$PATH tox --version
   script:
   # Install static buildbox-casd binary
-  - wget https://buildbox-casd-binaries.nyc3.cdn.digitaloceanspaces.com/buildbox-casd-x86_64-linux-20190919-bc3eeb4b.tar.xz
-  - tar -C /root/.local/bin -xf buildbox-casd-x86_64-linux-20190919-bc3eeb4b.tar.xz
+  - wget https://buildbox-casd-binaries.nyc3.cdn.digitaloceanspaces.com/buildbox-casd-x86_64-linux-20190930-f9862db1.tar.xz
+  - tar -C /root/.local/bin -xf buildbox-casd-x86_64-linux-20190930-f9862db1.tar.xz
 
   - PATH=/root/.local/bin:$PATH ${TEST_COMMAND}
   when: manual
@@ -275,7 +275,7 @@ docs:
 
 .overnight-tests: &overnight-tests-template
   stage: test
-  image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:30-master-83237142
+  image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:30-master-85428307
   variables:
     BST_EXT_URL: git+https://gitlab.com/BuildStream/bst-plugins-experimental.git
     BST_EXT_REF: 0.12.0-40-g7aa1423377629281decc455d1090964417c38f2e
diff --git a/src/buildstream/_cas/cascache.py b/src/buildstream/_cas/cascache.py
index 6002adc4a..97ee2edd7 100644
--- a/src/buildstream/_cas/cascache.py
+++ b/src/buildstream/_cas/cascache.py
@@ -421,6 +421,41 @@ class CASCache():
 
         return digest
 
+    # import_directory():
+    #
+    # Import directory tree into CAS.
+    #
+    # Args:
+    #     path (str): Path to directory to import
+    #
+    # Returns:
+    #     (Digest): The digest of the imported directory
+    #
+    def import_directory(self, path):
+        local_cas = self._get_local_cas()
+
+        request = local_cas_pb2.CaptureTreeRequest()
+        request.path.append(path)
+        response = local_cas.CaptureTree(request)
+
+        if len(response.responses) != 1:
+            raise CASCacheError("Expected 1 response from CaptureTree, got {}".format(len(response.responses)))
+
+        tree_response = response.responses[0]
+        if tree_response.status.code == code_pb2.RESOURCE_EXHAUSTED:
+            raise CASCacheError("Cache too full", reason="cache-too-full")
+        if tree_response.status.code != code_pb2.OK:
+            raise CASCacheError("Failed to capture tree {}: {}".format(path, tree_response.status.code))
+
+        treepath = self.objpath(tree_response.tree_digest)
+        tree = remote_execution_pb2.Tree()
+        with open(treepath, 'rb') as f:
+            tree.ParseFromString(f.read())
+
+        root_directory = tree.root.SerializeToString()
+
+        return utils._message_digest(root_directory)
+
     # set_ref():
     #
     # Create or replace a ref.
diff --git a/src/buildstream/storage/_casbaseddirectory.py b/src/buildstream/storage/_casbaseddirectory.py
index 66b7a7259..3786f25b6 100644
--- a/src/buildstream/storage/_casbaseddirectory.py
+++ b/src/buildstream/storage/_casbaseddirectory.py
@@ -253,52 +253,6 @@ class CasBasedDirectory(Directory):
             fileListResult.overwritten.append(relative_pathname)
             return True
 
-    def _import_files_from_directory(self, source_directory, filter_callback,
-                                     *, path_prefix="", result,
-                                     can_link=False):
-        """ Import files from a traditional directory. """
-
-        for direntry in os.scandir(source_directory):
-            # The destination filename, relative to the root where the import started
-            relative_pathname = os.path.join(path_prefix, direntry.name)
-
-            is_dir = direntry.is_dir(follow_symlinks=False)
-
-            if is_dir:
-                src_subdir = os.path.join(source_directory, direntry.name)
-
-                try:
-                    create_subdir = direntry.name not in self.index
-                    dest_subdir = self.descend(direntry.name, create=create_subdir)
-                except VirtualDirectoryError:
-                    filetype = self.index[direntry.name].type
-                    raise VirtualDirectoryError('Destination is a {}, not a directory: /{}'
-                                                .format(filetype, relative_pathname))
-
-                dest_subdir._import_files_from_directory(
-                    src_subdir, filter_callback,
-                    path_prefix=relative_pathname, result=result,
-                    can_link=can_link)
-
-            if filter_callback and not filter_callback(relative_pathname):
-                if is_dir and create_subdir and dest_subdir.is_empty():
-                    # Complete subdirectory has been filtered out, remove it
-                    self.delete_entry(direntry.name)
-
-                # Entry filtered out, move to next
-                continue
-
-            if direntry.is_file(follow_symlinks=False):
-                if self._check_replacement(direntry.name, relative_pathname, result):
-                    self._add_file(source_directory, direntry.name,
-                                   modified=relative_pathname in result.overwritten,
-                                   can_link=can_link)
-                    result.files_written.append(relative_pathname)
-            elif direntry.is_symlink():
-                if self._check_replacement(direntry.name, relative_pathname, result):
-                    self._copy_link_from_filesystem(source_directory, direntry.name)
-                    result.files_written.append(relative_pathname)
-
     def _partial_import_cas_into_cas(self, source_directory, filter_callback, *, path_prefix="", origin=None, result):
         """ Import files from a CAS-based directory. """
         if origin is None:
@@ -382,16 +336,19 @@ class CasBasedDirectory(Directory):
         result = FileListResult()
 
         if isinstance(external_pathspec, FileBasedDirectory):
-            source_directory = external_pathspec._get_underlying_directory()
-            self._import_files_from_directory(source_directory, filter_callback,
-                                              result=result, can_link=can_link)
-        elif isinstance(external_pathspec, str):
-            source_directory = external_pathspec
-            self._import_files_from_directory(source_directory, filter_callback,
-                                              result=result, can_link=can_link)
-        else:
-            assert isinstance(external_pathspec, CasBasedDirectory)
-            self._partial_import_cas_into_cas(external_pathspec, filter_callback, result=result)
+            external_pathspec = external_pathspec._get_underlying_directory()
+
+        if isinstance(external_pathspec, str):
+            # Import files from local filesystem by first importing complete
+            # directory into CAS (using buildbox-casd) and then importing its
+            # content into this CasBasedDirectory using CAS-to-CAS import
+            # to write the report, handle possible conflicts (if the target
+            # directory is not empty) and apply the optional filter.
+            digest = self.cas_cache.import_directory(external_pathspec)
+            external_pathspec = CasBasedDirectory(self.cas_cache, digest=digest)
+
+        assert isinstance(external_pathspec, CasBasedDirectory)
+        self._partial_import_cas_into_cas(external_pathspec, filter_callback, result=result)
 
         # TODO: No notice is taken of report_written or update_mtime.
         # Current behaviour is to fully populate the report, which is inefficient,