From 23c66d2bd397b84c8905fa16027e0ba163d1b4cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrg=20Billeter?= Date: Mon, 30 Sep 2019 09:49:58 +0200 Subject: .gitlab-ci.yml: Update docker images for buildbox-casd --- .gitlab-ci.yml | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 34067613d..cfbad44b3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,7 @@ include: - template: Code-Quality.gitlab-ci.yml -image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-debian:9-master-83237142 +image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-debian:9-master-85428307 cache: key: "$CI_JOB_NAME-" @@ -54,24 +54,24 @@ variables: - .coverage-reports tests-debian-9: - image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-debian:9-master-83237142 + image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-debian:9-master-85428307 <<: *tests tests-fedora-29: - image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:29-master-83237142 + image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:29-master-85428307 <<: *tests tests-fedora-30: - image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:30-master-83237142 + image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:30-master-85428307 <<: *tests tests-ubuntu-18.04: - image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-ubuntu:18.04-master-83237142 + image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-ubuntu:18.04-master-85428307 <<: *tests tests-centos-7.6: <<: *tests - image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-centos:7.6.1810-master-83237142 + image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-centos:7.6.1810-master-85428307 overnight-fedora-30-aarch64: image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:aarch64-30-master-59168197 @@ -87,7 +87,7 @@ overnight-fedora-30-aarch64: tests-unix: # Use fedora here, to a) run a test on fedora and b) ensure that we # can get rid of ostree - this is not possible with debian-8 - image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:29-master-83237142 + image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:29-master-85428307 <<: *tests variables: BST_FORCE_SANDBOX: "chroot" @@ -104,7 +104,7 @@ tests-unix: - ${TEST_COMMAND} tests-buildbox: - image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:29-master-83237142 + image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:29-master-85428307 <<: *tests variables: BST_FORCE_SANDBOX: "buildbox" @@ -134,7 +134,7 @@ tests-buildbox: tests-fedora-missing-deps: # Ensure that tests behave nicely while missing bwrap and ostree - image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:29-master-83237142 + image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:29-master-85428307 <<: *tests script: @@ -153,7 +153,7 @@ tests-fedora-update-deps: # Check if the tests pass after updating requirements to their latest # allowed version. allow_failure: true - image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:29-master-83237142 + image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:29-master-85428307 <<: *tests script: @@ -167,7 +167,7 @@ tests-fedora-update-deps: tests-remote-execution: allow_failure: true - image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:29-master-83237142 + image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:29-master-85428307 <<: *tests before_script: - dnf install -y docker docker-compose @@ -190,7 +190,7 @@ tests-remote-execution: PYTEST_ARGS: "--color=yes --remote-execution" tests-spawn-multiprocessing-start-method: - image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:29-master-83237142 + image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:29-master-85428307 <<: *tests variables: BST_FORCE_START_METHOD: "spawn" @@ -230,8 +230,8 @@ tests-wsl-master: - PATH=/root/.local/bin:$PATH tox --version script: # Install static buildbox-casd binary - - wget https://buildbox-casd-binaries.nyc3.cdn.digitaloceanspaces.com/buildbox-casd-x86_64-linux-20190919-bc3eeb4b.tar.xz - - tar -C /root/.local/bin -xf buildbox-casd-x86_64-linux-20190919-bc3eeb4b.tar.xz + - wget https://buildbox-casd-binaries.nyc3.cdn.digitaloceanspaces.com/buildbox-casd-x86_64-linux-20190930-f9862db1.tar.xz + - tar -C /root/.local/bin -xf buildbox-casd-x86_64-linux-20190930-f9862db1.tar.xz - PATH=/root/.local/bin:$PATH ${TEST_COMMAND} only: @@ -250,8 +250,8 @@ tests-wsl-non-master: - PATH=/root/.local/bin:$PATH tox --version script: # Install static buildbox-casd binary - - wget https://buildbox-casd-binaries.nyc3.cdn.digitaloceanspaces.com/buildbox-casd-x86_64-linux-20190919-bc3eeb4b.tar.xz - - tar -C /root/.local/bin -xf buildbox-casd-x86_64-linux-20190919-bc3eeb4b.tar.xz + - wget https://buildbox-casd-binaries.nyc3.cdn.digitaloceanspaces.com/buildbox-casd-x86_64-linux-20190930-f9862db1.tar.xz + - tar -C /root/.local/bin -xf buildbox-casd-x86_64-linux-20190930-f9862db1.tar.xz - PATH=/root/.local/bin:$PATH ${TEST_COMMAND} when: manual @@ -275,7 +275,7 @@ docs: .overnight-tests: &overnight-tests-template stage: test - image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:30-master-83237142 + image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:30-master-85428307 variables: BST_EXT_URL: git+https://gitlab.com/BuildStream/bst-plugins-experimental.git BST_EXT_REF: 0.12.0-40-g7aa1423377629281decc455d1090964417c38f2e -- cgit v1.2.1 From 4c57dc0afb87b1f3be1e2abe87063326d27600cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrg=20Billeter?= Date: Tue, 24 Sep 2019 17:11:46 +0200 Subject: cascache.py: Add import_directory() method This allows import of a whole directory tree with a single gRPC call to buildbox-casd CaptureTree() instead of importing files one at a time. --- src/buildstream/_cas/cascache.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/buildstream/_cas/cascache.py b/src/buildstream/_cas/cascache.py index 6002adc4a..97ee2edd7 100644 --- a/src/buildstream/_cas/cascache.py +++ b/src/buildstream/_cas/cascache.py @@ -421,6 +421,41 @@ class CASCache(): return digest + # import_directory(): + # + # Import directory tree into CAS. + # + # Args: + # path (str): Path to directory to import + # + # Returns: + # (Digest): The digest of the imported directory + # + def import_directory(self, path): + local_cas = self._get_local_cas() + + request = local_cas_pb2.CaptureTreeRequest() + request.path.append(path) + response = local_cas.CaptureTree(request) + + if len(response.responses) != 1: + raise CASCacheError("Expected 1 response from CaptureTree, got {}".format(len(response.responses))) + + tree_response = response.responses[0] + if tree_response.status.code == code_pb2.RESOURCE_EXHAUSTED: + raise CASCacheError("Cache too full", reason="cache-too-full") + if tree_response.status.code != code_pb2.OK: + raise CASCacheError("Failed to capture tree {}: {}".format(path, tree_response.status.code)) + + treepath = self.objpath(tree_response.tree_digest) + tree = remote_execution_pb2.Tree() + with open(treepath, 'rb') as f: + tree.ParseFromString(f.read()) + + root_directory = tree.root.SerializeToString() + + return utils._message_digest(root_directory) + # set_ref(): # # Create or replace a ref. -- cgit v1.2.1 From aec623ede822d83fa5822a263177e4a8a3d0f056 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrg=20Billeter?= Date: Tue, 24 Sep 2019 17:11:50 +0200 Subject: _casbaseddirectory.py: Use CASCache.import_directory() --- src/buildstream/storage/_casbaseddirectory.py | 69 +++++---------------------- 1 file changed, 13 insertions(+), 56 deletions(-) diff --git a/src/buildstream/storage/_casbaseddirectory.py b/src/buildstream/storage/_casbaseddirectory.py index 66b7a7259..3786f25b6 100644 --- a/src/buildstream/storage/_casbaseddirectory.py +++ b/src/buildstream/storage/_casbaseddirectory.py @@ -253,52 +253,6 @@ class CasBasedDirectory(Directory): fileListResult.overwritten.append(relative_pathname) return True - def _import_files_from_directory(self, source_directory, filter_callback, - *, path_prefix="", result, - can_link=False): - """ Import files from a traditional directory. """ - - for direntry in os.scandir(source_directory): - # The destination filename, relative to the root where the import started - relative_pathname = os.path.join(path_prefix, direntry.name) - - is_dir = direntry.is_dir(follow_symlinks=False) - - if is_dir: - src_subdir = os.path.join(source_directory, direntry.name) - - try: - create_subdir = direntry.name not in self.index - dest_subdir = self.descend(direntry.name, create=create_subdir) - except VirtualDirectoryError: - filetype = self.index[direntry.name].type - raise VirtualDirectoryError('Destination is a {}, not a directory: /{}' - .format(filetype, relative_pathname)) - - dest_subdir._import_files_from_directory( - src_subdir, filter_callback, - path_prefix=relative_pathname, result=result, - can_link=can_link) - - if filter_callback and not filter_callback(relative_pathname): - if is_dir and create_subdir and dest_subdir.is_empty(): - # Complete subdirectory has been filtered out, remove it - self.delete_entry(direntry.name) - - # Entry filtered out, move to next - continue - - if direntry.is_file(follow_symlinks=False): - if self._check_replacement(direntry.name, relative_pathname, result): - self._add_file(source_directory, direntry.name, - modified=relative_pathname in result.overwritten, - can_link=can_link) - result.files_written.append(relative_pathname) - elif direntry.is_symlink(): - if self._check_replacement(direntry.name, relative_pathname, result): - self._copy_link_from_filesystem(source_directory, direntry.name) - result.files_written.append(relative_pathname) - def _partial_import_cas_into_cas(self, source_directory, filter_callback, *, path_prefix="", origin=None, result): """ Import files from a CAS-based directory. """ if origin is None: @@ -382,16 +336,19 @@ class CasBasedDirectory(Directory): result = FileListResult() if isinstance(external_pathspec, FileBasedDirectory): - source_directory = external_pathspec._get_underlying_directory() - self._import_files_from_directory(source_directory, filter_callback, - result=result, can_link=can_link) - elif isinstance(external_pathspec, str): - source_directory = external_pathspec - self._import_files_from_directory(source_directory, filter_callback, - result=result, can_link=can_link) - else: - assert isinstance(external_pathspec, CasBasedDirectory) - self._partial_import_cas_into_cas(external_pathspec, filter_callback, result=result) + external_pathspec = external_pathspec._get_underlying_directory() + + if isinstance(external_pathspec, str): + # Import files from local filesystem by first importing complete + # directory into CAS (using buildbox-casd) and then importing its + # content into this CasBasedDirectory using CAS-to-CAS import + # to write the report, handle possible conflicts (if the target + # directory is not empty) and apply the optional filter. + digest = self.cas_cache.import_directory(external_pathspec) + external_pathspec = CasBasedDirectory(self.cas_cache, digest=digest) + + assert isinstance(external_pathspec, CasBasedDirectory) + self._partial_import_cas_into_cas(external_pathspec, filter_callback, result=result) # TODO: No notice is taken of report_written or update_mtime. # Current behaviour is to fully populate the report, which is inefficient, -- cgit v1.2.1