diff options
author | bst-marge-bot <marge-bot@buildstream.build> | 2019-07-11 17:03:59 +0000 |
---|---|---|
committer | bst-marge-bot <marge-bot@buildstream.build> | 2019-07-11 17:03:59 +0000 |
commit | 24426ebe31fc2ad297b352a1d332c7cf158ef5c2 (patch) | |
tree | 1c64a7ca6b5fd89528e55ac5ae1a8af7874ff0e0 | |
parent | 199f080fac88fac1277541baec9def1c13afd048 (diff) | |
parent | af9530c66c7392abde791dea1ad52cc657d31004 (diff) | |
download | buildstream-24426ebe31fc2ad297b352a1d332c7cf158ef5c2.tar.gz |
Merge branch 'raoul/1043-disabling-blob-fetching' into 'master'
Configuration option for disabling blob fetching with RE
Closes #1043
See merge request BuildStream/buildstream!1402
-rw-r--r-- | doc/source/using_config.rst | 8 | ||||
-rw-r--r-- | src/buildstream/_context.py | 16 | ||||
-rw-r--r-- | src/buildstream/_stream.py | 8 | ||||
-rw-r--r-- | src/buildstream/element.py | 6 | ||||
-rw-r--r-- | tests/remoteexecution/partial.py | 17 |
5 files changed, 43 insertions, 12 deletions
diff --git a/doc/source/using_config.rst b/doc/source/using_config.rst index 2582e711f..8a6b9ac76 100644 --- a/doc/source/using_config.rst +++ b/doc/source/using_config.rst @@ -153,8 +153,11 @@ the user configuration. Remote execution ~~~~~~~~~~~~~~~~ -The same configuration for :ref:`remote execution <project_remote_execution>` -in ``project.conf`` can be provided in the user configuation. +The configuration for :ref:`remote execution <project_remote_execution>` +in ``project.conf`` can be provided in the user configuation. The global +configuration also has a ``pull-artifact-files`` option, which specifies when +remote execution is being performed whether to pull file blobs of artifacts, or +just the directory trees required to perform remote builds. There is only one remote execution configuration used per project. @@ -178,6 +181,7 @@ configuration will be used as fallback. action-cache-service: url: http://cache.flalback.example.com:50052 instance-name: main + pull-artifact-files: True 2. Project override: diff --git a/src/buildstream/_context.py b/src/buildstream/_context.py index 52e4c3db9..2bdf5b0b4 100644 --- a/src/buildstream/_context.py +++ b/src/buildstream/_context.py @@ -129,6 +129,9 @@ class Context(): # Whether or not to attempt to pull build trees globally self.pull_buildtrees = None + # Whether to pull the files of an artifact when doing remote execution + self.pull_artifact_files = None + # Whether or not to cache build trees on artifact creation self.cache_buildtrees = None @@ -258,6 +261,19 @@ class Context(): # Load source cache config self.source_cache_specs = SourceCache.specs_from_config_node(defaults) + # Load remote execution config getting pull-artifact-files from it + remote_execution = _yaml.node_get(defaults, dict, 'remote-execution', default_value=None) + if remote_execution: + self.pull_artifact_files = _yaml.node_get( + remote_execution, bool, 'pull-artifact-files', default_value=True) + # This stops it being used in the remote service set up + _yaml.node_del(remote_execution, 'pull-artifact-files', safe=True) + # Don't pass the remote execution settings if that was the only option + if _yaml.node_keys(remote_execution) == []: + _yaml.node_del(defaults, 'remote-execution') + else: + self.pull_artifact_files = True + self.remote_execution_specs = SandboxRemote.specs_from_config_node(defaults) # Load pull build trees configuration diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index 7e68f1afd..705e0a5d3 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -272,8 +272,12 @@ class Stream(): # Remote execution is configured for all projects. # Require artifact files only for target elements and their runtime dependencies. self._context.set_artifact_files_optional() - for element in self.targets: - element._set_artifact_files_required() + + # fetch blobs of targets if options set + if self._context.pull_artifact_files: + scope = Scope.ALL if build_all else Scope.RUN + for element in self.targets: + element._set_artifact_files_required(scope=scope) # Now construct the queues # diff --git a/src/buildstream/element.py b/src/buildstream/element.py index 758a0b97d..ccb373f1f 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -1529,7 +1529,7 @@ class Element(Plugin): # Mark artifact files for this element and its runtime dependencies as # required in the local cache. # - def _set_artifact_files_required(self): + def _set_artifact_files_required(self, scope=Scope.RUN): if self.__artifact_files_required: # Already done return @@ -1537,8 +1537,8 @@ class Element(Plugin): self.__artifact_files_required = True # Request artifact files of runtime dependencies - for dep in self.dependencies(Scope.RUN, recurse=False): - dep._set_artifact_files_required() + for dep in self.dependencies(scope, recurse=False): + dep._set_artifact_files_required(scope=scope) # _artifact_files_required(): # diff --git a/tests/remoteexecution/partial.py b/tests/remoteexecution/partial.py index a684bda18..bf5106f8d 100644 --- a/tests/remoteexecution/partial.py +++ b/tests/remoteexecution/partial.py @@ -23,7 +23,8 @@ DATA_DIR = os.path.join( # Test that `bst build` does not download file blobs of a build-only dependency # to the local cache. @pytest.mark.datafiles(DATA_DIR) -def test_build_dependency_partial_local_cas(cli, datafiles): +@pytest.mark.parametrize('pull_artifact_files', [True, False]) +def test_build_dependency_partial_local_cas(cli, datafiles, pull_artifact_files): project = str(datafiles) element_name = 'no-runtime-deps.bst' builddep_element_name = 'autotools/amhello.bst' @@ -33,16 +34,22 @@ def test_build_dependency_partial_local_cas(cli, datafiles): services = cli.ensure_services() assert set(services) == set(['action-cache', 'execution', 'storage']) + # configure pull blobs + cli.config['remote-execution']['pull-artifact-files'] = pull_artifact_files + result = cli.run(project=project, args=['build', element_name]) result.assert_success() - # Verify that the target element is available in local cache + # Verify artifact is pulled bar files when ensure artifact files is set result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout]) - result.assert_success() - assert_contains(checkout, ['/test']) + if pull_artifact_files: + result.assert_success() + assert_contains(checkout, ['/test']) + else: + result.assert_main_error(ErrorDomain.STREAM, 'uncached-checkout-attempt') - # Verify that the build-only dependency is not (complete) in the local cache + # Verify build dependencies is pulled for ALL and BUILD result = cli.run(project=project, args=['artifact', 'checkout', builddep_element_name, '--directory', builddep_checkout]) result.assert_main_error(ErrorDomain.STREAM, 'uncached-checkout-attempt') |