diff options
author | James Ennis <james.ennis@codethink.co.uk> | 2019-08-01 12:50:46 +0100 |
---|---|---|
committer | James Ennis <james.ennis@codethink.com> | 2019-08-06 13:03:47 +0000 |
commit | ef778ca58d13be60faad37d55a4a64918c1b3d00 (patch) | |
tree | c6dc8ab2f03662d91416450fb0456eac92bcd862 | |
parent | ef757b5e8ee7d1bfab8750d4a87cbc374eda295e (diff) | |
download | buildstream-ef778ca58d13be60faad37d55a4a64918c1b3d00.tar.gz |
plugins/elements/junction.py: Add 'ignore-junction-remotes' option
This option allows us to completely ignore remotes defined in the
subproject. As a user, we may not wish to interact with subproject
remotes as we may not be able to trust them, for example.
An appropriate test and some documentation has also been added.
-rw-r--r-- | src/buildstream/_project.py | 8 | ||||
-rw-r--r-- | src/buildstream/plugins/elements/junction.py | 11 | ||||
-rw-r--r-- | tests/artifactcache/junctions.py | 62 |
3 files changed, 77 insertions, 4 deletions
diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py index 3a5923f80..dff101582 100644 --- a/src/buildstream/_project.py +++ b/src/buildstream/_project.py @@ -647,9 +647,13 @@ class Project(): # If there is a junction Element which specifies that we want to remotely cache # its elements, append the junction's remotes to the artifact cache specs list - if self.junction and self.junction.cache_junction_elements: + if self.junction: parent = self.junction._get_project() - self.artifact_cache_specs = parent.artifact_cache_specs + self.artifact_cache_specs + if self.junction.cache_junction_elements: + self.artifact_cache_specs = parent.artifact_cache_specs + self.artifact_cache_specs + + if self.junction.ignore_junction_remotes: + self.artifact_cache_specs = [] # Load source caches with pull/push config self.source_cache_specs = SourceCache.specs_from_config_node(config, self.directory) diff --git a/src/buildstream/plugins/elements/junction.py b/src/buildstream/plugins/elements/junction.py index 235702b07..aec32516b 100644 --- a/src/buildstream/plugins/elements/junction.py +++ b/src/buildstream/plugins/elements/junction.py @@ -59,12 +59,18 @@ Overview # should interact with project remotes (default: False). cache-junction-elements: False + # Optionally ignore junction remotes, this means that BuildStream + # will not attempt to pull artifacts from the junction project's + # remote(s) (default: False). + ignore-junction-remotes: False + .. note:: The configuration option to allow specifying junction targets is available since :ref:`format version 24 <project_format_version>` and the configuration - option allowing junction project elements to interact with parent remotes is - available since :ref:`format version 25 <project_format_version>`. + options allowing for junction project elements to interact with parent remotes + or to completely ignore junction project remotes are available since + :ref:`format version 25 <project_format_version>`. .. note:: @@ -187,6 +193,7 @@ class JunctionElement(Element): self.target_element = None self.target_junction = None self.cache_junction_elements = node.get_bool('cache-junction-elements', default=False) + self.ignore_junction_remotes = node.get_bool('ignore-junction-remotes', default=False) def preflight(self): # "target" cannot be used in conjunction with: diff --git a/tests/artifactcache/junctions.py b/tests/artifactcache/junctions.py index bfddcedac..1fafb11f1 100644 --- a/tests/artifactcache/junctions.py +++ b/tests/artifactcache/junctions.py @@ -133,3 +133,65 @@ def test_caching_junction_elements(cli, tmpdir, datafiles): assert_not_shared(cli, base_share, project, 'target.bst', project_name='parent') assert_not_shared(cli, base_share, project, 'app.bst', project_name='parent') assert_shared(cli, base_share, base_project, 'base-element.bst', project_name='base') + + +@pytest.mark.datafiles(DATA_DIR) +def test_ignore_junction_remotes(cli, tmpdir, datafiles): + project = os.path.join(str(datafiles), 'parent') + base_project = os.path.join(str(project), 'base') + + # Load the junction element + junction_element = os.path.join(project, 'base.bst') + junction_data = _yaml.roundtrip_load(junction_element) + + with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare-parent')) as share,\ + create_artifact_share(os.path.join(str(tmpdir), 'artifactshare-base')) as base_share: + + # Immediately declare the artifact caches in the appropriate project configs + project_set_artifacts(project, share.repo) + project_set_artifacts(base_project, base_share.repo) + + # Build and populate the project remotes with their respective elements + result = cli.run(project=project, args=['build', 'target.bst']) + assert result.exit_code == 0 + + # And finally assert that the artifacts are in the right shares + # + # The parent project's cache should only contain project elements + assert_shared(cli, share, project, 'target.bst', project_name='parent') + assert_shared(cli, share, project, 'app.bst', project_name='parent') + assert_not_shared(cli, share, base_project, 'base-element.bst', project_name='base') + + # The junction project's cache should only contain elements in the junction project + assert_not_shared(cli, base_share, project, 'target.bst', project_name='parent') + assert_not_shared(cli, base_share, project, 'app.bst', project_name='parent') + assert_shared(cli, base_share, base_project, 'base-element.bst', project_name='base') + + # Ensure that, from now on, we ignore junction element remotes + junction_data['config'] = {"ignore-junction-remotes": True} + _yaml.roundtrip_dump(junction_data, junction_element) + + # Now delete everything from the local cache and try to + # redownload from the shares. + # + cas = os.path.join(cli.directory, 'cas') + shutil.rmtree(cas) + artifact_dir = os.path.join(cli.directory, 'artifacts') + shutil.rmtree(artifact_dir) + + # Assert that nothing is cached locally anymore + state = cli.get_element_state(project, 'target.bst') + assert state != 'cached' + state = cli.get_element_state(base_project, 'base-element.bst') + assert state != 'cached' + + # Now try bst artifact pull + result = cli.run(project=project, args=['artifact', 'pull', '--deps', 'all', 'target.bst']) + assert result.exit_code == 0 + + # And assert that they are again in the local cache, without having built + state = cli.get_element_state(project, 'target.bst') + assert state == 'cached' + # We shouldn't be able to download base-element! + state = cli.get_element_state(base_project, 'base-element.bst') + assert state != 'cached' |