From 7c03094ad35d89f59a750a8be7c13e6c592e306d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrg=20Billeter?= Date: Wed, 1 Jul 2020 19:23:20 +0200 Subject: tests/testutils/artifactshare.py: Fix sourcedir --- tests/testutils/artifactshare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testutils/artifactshare.py b/tests/testutils/artifactshare.py index a15a7c27e..e471d7989 100644 --- a/tests/testutils/artifactshare.py +++ b/tests/testutils/artifactshare.py @@ -120,7 +120,7 @@ class ArtifactShare(BaseArtifactShare): os.makedirs(self.repodir) self.artifactdir = os.path.join(self.repodir, "artifacts", "refs") os.makedirs(self.artifactdir) - self.sourcedir = os.path.join(self.repodir, "source_protos", "refs") + self.sourcedir = os.path.join(self.repodir, "source_protos") os.makedirs(self.sourcedir) logdir = os.path.join(self.directory, "logs") if casd else None -- cgit v1.2.1 From 8745864ceea5da0ee984cce89791f452c0dadcbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrg=20Billeter?= Date: Wed, 1 Jul 2020 16:34:45 +0200 Subject: Add `bst source push` command --- doc/source/using_commands.rst | 7 ++++++ src/buildstream/_frontend/cli.py | 52 ++++++++++++++++++++++++++++++++++++++++ src/buildstream/_stream.py | 42 ++++++++++++++++++++++++++++++++ tests/frontend/completions.py | 1 + 4 files changed, 102 insertions(+) diff --git a/doc/source/using_commands.rst b/doc/source/using_commands.rst index 276a17ce6..27fe692d3 100644 --- a/doc/source/using_commands.rst +++ b/doc/source/using_commands.rst @@ -144,6 +144,13 @@ Source subcommands ---- +.. _invoking_source_push: + +.. click:: buildstream._frontend.cli:source_push + :prog: bst source push + +---- + .. _invoking_source_checkout: .. click:: buildstream._frontend.cli:source_checkout diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index d5fa47091..0adfdd84a 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -819,6 +819,58 @@ def source_fetch(app, elements, deps, except_, remote): app.stream.fetch(elements, selection=deps, except_targets=except_, remote=remote) +################################################################## +# Source Push Command # +################################################################## +@source.command(name="push", short_help="Push sources in a pipeline") +@click.option( + "--deps", + "-d", + default=_PipelineSelection.NONE, + show_default=True, + type=FastEnumType( + _PipelineSelection, + [ + _PipelineSelection.NONE, + _PipelineSelection.PLAN, + _PipelineSelection.BUILD, + _PipelineSelection.RUN, + _PipelineSelection.ALL, + ], + ), + help="The dependencies to push", +) +@click.option( + "--remote", "-r", default=None, help="The URL of the remote source cache (defaults to the first configured cache)" +) +@click.argument("elements", nargs=-1, type=click.Path(readable=False)) +@click.pass_obj +def source_push(app, elements, deps, remote): + """Push sources required to build the pipeline + + Specifying no elements will result in pushing the sources of the default + targets of the project. If no default targets are configured, sources of + all project elements will be pushed. + + When this command is executed from a workspace directory, the default + is to push the sources of the workspace element. + + Specify `--deps` to control which sources to fetch: + + \b + none: No dependencies, just the element itself + plan: Only dependencies required for the build plan + run: Runtime dependencies, including the element itself + build: Build time dependencies, excluding the element itself + all: All dependencies + """ + with app.initialized(session_name="Push"): + if not elements: + elements = app.project.get_default_targets() + + app.stream.source_push(elements, selection=deps, remote=remote) + + ################################################################## # Source Track Command # ################################################################## diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index 3d646a756..989a00db7 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -376,6 +376,48 @@ class Stream: self._enqueue_plan(elements, queue=track_queue) self._run() + # source_push() + # + # Push sources. + # + # Args: + # targets (list of str): Targets to push + # selection (_PipelineSelection): The selection mode for the specified targets + # remote (str): The URL of a specific remote server to push to, or None + # + # If `remote` specified as None, then regular configuration will be used + # to determine where to push sources to. + # + # If any of the given targets are missing their expected sources, + # a fetch queue will be created if user context and available remotes allow for + # attempting to fetch them. + # + def source_push(self, targets, *, selection=_PipelineSelection.NONE, remote=None): + + use_source_config = True + if remote: + use_source_config = False + + elements = self._load( + targets, + selection=selection, + use_source_config=use_source_config, + source_remote_url=remote, + load_refs=True, + ) + + if not self._sourcecache.has_push_remotes(): + raise StreamError("No source caches available for pushing sources") + + self._pipeline.assert_consistent(elements) + + self._add_queue(FetchQueue(self._scheduler, skip_cached=True)) + + self._add_queue(SourcePushQueue(self._scheduler)) + + self._enqueue_plan(elements) + self._run() + # pull() # # Pulls artifacts from remote artifact server(s) diff --git a/tests/frontend/completions.py b/tests/frontend/completions.py index a5e3c8ed3..952ed177b 100644 --- a/tests/frontend/completions.py +++ b/tests/frontend/completions.py @@ -44,6 +44,7 @@ MAIN_OPTIONS = [ SOURCE_COMMANDS = [ "checkout ", "fetch ", + "push ", "track ", ] -- cgit v1.2.1 From 9c41a3c0e3fc1940f5a9015842a41dd0514e114e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrg=20Billeter?= Date: Wed, 1 Jul 2020 18:36:30 +0200 Subject: Regenerate man pages --- man/bst-artifact-checkout.1 | 2 +- man/bst-artifact-delete.1 | 2 +- man/bst-artifact-list-contents.1 | 2 +- man/bst-artifact-log.1 | 2 +- man/bst-artifact-pull.1 | 2 +- man/bst-artifact-push.1 | 2 +- man/bst-artifact-server.1 | 2 +- man/bst-artifact-show.1 | 2 +- man/bst-artifact.1 | 2 +- man/bst-build.1 | 2 +- man/bst-help.1 | 2 +- man/bst-init.1 | 2 +- man/bst-shell.1 | 2 +- man/bst-show.1 | 2 +- man/bst-source-checkout.1 | 2 +- man/bst-source-fetch.1 | 2 +- man/bst-source-push.1 | 31 +++++++++++++++++++++++++++++++ man/bst-source-track.1 | 2 +- man/bst-source.1 | 6 +++++- man/bst-workspace-close.1 | 2 +- man/bst-workspace-list.1 | 2 +- man/bst-workspace-open.1 | 2 +- man/bst-workspace-reset.1 | 2 +- man/bst-workspace.1 | 2 +- man/bst.1 | 2 +- 25 files changed, 59 insertions(+), 24 deletions(-) create mode 100644 man/bst-source-push.1 diff --git a/man/bst-artifact-checkout.1 b/man/bst-artifact-checkout.1 index 51f537f69..e992c6ccd 100644 --- a/man/bst-artifact-checkout.1 +++ b/man/bst-artifact-checkout.1 @@ -1,4 +1,4 @@ -.TH "BST ARTIFACT CHECKOUT" "1" "2020-05-07" "" "bst artifact checkout Manual" +.TH "BST ARTIFACT CHECKOUT" "1" "2020-07-01" "" "bst artifact checkout Manual" .SH NAME bst\-artifact\-checkout \- Checkout contents of an artifact .SH SYNOPSIS diff --git a/man/bst-artifact-delete.1 b/man/bst-artifact-delete.1 index 8b92499f8..b1dc10607 100644 --- a/man/bst-artifact-delete.1 +++ b/man/bst-artifact-delete.1 @@ -1,4 +1,4 @@ -.TH "BST ARTIFACT DELETE" "1" "2020-05-07" "" "bst artifact delete Manual" +.TH "BST ARTIFACT DELETE" "1" "2020-07-01" "" "bst artifact delete Manual" .SH NAME bst\-artifact\-delete \- Remove artifacts from the local cache .SH SYNOPSIS diff --git a/man/bst-artifact-list-contents.1 b/man/bst-artifact-list-contents.1 index b53df3073..b0160d75c 100644 --- a/man/bst-artifact-list-contents.1 +++ b/man/bst-artifact-list-contents.1 @@ -1,4 +1,4 @@ -.TH "BST ARTIFACT LIST-CONTENTS" "1" "2020-05-07" "" "bst artifact list-contents Manual" +.TH "BST ARTIFACT LIST-CONTENTS" "1" "2020-07-01" "" "bst artifact list-contents Manual" .SH NAME bst\-artifact\-list-contents \- List the contents of an artifact .SH SYNOPSIS diff --git a/man/bst-artifact-log.1 b/man/bst-artifact-log.1 index 84153fc38..58b3cec23 100644 --- a/man/bst-artifact-log.1 +++ b/man/bst-artifact-log.1 @@ -1,4 +1,4 @@ -.TH "BST ARTIFACT LOG" "1" "2020-05-07" "" "bst artifact log Manual" +.TH "BST ARTIFACT LOG" "1" "2020-07-01" "" "bst artifact log Manual" .SH NAME bst\-artifact\-log \- Show logs of artifacts .SH SYNOPSIS diff --git a/man/bst-artifact-pull.1 b/man/bst-artifact-pull.1 index d58346aac..15ad9e726 100644 --- a/man/bst-artifact-pull.1 +++ b/man/bst-artifact-pull.1 @@ -1,4 +1,4 @@ -.TH "BST ARTIFACT PULL" "1" "2020-05-07" "" "bst artifact pull Manual" +.TH "BST ARTIFACT PULL" "1" "2020-07-01" "" "bst artifact pull Manual" .SH NAME bst\-artifact\-pull \- Pull a built artifact .SH SYNOPSIS diff --git a/man/bst-artifact-push.1 b/man/bst-artifact-push.1 index 88cf96abf..9d71aacf8 100644 --- a/man/bst-artifact-push.1 +++ b/man/bst-artifact-push.1 @@ -1,4 +1,4 @@ -.TH "BST ARTIFACT PUSH" "1" "2020-05-07" "" "bst artifact push Manual" +.TH "BST ARTIFACT PUSH" "1" "2020-07-01" "" "bst artifact push Manual" .SH NAME bst\-artifact\-push \- Push a built artifact .SH SYNOPSIS diff --git a/man/bst-artifact-server.1 b/man/bst-artifact-server.1 index 7d3735847..f3af6e705 100644 --- a/man/bst-artifact-server.1 +++ b/man/bst-artifact-server.1 @@ -1,4 +1,4 @@ -.TH "BST-ARTIFACT-SERVER" "1" "2020-05-07" "" "bst-artifact-server Manual" +.TH "BST-ARTIFACT-SERVER" "1" "2020-07-01" "" "bst-artifact-server Manual" .SH NAME bst-artifact-server \- CAS Artifact Server .SH SYNOPSIS diff --git a/man/bst-artifact-show.1 b/man/bst-artifact-show.1 index 660c9dc93..cff3f4954 100644 --- a/man/bst-artifact-show.1 +++ b/man/bst-artifact-show.1 @@ -1,4 +1,4 @@ -.TH "BST ARTIFACT SHOW" "1" "2020-05-07" "" "bst artifact show Manual" +.TH "BST ARTIFACT SHOW" "1" "2020-07-01" "" "bst artifact show Manual" .SH NAME bst\-artifact\-show \- Show the cached state of artifacts .SH SYNOPSIS diff --git a/man/bst-artifact.1 b/man/bst-artifact.1 index 3bdbddadb..bb1feade0 100644 --- a/man/bst-artifact.1 +++ b/man/bst-artifact.1 @@ -1,4 +1,4 @@ -.TH "BST ARTIFACT" "1" "2020-05-07" "" "bst artifact Manual" +.TH "BST ARTIFACT" "1" "2020-07-01" "" "bst artifact Manual" .SH NAME bst\-artifact \- Manipulate cached artifacts. .SH SYNOPSIS diff --git a/man/bst-build.1 b/man/bst-build.1 index 34ed95e41..2842089aa 100644 --- a/man/bst-build.1 +++ b/man/bst-build.1 @@ -1,4 +1,4 @@ -.TH "BST BUILD" "1" "2020-05-07" "" "bst build Manual" +.TH "BST BUILD" "1" "2020-07-01" "" "bst build Manual" .SH NAME bst\-build \- Build elements in a pipeline .SH SYNOPSIS diff --git a/man/bst-help.1 b/man/bst-help.1 index a5a72983a..459c91aeb 100644 --- a/man/bst-help.1 +++ b/man/bst-help.1 @@ -1,4 +1,4 @@ -.TH "BST HELP" "1" "2020-05-07" "" "bst help Manual" +.TH "BST HELP" "1" "2020-07-01" "" "bst help Manual" .SH NAME bst\-help \- Print usage information .SH SYNOPSIS diff --git a/man/bst-init.1 b/man/bst-init.1 index cc2735799..91dc924bb 100644 --- a/man/bst-init.1 +++ b/man/bst-init.1 @@ -1,4 +1,4 @@ -.TH "BST INIT" "1" "2020-05-07" "" "bst init Manual" +.TH "BST INIT" "1" "2020-07-01" "" "bst init Manual" .SH NAME bst\-init \- Initialize a new BuildStream project .SH SYNOPSIS diff --git a/man/bst-shell.1 b/man/bst-shell.1 index 33474bc75..54e694721 100644 --- a/man/bst-shell.1 +++ b/man/bst-shell.1 @@ -1,4 +1,4 @@ -.TH "BST SHELL" "1" "2020-05-07" "" "bst shell Manual" +.TH "BST SHELL" "1" "2020-07-01" "" "bst shell Manual" .SH NAME bst\-shell \- Shell into an element's sandbox environment .SH SYNOPSIS diff --git a/man/bst-show.1 b/man/bst-show.1 index d100c7c8e..a4ded7204 100644 --- a/man/bst-show.1 +++ b/man/bst-show.1 @@ -1,4 +1,4 @@ -.TH "BST SHOW" "1" "2020-05-07" "" "bst show Manual" +.TH "BST SHOW" "1" "2020-07-01" "" "bst show Manual" .SH NAME bst\-show \- Show elements in the pipeline .SH SYNOPSIS diff --git a/man/bst-source-checkout.1 b/man/bst-source-checkout.1 index f843baa3c..e0a806951 100644 --- a/man/bst-source-checkout.1 +++ b/man/bst-source-checkout.1 @@ -1,4 +1,4 @@ -.TH "BST SOURCE CHECKOUT" "1" "2020-05-07" "" "bst source checkout Manual" +.TH "BST SOURCE CHECKOUT" "1" "2020-07-01" "" "bst source checkout Manual" .SH NAME bst\-source\-checkout \- Checkout sources of an element .SH SYNOPSIS diff --git a/man/bst-source-fetch.1 b/man/bst-source-fetch.1 index 3059befb7..a0a832010 100644 --- a/man/bst-source-fetch.1 +++ b/man/bst-source-fetch.1 @@ -1,4 +1,4 @@ -.TH "BST SOURCE FETCH" "1" "2020-05-07" "" "bst source fetch Manual" +.TH "BST SOURCE FETCH" "1" "2020-07-01" "" "bst source fetch Manual" .SH NAME bst\-source\-fetch \- Fetch sources in a pipeline .SH SYNOPSIS diff --git a/man/bst-source-push.1 b/man/bst-source-push.1 new file mode 100644 index 000000000..edb2996c3 --- /dev/null +++ b/man/bst-source-push.1 @@ -0,0 +1,31 @@ +.TH "BST SOURCE PUSH" "1" "2020-07-01" "" "bst source push Manual" +.SH NAME +bst\-source\-push \- Push sources in a pipeline +.SH SYNOPSIS +.B bst source push +[OPTIONS] [ELEMENTS]... +.SH DESCRIPTION +Push sources required to build the pipeline +.PP +Specifying no elements will result in pushing the sources of the default +targets of the project. If no default targets are configured, sources of +all project elements will be pushed. +.PP +When this command is executed from a workspace directory, the default +is to push the sources of the workspace element. +.PP +Specify `--deps` to control which sources to fetch: +.PP + + none: No dependencies, just the element itself + plan: Only dependencies required for the build plan + run: Runtime dependencies, including the element itself + build: Build time dependencies, excluding the element itself + all: All dependencies +.SH OPTIONS +.TP +\fB\-d,\fP \-\-deps [none|plan|build|run|all] +The dependencies to push [default: none] +.TP +\fB\-r,\fP \-\-remote TEXT +The URL of the remote source cache (defaults to the first configured cache) diff --git a/man/bst-source-track.1 b/man/bst-source-track.1 index 6863efc45..ed5236eea 100644 --- a/man/bst-source-track.1 +++ b/man/bst-source-track.1 @@ -1,4 +1,4 @@ -.TH "BST SOURCE TRACK" "1" "2020-05-07" "" "bst source track Manual" +.TH "BST SOURCE TRACK" "1" "2020-07-01" "" "bst source track Manual" .SH NAME bst\-source\-track \- Track new source references .SH SYNOPSIS diff --git a/man/bst-source.1 b/man/bst-source.1 index adabef759..f53f26b7a 100644 --- a/man/bst-source.1 +++ b/man/bst-source.1 @@ -1,4 +1,4 @@ -.TH "BST SOURCE" "1" "2020-05-07" "" "bst source Manual" +.TH "BST SOURCE" "1" "2020-07-01" "" "bst source Manual" .SH NAME bst\-source \- Manipulate sources for an element .SH SYNOPSIS @@ -12,6 +12,10 @@ Manipulate sources for an element Fetch sources in a pipeline See \fBbst source-fetch(1)\fP for full documentation on the \fBfetch\fP command. .PP +\fBpush\fP + Push sources in a pipeline + See \fBbst source-push(1)\fP for full documentation on the \fBpush\fP command. +.PP \fBtrack\fP Track new source references See \fBbst source-track(1)\fP for full documentation on the \fBtrack\fP command. diff --git a/man/bst-workspace-close.1 b/man/bst-workspace-close.1 index 63e071507..b628ae4e7 100644 --- a/man/bst-workspace-close.1 +++ b/man/bst-workspace-close.1 @@ -1,4 +1,4 @@ -.TH "BST WORKSPACE CLOSE" "1" "2020-05-07" "" "bst workspace close Manual" +.TH "BST WORKSPACE CLOSE" "1" "2020-07-01" "" "bst workspace close Manual" .SH NAME bst\-workspace\-close \- Close workspaces .SH SYNOPSIS diff --git a/man/bst-workspace-list.1 b/man/bst-workspace-list.1 index bad96bc72..8dc26eb3e 100644 --- a/man/bst-workspace-list.1 +++ b/man/bst-workspace-list.1 @@ -1,4 +1,4 @@ -.TH "BST WORKSPACE LIST" "1" "2020-05-07" "" "bst workspace list Manual" +.TH "BST WORKSPACE LIST" "1" "2020-07-01" "" "bst workspace list Manual" .SH NAME bst\-workspace\-list \- List open workspaces .SH SYNOPSIS diff --git a/man/bst-workspace-open.1 b/man/bst-workspace-open.1 index 34993b841..f1acb69d1 100644 --- a/man/bst-workspace-open.1 +++ b/man/bst-workspace-open.1 @@ -1,4 +1,4 @@ -.TH "BST WORKSPACE OPEN" "1" "2020-05-07" "" "bst workspace open Manual" +.TH "BST WORKSPACE OPEN" "1" "2020-07-01" "" "bst workspace open Manual" .SH NAME bst\-workspace\-open \- Open a new workspace .SH SYNOPSIS diff --git a/man/bst-workspace-reset.1 b/man/bst-workspace-reset.1 index e0e86e87a..7ea1c99e6 100644 --- a/man/bst-workspace-reset.1 +++ b/man/bst-workspace-reset.1 @@ -1,4 +1,4 @@ -.TH "BST WORKSPACE RESET" "1" "2020-05-07" "" "bst workspace reset Manual" +.TH "BST WORKSPACE RESET" "1" "2020-07-01" "" "bst workspace reset Manual" .SH NAME bst\-workspace\-reset \- Reset a workspace to its original state .SH SYNOPSIS diff --git a/man/bst-workspace.1 b/man/bst-workspace.1 index 1f881e20b..c27133b77 100644 --- a/man/bst-workspace.1 +++ b/man/bst-workspace.1 @@ -1,4 +1,4 @@ -.TH "BST WORKSPACE" "1" "2020-05-07" "" "bst workspace Manual" +.TH "BST WORKSPACE" "1" "2020-07-01" "" "bst workspace Manual" .SH NAME bst\-workspace \- Manipulate developer workspaces .SH SYNOPSIS diff --git a/man/bst.1 b/man/bst.1 index c1ca5c88c..4ea843548 100644 --- a/man/bst.1 +++ b/man/bst.1 @@ -1,4 +1,4 @@ -.TH "BST" "1" "2020-05-07" "" "bst Manual" +.TH "BST" "1" "2020-07-01" "" "bst Manual" .SH NAME bst \- Build and manipulate BuildStream projects... .SH SYNOPSIS -- cgit v1.2.1 From 03555170771862f5bae459c7a0c035c7769fb025 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrg=20Billeter?= Date: Mon, 6 Jul 2020 09:42:04 +0200 Subject: NEWS: Add note for `bst source push` --- NEWS | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/NEWS b/NEWS index 09c627dc7..f82483e6e 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,14 @@ +============ +(unreleased) +============ + +CLI +--- + + o Add `bst source push` subcommand. This command pushes element sources to a + remote source cache. + + ================== buildstream 1.93.4 ================== -- cgit v1.2.1 From 4a2365378e06a5800f62a30a61f8b05a58202e45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrg=20Billeter?= Date: Wed, 1 Jul 2020 17:31:48 +0200 Subject: tests/sourcecache/fetch.py: Use `bst source push` to replace hack --- tests/sourcecache/fetch.py | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/tests/sourcecache/fetch.py b/tests/sourcecache/fetch.py index 889de62f0..1e54a98a5 100644 --- a/tests/sourcecache/fetch.py +++ b/tests/sourcecache/fetch.py @@ -35,15 +35,6 @@ from tests.testutils import create_artifact_share, dummy_context DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") -def move_local_cas_to_remote_source_share(local, remote): - shutil.rmtree(os.path.join(remote, "repo", "cas")) - shutil.rmtree(os.path.join(remote, "repo", "source_protos")) - shutil.move(os.path.join(local, "source_protos"), os.path.join(remote, "repo")) - shutil.move(os.path.join(local, "cas"), os.path.join(remote, "repo")) - shutil.rmtree(os.path.join(local, "sources")) - shutil.rmtree(os.path.join(local, "artifacts")) - - def create_test_element(tmpdir, project_dir): repo = create_repo("git", str(tmpdir)) ref = repo.create(os.path.join(project_dir, "files")) @@ -98,11 +89,17 @@ def test_source_fetch(cli, tmpdir, datafiles): sourcecache = context.sourcecache digest = sourcecache.export(source)._get_digest() - move_local_cas_to_remote_source_share(str(cache_dir), share.directory) + # Push the source to the remote + res = cli.run(project=project_dir, args=["source", "push", "--remote", share.repo, element_name]) + res.assert_success() - # check the share has the object + # check the share has the proto and the object + assert share.get_source_proto(source._get_source_name()) assert share.has_object(digest) + # Delete the source locally + shutil.rmtree(os.path.join(str(cache_dir), "sources")) + shutil.rmtree(os.path.join(str(cache_dir), "cas")) state = cli.get_element_state(project_dir, element_name) assert state == "fetch needed" @@ -111,6 +108,12 @@ def test_source_fetch(cli, tmpdir, datafiles): res.assert_success() assert "Pulled source" in res.stderr + with context_with_source_cache(cli, cache_dir, share, tmpdir) as context: + project = Project(project_dir, context) + project.ensure_fully_loaded() + + element = project.load_elements([element_name])[0] + # check that we have the source in the cas now and it's not fetched assert element._has_all_sources_in_source_cache() assert os.listdir(os.path.join(str(tmpdir), "cache", "sources", "git")) == [] @@ -206,13 +209,18 @@ def test_source_pull_partial_fallback_fetch(cli, tmpdir, datafiles): sourcecache = context.sourcecache digest = sourcecache.export(source)._get_digest() - move_local_cas_to_remote_source_share(str(cache_dir), share.directory) + # Push the source to the remote + res = cli.run(project=project_dir, args=["source", "push", "--remote", share.repo, element_name]) + res.assert_success() # Remove the cas content, only keep the proto and such around shutil.rmtree(os.path.join(str(tmpdir), "sourceshare", "repo", "cas", "objects")) # check the share doesn't have the object assert not share.has_object(digest) + # Delete the source locally + shutil.rmtree(os.path.join(str(cache_dir), "sources")) + shutil.rmtree(os.path.join(str(cache_dir), "cas")) state = cli.get_element_state(project_dir, element_name) assert state == "fetch needed" -- cgit v1.2.1 From 74c044d31ba0fb122996e3f9272d4b9a22cc98bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrg=20Billeter?= Date: Wed, 1 Jul 2020 18:24:51 +0200 Subject: tests/frontend/remote-caches.py: Split test_source_artifact_caches Use `bst source push` in a new test instead of manually deleting parts of a remote cache. --- tests/frontend/remote-caches.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/tests/frontend/remote-caches.py b/tests/frontend/remote-caches.py index b112e0882..ebafddf85 100644 --- a/tests/frontend/remote-caches.py +++ b/tests/frontend/remote-caches.py @@ -69,12 +69,34 @@ def test_source_artifact_caches(cli, tmpdir, datafiles): assert "Pulled artifact " in res.stderr assert "Pulled source " not in res.stderr - # remove the artifact from the repo and check it pulls sources, builds + +@pytest.mark.datafiles(DATA_DIR) +def test_source_cache_empty_artifact_cache(cli, tmpdir, datafiles): + cachedir = os.path.join(str(tmpdir), "cache") + project_dir = str(datafiles) + element_path = os.path.join(project_dir, "elements") + + with create_artifact_share(os.path.join(str(tmpdir), "share")) as share: + user_config_file = str(tmpdir.join("buildstream.conf")) + user_config = { + "scheduler": {"pushers": 1}, + "source-caches": {"url": share.repo, "push": True,}, + "artifacts": {"url": share.repo, "push": True,}, + "cachedir": cachedir, + } + _yaml.roundtrip_dump(user_config, file=user_config_file) + cli.configure(user_config) + + create_element_size("repo.bst", project_dir, element_path, [], 10000) + + res = cli.run(project=project_dir, args=["source", "push", "repo.bst"]) + res.assert_success() + assert "Pushed source " in res.stderr + + # delete local sources and check it pulls sources, builds # and then pushes the artifacts shutil.rmtree(os.path.join(cachedir, "cas")) - shutil.rmtree(os.path.join(cachedir, "artifacts")) - print(os.listdir(os.path.join(share.repodir, "artifacts", "refs"))) - shutil.rmtree(os.path.join(share.repodir, "artifacts", "refs", "test")) + shutil.rmtree(os.path.join(cachedir, "sources")) res = cli.run(project=project_dir, args=["build", "repo.bst"]) res.assert_success() -- cgit v1.2.1