diff options
author | bst-marge-bot <marge-bot@buildstream.build> | 2019-11-04 12:09:34 +0000 |
---|---|---|
committer | bst-marge-bot <marge-bot@buildstream.build> | 2019-11-04 12:09:34 +0000 |
commit | 101a0d95b3afcbecf88b1b68ce4840fdaf0c4574 (patch) | |
tree | 750a9b88a7cd22c9907a880cb47fcbcf5f245f49 | |
parent | 66045d9c817992dbce855d4217ff00970ae7ff79 (diff) | |
parent | 9765eba16cd097db1e2496dbdca1e313d59bbe49 (diff) | |
download | buildstream-101a0d95b3afcbecf88b1b68ce4840fdaf0c4574.tar.gz |
Merge branch 'chandan/remove-build-track' into 'master'
frontend: Remove tracking options from `bst build`
See merge request BuildStream/buildstream!1671
34 files changed, 210 insertions, 727 deletions
@@ -2,6 +2,21 @@ (unreleased) ============ +CLI +--- + + o BREAKING CHANGE: `bst build` no longer accepts any options related to + tracking. Please use `bst source track` separately prior to running + `bst build`, if you need similar functionality. The full list of removed + options is as follows: + + * `--track` + * `--track-all` + * `--track-except` + * `--track-cross-junctions` / `-J` + * `--track-save` + + ================== buildstream 1.91.2 ================== diff --git a/man/bst-artifact-checkout.1 b/man/bst-artifact-checkout.1 index 138b7ab54..8b0a6e14f 100644 --- a/man/bst-artifact-checkout.1 +++ b/man/bst-artifact-checkout.1 @@ -1,9 +1,9 @@ -.TH "BST ARTIFACT CHECKOUT" "1" "14-Jul-2019" "" "bst artifact checkout Manual" +.TH "BST ARTIFACT CHECKOUT" "1" "31-Oct-2019" "" "bst artifact checkout Manual" .SH NAME bst\-artifact\-checkout \- Checkout contents of an artifact .SH SYNOPSIS .B bst artifact checkout -[OPTIONS] [ELEMENT] +[OPTIONS] [TARGET] .SH DESCRIPTION Checkout contents of an artifact .PP @@ -14,8 +14,8 @@ is to checkout the artifact of the workspace element. \fB\-f,\fP \-\-force Allow files to be overwritten .TP -\fB\-d,\fP \-\-deps [run|build|none] -The dependencies to checkout (default: run) +\fB\-d,\fP \-\-deps [run|build|none|all] +The dependencies to checkout [default: run] .TP \fB\-\-integrate\fP / \-\-no\-integrate Whether to run integration commands @@ -26,8 +26,11 @@ Checkout hardlinks instead of copying if possible \fB\-\-tar\fP LOCATION Create a tarball from the artifact contents instead of a file tree. If LOCATION is '-', the tarball will be dumped to the standard output. .TP +\fB\-\-compression\fP [gz|xz|bz2] +The compression option of the tarball created. +.TP \fB\-\-pull\fP -Whether to pull the artifact if it's missing or incomplete. +Pull the artifact if it's missing or incomplete. .TP \fB\-\-directory\fP DIRECTORY The directory to checkout the artifact to diff --git a/man/bst-artifact-delete.1 b/man/bst-artifact-delete.1 index aa1bb34e4..d3d8394b7 100644 --- a/man/bst-artifact-delete.1 +++ b/man/bst-artifact-delete.1 @@ -1,4 +1,4 @@ -.TH "BST ARTIFACT DELETE" "1" "14-Jul-2019" "" "bst artifact delete Manual" +.TH "BST ARTIFACT DELETE" "1" "31-Oct-2019" "" "bst artifact delete Manual" .SH NAME bst\-artifact\-delete \- Remove artifacts from the local cache .SH SYNOPSIS @@ -8,5 +8,5 @@ bst\-artifact\-delete \- Remove artifacts from the local cache Remove artifacts from the local cache .SH OPTIONS .TP -\fB\-\-no\-prune\fP -Do not prune the local cache of unreachable refs +\fB\-d,\fP \-\-deps [none|run|build|all] +The dependencies to delete [default: none] diff --git a/man/bst-artifact-list-contents.1 b/man/bst-artifact-list-contents.1 new file mode 100644 index 000000000..90949af6a --- /dev/null +++ b/man/bst-artifact-list-contents.1 @@ -0,0 +1,15 @@ +.TH "BST ARTIFACT LIST-CONTENTS" "1" "31-Oct-2019" "" "bst artifact list-contents Manual" +.SH NAME +bst\-artifact\-list-contents \- List the contents of an artifact +.SH SYNOPSIS +.B bst artifact list-contents +[OPTIONS] [ARTIFACTS]... +.SH DESCRIPTION +List the contents of an artifact. +.PP +Note that 'artifacts' can be element names, which must end in '.bst', +or artifact references, which must be in the format `<project_name>/<element>/<key>`. +.SH OPTIONS +.TP +\fB\-l,\fP \-\-long +Provide more information about the contents of the artifact. diff --git a/man/bst-artifact-log.1 b/man/bst-artifact-log.1 index 34205e3f8..e7d646602 100644 --- a/man/bst-artifact-log.1 +++ b/man/bst-artifact-log.1 @@ -1,36 +1,12 @@ -.TH "BST ARTIFACT LOG" "1" "14-Jul-2019" "" "bst artifact log Manual" +.TH "BST ARTIFACT LOG" "1" "31-Oct-2019" "" "bst artifact log Manual" .SH NAME bst\-artifact\-log \- Show logs of artifacts .SH SYNOPSIS .B bst artifact log [OPTIONS] [ARTIFACTS]... .SH DESCRIPTION -Show logs of artifacts. -.PP -Note that 'artifacts' can be element references like "hello.bst", and they -can also be artifact references. You may use shell-style wildcards for -either. -.PP -Here are some examples of element references: -.PP - -- `hello.bst` -- `*.bst` -.PP -Note that element references must end with '.bst' to distinguish them from -artifact references. Anything that does not end in '.bst' is an artifact -ref. -.PP -Artifact references follow the format `<project_name>/<element>/<key>`. -Note that 'element' is without the `.bst` extension. -.PP -Here are some examples of artifact references: -.PP - -- `myproject/hello/*` -- `myproject/*` -- `*` -- `myproject/hello/827637*` -- `myproject/he*/827637*` -- `myproject/he??o/827637*` -- `m*/h*/8276376b077eda104c812e6ec2f488c7c9eea211ce572c83d734c10bf241209f` +Show build logs of artifacts +.SH OPTIONS +.TP +\fB\-\-out\fP PATH +Output logs to individual files in the specified path. If absent, logs are written to stdout. diff --git a/man/bst-artifact-pull.1 b/man/bst-artifact-pull.1 index 6b9262570..f768ca0d7 100644 --- a/man/bst-artifact-pull.1 +++ b/man/bst-artifact-pull.1 @@ -1,9 +1,9 @@ -.TH "BST ARTIFACT PULL" "1" "14-Jul-2019" "" "bst artifact pull Manual" +.TH "BST ARTIFACT PULL" "1" "31-Oct-2019" "" "bst artifact pull Manual" .SH NAME bst\-artifact\-pull \- Pull a built artifact .SH SYNOPSIS .B bst artifact pull -[OPTIONS] [ELEMENTS]... +[OPTIONS] [ARTIFACTS]... .SH DESCRIPTION Pull a built artifact from the configured remote artifact cache. .PP @@ -26,7 +26,7 @@ Specify `--deps` to control which artifacts to pull: .SH OPTIONS .TP \fB\-d,\fP \-\-deps [none|all] -The dependency artifacts to pull (default: none) +The dependency artifacts to pull [default: none] .TP \fB\-r,\fP \-\-remote TEXT The URL of the remote cache (defaults to the first configured cache) diff --git a/man/bst-artifact-push.1 b/man/bst-artifact-push.1 index 6894a5be2..69280ff26 100644 --- a/man/bst-artifact-push.1 +++ b/man/bst-artifact-push.1 @@ -1,9 +1,9 @@ -.TH "BST ARTIFACT PUSH" "1" "14-Jul-2019" "" "bst artifact push Manual" +.TH "BST ARTIFACT PUSH" "1" "31-Oct-2019" "" "bst artifact push Manual" .SH NAME bst\-artifact\-push \- Push a built artifact .SH SYNOPSIS .B bst artifact push -[OPTIONS] [ELEMENTS]... +[OPTIONS] [ARTIFACTS]... .SH DESCRIPTION Push a built artifact to a remote artifact cache. .PP @@ -29,7 +29,7 @@ Specify `--deps` to control which artifacts to push: .SH OPTIONS .TP \fB\-d,\fP \-\-deps [none|all] -The dependencies to push (default: none) +The dependencies to push [default: none] .TP \fB\-r,\fP \-\-remote TEXT The URL of the remote cache (defaults to the first configured cache) diff --git a/man/bst-artifact-server.1 b/man/bst-artifact-server.1 index ed5c0c096..1f84a4924 100644 --- a/man/bst-artifact-server.1 +++ b/man/bst-artifact-server.1 @@ -1,4 +1,4 @@ -.TH "BST-ARTIFACT-SERVER" "1" "14-Jul-2019" "" "bst-artifact-server Manual" +.TH "BST-ARTIFACT-SERVER" "1" "31-Oct-2019" "" "bst-artifact-server Manual" .SH NAME bst-artifact-server \- CAS Artifact Server .SH SYNOPSIS @@ -21,8 +21,8 @@ Public client certificates for TLS (PEM-encoded) \fB\-\-enable\-push\fP Allow clients to upload blobs and update artifact cache .TP -\fB\-\-head\-room\-min\fP INTEGER -Disk head room minimum in bytes +\fB\-\-quota\fP INTEGER +Maximum disk usage in bytes [default: 10000000000.0] .TP -\fB\-\-head\-room\-max\fP INTEGER -Disk head room maximum in bytes +\fB\-\-index\-only\fP +Only provide the BuildStream artifact and source services ("index"), not the CAS ("storage") diff --git a/man/bst-artifact-show.1 b/man/bst-artifact-show.1 new file mode 100644 index 000000000..358367b33 --- /dev/null +++ b/man/bst-artifact-show.1 @@ -0,0 +1,12 @@ +.TH "BST ARTIFACT SHOW" "1" "31-Oct-2019" "" "bst artifact show Manual" +.SH NAME +bst\-artifact\-show \- Show the cached state of artifacts +.SH SYNOPSIS +.B bst artifact show +[OPTIONS] [ARTIFACTS]... +.SH DESCRIPTION +show the cached state of artifacts +.SH OPTIONS +.TP +\fB\-d,\fP \-\-deps [build|run|all|none] +The dependencies we also want to show [default: none] diff --git a/man/bst-artifact.1 b/man/bst-artifact.1 index d5b0a4797..d2ac2dc02 100644 --- a/man/bst-artifact.1 +++ b/man/bst-artifact.1 @@ -1,13 +1,37 @@ -.TH "BST ARTIFACT" "1" "14-Jul-2019" "" "bst artifact Manual" +.TH "BST ARTIFACT" "1" "31-Oct-2019" "" "bst artifact Manual" .SH NAME -bst\-artifact \- Manipulate cached artifacts +bst\-artifact \- Manipulate cached artifacts. .SH SYNOPSIS .B bst artifact [OPTIONS] COMMAND [ARGS]... .SH DESCRIPTION Manipulate cached artifacts +.PP +Some subcommands take artifact references as arguments. Artifacts +can be specified in two ways: +.PP + +- artifact refs: triples of the form <project name>/<element name>/<cache key> +- element paths +.PP +When elements are given, the artifact corresponding to the element is used. +.PP +The commands also support shell-style wildcard expansion: `?` matches a +single character, and `*` matches zero or more. The patterns are matched +against artifact refs by default. If the pattern ends with `.bst` then +it matches element paths instead. Some example arguments are: +.PP + +- `myproject/hello/8276376b077eda104c812e6ec2f488c7c9eea211ce572c83d734c10bf241209f` +- `myproject/he*/827637*` +- `*.bst` (all elements) +- `myproject/*` (all artifacts from myproject) .SH COMMANDS .PP +\fBshow\fP + Show the cached state of artifacts + See \fBbst artifact-show(1)\fP for full documentation on the \fBshow\fP command. +.PP \fBcheckout\fP Checkout contents of an artifact See \fBbst artifact-checkout(1)\fP for full documentation on the \fBcheckout\fP command. @@ -24,6 +48,10 @@ Manipulate cached artifacts Show logs of artifacts See \fBbst artifact-log(1)\fP for full documentation on the \fBlog\fP command. .PP +\fBlist-contents\fP + List the contents of an artifact + See \fBbst artifact-list-contents(1)\fP for full documentation on the \fBlist-contents\fP command. +.PP \fBdelete\fP Remove artifacts from the local cache See \fBbst artifact-delete(1)\fP for full documentation on the \fBdelete\fP command. diff --git a/man/bst-build.1 b/man/bst-build.1 index 1eab130a4..ebd6eaf3e 100644 --- a/man/bst-build.1 +++ b/man/bst-build.1 @@ -1,4 +1,4 @@ -.TH "BST BUILD" "1" "14-Jul-2019" "" "bst build Manual" +.TH "BST BUILD" "1" "31-Oct-2019" "" "bst build Manual" .SH NAME bst\-build \- Build elements in a pipeline .SH SYNOPSIS @@ -13,25 +13,16 @@ elements will be built. .PP When this command is executed from a workspace directory, the default is to build the workspace element. +.PP +Specify `--deps` to control which dependencies to build: +.PP + + plan: Only dependencies required for the build plan + all: All dependencies .SH OPTIONS .TP -\fB\-\-all\fP -Build elements that would not be needed for the current build plan -.TP -\fB\-\-track\fP PATH -Specify elements to track during the build. Can be used repeatedly to specify multiple elements -.TP -\fB\-\-track\-all\fP -Track all elements in the pipeline -.TP -\fB\-\-track\-except\fP PATH -Except certain dependencies from tracking -.TP -\fB\-J,\fP \-\-track\-cross\-junctions -Allow tracking to cross junction boundaries -.TP -\fB\-\-track\-save\fP -Deprecated: This is ignored +\fB\-d,\fP \-\-deps [plan|all] +The dependencies to build .TP \fB\-r,\fP \-\-remote TEXT The URL of the remote cache (defaults to the first configured cache) diff --git a/man/bst-help.1 b/man/bst-help.1 index df6caf815..92928e519 100644 --- a/man/bst-help.1 +++ b/man/bst-help.1 @@ -1,4 +1,4 @@ -.TH "BST HELP" "1" "14-Jul-2019" "" "bst help Manual" +.TH "BST HELP" "1" "31-Oct-2019" "" "bst help Manual" .SH NAME bst\-help \- Print usage information .SH SYNOPSIS diff --git a/man/bst-init.1 b/man/bst-init.1 index 58c4e1311..a9e9d24ea 100644 --- a/man/bst-init.1 +++ b/man/bst-init.1 @@ -1,4 +1,4 @@ -.TH "BST INIT" "1" "14-Jul-2019" "" "bst init Manual" +.TH "BST INIT" "1" "31-Oct-2019" "" "bst init Manual" .SH NAME bst\-init \- Initialize a new BuildStream project .SH SYNOPSIS @@ -18,10 +18,10 @@ interactive session. The project name to use .TP \fB\-\-format\-version\fP INTEGER -The required format version (default: 24) +The required format version [default: 25] .TP \fB\-\-element\-path\fP PATH -The subdirectory to store elements in (default: elements) +The subdirectory to store elements in [default: elements] .TP \fB\-f,\fP \-\-force Allow overwriting an existing project.conf diff --git a/man/bst-shell.1 b/man/bst-shell.1 index a6e8956da..d32eb613c 100644 --- a/man/bst-shell.1 +++ b/man/bst-shell.1 @@ -1,4 +1,4 @@ -.TH "BST SHELL" "1" "14-Jul-2019" "" "bst shell Manual" +.TH "BST SHELL" "1" "31-Oct-2019" "" "bst shell Manual" .SH NAME bst\-shell \- Shell into an element's sandbox environment .SH SYNOPSIS @@ -44,7 +44,7 @@ Mount a file or directory into the sandbox Create an isolated build sandbox .TP \fB\-t,\fP \-\-use\-buildtree [ask|try|always|never] -Defaults to ask but if set to always the function will fail if a build tree is not available +Use a buildtree. If `always` is set, will always fail to build if a buildtree is not available. [default: ask] .TP \fB\-\-pull\fP Attempt to pull missing or incomplete artifacts diff --git a/man/bst-show.1 b/man/bst-show.1 index ce74604a5..0d66f73c1 100644 --- a/man/bst-show.1 +++ b/man/bst-show.1 @@ -1,4 +1,4 @@ -.TH "BST SHOW" "1" "14-Jul-2019" "" "bst show Manual" +.TH "BST SHOW" "1" "31-Oct-2019" "" "bst show Manual" .SH NAME bst\-show \- Show elements in the pipeline .SH SYNOPSIS @@ -66,10 +66,10 @@ If you want to use a newline in a format string in bash, use the '$' modifier: Except certain dependencies .TP \fB\-d,\fP \-\-deps [none|plan|run|build|all] -The dependencies to show (default: all) +The dependencies to show [default: all] .TP \fB\-\-order\fP [stage|alpha] -Staging or alphabetic ordering of dependencies +Staging or alphabetic ordering of dependencies [default: stage] .TP \fB\-f,\fP \-\-format FORMAT Format string for each element diff --git a/man/bst-source-checkout.1 b/man/bst-source-checkout.1 index 4a3e1edd9..6af45c720 100644 --- a/man/bst-source-checkout.1 +++ b/man/bst-source-checkout.1 @@ -1,9 +1,9 @@ -.TH "BST SOURCE CHECKOUT" "1" "14-Jul-2019" "" "bst source checkout Manual" +.TH "BST SOURCE CHECKOUT" "1" "31-Oct-2019" "" "bst source checkout Manual" .SH NAME -bst\-source\-checkout \- Checkout sources for an element +bst\-source\-checkout \- Checkout sources of an element .SH SYNOPSIS .B bst source checkout -[OPTIONS] [ELEMENT] [LOCATION] +[OPTIONS] [ELEMENT] .SH DESCRIPTION Checkout sources of an element to the specified location .PP @@ -18,10 +18,16 @@ Allow files to be overwritten Except certain dependencies .TP \fB\-d,\fP \-\-deps [build|none|run|all] -The dependencies whose sources to checkout (default: none) +The dependencies whose sources to checkout [default: none] .TP -\fB\-\-tar\fP -Create a tarball from the element's sources instead of a file tree. +\fB\-\-tar\fP LOCATION +Create a tarball containing the sources instead of a file tree. +.TP +\fB\-\-compression\fP [gz|xz|bz2] +The compression option of the tarball created. .TP \fB\-\-include\-build\-scripts\fP .PP +.TP +\fB\-\-directory\fP DIRECTORY +The directory to checkout the sources to diff --git a/man/bst-source-fetch.1 b/man/bst-source-fetch.1 index e25ec9175..e53e2be53 100644 --- a/man/bst-source-fetch.1 +++ b/man/bst-source-fetch.1 @@ -1,4 +1,4 @@ -.TH "BST SOURCE FETCH" "1" "14-Jul-2019" "" "bst source fetch Manual" +.TH "BST SOURCE FETCH" "1" "31-Oct-2019" "" "bst source fetch Manual" .SH NAME bst\-source\-fetch \- Fetch sources in a pipeline .SH SYNOPSIS @@ -31,7 +31,7 @@ Specify `--deps` to control which sources to fetch: Except certain dependencies from fetching .TP \fB\-d,\fP \-\-deps [none|plan|all] -The dependencies to fetch (default: plan) +The dependencies to fetch [default: plan] .TP \fB\-\-track\fP Track new source references before fetching diff --git a/man/bst-source-track.1 b/man/bst-source-track.1 index 6ce8e7a1d..12171553b 100644 --- a/man/bst-source-track.1 +++ b/man/bst-source-track.1 @@ -1,4 +1,4 @@ -.TH "BST SOURCE TRACK" "1" "14-Jul-2019" "" "bst source track Manual" +.TH "BST SOURCE TRACK" "1" "31-Oct-2019" "" "bst source track Manual" .SH NAME bst\-source\-track \- Track new source references .SH SYNOPSIS @@ -31,7 +31,7 @@ Specify `--deps` to control which sources to track: Except certain dependencies from tracking .TP \fB\-d,\fP \-\-deps [none|all] -The dependencies to track (default: none) +The dependencies to track [default: none] .TP \fB\-J,\fP \-\-cross\-junctions Allow crossing junction boundaries diff --git a/man/bst-source.1 b/man/bst-source.1 index 2598d4d1d..447fb7f06 100644 --- a/man/bst-source.1 +++ b/man/bst-source.1 @@ -1,4 +1,4 @@ -.TH "BST SOURCE" "1" "14-Jul-2019" "" "bst source Manual" +.TH "BST SOURCE" "1" "31-Oct-2019" "" "bst source Manual" .SH NAME bst\-source \- Manipulate sources for an element .SH SYNOPSIS @@ -17,5 +17,5 @@ Manipulate sources for an element See \fBbst source-track(1)\fP for full documentation on the \fBtrack\fP command. .PP \fBcheckout\fP - Checkout sources for an element + Checkout sources of an element See \fBbst source-checkout(1)\fP for full documentation on the \fBcheckout\fP command. diff --git a/man/bst-workspace-close.1 b/man/bst-workspace-close.1 index 0bcf74bf7..1abbe1a5a 100644 --- a/man/bst-workspace-close.1 +++ b/man/bst-workspace-close.1 @@ -1,4 +1,4 @@ -.TH "BST WORKSPACE CLOSE" "1" "14-Jul-2019" "" "bst workspace close Manual" +.TH "BST WORKSPACE CLOSE" "1" "31-Oct-2019" "" "bst workspace close Manual" .SH NAME bst\-workspace\-close \- Close workspaces .SH SYNOPSIS diff --git a/man/bst-workspace-list.1 b/man/bst-workspace-list.1 index 199ff3461..630efebc4 100644 --- a/man/bst-workspace-list.1 +++ b/man/bst-workspace-list.1 @@ -1,4 +1,4 @@ -.TH "BST WORKSPACE LIST" "1" "14-Jul-2019" "" "bst workspace list Manual" +.TH "BST WORKSPACE LIST" "1" "31-Oct-2019" "" "bst workspace list Manual" .SH NAME bst\-workspace\-list \- List open workspaces .SH SYNOPSIS diff --git a/man/bst-workspace-open.1 b/man/bst-workspace-open.1 index cb59cbfb6..4ccf010af 100644 --- a/man/bst-workspace-open.1 +++ b/man/bst-workspace-open.1 @@ -1,4 +1,4 @@ -.TH "BST WORKSPACE OPEN" "1" "14-Jul-2019" "" "bst workspace open Manual" +.TH "BST WORKSPACE OPEN" "1" "31-Oct-2019" "" "bst workspace open Manual" .SH NAME bst\-workspace\-open \- Open a new workspace .SH SYNOPSIS diff --git a/man/bst-workspace-reset.1 b/man/bst-workspace-reset.1 index 255840434..a9c0d2ce6 100644 --- a/man/bst-workspace-reset.1 +++ b/man/bst-workspace-reset.1 @@ -1,4 +1,4 @@ -.TH "BST WORKSPACE RESET" "1" "14-Jul-2019" "" "bst workspace reset Manual" +.TH "BST WORKSPACE RESET" "1" "31-Oct-2019" "" "bst workspace reset Manual" .SH NAME bst\-workspace\-reset \- Reset a workspace to its original state .SH SYNOPSIS diff --git a/man/bst-workspace.1 b/man/bst-workspace.1 index aab453ccc..ce5842a87 100644 --- a/man/bst-workspace.1 +++ b/man/bst-workspace.1 @@ -1,4 +1,4 @@ -.TH "BST WORKSPACE" "1" "14-Jul-2019" "" "bst workspace Manual" +.TH "BST WORKSPACE" "1" "31-Oct-2019" "" "bst workspace Manual" .SH NAME bst\-workspace \- Manipulate developer workspaces .SH SYNOPSIS @@ -1,4 +1,4 @@ -.TH "BST" "1" "14-Jul-2019" "" "bst Manual" +.TH "BST" "1" "31-Oct-2019" "" "bst Manual" .SH NAME bst \- Build and manipulate BuildStream projects... .SH SYNOPSIS @@ -104,7 +104,7 @@ Cache artifact build tree content on creation See \fBbst-workspace(1)\fP for full documentation on the \fBworkspace\fP command. .PP \fBartifact\fP - Manipulate cached artifacts + Manipulate cached artifacts. See \fBbst-artifact(1)\fP for full documentation on the \fBartifact\fP command. .PP \fBfetch\fP diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index 42d557615..67ea02d59 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -415,25 +415,12 @@ def init(app, project_name, format_version, element_path, force, target_director @click.option('--deps', '-d', default=None, type=click.Choice(['plan', 'all']), help='The dependencies to build') -@click.option('--track', 'track_', multiple=True, - type=click.Path(readable=False), - help="Specify elements to track during the build. Can be used " - "repeatedly to specify multiple elements") -@click.option('--track-all', is_flag=True, - help="Track all elements in the pipeline") -@click.option('--track-except', multiple=True, - type=click.Path(readable=False), - help="Except certain dependencies from tracking") -@click.option('--track-cross-junctions', '-J', is_flag=True, - help="Allow tracking to cross junction boundaries") -@click.option('--track-save', is_flag=True, - help="Deprecated: This is ignored") @click.option('--remote', '-r', default=None, help="The URL of the remote cache (defaults to the first configured cache)") @click.argument('elements', nargs=-1, type=click.Path(readable=False)) @click.pass_obj -def build(app, elements, deps, track_, track_save, track_all, track_except, track_cross_junctions, remote): +def build(app, elements, deps, remote): """Build elements in a pipeline Specifying no elements will result in building the default targets @@ -449,15 +436,6 @@ def build(app, elements, deps, track_, track_save, track_all, track_except, trac plan: Only dependencies required for the build plan all: All dependencies """ - - if (track_except or track_cross_junctions) and not (track_ or track_all): - click.echo("ERROR: The --track-except and --track-cross-junctions options " - "can only be used with --track or --track-all", err=True) - sys.exit(-1) - - if track_save: - click.echo("WARNING: --track-save is deprecated, saving is now unconditional", err=True) - with app.initialized(session_name="Build"): ignore_junction_targets = False @@ -469,14 +447,8 @@ def build(app, elements, deps, track_, track_save, track_all, track_except, trac # Junction elements cannot be built, exclude them from default targets ignore_junction_targets = True - if track_all: - track_ = elements - app.stream.build(elements, selection=deps, - track_targets=track_, - track_except=track_except, - track_cross_junctions=track_cross_junctions, ignore_junction_targets=ignore_junction_targets, remote=remote) diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index c7ada6eb6..c71bec80a 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -240,9 +240,6 @@ class Stream(): # Args: # targets (list of str): Targets to build # selection (PipelineSelection): The selection mode for the specified targets - # track_targets (list of str): Specified targets for tracking - # track_except (list of str): Specified targets to except from tracking - # track_cross_junctions (bool): Whether tracking should cross junction boundaries # ignore_junction_targets (bool): Whether junction targets should be filtered out # remote (str): The URL of a specific remote server to push to, or None # @@ -251,9 +248,6 @@ class Stream(): # def build(self, targets, *, selection=PipelineSelection.PLAN, - track_targets=None, - track_except=None, - track_cross_junctions=False, ignore_junction_targets=False, remote=None): @@ -261,21 +255,16 @@ class Stream(): if remote: use_config = False - elements, track_elements = \ - self._load(targets, track_targets, - selection=selection, track_selection=PipelineSelection.ALL, - track_except_targets=track_except, - track_cross_junctions=track_cross_junctions, + elements, _ = \ + self._load(targets, [], + selection=selection, ignore_junction_targets=ignore_junction_targets, use_artifact_config=use_config, artifact_remote_url=remote, use_source_config=True, dynamic_plan=True) - # Remove the tracking elements from the main targets - elements = self._pipeline.subtract_elements(elements, track_elements) - - # Assert that the elements we're not going to track are consistent + # Assert that the elements are consistent self._pipeline.assert_consistent(elements) if all(project.remote_execution_specs for project in self._context.get_projects()): @@ -292,10 +281,6 @@ class Stream(): # Now construct the queues # self._scheduler.clear_queues() - track_queue = None - if track_elements: - track_queue = TrackQueue(self._scheduler) - self._add_queue(track_queue, track=True) if self._artifacts.has_fetch_remotes(): self._add_queue(PullQueue(self._scheduler)) @@ -311,9 +296,6 @@ class Stream(): self._add_queue(SourcePushQueue(self._scheduler)) # Enqueue elements - # - if track_elements: - self._enqueue_plan(track_elements, queue=track_queue) self._enqueue_plan(elements) self._run() diff --git a/tests/artifactcache/expiry.py b/tests/artifactcache/expiry.py index d33034813..9ede1a8d3 100644 --- a/tests/artifactcache/expiry.py +++ b/tests/artifactcache/expiry.py @@ -29,7 +29,7 @@ from buildstream._cas import CASCache from buildstream._exceptions import ErrorDomain, LoadErrorReason from buildstream.testing import cli # pylint: disable=unused-import -from tests.testutils import create_element_size, update_element_size, wait_for_cache_granularity +from tests.testutils import create_element_size, wait_for_cache_granularity DATA_DIR = os.path.join( @@ -257,67 +257,6 @@ def test_never_delete_required(cli, datafiles): assert states['target.bst'] != 'cached' -# Assert that we never delete a dependency required for a build tree, -# even when the artifact cache was previously populated with -# artifacts we do not require, and the new build is run with dynamic tracking. -# -@pytest.mark.datafiles(DATA_DIR) -def test_never_delete_required_track(cli, datafiles): - project = str(datafiles) - element_path = 'elements' - - cli.configure({ - 'cache': { - 'quota': 10000000 - }, - 'scheduler': { - 'fetchers': 1, - 'builders': 1 - } - }) - - # Create a linear build tree - repo_dep1 = create_element_size('dep1.bst', project, element_path, [], 2000000) - repo_dep2 = create_element_size('dep2.bst', project, element_path, ['dep1.bst'], 2000000) - repo_dep3 = create_element_size('dep3.bst', project, element_path, ['dep2.bst'], 2000000) - repo_target = create_element_size('target.bst', project, element_path, ['dep3.bst'], 2000000) - - # This should all fit into the artifact cache - res = cli.run(project=project, args=['build', 'target.bst']) - res.assert_success() - - # They should all be cached - states = cli.get_element_states(project, ['target.bst']) - assert states['dep1.bst'] == 'cached' - assert states['dep2.bst'] == 'cached' - assert states['dep3.bst'] == 'cached' - assert states['target.bst'] == 'cached' - - # Now increase the size of all the elements - # - update_element_size('dep1.bst', project, repo_dep1, 8000000) - update_element_size('dep2.bst', project, repo_dep2, 8000000) - update_element_size('dep3.bst', project, repo_dep3, 8000000) - update_element_size('target.bst', project, repo_target, 8000000) - - # Now repeat the same test we did in test_never_delete_required(), - # except this time let's add dynamic tracking - # - res = cli.run(project=project, args=['build', '--track-all', 'target.bst']) - res.assert_main_error(ErrorDomain.STREAM, None) - res.assert_task_error(ErrorDomain.CAS, 'cache-too-full') - - # Expect the almost the same result that we did in test_never_delete_required() - # As the source will be downloaded first, we will be over the limit once - # the source for dep2.bst is downloaded - # - states = cli.get_element_states(project, ['target.bst']) - assert states['dep1.bst'] == 'cached' - assert states['dep2.bst'] == 'buildable' - assert states['dep3.bst'] != 'cached' - assert states['target.bst'] != 'cached' - - # Ensure that only valid cache quotas make it through the loading # process. # diff --git a/tests/frontend/buildtrack.py b/tests/frontend/buildtrack.py deleted file mode 100644 index ff3c53281..000000000 --- a/tests/frontend/buildtrack.py +++ /dev/null @@ -1,402 +0,0 @@ -# Pylint doesn't play well with fixtures and dependency injection from pytest -# pylint: disable=redefined-outer-name - -import os -import re -import shutil -import itertools - -import pytest - -from buildstream import _yaml -from buildstream.testing import create_repo -from buildstream.testing import cli # pylint: disable=unused-import -from buildstream._exceptions import ErrorDomain -from tests.testutils import generate_junction - -from . import configure_project - -# Project directory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project", -) - - -def create_element(repo, name, path, dependencies, ref=None): - element = { - 'kind': 'import', - 'sources': [ - repo.source_config(ref=ref) - ], - 'depends': dependencies - } - _yaml.roundtrip_dump(element, os.path.join(path, name)) - - -@pytest.mark.datafiles(os.path.join(DATA_DIR)) -@pytest.mark.parametrize("strict", [True, False], ids=["strict", "no-strict"]) -@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')]) -@pytest.mark.parametrize("track_targets,exceptions,tracked", [ - # Test with no exceptions - (['0.bst'], [], ['0.bst', '2.bst', '3.bst', '4.bst', '5.bst', '6.bst', '7.bst']), - (['3.bst'], [], ['3.bst', '4.bst', '5.bst', '6.bst']), - (['2.bst', '3.bst'], [], ['2.bst', '3.bst', '4.bst', '5.bst', '6.bst', '7.bst']), - - # Test excepting '2.bst' - (['0.bst'], ['2.bst'], ['0.bst', '3.bst', '4.bst', '5.bst', '6.bst']), - (['3.bst'], ['2.bst'], []), - (['2.bst', '3.bst'], ['2.bst'], ['3.bst', '4.bst', '5.bst', '6.bst']), - - # Test excepting '2.bst' and '3.bst' - (['0.bst'], ['2.bst', '3.bst'], ['0.bst']), - (['3.bst'], ['2.bst', '3.bst'], []), - (['2.bst', '3.bst'], ['2.bst', '3.bst'], []) -]) -def test_build_track(cli, datafiles, tmpdir, ref_storage, strict, - track_targets, exceptions, tracked): - project = str(datafiles) - dev_files_path = os.path.join(project, 'files', 'dev-files') - element_path = os.path.join(project, 'elements') - - repo = create_repo('git', str(tmpdir)) - ref = repo.create(dev_files_path) - - configure_project(project, { - 'ref-storage': ref_storage - }) - cli.configure({ - 'projects': { - 'test': { - 'strict': strict - } - } - }) - - create_elements = { - '0.bst': [ - '2.bst', - '3.bst' - ], - '2.bst': [ - '3.bst', - '7.bst' - ], - '3.bst': [ - '4.bst', - '5.bst', - '6.bst' - ], - '4.bst': [], - '5.bst': [], - '6.bst': [ - '5.bst' - ], - '7.bst': [] - } - - initial_project_refs = {} - for element, dependencies in create_elements.items(): - # Test the element inconsistency resolution by ensuring that - # only elements that aren't tracked have refs - if element in set(tracked): - # Elements which should not have a ref set - # - create_element(repo, element, element_path, dependencies) - elif ref_storage == 'project.refs': - # Store a ref in project.refs - # - create_element(repo, element, element_path, dependencies) - initial_project_refs[element] = [{'ref': ref}] - else: - # Store a ref in the element itself - # - create_element(repo, element, element_path, dependencies, ref=ref) - - # Generate initial project.refs - if ref_storage == 'project.refs': - project_refs = { - 'projects': { - 'test': initial_project_refs - } - } - _yaml.roundtrip_dump(project_refs, os.path.join(project, 'project.refs')) - - args = ['build'] - args += itertools.chain.from_iterable(zip(itertools.repeat('--track'), track_targets)) - args += itertools.chain.from_iterable(zip(itertools.repeat('--track-except'), exceptions)) - args += ['0.bst'] - - result = cli.run(project=project, silent=True, args=args) - result.assert_success() - - # Assert that the main target 0.bst is cached - assert cli.get_element_state(project, '0.bst') == 'cached' - - # Assert that we tracked exactly the elements we expected to - tracked_elements = result.get_tracked_elements() - assert set(tracked_elements) == set(tracked) - - # Delete element sources - source_dir = os.path.join(project, 'cache', 'sources') - shutil.rmtree(source_dir) - source_protos = os.path.join(project, 'cache', 'source_protos') - shutil.rmtree(source_protos) - - # Delete artifacts one by one and assert element states - for target in set(tracked): - cli.remove_artifact_from_cache(project, target) - - # Assert that it's tracked - assert cli.get_element_state(project, target) == 'fetch needed' - - # Assert there was a project.refs created, depending on the configuration - if ref_storage == 'project.refs': - assert os.path.exists(os.path.join(project, 'project.refs')) - else: - assert not os.path.exists(os.path.join(project, 'project.refs')) - - -# This tests a very specific scenario: -# -# o Local cache is empty -# o Strict mode is disabled -# o The build target has only build dependencies -# o The build is launched with --track-all -# -# In this scenario, we have encountered bugs where BuildStream returns -# successfully after tracking completes without ever pulling, fetching or -# building anything. -# -@pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("strict", [True, False], ids=["strict", "no-strict"]) -@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')]) -def test_build_track_all(cli, tmpdir, datafiles, strict, ref_storage): - project = os.path.join(datafiles.dirname, datafiles.basename) - subproject_path = os.path.join(project, 'files', 'sub-project') - subproject_element_path = os.path.join(project, 'files', 'sub-project', 'elements') - junction_path = os.path.join(project, 'elements', 'junction.bst') - element_path = os.path.join(project, 'elements') - dev_files_path = os.path.join(project, 'files', 'dev-files') - - configure_project(project, { - 'ref-storage': ref_storage - }) - cli.configure({ - 'projects': { - 'test': { - 'strict': strict - } - } - }) - - # We need a repo for real trackable elements - repo = create_repo('git', str(tmpdir)) - ref = repo.create(dev_files_path) - - # Create a trackable element to depend on the cross junction element, - # this one has it's ref resolved already - create_element(repo, 'sub-target.bst', subproject_element_path, ['import-etc.bst'], ref=ref) - - # Create a trackable element to depend on the cross junction element - create_element(repo, 'target.bst', element_path, [ - { - 'junction': 'junction.bst', - 'filename': 'sub-target.bst' - } - ]) - - # Create a repo to hold the subproject and generate a junction element for it - generate_junction(tmpdir, subproject_path, junction_path, store_ref=False) - - # Now create a compose element at the top level - element = { - 'kind': 'compose', - 'depends': [ - { - 'filename': 'target.bst', - 'type': 'build' - } - ] - } - _yaml.roundtrip_dump(element, os.path.join(element_path, 'composed.bst')) - - # Track the junction itself first. - result = cli.run(project=project, args=['source', 'track', 'junction.bst']) - result.assert_success() - - # Build it with --track-all - result = cli.run(project=project, silent=True, args=['build', '--track-all', 'composed.bst']) - result.assert_success() - - # Assert that the main target is cached as a result - assert cli.get_element_state(project, 'composed.bst') == 'cached' - - -@pytest.mark.datafiles(os.path.join(DATA_DIR)) -@pytest.mark.parametrize("track_targets,exceptions,tracked", [ - # Test with no exceptions - (['0.bst'], [], ['0.bst', '2.bst', '3.bst', '4.bst', '5.bst', '6.bst', '7.bst']), - (['3.bst'], [], ['3.bst', '4.bst', '5.bst', '6.bst']), - (['2.bst', '3.bst'], [], ['2.bst', '3.bst', '4.bst', '5.bst', '6.bst', '7.bst']), - - # Test excepting '2.bst' - (['0.bst'], ['2.bst'], ['0.bst', '3.bst', '4.bst', '5.bst', '6.bst']), - (['3.bst'], ['2.bst'], []), - (['2.bst', '3.bst'], ['2.bst'], ['3.bst', '4.bst', '5.bst', '6.bst']), - - # Test excepting '2.bst' and '3.bst' - (['0.bst'], ['2.bst', '3.bst'], ['0.bst']), - (['3.bst'], ['2.bst', '3.bst'], []), - (['2.bst', '3.bst'], ['2.bst', '3.bst'], []) -]) -def test_build_track_update(cli, datafiles, tmpdir, track_targets, - exceptions, tracked): - project = str(datafiles) - dev_files_path = os.path.join(project, 'files', 'dev-files') - element_path = os.path.join(project, 'elements') - - repo = create_repo('git', str(tmpdir)) - ref = repo.create(dev_files_path) - - create_elements = { - '0.bst': [ - '2.bst', - '3.bst' - ], - '2.bst': [ - '3.bst', - '7.bst' - ], - '3.bst': [ - '4.bst', - '5.bst', - '6.bst' - ], - '4.bst': [], - '5.bst': [], - '6.bst': [ - '5.bst' - ], - '7.bst': [] - } - for element, dependencies in create_elements.items(): - # We set a ref for all elements, so that we ensure that we - # only track exactly those elements that we want to track, - # even if others can be tracked - create_element(repo, element, element_path, dependencies, ref=ref) - repo.add_commit() - - args = ['build'] - args += itertools.chain.from_iterable(zip(itertools.repeat('--track'), track_targets)) - args += itertools.chain.from_iterable(zip(itertools.repeat('--track-except'), exceptions)) - args += ['0.bst'] - - result = cli.run(project=project, silent=True, args=args) - tracked_elements = result.get_tracked_elements() - - assert set(tracked_elements) == set(tracked) - - -@pytest.mark.datafiles(os.path.join(DATA_DIR)) -@pytest.mark.parametrize("track_targets,exceptions", [ - # Test tracking the main target element, but excepting some of its - # children - (['0.bst'], ['6.bst']), - - # Test only tracking a child element - (['3.bst'], []), -]) -def test_build_track_inconsistent(cli, datafiles, tmpdir, - track_targets, exceptions): - project = str(datafiles) - dev_files_path = os.path.join(project, 'files', 'dev-files') - element_path = os.path.join(project, 'elements') - - repo = create_repo('git', str(tmpdir)) - repo.create(dev_files_path) - - create_elements = { - '0.bst': [ - '2.bst', - '3.bst' - ], - '2.bst': [ - '3.bst', - '7.bst' - ], - '3.bst': [ - '4.bst', - '5.bst', - '6.bst' - ], - '4.bst': [], - '5.bst': [], - '6.bst': [ - '5.bst' - ], - '7.bst': [] - } - for element, dependencies in create_elements.items(): - # We don't add refs so that all elements *have* to be tracked - create_element(repo, element, element_path, dependencies) - - args = ['build'] - args += itertools.chain.from_iterable(zip(itertools.repeat('--track'), track_targets)) - args += itertools.chain.from_iterable(zip(itertools.repeat('--track-except'), exceptions)) - args += ['0.bst'] - - result = cli.run(project=project, args=args, silent=True) - result.assert_main_error(ErrorDomain.PIPELINE, "inconsistent-pipeline") - - -# Assert that if a build element has a dependency in the tracking -# queue it does not start building before tracking finishes. -@pytest.mark.datafiles(os.path.join(DATA_DIR)) -@pytest.mark.parametrize("strict", ['--strict', '--no-strict']) -def test_build_track_track_first(cli, datafiles, tmpdir, strict): - project = str(datafiles) - dev_files_path = os.path.join(project, 'files', 'dev-files') - element_path = os.path.join(project, 'elements') - - repo = create_repo('git', str(tmpdir)) - ref = repo.create(dev_files_path) - - create_elements = { - '0.bst': [ - '1.bst' - ], - '1.bst': [], - '2.bst': [ - '0.bst' - ] - } - for element, dependencies in create_elements.items(): - # We set a ref so that 0.bst can already be built even if - # 1.bst has not been tracked yet. - create_element(repo, element, element_path, dependencies, ref=ref) - repo.add_commit() - - # Build 1.bst and 2.bst first so we have an artifact for them - args = [strict, 'build', '2.bst'] - result = cli.run(args=args, project=project, silent=True) - result.assert_success() - - # Test building 0.bst while tracking 1.bst - cli.remove_artifact_from_cache(project, '0.bst') - - args = [strict, 'build', '--track', '1.bst', '2.bst'] - result = cli.run(args=args, project=project, silent=True) - result.assert_success() - - # Assert that 1.bst successfully tracks before 0.bst builds - track_messages = re.finditer(r'\[track:1.bst\s*]', result.stderr) - build_0 = re.search(r'\[\s*build:0.bst\s*] START', result.stderr).start() - assert all(track_message.start() < build_0 for track_message in track_messages) - - # Assert that 2.bst is *only* rebuilt if we are in strict mode - build_2 = re.search(r'\[\s*build:2.bst\s*] START', result.stderr) - if strict == '--strict': - assert build_2 is not None - else: - assert build_2 is None diff --git a/tests/frontend/completions.py b/tests/frontend/completions.py index c88d0e0b9..5df873666 100644 --- a/tests/frontend/completions.py +++ b/tests/frontend/completions.py @@ -147,10 +147,6 @@ def test_commands(cli, cmd, word_idx, expected): # Test that options of subcommands also complete ('bst --no-colors build -', 3, ['--deps ', '-d ', - '--track ', '--track-all ', - '--track-except ', - '--track-cross-junctions ', '-J ', - '--track-save ', '--remote ', '-r ']), # Test the behavior of completing after an option that has a diff --git a/tests/frontend/pull.py b/tests/frontend/pull.py index 2dee10ac0..234f1133d 100644 --- a/tests/frontend/pull.py +++ b/tests/frontend/pull.py @@ -264,52 +264,6 @@ def test_push_pull_non_strict(cli, tmpdir, datafiles): assert cli.get_element_state(project, 'target.bst') == 'cached' -# Regression test for https://gitlab.com/BuildStream/buildstream/issues/202 -@pytest.mark.datafiles(DATA_DIR) -def test_push_pull_track_non_strict(cli, tmpdir, datafiles): - project = str(datafiles) - - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: - - # First build the target element and push to the remote. - cli.configure({ - 'artifacts': {'url': share.repo, 'push': True}, - 'projects': { - 'test': {'strict': False} - } - }) - result = cli.run(project=project, args=['build', 'target.bst']) - result.assert_success() - assert cli.get_element_state(project, 'target.bst') == 'cached' - - # Assert that everything is now cached in the remote. - all_elements = {'target.bst', 'import-bin.bst', 'import-dev.bst', 'compose-all.bst'} - for element_name in all_elements: - assert_shared(cli, share, project, element_name) - - # Now we've pushed, delete the user's local artifact cache - # directory and try to redownload it from the share - # - casdir = os.path.join(cli.directory, 'cas') - shutil.rmtree(casdir) - artifactdir = os.path.join(cli.directory, 'artifacts') - shutil.rmtree(artifactdir) - - # Assert that nothing is cached locally anymore - for element_name in all_elements: - assert cli.get_element_state(project, element_name) != 'cached' - - # Now try bst build with tracking and pulling. - # Tracking will be skipped for target.bst as it doesn't have any sources. - # With the non-strict build plan target.bst immediately enters the pull queue. - # However, pulling has to be deferred until the dependencies have been - # tracked as the strict cache key needs to be calculated before querying - # the caches. - result = cli.run(project=project, args=['build', '--track-all', '--deps', 'all', 'target.bst']) - result.assert_success() - assert set(result.get_pulled_elements()) == all_elements - - @pytest.mark.datafiles(DATA_DIR) def test_push_pull_cross_junction(cli, tmpdir, datafiles): project = str(datafiles) diff --git a/tests/frontend/workspace.py b/tests/frontend/workspace.py index b53a62da8..43cd6f381 100644 --- a/tests/frontend/workspace.py +++ b/tests/frontend/workspace.py @@ -67,7 +67,7 @@ class WorkspaceCreator(): self.workspace_cmd = os.path.join(self.project_path, 'workspace_cmd') - def create_workspace_element(self, kind, track, suffix='', workspace_dir=None, + def create_workspace_element(self, kind, suffix='', workspace_dir=None, element_attrs=None): element_name = 'workspace-test-{}{}.bst'.format(kind, suffix) element_path = os.path.join(self.project_path, 'elements') @@ -80,8 +80,6 @@ class WorkspaceCreator(): # the bin files, and then collect the initial ref. repo = create_repo(kind, str(self.tmpdir)) ref = repo.create(self.bin_files_path) - if track: - ref = None # Write out our test target element = { @@ -96,7 +94,7 @@ class WorkspaceCreator(): os.path.join(element_path, element_name)) return element_name, element_path, workspace_dir - def create_workspace_elements(self, kinds, track, suffixs=None, workspace_dir_usr=None, + def create_workspace_elements(self, kinds, suffixs=None, workspace_dir_usr=None, element_attrs=None): element_tuples = [] @@ -109,33 +107,28 @@ class WorkspaceCreator(): for suffix, kind in zip(suffixs, kinds): element_name, _, workspace_dir = \ - self.create_workspace_element(kind, track, suffix, workspace_dir_usr, + self.create_workspace_element(kind, suffix, workspace_dir_usr, element_attrs) element_tuples.append((element_name, workspace_dir)) - # Assert that there is no reference, a track & fetch is needed + # Assert that there is a fetch is needed states = self.cli.get_element_states(self.project_path, [ e for e, _ in element_tuples ]) - if track: - assert not any(states[e] != 'no reference' for e, _ in element_tuples) - else: - assert not any(states[e] != 'fetch needed' for e, _ in element_tuples) + assert not any(states[e] != 'fetch needed' for e, _ in element_tuples) return element_tuples - def open_workspaces(self, kinds, track, suffixs=None, workspace_dir=None, + def open_workspaces(self, kinds, suffixs=None, workspace_dir=None, element_attrs=None, no_checkout=False): - element_tuples = self.create_workspace_elements(kinds, track, suffixs, workspace_dir, + element_tuples = self.create_workspace_elements(kinds, suffixs, workspace_dir, element_attrs) os.makedirs(self.workspace_cmd, exist_ok=True) # Now open the workspace, this should have the effect of automatically # tracking & fetching the source from the repo. args = ['workspace', 'open'] - if track: - args.append('--track') if no_checkout: args.append('--no-checkout') if workspace_dir is not None: @@ -163,10 +156,10 @@ class WorkspaceCreator(): return element_tuples -def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', workspace_dir=None, +def open_workspace(cli, tmpdir, datafiles, kind, suffix='', workspace_dir=None, project_path=None, element_attrs=None, no_checkout=False): workspace_object = WorkspaceCreator(cli, tmpdir, datafiles, project_path) - workspaces = workspace_object.open_workspaces((kind, ), track, (suffix, ), workspace_dir, + workspaces = workspace_object.open_workspaces((kind, ), (suffix, ), workspace_dir, element_attrs, no_checkout) assert len(workspaces) == 1 element_name, workspace = workspaces[0] @@ -175,7 +168,7 @@ def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', workspace_dir @pytest.mark.datafiles(DATA_DIR) def test_open_bzr_customize(cli, tmpdir, datafiles): - element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "bzr", False) + element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "bzr") # Check that the .bzr dir exists bzrdir = os.path.join(workspace, ".bzr") @@ -195,7 +188,7 @@ def test_open_bzr_customize(cli, tmpdir, datafiles): def test_open_multi(cli, tmpdir, datafiles): workspace_object = WorkspaceCreator(cli, tmpdir, datafiles) - workspaces = workspace_object.open_workspaces(repo_kinds, False) + workspaces = workspace_object.open_workspaces(repo_kinds) for (elname, workspace), kind in zip(workspaces, repo_kinds): assert kind in elname @@ -214,7 +207,7 @@ def test_open_multi(cli, tmpdir, datafiles): def test_open_multi_unwritable(cli, tmpdir, datafiles): workspace_object = WorkspaceCreator(cli, tmpdir, datafiles) - element_tuples = workspace_object.create_workspace_elements(repo_kinds, False, repo_kinds) + element_tuples = workspace_object.create_workspace_elements(repo_kinds, repo_kinds) os.makedirs(workspace_object.workspace_cmd, exist_ok=True) # Now open the workspace, this should have the effect of automatically @@ -241,7 +234,7 @@ def test_open_multi_unwritable(cli, tmpdir, datafiles): def test_open_multi_with_directory(cli, tmpdir, datafiles): workspace_object = WorkspaceCreator(cli, tmpdir, datafiles) - element_tuples = workspace_object.create_workspace_elements(repo_kinds, False, repo_kinds) + element_tuples = workspace_object.create_workspace_elements(repo_kinds, repo_kinds) os.makedirs(workspace_object.workspace_cmd, exist_ok=True) # Now open the workspace, this should have the effect of automatically @@ -261,7 +254,7 @@ def test_open_defaultlocation(cli, tmpdir, datafiles): workspace_object = WorkspaceCreator(cli, tmpdir, datafiles) # pylint: disable=unbalanced-tuple-unpacking - ((element_name, workspace_dir), ) = workspace_object.create_workspace_elements(['git'], False, ['git']) + ((element_name, workspace_dir), ) = workspace_object.create_workspace_elements(['git'], ['git']) os.makedirs(workspace_object.workspace_cmd, exist_ok=True) # Now open the workspace, this should have the effect of automatically @@ -294,7 +287,7 @@ def test_open_defaultlocation_exists(cli, tmpdir, datafiles): workspace_object = WorkspaceCreator(cli, tmpdir, datafiles) # pylint: disable=unbalanced-tuple-unpacking - ((element_name, workspace_dir), ) = workspace_object.create_workspace_elements(['git'], False, ['git']) + ((element_name, workspace_dir), ) = workspace_object.create_workspace_elements(['git'], ['git']) os.makedirs(workspace_object.workspace_cmd, exist_ok=True) with open(workspace_dir, 'w') as fl: @@ -318,12 +311,12 @@ def test_open_defaultlocation_exists(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_open_track(cli, tmpdir, datafiles): - open_workspace(cli, tmpdir, datafiles, 'git', True) + open_workspace(cli, tmpdir, datafiles, 'git') @pytest.mark.datafiles(DATA_DIR) def test_open_force(cli, tmpdir, datafiles): - element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git', False) + element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git') # Close the workspace result = cli.run(project=project, args=[ @@ -343,7 +336,7 @@ def test_open_force(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_open_force_open(cli, tmpdir, datafiles): - element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git', False) + element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git') # Assert the workspace dir exists assert os.path.exists(workspace) @@ -358,7 +351,7 @@ def test_open_force_open(cli, tmpdir, datafiles): # Regression test for #1086. @pytest.mark.datafiles(DATA_DIR) def test_open_force_open_no_checkout(cli, tmpdir, datafiles): - element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git', False) + element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git') hello_path = os.path.join(workspace, 'hello.txt') # Assert the workspace dir exists @@ -382,7 +375,7 @@ def test_open_force_open_no_checkout(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_open_force_different_workspace(cli, tmpdir, datafiles): - _, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git', False, "-alpha") + _, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git', "-alpha") # Assert the workspace dir exists assert os.path.exists(workspace) @@ -392,7 +385,7 @@ def test_open_force_different_workspace(cli, tmpdir, datafiles): tmpdir = os.path.join(str(tmpdir), "-beta") shutil.move(hello_path, hello1_path) - element_name2, _, workspace2 = open_workspace(cli, tmpdir, datafiles, 'git', False, "-beta") + element_name2, _, workspace2 = open_workspace(cli, tmpdir, datafiles, 'git', "-beta") # Assert the workspace dir exists assert os.path.exists(workspace2) @@ -418,7 +411,7 @@ def test_open_force_different_workspace(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_close(cli, tmpdir, datafiles): - element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git', False) + element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git') # Close the workspace result = cli.run(project=project, args=[ @@ -434,7 +427,7 @@ def test_close(cli, tmpdir, datafiles): def test_close_external_after_move_project(cli, tmpdir, datafiles): workspace_dir = os.path.join(str(tmpdir), "workspace") project_path = os.path.join(str(tmpdir), 'initial_project') - element_name, _, _ = open_workspace(cli, tmpdir, datafiles, 'git', False, "", workspace_dir, project_path) + element_name, _, _ = open_workspace(cli, tmpdir, datafiles, 'git', "", workspace_dir, project_path) assert os.path.exists(workspace_dir) moved_dir = os.path.join(str(tmpdir), 'external_project') shutil.move(project_path, moved_dir) @@ -454,7 +447,7 @@ def test_close_external_after_move_project(cli, tmpdir, datafiles): def test_close_internal_after_move_project(cli, tmpdir, datafiles): initial_dir = os.path.join(str(tmpdir), 'initial_project') initial_workspace = os.path.join(initial_dir, 'workspace') - element_name, _, _ = open_workspace(cli, tmpdir, datafiles, 'git', False, + element_name, _, _ = open_workspace(cli, tmpdir, datafiles, 'git', workspace_dir=initial_workspace, project_path=initial_dir) moved_dir = os.path.join(str(tmpdir), 'internal_project') shutil.move(initial_dir, moved_dir) @@ -473,7 +466,7 @@ def test_close_internal_after_move_project(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_close_removed(cli, tmpdir, datafiles): - element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git', False) + element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git') # Remove it first, closing the workspace should work shutil.rmtree(workspace) @@ -490,7 +483,7 @@ def test_close_removed(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_close_nonexistant_element(cli, tmpdir, datafiles): - element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git', False) + element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git') element_path = os.path.join(datafiles.dirname, datafiles.basename, 'elements', element_name) # First brutally remove the element.bst file, ensuring that @@ -513,9 +506,9 @@ def test_close_multiple(cli, tmpdir, datafiles): tmpdir_alpha = os.path.join(str(tmpdir), 'alpha') tmpdir_beta = os.path.join(str(tmpdir), 'beta') alpha, project, workspace_alpha = open_workspace( - cli, tmpdir_alpha, datafiles, 'git', False, suffix='-alpha') + cli, tmpdir_alpha, datafiles, 'git', suffix='-alpha') beta, project, workspace_beta = open_workspace( - cli, tmpdir_beta, datafiles, 'git', False, suffix='-beta') + cli, tmpdir_beta, datafiles, 'git', suffix='-beta') # Close the workspaces result = cli.run(project=project, args=[ @@ -533,9 +526,9 @@ def test_close_all(cli, tmpdir, datafiles): tmpdir_alpha = os.path.join(str(tmpdir), 'alpha') tmpdir_beta = os.path.join(str(tmpdir), 'beta') _, project, workspace_alpha = open_workspace( - cli, tmpdir_alpha, datafiles, 'git', False, suffix='-alpha') + cli, tmpdir_alpha, datafiles, 'git', suffix='-alpha') _, project, workspace_beta = open_workspace( - cli, tmpdir_beta, datafiles, 'git', False, suffix='-beta') + cli, tmpdir_beta, datafiles, 'git', suffix='-beta') # Close the workspaces result = cli.run(project=project, args=[ @@ -551,7 +544,7 @@ def test_close_all(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_reset(cli, tmpdir, datafiles): # Open the workspace - element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git', False) + element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git') # Modify workspace shutil.rmtree(os.path.join(workspace, 'usr', 'bin')) @@ -572,7 +565,7 @@ def test_reset(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_reset_soft(cli, tmpdir, datafiles): # Open the workspace - element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git', False) + element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git') assert cli.get_element_state(project, element_name) == 'buildable' @@ -627,9 +620,9 @@ def test_reset_multiple(cli, tmpdir, datafiles): tmpdir_alpha = os.path.join(str(tmpdir), 'alpha') tmpdir_beta = os.path.join(str(tmpdir), 'beta') alpha, project, workspace_alpha = open_workspace( - cli, tmpdir_alpha, datafiles, 'git', False, suffix='-alpha') + cli, tmpdir_alpha, datafiles, 'git', suffix='-alpha') beta, project, workspace_beta = open_workspace( - cli, tmpdir_beta, datafiles, 'git', False, suffix='-beta') + cli, tmpdir_beta, datafiles, 'git', suffix='-beta') # Modify workspaces shutil.rmtree(os.path.join(workspace_alpha, 'usr', 'bin')) @@ -653,9 +646,9 @@ def test_reset_all(cli, tmpdir, datafiles): tmpdir_alpha = os.path.join(str(tmpdir), 'alpha') tmpdir_beta = os.path.join(str(tmpdir), 'beta') _, project, workspace_alpha = open_workspace( - cli, tmpdir_alpha, datafiles, 'git', False, suffix='-alpha') + cli, tmpdir_alpha, datafiles, 'git', suffix='-alpha') _, project, workspace_beta = open_workspace( - cli, tmpdir_beta, datafiles, 'git', False, suffix='-beta') + cli, tmpdir_beta, datafiles, 'git', suffix='-beta') # Modify workspaces shutil.rmtree(os.path.join(workspace_alpha, 'usr', 'bin')) @@ -675,7 +668,7 @@ def test_reset_all(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_list(cli, tmpdir, datafiles): - element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git', False) + element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git') # Now list the workspaces result = cli.run(project=project, args=[ @@ -785,7 +778,7 @@ def test_buildable_no_ref(cli, tmpdir, datafiles): @pytest.mark.parametrize("modification", [("addfile"), ("removefile"), ("modifyfile")]) @pytest.mark.parametrize("strict", [("strict"), ("non-strict")]) def test_detect_modifications(cli, tmpdir, datafiles, modification, strict): - element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git', False) + element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git') checkout = os.path.join(str(tmpdir), 'checkout') # Configure strict mode @@ -1055,7 +1048,7 @@ def test_list_supported_workspace(cli, tmpdir, datafiles, workspace_cfg, expecte @pytest.mark.datafiles(DATA_DIR) def test_inconsitent_pipeline_message(cli, tmpdir, datafiles): - element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git', False) + element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git') shutil.rmtree(workspace) @@ -1070,7 +1063,7 @@ def test_inconsitent_pipeline_message(cli, tmpdir, datafiles): def test_cache_key_workspace_in_dependencies(cli, tmpdir, datafiles, strict): checkout = os.path.join(str(tmpdir), 'checkout') element_name, project, workspace = open_workspace(cli, os.path.join(str(tmpdir), 'repo-a'), - datafiles, 'git', False) + datafiles, 'git') element_path = os.path.join(project, 'elements') back_dep_element_name = 'workspace-test-back-dep.bst' @@ -1153,7 +1146,7 @@ def test_multiple_failed_builds(cli, tmpdir, datafiles): } } element_name, project, _ = open_workspace(cli, tmpdir, datafiles, - "git", False, element_attrs=element_config) + "git", element_attrs=element_config) for _ in range(2): result = cli.run(project=project, args=["build", element_name]) @@ -1174,7 +1167,7 @@ def test_external_fetch(cli, datafiles, tmpdir_factory, subdir, guess_element): create_element_size(depend_element, str(datafiles), 'elements', [], 1024) element_name, project, workspace = open_workspace( - cli, tmpdir, datafiles, "git", False, no_checkout=True, + cli, tmpdir, datafiles, "git", no_checkout=True, element_attrs={'depends': [depend_element]} ) arg_elm = [element_name] if not guess_element else [] @@ -1201,7 +1194,7 @@ def test_external_fetch(cli, datafiles, tmpdir_factory, subdir, guess_element): def test_external_push_pull(cli, datafiles, tmpdir_factory, guess_element): # Pushing and pulling to/from an artifact cache works from an external workspace tmpdir = tmpdir_factory.mktemp('') - element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False) + element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git") arg_elm = [element_name] if not guess_element else [] with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: @@ -1226,7 +1219,7 @@ def test_external_push_pull(cli, datafiles, tmpdir_factory, guess_element): @pytest.mark.parametrize("guess_element", [True, False], ids=["guess", "no-guess"]) def test_external_track(cli, datafiles, tmpdir_factory, guess_element): tmpdir = tmpdir_factory.mktemp('') - element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False) + element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git") element_file = os.path.join(str(datafiles), 'elements', element_name) arg_elm = [element_name] if not guess_element else [] @@ -1264,8 +1257,8 @@ def test_external_open_other(cli, datafiles, tmpdir_factory): tmpdir1 = tmpdir_factory.mktemp('') tmpdir2 = tmpdir_factory.mktemp('') # Making use of the assumption that it's the same project in both invocations of open_workspace - _, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha") - beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta") + _, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", suffix="-alpha") + beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", suffix="-beta") # Closing the other element first, because I'm too lazy to create an # element without opening it @@ -1284,8 +1277,8 @@ def test_external_close_other(cli, datafiles, tmpdir_factory): tmpdir1 = tmpdir_factory.mktemp('') tmpdir2 = tmpdir_factory.mktemp('') # Making use of the assumption that it's the same project in both invocations of open_workspace - _, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha") - beta_element, _, _ = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta") + _, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", suffix="-alpha") + beta_element, _, _ = open_workspace(cli, tmpdir2, datafiles, "git", suffix="-beta") result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'close', beta_element]) result.assert_success() @@ -1299,8 +1292,8 @@ def test_external_close_self(cli, datafiles, tmpdir_factory, guess_element): tmpdir1 = tmpdir_factory.mktemp('') tmpdir2 = tmpdir_factory.mktemp('') # Making use of the assumption that it's the same project in both invocations of open_workspace - alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha") - _, _, _ = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta") + alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", suffix="-alpha") + _, _, _ = open_workspace(cli, tmpdir2, datafiles, "git", suffix="-beta") arg_elm = [alpha_element] if not guess_element else [] result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'close', *arg_elm]) @@ -1313,8 +1306,8 @@ def test_external_reset_other(cli, datafiles, tmpdir_factory): tmpdir1 = tmpdir_factory.mktemp('') tmpdir2 = tmpdir_factory.mktemp('') # Making use of the assumption that it's the same project in both invocations of open_workspace - _, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha") - beta_element, _, _ = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta") + _, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", suffix="-alpha") + beta_element, _, _ = open_workspace(cli, tmpdir2, datafiles, "git", suffix="-beta") result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'reset', beta_element]) result.assert_success() @@ -1323,7 +1316,7 @@ def test_external_reset_other(cli, datafiles, tmpdir_factory): @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("guess_element", [True, False], ids=["guess", "no-guess"]) def test_external_reset_self(cli, datafiles, tmpdir, guess_element): - element, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False) + element, project, workspace = open_workspace(cli, tmpdir, datafiles, "git") arg_elm = [element] if not guess_element else [] # Command succeeds @@ -1339,7 +1332,7 @@ def test_external_reset_self(cli, datafiles, tmpdir, guess_element): def test_external_list(cli, datafiles, tmpdir_factory): tmpdir = tmpdir_factory.mktemp('') # Making use of the assumption that it's the same project in both invocations of open_workspace - _, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False) + _, project, workspace = open_workspace(cli, tmpdir, datafiles, "git") result = cli.run(project=project, args=['-C', workspace, 'workspace', 'list']) result.assert_success() diff --git a/tests/remoteexecution/junction.py b/tests/remoteexecution/junction.py index 87d00a40d..db087fd90 100644 --- a/tests/remoteexecution/junction.py +++ b/tests/remoteexecution/junction.py @@ -107,9 +107,12 @@ def test_junction_build_remote(cli, tmpdir, datafiles): result = cli.run(project=project, args=['source', 'track', 'junction.bst']) result.assert_success() - # Build it with --track-all - result = cli.run(project=project, silent=True, args=[ - 'build', '--track-all', 'composed.bst']) + # track target to ensure we have refs + result = cli.run(project=project, args=['source', 'track', '--deps', 'all', 'composed.bst']) + result.assert_success() + + # build + result = cli.run(project=project, silent=True, args=['build', 'composed.bst']) result.assert_success() # Assert that the main target is cached as a result diff --git a/tests/sources/keytest.py b/tests/sources/keytest.py index 5b5ccaba0..d3eab8d6b 100644 --- a/tests/sources/keytest.py +++ b/tests/sources/keytest.py @@ -46,6 +46,6 @@ def test_generate_key(cli, datafiles): res.assert_success() assert cli.get_element_state(project_dir, "key-test.bst") == "fetch needed" - res = cli.run(project=project_dir, args=["build", "--track", "key-test.bst"]) + res = cli.run(project=project_dir, args=["build", "key-test.bst"]) res.assert_success() assert cli.get_element_state(project_dir, "key-test.bst") == "cached" |