From 1b24148fdadcafaf4b316cc45819a06d8ec42647 Mon Sep 17 00:00:00 2001 From: Chandan Singh Date: Wed, 26 Sep 2018 00:30:17 +0100 Subject: Ensure `--deps=none` option works for `bst checkout` Currently, `bst checkout --deps none` command always produces empty output. Fix this issue and add regression test for the same. - element_enums.py: Add Scope.NONE. - element.py: Ensure Scope.NONE works correctly in addition to Scope.RUN/Scope.ALL in Element.dependencies() and Element.search(). - tests/frontend/buildcheckout.py: Fix tests for `--deps none`. Fixes #670. --- buildstream/element.py | 26 +++++++++++++++----------- buildstream/types.py | 6 ++++++ tests/frontend/buildcheckout.py | 20 ++++++++------------ 3 files changed, 29 insertions(+), 23 deletions(-) diff --git a/buildstream/element.py b/buildstream/element.py index d7072dc8c..d94f2e8ad 100644 --- a/buildstream/element.py +++ b/buildstream/element.py @@ -438,7 +438,7 @@ class Element(Plugin): visited=visited, recursed=True) # Yeild self only at the end, after anything needed has been traversed - if should_yield and (recurse or recursed) and (scope in (Scope.ALL, Scope.RUN)): + if should_yield and (recurse or recursed) and scope != Scope.BUILD: yield self def search(self, scope, name): @@ -1353,17 +1353,21 @@ class Element(Plugin): if scope == Scope.BUILD: self.stage(sandbox) elif scope == Scope.RUN: - # Stage deps in the sandbox root if deps == 'run': - with self.timed_activity("Staging dependencies", silent_nested=True): - self.stage_dependency_artifacts(sandbox, scope) - - # Run any integration commands provided by the dependencies - # once they are all staged and ready - if integrate: - with self.timed_activity("Integrating sandbox"): - for dep in self.dependencies(scope): - dep.integrate(sandbox) + dependency_scope = Scope.RUN + else: + dependency_scope = Scope.NONE + + # Stage deps in the sandbox root + with self.timed_activity("Staging dependencies", silent_nested=True): + self.stage_dependency_artifacts(sandbox, dependency_scope) + + # Run any integration commands provided by the dependencies + # once they are all staged and ready + if integrate: + with self.timed_activity("Integrating sandbox"): + for dep in self.dependencies(dependency_scope): + dep.integrate(sandbox) yield sandbox diff --git a/buildstream/types.py b/buildstream/types.py index 7bc7a1664..3eb4a9110 100644 --- a/buildstream/types.py +++ b/buildstream/types.py @@ -48,6 +48,12 @@ class Scope(Enum): itself. """ + NONE = 4 + """Just the element itself, no dependencies. + + *Since: 1.4* + """ + class Consistency(): """Defines the various consistency states of a :class:`.Source`. diff --git a/tests/frontend/buildcheckout.py b/tests/frontend/buildcheckout.py index 1299fa190..d43bac863 100644 --- a/tests/frontend/buildcheckout.py +++ b/tests/frontend/buildcheckout.py @@ -90,9 +90,10 @@ def test_build_invalid_suffix_dep(datafiles, cli, strict, hardlinks): def test_build_checkout_deps(datafiles, cli, deps): project = os.path.join(datafiles.dirname, datafiles.basename) checkout = os.path.join(cli.directory, 'checkout') + element_name = "checkout-deps.bst" # First build it - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=['build', element_name]) result.assert_success() # Assert that after a successful build, the builddir is empty @@ -101,20 +102,15 @@ def test_build_checkout_deps(datafiles, cli, deps): assert not os.listdir(builddir) # Now check it out - result = cli.run(project=project, args=['checkout', 'target.bst', '--deps', deps, checkout]) + result = cli.run(project=project, args=['checkout', element_name, '--deps', deps, checkout]) result.assert_success() - # Check that the executable hello file is found in the checkout - filename = os.path.join(checkout, 'usr', 'bin', 'hello') - - if deps == "run": - assert os.path.exists(filename) - else: - assert not os.path.exists(filename) - - # Check that the executable hello file is found in the checkout - filename = os.path.join(checkout, 'usr', 'include', 'pony.h') + # Verify output of this element + filename = os.path.join(checkout, 'etc', 'buildstream', 'config') + assert os.path.exists(filename) + # Verify output of this element's runtime dependencies + filename = os.path.join(checkout, 'usr', 'bin', 'hello') if deps == "run": assert os.path.exists(filename) else: -- cgit v1.2.1 From 8ac1e7576f43f691b2f011d2655efe3ddb6984a3 Mon Sep 17 00:00:00 2001 From: Chandan Singh Date: Fri, 21 Sep 2018 23:20:56 +0100 Subject: Add `--deps build` option to `bst checkout` As discussed in https://mail.gnome.org/archives/buildstream-list/2018-September/msg00064.html, add `--deps build` option to `bst checkout`. This will allow users to checkout the all build dependencies of a given element using a single command. - _frontend/cli.py: Add `--deps build` option for `bst checkout`. - element.py: Support `deps='build'` in Element._prepare_sandbox(). - tests/frontend/buildcheckout.py: Ensure `--deps build` works as expected. --- buildstream/_frontend/cli.py | 2 +- buildstream/element.py | 5 ++++- tests/frontend/buildcheckout.py | 14 ++++++++++++-- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py index b845d00eb..59c917323 100644 --- a/buildstream/_frontend/cli.py +++ b/buildstream/_frontend/cli.py @@ -640,7 +640,7 @@ def shell(app, element, sysroot, mount, isolate, build_, command): @click.option('--force', '-f', default=False, is_flag=True, help="Allow files to be overwritten") @click.option('--deps', '-d', default='run', - type=click.Choice(['run', 'none']), + type=click.Choice(['run', 'build', 'none']), help='The dependencies to checkout (default: run)') @click.option('--integrate/--no-integrate', default=True, is_flag=True, help="Whether to run integration commands") diff --git a/buildstream/element.py b/buildstream/element.py index d94f2e8ad..2caac8d3c 100644 --- a/buildstream/element.py +++ b/buildstream/element.py @@ -1353,7 +1353,10 @@ class Element(Plugin): if scope == Scope.BUILD: self.stage(sandbox) elif scope == Scope.RUN: - if deps == 'run': + + if deps == 'build': + dependency_scope = Scope.BUILD + elif deps == 'run': dependency_scope = Scope.RUN else: dependency_scope = Scope.NONE diff --git a/tests/frontend/buildcheckout.py b/tests/frontend/buildcheckout.py index d43bac863..03bba0b26 100644 --- a/tests/frontend/buildcheckout.py +++ b/tests/frontend/buildcheckout.py @@ -86,7 +86,7 @@ def test_build_invalid_suffix_dep(datafiles, cli, strict, hardlinks): @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("deps", [("run"), ("none")]) +@pytest.mark.parametrize("deps", [("run"), ("none"), ("build")]) def test_build_checkout_deps(datafiles, cli, deps): project = os.path.join(datafiles.dirname, datafiles.basename) checkout = os.path.join(cli.directory, 'checkout') @@ -107,7 +107,17 @@ def test_build_checkout_deps(datafiles, cli, deps): # Verify output of this element filename = os.path.join(checkout, 'etc', 'buildstream', 'config') - assert os.path.exists(filename) + if deps == "build": + assert not os.path.exists(filename) + else: + assert os.path.exists(filename) + + # Verify output of this element's build dependencies + filename = os.path.join(checkout, 'usr', 'include', 'pony.h') + if deps == "build": + assert os.path.exists(filename) + else: + assert not os.path.exists(filename) # Verify output of this element's runtime dependencies filename = os.path.join(checkout, 'usr', 'bin', 'hello') -- cgit v1.2.1 From ac60f47a4ce439db785d11b4b31699d1034ca3e7 Mon Sep 17 00:00:00 2001 From: Benjamin Schubert Date: Mon, 12 Nov 2018 09:37:17 +0000 Subject: Refactor and simplify _prepare_sandbox for elements Before we would have a intricate logics with multiple arguments that might get ignored. This simplifies the design and introduces a bool `shell` instead of having two different variables concerned about scope --- buildstream/_frontend/cli.py | 10 +++++++++- buildstream/_stream.py | 6 +++--- buildstream/element.py | 20 ++++++-------------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py index 59c917323..ef0dadb13 100644 --- a/buildstream/_frontend/cli.py +++ b/buildstream/_frontend/cli.py @@ -657,16 +657,24 @@ def shell(app, element, sysroot, mount, isolate, build_, command): def checkout(app, element, location, force, deps, integrate, hardlinks, tar): """Checkout a built artifact to the specified location """ + from ..element import Scope if hardlinks and tar: click.echo("ERROR: options --hardlinks and --tar conflict", err=True) sys.exit(-1) + if deps == "run": + scope = Scope.RUN + elif deps == "build": + scope = Scope.BUILD + elif deps == "none": + scope = Scope.NONE + with app.initialized(): app.stream.checkout(element, location=location, force=force, - deps=deps, + scope=scope, integrate=integrate, hardlinks=hardlinks, tar=tar) diff --git a/buildstream/_stream.py b/buildstream/_stream.py index 2f9799178..6a3d2c7fc 100644 --- a/buildstream/_stream.py +++ b/buildstream/_stream.py @@ -370,7 +370,7 @@ class Stream(): # target (str): Target to checkout # location (str): Location to checkout the artifact to # force (bool): Whether files can be overwritten if necessary - # deps (str): The dependencies to checkout + # scope (str): The scope of dependencies to checkout # integrate (bool): Whether to run integration commands # hardlinks (bool): Whether checking out files hardlinked to # their artifacts is acceptable @@ -383,7 +383,7 @@ class Stream(): def checkout(self, target, *, location=None, force=False, - deps='run', + scope=Scope.RUN, integrate=True, hardlinks=False, tar=False): @@ -396,7 +396,7 @@ class Stream(): # Stage deps into a temporary sandbox first try: - with target._prepare_sandbox(Scope.RUN, None, deps=deps, + with target._prepare_sandbox(scope=scope, directory=None, integrate=integrate) as sandbox: # Copy or move the sandbox to the target directory diff --git a/buildstream/element.py b/buildstream/element.py index 2caac8d3c..2f724f49d 100644 --- a/buildstream/element.py +++ b/buildstream/element.py @@ -1339,7 +1339,7 @@ class Element(Plugin): # is used to stage things by the `bst checkout` codepath # @contextmanager - def _prepare_sandbox(self, scope, directory, deps='run', integrate=True): + def _prepare_sandbox(self, scope, directory, shell=False, integrate=True): # bst shell and bst checkout require a local sandbox. bare_directory = True if directory else False with self.__sandbox(directory, config=self.__sandbox_config, allow_remote=False, @@ -1350,26 +1350,18 @@ class Element(Plugin): # Stage something if we need it if not directory: - if scope == Scope.BUILD: + if shell and scope == Scope.BUILD: self.stage(sandbox) - elif scope == Scope.RUN: - - if deps == 'build': - dependency_scope = Scope.BUILD - elif deps == 'run': - dependency_scope = Scope.RUN - else: - dependency_scope = Scope.NONE - + else: # Stage deps in the sandbox root with self.timed_activity("Staging dependencies", silent_nested=True): - self.stage_dependency_artifacts(sandbox, dependency_scope) + self.stage_dependency_artifacts(sandbox, scope) # Run any integration commands provided by the dependencies # once they are all staged and ready if integrate: with self.timed_activity("Integrating sandbox"): - for dep in self.dependencies(dependency_scope): + for dep in self.dependencies(scope): dep.integrate(sandbox) yield sandbox @@ -1865,7 +1857,7 @@ class Element(Plugin): # If directory is not specified, one will be staged using scope def _shell(self, scope=None, directory=None, *, mounts=None, isolate=False, prompt=None, command=None): - with self._prepare_sandbox(scope, directory) as sandbox: + with self._prepare_sandbox(scope, directory, shell=True) as sandbox: environment = self.get_environment() environment = copy.copy(environment) flags = SandboxFlags.INTERACTIVE | SandboxFlags.ROOT_READ_ONLY -- cgit v1.2.1