From 4708ee6bf9a7c6046157c79d227778b7b0b167ae Mon Sep 17 00:00:00 2001 From: Antoine Wacheux Date: Wed, 23 May 2018 10:04:07 +0100 Subject: _artifactcache/ostreecache.py: Fix artifact cache initialization result tuple In case of failure, the tuple contained 4 elements instead of 3, causing BuildStream to crash. --- buildstream/_artifactcache/ostreecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'buildstream') diff --git a/buildstream/_artifactcache/ostreecache.py b/buildstream/_artifactcache/ostreecache.py index c802fc2e2..9fc12f8d0 100644 --- a/buildstream/_artifactcache/ostreecache.py +++ b/buildstream/_artifactcache/ostreecache.py @@ -227,7 +227,7 @@ class OSTreeCache(ArtifactCache): except Exception as e: # pylint: disable=broad-except # Whatever happens, we need to return it to the calling process # - q.put((str(e), None, None, None)) + q.put((str(e), None, None)) # Kick off all the initialization jobs one by one. # -- cgit v1.2.1 From 065f5ac71f3f9b8b04a4bea66a2c0f6db781d29c Mon Sep 17 00:00:00 2001 From: Tristan Maat Date: Thu, 31 May 2018 17:43:55 +0100 Subject: _context.py: Normalize user-defined paths --- buildstream/_context.py | 1 + 1 file changed, 1 insertion(+) (limited to 'buildstream') diff --git a/buildstream/_context.py b/buildstream/_context.py index bf7f49515..c0d49b245 100644 --- a/buildstream/_context.py +++ b/buildstream/_context.py @@ -161,6 +161,7 @@ class Context(): path = _yaml.node_get(defaults, str, directory) path = os.path.expanduser(path) path = os.path.expandvars(path) + path = os.path.normpath(path) setattr(self, directory, path) # Load artifact share configuration -- cgit v1.2.1 From 75fe8037343223d3bb674f3dc16db70327e74609 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6k=C3=A7en=20Nurlu?= Date: Fri, 1 Jun 2018 11:32:20 +0100 Subject: _sandboxbwrap.py: Fix post-bwrap cleanup behaviour The cleanup was supposed not to remove folders (`/dev`, `/tmp`, `/proc`) if they already existed before bwrap but it did the opposite: it tried to remove them if they existed before, and didn't remove them if they were created during bwrap. This was caused by a `not` clause, and this removes it. Fixes #379 --- buildstream/sandbox/_sandboxbwrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'buildstream') diff --git a/buildstream/sandbox/_sandboxbwrap.py b/buildstream/sandbox/_sandboxbwrap.py index d18cb9ec0..798bcb690 100644 --- a/buildstream/sandbox/_sandboxbwrap.py +++ b/buildstream/sandbox/_sandboxbwrap.py @@ -207,7 +207,7 @@ class SandboxBwrap(Sandbox): # Skip removal of directories which already existed before # launching bwrap - if not existing_basedirs[basedir]: + if existing_basedirs[basedir]: continue base_directory = os.path.join(root_mount_source, basedir) -- cgit v1.2.1 From 8d88b52a18b1adfdc04ebbd15027ac31c4d3408e Mon Sep 17 00:00:00 2001 From: Phillip Smyth Date: Thu, 24 May 2018 16:28:15 +0100 Subject: utils.py: Correcting a typo in safe_remove's comment --- buildstream/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'buildstream') diff --git a/buildstream/utils.py b/buildstream/utils.py index 8e7219631..70759dc12 100644 --- a/buildstream/utils.py +++ b/buildstream/utils.py @@ -307,7 +307,7 @@ def safe_remove(path): """Removes a file or directory This will remove a file if it exists, and will - remove a directory if the directory is not empty. + remove a directory if the directory is empty. Args: path (str): The path to remove -- cgit v1.2.1 From 1d694b28297e86b53dff07e49abd135f5c7709ed Mon Sep 17 00:00:00 2001 From: Chandan Singh Date: Tue, 29 May 2018 12:12:06 +0100 Subject: _artifactcache/pushreceive.py: Add Click type for CLI argument 'repo' The CLI for `bst-artifact-receive` expects a `repo` argument, which is supposed to be a directory, but Click currently expects it to be just any string. This results in stack traces like the one below when the argument provided is not a directory: $ ~/.local/bin/bst-artifact-receive --pull-url http://foo foobaz Traceback (most recent call last): File "/root/.local/bin/bst-artifact-receive", line 8, in sys.exit(receive_main()) ... File "/src/buildstream/buildstream/_artifactcache/pushreceive.py", line 581, in __init__ self.repo.open(None) GLib.Error: g-io-error-quark: /src/buildstream/43fref: opendir(/src/buildstream/foobaz): No such file or directory (1) Add types for this argument such that it throws better error messages when it receives bad arguments. With the Click types added, it will instead fail with messages like these: $ ~/.local/bin/bst-artifact-receive --pull-url http://foo foobaz Usage: bst-artifact-receive [OPTIONS] REPO Error: Invalid value for "repo": Directory "foobaz" does not exist. $ ~/.local/bin/bst-artifact-receive --pull-url http://foo setup.py Usage: bst-artifact-receive [OPTIONS] REPO Error: Invalid value for "repo": Directory "setup.py" is a file. Fixes #409. --- buildstream/_artifactcache/pushreceive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'buildstream') diff --git a/buildstream/_artifactcache/pushreceive.py b/buildstream/_artifactcache/pushreceive.py index 777065e18..ecedc6657 100644 --- a/buildstream/_artifactcache/pushreceive.py +++ b/buildstream/_artifactcache/pushreceive.py @@ -796,7 +796,7 @@ def push(repo, remote, branches, output): @click.option('--debug', '-d', is_flag=True, default=False, help="Debug mode") @click.option('--pull-url', type=str, required=True, help="Clients who try to pull over SSH will be redirected here") -@click.argument('repo') +@click.argument('repo', type=click.Path(file_okay=False, dir_okay=True, writable=True, exists=True)) def receive_main(verbose, debug, pull_url, repo): """A BuildStream sister program for receiving artifacts send to a shared artifact cache """ -- cgit v1.2.1 From 7f9216b3da17418e6b918a872896d98d490ae86a Mon Sep 17 00:00:00 2001 From: Tristan Van Berkom Date: Wed, 6 Jun 2018 13:20:24 -0400 Subject: _frontend/app.py: Remove unused variable at global scope --- buildstream/_frontend/app.py | 1 - 1 file changed, 1 deletion(-) (limited to 'buildstream') diff --git a/buildstream/_frontend/app.py b/buildstream/_frontend/app.py index fa07a9a54..d30b59265 100644 --- a/buildstream/_frontend/app.py +++ b/buildstream/_frontend/app.py @@ -40,7 +40,6 @@ from .._exceptions import BstError, StreamError, LoadError, LoadErrorReason, App from .._message import Message, MessageType, unconditional_messages from .._stream import Stream from .._versions import BST_FORMAT_VERSION -from .. import __version__ as build_stream_version from .. import _yaml # Import frontend assets -- cgit v1.2.1 From 5dc8ab983def6bdc1ab1f447a9fec8b675aa7310 Mon Sep 17 00:00:00 2001 From: Ed Baunton Date: Wed, 6 Jun 2018 14:51:36 +0100 Subject: Add a kind for Make --- buildstream/plugins/elements/make.py | 42 ++++++++++++++++++++++++++++++++++ buildstream/plugins/elements/make.yaml | 42 ++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 buildstream/plugins/elements/make.py create mode 100644 buildstream/plugins/elements/make.yaml (limited to 'buildstream') diff --git a/buildstream/plugins/elements/make.py b/buildstream/plugins/elements/make.py new file mode 100644 index 000000000..a6ba8576b --- /dev/null +++ b/buildstream/plugins/elements/make.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +# +# Copyright Bloomberg Finance LP +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . +# +# Authors: +# Ed Baunton + +"""Make build element + +This is a :mod:`BuildElement ` implementation for +using GNU make based build. + +Here is the default configuration for the ``make`` element in full: + + .. literalinclude:: ../../../buildstream/plugins/elements/make.yaml + :language: yaml +""" + +from buildstream import BuildElement + + +# Element implementation for the 'make' kind. +class MakeElement(BuildElement): + pass + + +# Plugin entry point +def setup(): + return MakeElement diff --git a/buildstream/plugins/elements/make.yaml b/buildstream/plugins/elements/make.yaml new file mode 100644 index 000000000..a8e63b5d5 --- /dev/null +++ b/buildstream/plugins/elements/make.yaml @@ -0,0 +1,42 @@ +# make default configurations + +variables: + make: make + make-install: make -j1 DESTDIR="%{install-root}" install + + # Set this if the sources cannot handle parallelization. + # + # notparallel: True + +config: + + # Commands for building the software + # + build-commands: + - | + %{make} + + # Commands for installing the software into a + # destination folder + # + install-commands: + - | + %{make-install} + + # Commands for stripping debugging information out of + # installed binaries + # + strip-commands: + - | + %{strip-binaries} + +# Use max-jobs CPUs for building and enable verbosity +environment: + MAKEFLAGS: -j%{max-jobs} + V: 1 + +# And dont consider MAKEFLAGS or V as something which may +# effect build output. +environment-nocache: +- MAKEFLAGS +- V -- cgit v1.2.1 From 5ddec56fed3f16fe1a7cf8f15b3384ec478b2bda Mon Sep 17 00:00:00 2001 From: Tristan Van Berkom Date: Wed, 6 Jun 2018 14:54:03 -0400 Subject: _versions.py: Bump format version to 9 For addition of new `make` plugin --- buildstream/_versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'buildstream') diff --git a/buildstream/_versions.py b/buildstream/_versions.py index 6b4e3bddb..a101106aa 100644 --- a/buildstream/_versions.py +++ b/buildstream/_versions.py @@ -24,7 +24,7 @@ # This version is bumped whenever enhancements are made # to the `project.conf` format or the core element format. # -BST_FORMAT_VERSION = 8 +BST_FORMAT_VERSION = 9 # The base BuildStream artifact version -- cgit v1.2.1 From 17fa7ba4fa673b484dcfccd00ef935f2ccca3976 Mon Sep 17 00:00:00 2001 From: Tristan Van Berkom Date: Wed, 6 Jun 2018 14:54:27 -0400 Subject: make plugin: Ammended documentation to note it's since version. --- buildstream/plugins/elements/make.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'buildstream') diff --git a/buildstream/plugins/elements/make.py b/buildstream/plugins/elements/make.py index a6ba8576b..37024926c 100644 --- a/buildstream/plugins/elements/make.py +++ b/buildstream/plugins/elements/make.py @@ -23,6 +23,10 @@ This is a :mod:`BuildElement ` implementation for using GNU make based build. +.. note:: + + The ``make`` element is available since :ref:`format version 9 ` + Here is the default configuration for the ``make`` element in full: .. literalinclude:: ../../../buildstream/plugins/elements/make.yaml -- cgit v1.2.1 From 55dc3b27ade5327d1723ed139e9e39cc47f9aae8 Mon Sep 17 00:00:00 2001 From: Tristan Van Berkom Date: Wed, 6 Jun 2018 14:59:00 -0400 Subject: make build element: Add support for PREFIX --- buildstream/plugins/elements/make.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'buildstream') diff --git a/buildstream/plugins/elements/make.yaml b/buildstream/plugins/elements/make.yaml index a8e63b5d5..1438bb52b 100644 --- a/buildstream/plugins/elements/make.yaml +++ b/buildstream/plugins/elements/make.yaml @@ -1,8 +1,8 @@ # make default configurations variables: - make: make - make-install: make -j1 DESTDIR="%{install-root}" install + make: make PREFIX="%{prefix}" + make-install: make -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install # Set this if the sources cannot handle parallelization. # -- cgit v1.2.1 From 7df956547696092cb07082bc4f6ec12f6585cdd0 Mon Sep 17 00:00:00 2001 From: Chandan Singh Date: Wed, 6 Jun 2018 18:25:24 +0100 Subject: _loader/loader.py: Report element-path when failing to load elements It can be confusing, especially to new BuildStream users, that the CLI expects targets to be specified relative to element-path and not the current directory. Previously, the CLI would give a generic message stating that the file could not be found but it was not obvious that it was looking in the `element-path` directory. Explicitly print the element-path in the summary. Also, try to check if the specified element exists in the elements directory and print a hint to use the element-path relative paths if that's the case. Fixes #396. This is is also related to #341. This commit aims to tackle that issue by trying to educate users about element-path. --- buildstream/_loader/loader.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) (limited to 'buildstream') diff --git a/buildstream/_loader/loader.py b/buildstream/_loader/loader.py index e0ceb4fb9..1f3da35b9 100644 --- a/buildstream/_loader/loader.py +++ b/buildstream/_loader/loader.py @@ -207,7 +207,23 @@ class Loader(): # Load the data and process any conditional statements therein fullpath = os.path.join(self._basedir, filename) - node = _yaml.load(fullpath, shortname=filename, copy_tree=rewritable) + try: + node = _yaml.load(fullpath, shortname=filename, copy_tree=rewritable) + except LoadError as e: + if e.reason == LoadErrorReason.MISSING_FILE: + # If we can't find the file, try to suggest plausible + # alternatives by stripping the element-path from the given + # filename, and verifying that it exists. + message = "Could not find element '{}' in elements directory '{}'".format(filename, self._basedir) + detail = None + elements_dir = os.path.relpath(self._basedir, self.project.directory) + element_relpath = os.path.relpath(filename, elements_dir) + if filename.startswith(elements_dir) and os.path.exists(os.path.join(self._basedir, element_relpath)): + detail = "Did you mean '{}'?".format(element_relpath) + raise LoadError(LoadErrorReason.MISSING_FILE, + message, detail=detail) from e + else: + raise self._options.process_node(node) element = LoadElement(node, filename, self) -- cgit v1.2.1 From a5ff465d9225ad894435f95ec03f6ee21968fcb1 Mon Sep 17 00:00:00 2001 From: Chandan Singh Date: Sun, 22 Apr 2018 03:16:03 +0100 Subject: _project.py: Allow running bst commands from subdirectories of project root When initializing the project, BuildStream will continue searching for project.conf in parent directories in case it is not found in the current directory. Fixes #368. --- buildstream/_project.py | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) (limited to 'buildstream') diff --git a/buildstream/_project.py b/buildstream/_project.py index 5344e954e..25ffaf6d2 100644 --- a/buildstream/_project.py +++ b/buildstream/_project.py @@ -40,6 +40,9 @@ from ._workspaces import Workspaces # The separator we use for user specified aliases _ALIAS_SEPARATOR = ':' +# Project Configuration file +_PROJECT_CONF_FILE = 'project.conf' + # HostMount() # @@ -75,7 +78,7 @@ class Project(): self.name = None # The project directory - self.directory = os.path.abspath(directory) + self.directory = self._ensure_project_dir(directory) # Absolute path to where elements are loaded from within the project self.element_path = None @@ -211,7 +214,7 @@ class Project(): def _load(self): # Load builtin default - projectfile = os.path.join(self.directory, "project.conf") + projectfile = os.path.join(self.directory, _PROJECT_CONF_FILE) config = _yaml.load(_site.default_project_config) # Load project local config and override the builtin @@ -458,3 +461,27 @@ class Project(): # paths are passed in relative to the project, but must be absolute origin_dict['path'] = os.path.join(self.directory, origin_dict['path']) destination.append(origin_dict) + + # _ensure_project_dir() + # + # Returns path of the project directory, if a configuration file is found + # in given directory or any of its parent directories. + # + # Args: + # directory (str) - directory from where the command was invoked + # + # Raises: + # LoadError if project.conf is not found + # + def _ensure_project_dir(self, directory): + directory = os.path.abspath(directory) + while not os.path.isfile(os.path.join(directory, _PROJECT_CONF_FILE)): + parent_dir = os.path.dirname(directory) + if directory == parent_dir: + raise LoadError( + LoadErrorReason.MISSING_PROJECT_CONF, + '{} not found in current directory or any of its parent directories' + .format(_PROJECT_CONF_FILE)) + directory = parent_dir + + return directory -- cgit v1.2.1 From 69e594433df417324f4015c0364088acfacc1a72 Mon Sep 17 00:00:00 2001 From: Chandan Singh Date: Wed, 6 Jun 2018 19:21:14 +0100 Subject: _frontend/cli.py: Try to autocomplete element paths when running from a subdirectory The previous commit added support for running bst commands form subdirectories of the project root. Make autocomplete also work in a similar way. --- buildstream/_frontend/cli.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) (limited to 'buildstream') diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py index 41e97cb0e..5ed967d9d 100644 --- a/buildstream/_frontend/cli.py +++ b/buildstream/_frontend/cli.py @@ -20,6 +20,18 @@ def complete_target(args, incomplete): :return: all the possible user-specified completions for the param """ + project_conf = 'project.conf' + + def ensure_project_dir(directory): + directory = os.path.abspath(directory) + while not os.path.isfile(os.path.join(directory, project_conf)): + parent_dir = os.path.dirname(directory) + if directory == parent_dir: + break + directory = parent_dir + + return directory + # First resolve the directory, in case there is an # active --directory/-C option # @@ -35,10 +47,14 @@ def complete_target(args, incomplete): if idx >= 0 and len(args) > idx + 1: base_directory = args[idx + 1] + else: + # Check if this directory or any of its parent directories + # contain a project config file + base_directory = ensure_project_dir(base_directory) # Now parse the project.conf just to find the element path, # this is unfortunately a bit heavy. - project_file = os.path.join(base_directory, 'project.conf') + project_file = os.path.join(base_directory, project_conf) try: project = _yaml.load(project_file) except LoadError: -- cgit v1.2.1 From e36001f92d1ac3e3e40863e2683cd2baba2175fa Mon Sep 17 00:00:00 2001 From: Ed Baunton <> Date: Thu, 7 Jun 2018 14:43:27 +0100 Subject: Make `bst help` work --- buildstream/_frontend/cli.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'buildstream') diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py index 5ed967d9d..bc17fed87 100644 --- a/buildstream/_frontend/cli.py +++ b/buildstream/_frontend/cli.py @@ -189,6 +189,10 @@ def cli(context, **kwargs): context.obj = App.create(dict(kwargs)) context.call_on_close(context.obj.cleanup) +@cli.command(short_help="Print usage information") +@click.pass_context +def help(ctx): + click.echo(ctx.parent.get_help()) ################################################################## # Init Command # -- cgit v1.2.1 From b25c31ea358986fdc5f95cf607fc75f00fb5fb78 Mon Sep 17 00:00:00 2001 From: Ed Baunton Date: Thu, 7 Jun 2018 16:02:04 +0100 Subject: Implement bst help --- buildstream/_frontend/cli.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) (limited to 'buildstream') diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py index bc17fed87..9cd0a871c 100644 --- a/buildstream/_frontend/cli.py +++ b/buildstream/_frontend/cli.py @@ -189,10 +189,22 @@ def cli(context, **kwargs): context.obj = App.create(dict(kwargs)) context.call_on_close(context.obj.cleanup) -@cli.command(short_help="Print usage information") + +################################################################## +# Help Command # +################################################################## +@cli.command(name="help", short_help="Print usage information", + context_settings={"help_option_names": []}) +@click.argument("arg", nargs=-1) @click.pass_context -def help(ctx): - click.echo(ctx.parent.get_help()) +def help_command(ctx, **kwargs): + click.echo(ctx.parent.get_help(), err=True) + # TODO support bst help but currently + # seems non obvious how to do this with click. + if kwargs["arg"]: + click.echo("\n{} {} --help for more usage on a specific command\n".format( + ctx.parent.info_name, " ".join(kwargs["arg"])), err=True) + ################################################################## # Init Command # -- cgit v1.2.1 From 8234e9c7d7f989ba86e4da4acecd92f8732b003c Mon Sep 17 00:00:00 2001 From: Tristan Van Berkom Date: Thu, 7 Jun 2018 13:55:46 -0400 Subject: _frontend/cli.py: Allow specifying commands in `bst help` o This supports deeply nested commands as well as shallow commands o Automated support for bash completions included --- buildstream/_frontend/cli.py | 71 ++++++++++++++++++++++++++++++++++----- buildstream/_frontend/complete.py | 9 ++--- 2 files changed, 67 insertions(+), 13 deletions(-) (limited to 'buildstream') diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py index 9cd0a871c..143d59aa4 100644 --- a/buildstream/_frontend/cli.py +++ b/buildstream/_frontend/cli.py @@ -12,6 +12,45 @@ from .complete import main_bashcomplete, complete_path, CompleteUnhandled # Override of click's main entry point # ################################################################## +# search_command() +# +# Helper function to get a command and context object +# for a given command. +# +# Args: +# commands (list): A list of command words following `bst` invocation +# context (click.Context): An existing toplevel context, or None +# +# Returns: +# context (click.Context): The context of the associated command, or None +# +def search_command(args, *, context=None): + if context is None: + context = cli.make_context('bst', args, resilient_parsing=True) + + # Loop into the deepest command + command = cli + command_ctx = context + for cmd in args: + command = command_ctx.command.get_command(command_ctx, cmd) + if command is None: + return None + command_ctx = command.make_context(command.name, [command.name], + parent=command_ctx, + resilient_parsing=True) + + return command_ctx + + +# Completion for completing command names as help arguments +def complete_commands(cmd, args, incomplete): + command_ctx = search_command(args[1:]) + if command_ctx and command_ctx.command and isinstance(command_ctx.command, click.MultiCommand): + return [subcommand + " " for subcommand in command_ctx.command.list_commands(command_ctx)] + + return [] + + # Special completion for completing the bst elements in a project dir def complete_target(args, incomplete): """ @@ -73,7 +112,7 @@ def complete_target(args, incomplete): return complete_path("File", incomplete, base_directory=base_directory) -def override_completions(cmd_param, args, incomplete): +def override_completions(cmd, cmd_param, args, incomplete): """ :param cmd_param: command definition :param args: full list of args typed before the incomplete arg @@ -81,6 +120,9 @@ def override_completions(cmd_param, args, incomplete): :return: all the possible user-specified completions for the param """ + if cmd.name == 'help': + return complete_commands(cmd, args, incomplete) + # We can't easily extend click's data structures without # modifying click itself, so just do some weak special casing # right here and select which parameters we want to handle specially. @@ -195,15 +237,26 @@ def cli(context, **kwargs): ################################################################## @cli.command(name="help", short_help="Print usage information", context_settings={"help_option_names": []}) -@click.argument("arg", nargs=-1) +@click.argument("command", nargs=-1, metavar='COMMAND') @click.pass_context -def help_command(ctx, **kwargs): - click.echo(ctx.parent.get_help(), err=True) - # TODO support bst help but currently - # seems non obvious how to do this with click. - if kwargs["arg"]: - click.echo("\n{} {} --help for more usage on a specific command\n".format( - ctx.parent.info_name, " ".join(kwargs["arg"])), err=True) +def help_command(ctx, command): + """Print usage information about a given command + """ + command_ctx = search_command(command, context=ctx.parent) + if not command_ctx: + click.echo("Not a valid command: '{} {}'" + .format(ctx.parent.info_name, " ".join(command)), err=True) + sys.exit(-1) + + click.echo(command_ctx.command.get_help(command_ctx), err=True) + + # Hint about available sub commands + if isinstance(command_ctx.command, click.MultiCommand): + detail = " " + if command: + detail = " {} ".format(" ".join(command)) + click.echo("\nFor usage on a specific command: {} help{}COMMAND" + .format(ctx.parent.info_name, detail), err=True) ################################################################## diff --git a/buildstream/_frontend/complete.py b/buildstream/_frontend/complete.py index fa986ee6b..4a367e62a 100644 --- a/buildstream/_frontend/complete.py +++ b/buildstream/_frontend/complete.py @@ -209,7 +209,7 @@ def is_incomplete_argument(current_params, cmd_param): return False -def get_user_autocompletions(args, incomplete, cmd_param, override): +def get_user_autocompletions(args, incomplete, cmd, cmd_param, override): """ :param args: full list of args typed before the incomplete arg :param incomplete: the incomplete text of the arg to autocomplete @@ -222,7 +222,8 @@ def get_user_autocompletions(args, incomplete, cmd_param, override): # Use the type specific default completions unless it was overridden try: - return override(cmd_param=cmd_param, + return override(cmd=cmd, + cmd_param=cmd_param, args=args, incomplete=incomplete) except CompleteUnhandled: @@ -268,14 +269,14 @@ def get_choices(cli, prog_name, args, incomplete, override): # completion for option values by choices for cmd_param in ctx.command.params: if isinstance(cmd_param, Option) and is_incomplete_option(all_args, cmd_param): - choices.extend(get_user_autocompletions(all_args, incomplete, cmd_param, override)) + choices.extend(get_user_autocompletions(all_args, incomplete, ctx.command, cmd_param, override)) found_param = True break if not found_param: # completion for argument values by choices for cmd_param in ctx.command.params: if isinstance(cmd_param, Argument) and is_incomplete_argument(ctx.params, cmd_param): - choices.extend(get_user_autocompletions(all_args, incomplete, cmd_param, override)) + choices.extend(get_user_autocompletions(all_args, incomplete, ctx.command, cmd_param, override)) found_param = True break -- cgit v1.2.1 From 4632cb039d543d78a9a8956c605fa3ed40f3f035 Mon Sep 17 00:00:00 2001 From: Tristan Maat Date: Fri, 23 Mar 2018 17:26:32 +0000 Subject: _ostree.py: Reintroduce remove() --- buildstream/_ostree.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) (limited to 'buildstream') diff --git a/buildstream/_ostree.py b/buildstream/_ostree.py index dfa7567de..246b54f51 100644 --- a/buildstream/_ostree.py +++ b/buildstream/_ostree.py @@ -225,6 +225,42 @@ def exists(repo, ref): return has_object +# remove(): +# +# Removes the given commit or symbolic ref from the repo. +# +# Args: +# repo (OSTree.Repo): The repo +# ref (str): A commit checksum or symbolic ref +# defer_prune (bool): Whether to defer pruning to the caller. NOTE: +# The space won't be freed until you manually +# call repo.prune. +# +# Returns: +# (int|None) The amount of space pruned from the repository in +# Bytes, or None if defer_prune is True +# +def remove(repo, ref, *, defer_prune=False): + + # Get the commit checksum, this will: + # + # o Return a commit checksum if ref is a symbolic branch + # o Return the same commit checksum if ref is a valid commit checksum + # o Return None if the ostree repo doesnt know this ref. + # + check = checksum(repo, ref) + if check is None: + raise OSTreeError("Could not find artifact for ref '{}'".format(ref)) + + repo.set_ref_immediate(None, ref, None) + + if not defer_prune: + _, _, _, pruned = repo.prune(OSTree.RepoPruneFlags.REFS_ONLY, -1) + return pruned + + return None + + # checksum(): # # Returns the commit checksum for a given symbolic ref, -- cgit v1.2.1 From 60a29f687c520fd7b17f4b98ab0a033c41e31fc5 Mon Sep 17 00:00:00 2001 From: Tristan Maat Date: Sat, 24 Mar 2018 17:56:56 +0000 Subject: _ostree.py: Introduce _list_all_refs() and list_artifacts() The unused list_remote_refs() function has also been removed as part of this commit. --- buildstream/_ostree.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) (limited to 'buildstream') diff --git a/buildstream/_ostree.py b/buildstream/_ostree.py index 246b54f51..e40df5fb5 100644 --- a/buildstream/_ostree.py +++ b/buildstream/_ostree.py @@ -552,3 +552,47 @@ def configure_remote(repo, remote, url, key_url=None): repo.remote_gpg_import(remote, stream, None, 0, None) except GLib.GError as e: raise OSTreeError("Failed to add gpg key from url '{}': {}".format(key_url, e.message)) from e + + +# list_artifacts(): +# +# List cached artifacts in Least Recently Modified (LRM) order. +# +# Returns: +# (list) - A list of refs in LRM order +# +def list_artifacts(repo): + # string of: /path/to/repo/refs/heads + ref_heads = os.path.join(repo.get_path().get_path(), 'refs', 'heads') + + # obtain list of // + refs = _list_all_refs(repo).keys() + + mtimes = [] + for ref in refs: + ref_path = os.path.join(ref_heads, ref) + if os.path.exists(ref_path): + # Obtain the mtime (the time a file was last modified) + mtimes.append(os.path.getmtime(ref_path)) + + # NOTE: Sorted will sort from earliest to latest, thus the + # first element of this list will be the file modified earliest. + return [ref for _, ref in sorted(zip(mtimes, refs))] + + +# _list_all_refs(): +# +# Create a list of all refs. +# +# Args: +# repo (OSTree.Repo): The repo +# +# Returns: +# (dict): A dict of refs to checksums. +# +def _list_all_refs(repo): + try: + _, refs = repo.list_refs(None) + return refs + except GLib.GError as e: + raise OSTreeError(message=e.message) from e -- cgit v1.2.1 From b7191fda9eb66229f7fdb62e7dd9e9b448270615 Mon Sep 17 00:00:00 2001 From: James Ennis Date: Wed, 18 Apr 2018 12:42:52 +0100 Subject: pushreceive.py: Ensure there is a repopath --- buildstream/_artifactcache/pushreceive.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'buildstream') diff --git a/buildstream/_artifactcache/pushreceive.py b/buildstream/_artifactcache/pushreceive.py index ecedc6657..209aef620 100644 --- a/buildstream/_artifactcache/pushreceive.py +++ b/buildstream/_artifactcache/pushreceive.py @@ -576,6 +576,9 @@ class OSTreeReceiver(object): if self.repopath is None: self.repo = OSTree.Repo.new_default() + self.repopath = self.repo.get_path().get_path() + # NOTE: OSTree.Repo.get_path() returns Gio.File + # Gio.File.get_path() returns a string of the pathway else: self.repo = OSTree.Repo.new(Gio.File.new_for_path(self.repopath)) self.repo.open(None) -- cgit v1.2.1 From 58adbe0b3ca29e1d88d2003abca5192381512054 Mon Sep 17 00:00:00 2001 From: James Ennis Date: Wed, 18 Apr 2018 15:01:37 +0100 Subject: pushreceive.py: Remove LRP artifacts from cache, introduce clean_up_cache() This fixes #136 --- buildstream/_artifactcache/pushreceive.py | 57 +++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 3 deletions(-) (limited to 'buildstream') diff --git a/buildstream/_artifactcache/pushreceive.py b/buildstream/_artifactcache/pushreceive.py index 209aef620..f8fd76d79 100644 --- a/buildstream/_artifactcache/pushreceive.py +++ b/buildstream/_artifactcache/pushreceive.py @@ -33,6 +33,7 @@ from urllib.parse import urlparse import click import gi +from .. import _ostree from .. import _signals # nopep8 from .._profile import Topics, profile_start, profile_end @@ -277,10 +278,14 @@ class PushMessageReader(object): _, args = self.receive([PushCommandType.update]) return args - def receive_putobjects(self, repo): - + def receive_putobjects(self, repo, repopath): received_objects = [] + # Determine the available disk space, in bytes, of the file system + # which mounts the repo + stats = os.statvfs(repopath) + free_disk_space = stats.f_bfree * stats.f_bsize + # Open a TarFile for reading uncompressed tar from a stream tar = tarfile.TarFile.open(mode='r|', fileobj=self.file) @@ -288,6 +293,7 @@ class PushMessageReader(object): # # This should block while tar.next() reads the next # tar object from the stream. + buffer_ = int(2e9) while True: filepos = tar.fileobj.tell() tar_info = tar.next() @@ -300,7 +306,16 @@ class PushMessageReader(object): tar.fileobj.read(512) break + # obtain size of tar object in bytes + artifact_size = tar_info.size + + if artifact_size > free_disk_space - buffer_: + # Clean up the cache with a buffer of 2GB + removed_size = clean_up_cache(repo, artifact_size, free_disk_space, buffer_) + free_disk_space += removed_size + tar.extract(tar_info, self.tmpdir) + free_disk_space -= artifact_size received_objects.append(tar_info.name) # Finished with this stream @@ -645,7 +660,7 @@ class OSTreeReceiver(object): return 0 # Receive the actual objects - received_objects = self.reader.receive_putobjects(self.repo) + received_objects = self.reader.receive_putobjects(self.repo, self.repopath) # Ensure that pusher has sent all objects self.reader.receive_done() @@ -794,6 +809,42 @@ def push(repo, remote, branches, output): return False +# clean_up_cache() +# +# Keep removing Least Recently Pushed (LRP) artifacts in a cache until there +# is enough space for the incoming artifact +# +# Args: +# repo: OSTree.Repo object +# free_disk_space: The available disk space on the file system in bytes +# artifact_size: The size of the artifact in bytes +# buffer_: The amount of headroom we want on disk. +# +# Returns: +# int: The total bytes removed on the filesystem +# +def clean_up_cache(repo, artifact_size, free_disk_space, buffer_): + # obtain a list of LRP artifacts + LRP_artifacts = _ostree.list_artifacts(repo) + + removed_size = 0 # in bytes + while artifact_size - removed_size > free_disk_space - buffer_: + try: + to_remove = LRP_artifacts.pop(0) # The first element in the list is the LRP artifact + except IndexError: + logging.info("There are no more artifacts left in the cache. Adding artifact...") + break + + removed_size += _ostree.remove(repo, to_remove, defer_prune=False) + + if removed_size > 0: + logging.info("Successfully removed {} bytes from the cache".format(removed_size)) + else: + logging.info("No artifacts were removed from the cache.") + + return removed_size + + @click.command(short_help="Receive pushed artifacts over ssh") @click.option('--verbose', '-v', is_flag=True, default=False, help="Verbose mode") @click.option('--debug', '-d', is_flag=True, default=False, help="Debug mode") -- cgit v1.2.1 From b8a52cbdcf30c92c854edf5b8305fadc99a0f035 Mon Sep 17 00:00:00 2001 From: James Ennis Date: Fri, 20 Apr 2018 17:20:35 +0100 Subject: pushreceive.py: Add comments to code and name variables more sensibly --- buildstream/_artifactcache/pushreceive.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'buildstream') diff --git a/buildstream/_artifactcache/pushreceive.py b/buildstream/_artifactcache/pushreceive.py index f8fd76d79..e17e1de9a 100644 --- a/buildstream/_artifactcache/pushreceive.py +++ b/buildstream/_artifactcache/pushreceive.py @@ -113,7 +113,7 @@ class PushMessageWriter(object): def __init__(self, file, byteorder=sys.byteorder): self.file = file self.byteorder = byteorder - self.msg_byteorder = python_to_msg_byteorder(self.byteorder) + self.msg_byteorder = python_to_msg_byteorder(self.byteorder) # 'l' or 'B' def encode_header(self, cmdtype, size): header = self.msg_byteorder.encode() + \ @@ -388,6 +388,7 @@ def foo_run(func, args, stdin_fd, stdout_fd, stderr_fd): class ProcessWithPipes(object): def __init__(self, func, args, *, stderr=None): + # Create a pipe and return a pair of file descriptors (r, w) r0, w0 = os.pipe() r1, w1 = os.pipe() if stderr is None: @@ -432,10 +433,12 @@ class OSTreePusher(object): # Enumerate branches to push if branches is None: + # obtain a dict of 'refs': 'checksums' _, self.refs = self.repo.list_refs(None, None) else: self.refs = {} for branch in branches: + # branch is a ref, now find its checksum (i.e. rev) _, rev = self.repo.resolve_rev(branch, False) self.refs[branch] = rev @@ -454,6 +457,9 @@ class OSTreePusher(object): logging.info('Executing {}'.format(' '.join(ssh_cmd))) if self.remote_host: + # subprocess.Popen(args, bufsize=-1,...) + # Executes a child program in a new process which returns an open file + # object connected to the pipe. self.ssh = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=self.output, @@ -530,6 +536,8 @@ class OSTreePusher(object): for branch, rev in self.refs.items(): remote_rev = remote_refs.get(branch, '0' * 64) if rev != remote_rev: + # if the checksums for a branch aren't equal add a tuple of + # the remote_rev and local rev to a new dictionary. update_refs[branch] = remote_rev, rev if not update_refs: logging.info('Nothing to update') @@ -550,9 +558,12 @@ class OSTreePusher(object): commits = set() exc_info = None ref_count = 0 + + # update the remote checksum with the local one for branch, revs in update_refs.items(): logging.info('Updating {} {} to {}'.format(branch, revs[0], revs[1])) try: + # obtain a set of the commits needed to be pushed self.needed_commits(revs[0], revs[1], commits) ref_count += 1 except PushExistsException: @@ -564,6 +575,7 @@ class OSTreePusher(object): raise exc_info[0].with_traceback(exc_info[1], exc_info[2]) logging.info('Enumerating objects to send') + # obtain a set of the objects which need to be pushed to the server objects = self.needed_objects(commits) # Send all the objects to receiver, checking status after each -- cgit v1.2.1 From b823a616c555a7d4c0cb93416d78db6053a06667 Mon Sep 17 00:00:00 2001 From: James Ennis Date: Wed, 30 May 2018 15:56:41 +0100 Subject: pushreceive.py: Ensure huge artifacts are not pushed --- buildstream/_artifactcache/pushreceive.py | 37 ++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) (limited to 'buildstream') diff --git a/buildstream/_artifactcache/pushreceive.py b/buildstream/_artifactcache/pushreceive.py index e17e1de9a..ab74e80ea 100644 --- a/buildstream/_artifactcache/pushreceive.py +++ b/buildstream/_artifactcache/pushreceive.py @@ -56,6 +56,11 @@ class PushExistsException(Exception): pass +# Trying to push an artifact that is too large +class ArtifactTooLargeException(Exception): + pass + + class PushCommandType(Enum): info = 0 update = 1 @@ -285,6 +290,7 @@ class PushMessageReader(object): # which mounts the repo stats = os.statvfs(repopath) free_disk_space = stats.f_bfree * stats.f_bsize + total_disk_space = stats.f_blocks * stats.f_bsize # Open a TarFile for reading uncompressed tar from a stream tar = tarfile.TarFile.open(mode='r|', fileobj=self.file) @@ -309,6 +315,11 @@ class PushMessageReader(object): # obtain size of tar object in bytes artifact_size = tar_info.size + if artifact_size > total_disk_space - buffer_: + raise ArtifactTooLargeException("Artifact of size: {} is too large for " + "the filesystem which mounts the remote " + "cache".format(artifact_size)) + if artifact_size > free_disk_space - buffer_: # Clean up the cache with a buffer of 2GB removed_size = clean_up_cache(repo, artifact_size, free_disk_space, buffer_) @@ -579,7 +590,13 @@ class OSTreePusher(object): objects = self.needed_objects(commits) # Send all the objects to receiver, checking status after each - self.writer.send_putobjects(self.repo, objects) + try: + self.writer.send_putobjects(self.repo, objects) + except BrokenPipeError: + # If the remote closes, we receive a BrokenPipeError + # Return 1 to notify the frontend that something went + # wrong on the server. + return 1 # Inform receiver that all objects have been sent self.writer.send_done() @@ -626,8 +643,11 @@ class OSTreeReceiver(object): def run(self): try: exit_code = self.do_run() - self.close() - return exit_code + except ArtifactTooLargeException: + logging.warning("The artifact was too large for the filesystem which mounts " + "the remote cache.") + exit_code = 0 + except: # BLIND EXCEPT - Just abort if we receive any exception, this # can be a broken pipe, a tarfile read error when the remote @@ -635,6 +655,9 @@ class OSTreeReceiver(object): self.close() raise + self.close() + return exit_code + def do_run(self): # Receive remote info args = self.reader.receive_info() @@ -844,8 +867,12 @@ def clean_up_cache(repo, artifact_size, free_disk_space, buffer_): try: to_remove = LRP_artifacts.pop(0) # The first element in the list is the LRP artifact except IndexError: - logging.info("There are no more artifacts left in the cache. Adding artifact...") - break + # This exception is caught if there are no more artifacts in the list + # LRP_artifacts. This means the the artifact is too large for the filesystem + # so we abort the process + raise ArtifactTooLargeException("Artifact of size {} is too large for " + "the filesystem which mounts the remote " + "cache".format(artifact_size)) removed_size += _ostree.remove(repo, to_remove, defer_prune=False) -- cgit v1.2.1 From a231d41bcbcf4ad32b87968fa92de9c1637201f8 Mon Sep 17 00:00:00 2001 From: James Ennis Date: Mon, 4 Jun 2018 11:02:08 +0100 Subject: pushreceive.py: Abstract the buffer_ --- buildstream/_artifactcache/pushreceive.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) (limited to 'buildstream') diff --git a/buildstream/_artifactcache/pushreceive.py b/buildstream/_artifactcache/pushreceive.py index ab74e80ea..d4b378cb7 100644 --- a/buildstream/_artifactcache/pushreceive.py +++ b/buildstream/_artifactcache/pushreceive.py @@ -289,8 +289,9 @@ class PushMessageReader(object): # Determine the available disk space, in bytes, of the file system # which mounts the repo stats = os.statvfs(repopath) - free_disk_space = stats.f_bfree * stats.f_bsize - total_disk_space = stats.f_blocks * stats.f_bsize + buffer_ = int(2e9) # Add a 2 GB buffer + free_disk_space = (stats.f_bfree * stats.f_bsize) - buffer_ + total_disk_space = (stats.f_blocks * stats.f_bsize) - buffer_ # Open a TarFile for reading uncompressed tar from a stream tar = tarfile.TarFile.open(mode='r|', fileobj=self.file) @@ -299,7 +300,6 @@ class PushMessageReader(object): # # This should block while tar.next() reads the next # tar object from the stream. - buffer_ = int(2e9) while True: filepos = tar.fileobj.tell() tar_info = tar.next() @@ -315,14 +315,14 @@ class PushMessageReader(object): # obtain size of tar object in bytes artifact_size = tar_info.size - if artifact_size > total_disk_space - buffer_: + if artifact_size > total_disk_space: raise ArtifactTooLargeException("Artifact of size: {} is too large for " "the filesystem which mounts the remote " "cache".format(artifact_size)) - if artifact_size > free_disk_space - buffer_: + if artifact_size > free_disk_space: # Clean up the cache with a buffer of 2GB - removed_size = clean_up_cache(repo, artifact_size, free_disk_space, buffer_) + removed_size = clean_up_cache(repo, artifact_size, free_disk_space) free_disk_space += removed_size tar.extract(tar_info, self.tmpdir) @@ -853,17 +853,16 @@ def push(repo, remote, branches, output): # repo: OSTree.Repo object # free_disk_space: The available disk space on the file system in bytes # artifact_size: The size of the artifact in bytes -# buffer_: The amount of headroom we want on disk. # # Returns: # int: The total bytes removed on the filesystem # -def clean_up_cache(repo, artifact_size, free_disk_space, buffer_): +def clean_up_cache(repo, artifact_size, free_disk_space): # obtain a list of LRP artifacts LRP_artifacts = _ostree.list_artifacts(repo) removed_size = 0 # in bytes - while artifact_size - removed_size > free_disk_space - buffer_: + while artifact_size - removed_size > free_disk_space: try: to_remove = LRP_artifacts.pop(0) # The first element in the list is the LRP artifact except IndexError: -- cgit v1.2.1 From de194dad617ef9c35fd03ac6bd8622ebdfa7eb77 Mon Sep 17 00:00:00 2001 From: Tristan Van Berkom Date: Thu, 7 Jun 2018 16:38:06 -0400 Subject: _artifactcache/pushreceive.py: Cleanup reported error when receiving oversized artifacts This user facing string was redundantly declared in two places, only the message when catching the error was ever printed. --- buildstream/_artifactcache/pushreceive.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'buildstream') diff --git a/buildstream/_artifactcache/pushreceive.py b/buildstream/_artifactcache/pushreceive.py index d4b378cb7..41dacf33f 100644 --- a/buildstream/_artifactcache/pushreceive.py +++ b/buildstream/_artifactcache/pushreceive.py @@ -643,9 +643,8 @@ class OSTreeReceiver(object): def run(self): try: exit_code = self.do_run() - except ArtifactTooLargeException: - logging.warning("The artifact was too large for the filesystem which mounts " - "the remote cache.") + except ArtifactTooLargeException as e: + logging.warning(str(e)) exit_code = 0 except: -- cgit v1.2.1 From 8f2bf4e6e97f3ff859653c6cf60010ce9a2ebca0 Mon Sep 17 00:00:00 2001 From: Tristan Van Berkom Date: Fri, 8 Jun 2018 15:48:11 -0400 Subject: autotools plugin: Dont regenerate existing configure scripts This closes #256 --- buildstream/plugins/elements/autotools.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'buildstream') diff --git a/buildstream/plugins/elements/autotools.yaml b/buildstream/plugins/elements/autotools.yaml index 97ab0664c..021d3815c 100644 --- a/buildstream/plugins/elements/autotools.yaml +++ b/buildstream/plugins/elements/autotools.yaml @@ -4,10 +4,12 @@ variables: autogen: | export NOCONFIGURE=1; - if [ -e autogen ]; then ./autogen; - elif [ -e autogen.sh ]; then ./autogen.sh; - elif [ -e bootstrap ]; then ./bootstrap; - elif [ -e bootstrap.sh ]; then ./bootstrap.sh; + + if [ -x %{conf-cmd} ]; then true; + elif [ -x autogen ]; then ./autogen; + elif [ -x autogen.sh ]; then ./autogen.sh; + elif [ -x bootstrap ]; then ./bootstrap; + elif [ -x bootstrap.sh ]; then ./bootstrap.sh; else autoreconf -ivf; fi -- cgit v1.2.1 From af10c1ba62ebf030511ccb38ae58b7baffa4032f Mon Sep 17 00:00:00 2001 From: Valentin David Date: Fri, 4 May 2018 16:26:50 +0200 Subject: Interpret names as colon separated junction path in loader. 'a.bst:b.bst' gets interpreted as 'b.bst' from junction 'a.bst'. Part of #359. --- buildstream/_loader/loader.py | 47 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 6 deletions(-) (limited to 'buildstream') diff --git a/buildstream/_loader/loader.py b/buildstream/_loader/loader.py index 1f3da35b9..9e4406bc6 100644 --- a/buildstream/_loader/loader.py +++ b/buildstream/_loader/loader.py @@ -107,9 +107,13 @@ class Loader(): # First pass, recursively load files and populate our table of LoadElements # + deps = [] + for target in self._targets: profile_start(Topics.LOAD_PROJECT, target) - self._load_file(target, rewritable, ticker) + junction, name, loader = self._parse_name(target, rewritable, ticker) + loader._load_file(name, rewritable, ticker) + deps.append(Dependency(name, junction=junction)) profile_end(Topics.LOAD_PROJECT, target) # @@ -119,7 +123,8 @@ class Loader(): # Set up a dummy element that depends on all top-level targets # to resolve potential circular dependencies between them DummyTarget = namedtuple('DummyTarget', ['name', 'full_name', 'deps']) - dummy = DummyTarget(name='', full_name='', deps=[Dependency(e) for e in self._targets]) + + dummy = DummyTarget(name='', full_name='', deps=deps) self._elements[''] = dummy profile_key = "_".join(t for t in self._targets) @@ -127,17 +132,20 @@ class Loader(): self._check_circular_deps('') profile_end(Topics.CIRCULAR_CHECK, profile_key) + ret = [] # # Sort direct dependencies of elements by their dependency ordering # for target in self._targets: profile_start(Topics.SORT_DEPENDENCIES, target) - self._sort_dependencies(target) + junction, name, loader = self._parse_name(target, rewritable, ticker) + loader._sort_dependencies(name) profile_end(Topics.SORT_DEPENDENCIES, target) + # Finally, wrap what we have into LoadElements and return the target + # + ret.append(loader._collect_element(name)) - # Finally, wrap what we have into LoadElements and return the target - # - return [self._collect_element(target) for target in self._targets] + return ret # cleanup(): # @@ -554,3 +562,30 @@ class Loader(): return self._loaders[dep.junction] else: return self + + # _parse_name(): + # + # Get junction and base name of element along with loader for the sub-project + # + # Args: + # name (str): Name of target + # rewritable (bool): Whether the loaded files should be rewritable + # this is a bit more expensive due to deep copies + # ticker (callable): An optional function for tracking load progress + # + # Returns: + # (tuple): - (str): name of the junction element + # - (str): name of the element + # - (Loader): loader for sub-project + # + def _parse_name(self, name, rewritable, ticker): + # We allow to split only once since deep junctions names are forbidden. + # Users who want to refer to elements in sub-sub-projects are required + # to create junctions on the top level project. + junction_path = name.rsplit(':', 1) + if len(junction_path) == 1: + return None, junction_path[-1], self + else: + self._load_file(junction_path[-2], rewritable, ticker) + loader = self._get_loader(junction_path[-2], rewritable=rewritable, ticker=ticker) + return junction_path[-2], junction_path[-1], loader -- cgit v1.2.1 From ccec163b06193b3c21ab0d571d76b72856f0ea55 Mon Sep 17 00:00:00 2001 From: Valentin David Date: Fri, 11 May 2018 18:04:38 +0200 Subject: Reword uses of project in Workspaces. Make it clear we expect the top-level project here as we use it to resolve paths relative to project directory. --- buildstream/_workspaces.py | 52 ++++++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 22 deletions(-) (limited to 'buildstream') diff --git a/buildstream/_workspaces.py b/buildstream/_workspaces.py index f6cdeb88b..41b4e9f05 100644 --- a/buildstream/_workspaces.py +++ b/buildstream/_workspaces.py @@ -46,7 +46,7 @@ _WORKSPACE_MEMBERS = [ # methods. # # Args: -# project (Project): The project this workspace is part of +# toplevel_project (Project): Top project. Will be used for resolving relative workspace paths. # path (str): The path that should host this workspace # last_successful (str): The key of the last successful build of this workspace # running_files (dict): A dict mapping dependency elements to files @@ -54,13 +54,13 @@ _WORKSPACE_MEMBERS = [ # made obsolete with failed build artifacts. # class Workspace(): - def __init__(self, project, *, last_successful=None, path=None, prepared=False, running_files=None): + def __init__(self, toplevel_project, *, last_successful=None, path=None, prepared=False, running_files=None): self.prepared = prepared self.last_successful = last_successful self.path = path self.running_files = running_files if running_files is not None else {} - self._project = project + self._toplevel_project = toplevel_project self._key = None # to_dict() @@ -81,17 +81,17 @@ class Workspace(): # when loading from a YAML file. # # Args: - # project (Project): The Project to load this for + # toplevel_project (Project): Top project. Will be used for resolving relative workspace paths. # dictionary: A simple dictionary object # # Returns: # (Workspace): A newly instantiated Workspace # @classmethod - def from_dict(cls, project, dictionary): + def from_dict(cls, toplevel_project, dictionary): # Just pass the dictionary as kwargs - return cls(project, **dictionary) + return cls(toplevel_project, **dictionary) # differs() # @@ -201,7 +201,7 @@ class Workspace(): # Returns: The absolute path of the element's workspace. # def get_absolute_path(self): - return os.path.join(self._project.directory, self.path) + return os.path.join(self._toplevel_project.directory, self.path) # Workspaces() @@ -209,11 +209,12 @@ class Workspace(): # A class to manage Workspaces for multiple elements. # # Args: -# project (Project): The project the workspaces should be associated to +# toplevel_project (Project): Top project used to resolve paths. # class Workspaces(): - def __init__(self, project): - self._project = project + def __init__(self, toplevel_project): + self._toplevel_project = toplevel_project + self._bst_directory = os.path.join(toplevel_project.directory, ".bst") self._workspaces = self._load_config() # list() @@ -236,7 +237,7 @@ class Workspaces(): # path (str) - The path in which the workspace should be kept # def create_workspace(self, element_name, path): - self._workspaces[element_name] = Workspace(self._project, path=path) + self._workspaces[element_name] = Workspace(self._toplevel_project, path=path) return self._workspaces[element_name] @@ -270,7 +271,7 @@ class Workspaces(): def update_workspace(self, element_name, workspace_dict): assert element_name in self._workspaces - workspace = Workspace.from_dict(self._project, workspace_dict) + workspace = Workspace.from_dict(self._toplevel_project, workspace_dict) if self._workspaces[element_name].differs(workspace): self._workspaces[element_name] = workspace return True @@ -305,9 +306,9 @@ class Workspaces(): for element, workspace in _yaml.node_items(self._workspaces) } } - os.makedirs(os.path.join(self._project.directory, ".bst"), exist_ok=True) + os.makedirs(self._bst_directory, exist_ok=True) _yaml.dump(_yaml.node_sanitize(config), - os.path.join(self._project.directory, ".bst", "workspaces.yml")) + self._get_filename()) # _load_config() # @@ -319,7 +320,7 @@ class Workspaces(): # Raises: LoadError if there was a problem with the workspace config # def _load_config(self): - workspace_file = os.path.join(self._project.directory, ".bst", "workspaces.yml") + workspace_file = self._get_filename() try: node = _yaml.load(workspace_file) except LoadError as e: @@ -361,8 +362,7 @@ class Workspaces(): "This is not supported anymore.\n" + \ "Please remove this element from '{}'." raise LoadError(LoadErrorReason.INVALID_DATA, - detail.format(element, - os.path.join(self._project.directory, ".bst", "workspaces.yml"))) + detail.format(element, self._get_filename())) workspaces[element] = sources[0][1] @@ -371,13 +371,13 @@ class Workspaces(): "Workspace config is in unexpected format.") res = { - element: Workspace(self._project, path=config) + element: Workspace(self._toplevel_project, path=config) for element, config in _yaml.node_items(workspaces) } elif version >= 1 and version <= BST_WORKSPACE_FORMAT_VERSION: workspaces = _yaml.node_get(workspaces, dict, "workspaces", default_value={}) - res = {element: self._load_workspace(self._project, node) + res = {element: self._load_workspace(node) for element, node in _yaml.node_items(workspaces)} else: @@ -394,16 +394,24 @@ class Workspaces(): # # Args: # node: A YAML Node - # project (Project): The Project to load this for # # Returns: # (Workspace): A newly instantiated Workspace # - def _load_workspace(self, project, node): + def _load_workspace(self, node): dictionary = { 'prepared': _yaml.node_get(node, bool, 'prepared', default_value=False), 'path': _yaml.node_get(node, str, 'path'), 'last_successful': _yaml.node_get(node, str, 'last_successful', default_value=None), 'running_files': _yaml.node_get(node, dict, 'running_files', default_value=None), } - return Workspace.from_dict(self._project, dictionary) + return Workspace.from_dict(self._toplevel_project, dictionary) + + # _get_filename(): + # + # Get the workspaces.yml file path. + # + # Returns: + # (str): The path to workspaces.yml file. + def _get_filename(self): + return os.path.join(self._bst_directory, "workspaces.yml") -- cgit v1.2.1 From 130bfbb84e0de2c2289b9dd708b3f79682d140f4 Mon Sep 17 00:00:00 2001 From: Valentin David Date: Mon, 7 May 2018 19:02:06 +0200 Subject: Handle cross junction elements in workspaces. Workspaces are now index by colon separated junction path. This now allows to create workspaces for elements in external projects. Workspaces are owned by context instead of root project. However it is initialized once top-level project is registered as we need to resolve paths relatively to this top-level project. Part of #359. --- buildstream/_context.py | 7 +++++++ buildstream/_frontend/cli.py | 4 ++-- buildstream/_project.py | 5 ----- buildstream/_scheduler/queue.py | 7 ++++--- buildstream/_stream.py | 32 +++++++++++++++++++------------- buildstream/element.py | 12 +++++------- 6 files changed, 37 insertions(+), 30 deletions(-) (limited to 'buildstream') diff --git a/buildstream/_context.py b/buildstream/_context.py index c0d49b245..114ac9e86 100644 --- a/buildstream/_context.py +++ b/buildstream/_context.py @@ -30,6 +30,7 @@ from ._exceptions import LoadError, LoadErrorReason, BstError from ._message import Message, MessageType from ._profile import Topics, profile_start, profile_end from ._artifactcache import ArtifactCache +from ._workspaces import Workspaces # Context() @@ -113,6 +114,7 @@ class Context(): self._message_depth = deque() self._projects = [] self._project_overrides = {} + self._workspaces = None # load() # @@ -219,6 +221,8 @@ class Context(): # project (Project): The project to add # def add_project(self, project): + if not self._projects: + self._workspaces = Workspaces(project) self._projects.append(project) # get_projects(): @@ -242,6 +246,9 @@ class Context(): def get_toplevel_project(self): return self._projects[0] + def get_workspaces(self): + return self._workspaces + # get_overrides(): # # Fetch the override dictionary for the active project. This returns diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py index 143d59aa4..465124557 100644 --- a/buildstream/_frontend/cli.py +++ b/buildstream/_frontend/cli.py @@ -711,7 +711,7 @@ def workspace_close(app, remove_dir, all_, elements): sys.exit(0) if all_: - elements = [element_name for element_name, _ in app.project.workspaces.list()] + elements = [element_name for element_name, _ in app.context.get_workspaces().list()] elements = app.stream.redirect_element_names(elements) @@ -763,7 +763,7 @@ def workspace_reset(app, soft, track_, all_, elements): sys.exit(-1) if all_: - elements = tuple(element_name for element_name, _ in app.project.workspaces.list()) + elements = tuple(element_name for element_name, _ in app.context.get_workspaces().list()) app.stream.workspace_reset(elements, soft=soft, track_first=track_) diff --git a/buildstream/_project.py b/buildstream/_project.py index 25ffaf6d2..9f42bf613 100644 --- a/buildstream/_project.py +++ b/buildstream/_project.py @@ -34,7 +34,6 @@ from ._elementfactory import ElementFactory from ._sourcefactory import SourceFactory from ._projectrefs import ProjectRefs, ProjectRefStorage from ._versions import BST_FORMAT_VERSION -from ._workspaces import Workspaces # The separator we use for user specified aliases @@ -87,7 +86,6 @@ class Project(): self.refs = ProjectRefs(self.directory, 'project.refs') self.junction_refs = ProjectRefs(self.directory, 'junction.refs') - self.workspaces = None # Workspaces self.options = None # OptionPool self.junction = junction # The junction Element object, if this is a subproject self.fail_on_overlap = False # Whether overlaps are treated as errors @@ -301,9 +299,6 @@ class Project(): # Load artifacts pull/push configuration for this project self.artifact_cache_specs = ArtifactCache.specs_from_config_node(config) - # Workspace configurations - self.workspaces = Workspaces(self) - # Plugin origins and versions origins = _yaml.node_get(config, list, 'plugins', default_value=[]) for origin in origins: diff --git a/buildstream/_scheduler/queue.py b/buildstream/_scheduler/queue.py index 7c4ad6919..083822364 100644 --- a/buildstream/_scheduler/queue.py +++ b/buildstream/_scheduler/queue.py @@ -269,10 +269,11 @@ class Queue(): # Handle any workspace modifications now # if job.workspace_dict: - project = element._get_project() - if project.workspaces.update_workspace(element.name, job.workspace_dict): + context = element._get_context() + workspaces = context.get_workspaces() + if workspaces.update_workspace(element._get_full_name(), job.workspace_dict): try: - project.workspaces.save_config() + workspaces.save_config() except BstError as e: self._message(element, MessageType.ERROR, "Error saving workspaces", detail=str(e)) except Exception as e: # pylint: disable=broad-except diff --git a/buildstream/_stream.py b/buildstream/_stream.py index f2806b4c8..c2fce58c0 100644 --- a/buildstream/_stream.py +++ b/buildstream/_stream.py @@ -434,8 +434,10 @@ class Stream(): detail += " \n".join(build_depends) raise StreamError("The given element has no sources", detail=detail) + workspaces = self._context.get_workspaces() + # Check for workspace config - workspace = self._project.workspaces.get_workspace(target.name) + workspace = workspaces.get_workspace(target._get_full_name()) if workspace: raise StreamError("Workspace '{}' is already defined at: {}" .format(target.name, workspace.path)) @@ -460,13 +462,13 @@ class Stream(): except OSError as e: raise StreamError("Failed to create workspace directory: {}".format(e)) from e - self._project.workspaces.create_workspace(target.name, workdir) + workspaces.create_workspace(target._get_full_name(), workdir) if not no_checkout: with target.timed_activity("Staging sources to {}".format(directory)): target._open_workspace() - self._project.workspaces.save_config() + workspaces.save_config() self._message(MessageType.INFO, "Saved workspace configuration") # workspace_close @@ -478,7 +480,8 @@ class Stream(): # remove_dir (bool): Whether to remove the associated directory # def workspace_close(self, element_name, *, remove_dir): - workspace = self._project.workspaces.get_workspace(element_name) + workspaces = self._context.get_workspaces() + workspace = workspaces.get_workspace(element_name) # Remove workspace directory if prompted if remove_dir: @@ -491,8 +494,8 @@ class Stream(): .format(workspace.path, e)) from e # Delete the workspace and save the configuration - self._project.workspaces.delete_workspace(element_name) - self._project.workspaces.save_config() + workspaces.delete_workspace(element_name) + workspaces.save_config() self._message(MessageType.INFO, "Closed workspace for {}".format(element_name)) # workspace_reset @@ -525,8 +528,10 @@ class Stream(): if track_first: self._fetch(elements, track_elements=track_elements) + workspaces = self._context.get_workspaces() + for element in elements: - workspace = self._project.workspaces.get_workspace(element.name) + workspace = workspaces.get_workspace(element._get_full_name()) if soft: workspace.prepared = False @@ -542,15 +547,15 @@ class Stream(): raise StreamError("Could not remove '{}': {}" .format(workspace.path, e)) from e - self._project.workspaces.delete_workspace(element.name) - self._project.workspaces.create_workspace(element.name, workspace.path) + workspaces.delete_workspace(element._get_full_name()) + workspaces.create_workspace(element._get_full_name(), workspace.path) with element.timed_activity("Staging sources to {}".format(workspace.path)): element._open_workspace() self._message(MessageType.INFO, "Reset workspace for {} at: {}".format(element.name, workspace.path)) - self._project.workspaces.save_config() + workspaces.save_config() # workspace_exists # @@ -566,11 +571,12 @@ class Stream(): # True if there are any existing workspaces. # def workspace_exists(self, element_name=None): + workspaces = self._context.get_workspaces() if element_name: - workspace = self._project.workspaces.get_workspace(element_name) + workspace = workspaces.get_workspace(element_name) if workspace: return True - elif any(self._project.workspaces.list()): + elif any(workspaces.list()): return True return False @@ -581,7 +587,7 @@ class Stream(): # def workspace_list(self): workspaces = [] - for element_name, workspace_ in self._project.workspaces.list(): + for element_name, workspace_ in self._context.get_workspaces().list(): workspace_detail = { 'element': element_name, 'directory': workspace_.path, diff --git a/buildstream/element.py b/buildstream/element.py index 832f0dd93..796d79d66 100644 --- a/buildstream/element.py +++ b/buildstream/element.py @@ -673,7 +673,6 @@ class Element(Plugin): overlaps = OrderedDict() files_written = {} old_dep_keys = {} - project = self._get_project() workspace = self._get_workspace() if self.__can_build_incrementally() and workspace.last_successful: @@ -702,7 +701,7 @@ class Element(Plugin): # In case we are running `bst shell`, this happens in the # main process and we need to update the workspace config if utils._is_main_process(): - project.workspaces.save_config() + self._get_context().get_workspaces().save_config() result = dep.stage_artifact(sandbox, path=path, @@ -1393,12 +1392,11 @@ class Element(Plugin): # For this reason, it is safe to update and # save the workspaces configuration # - project = self._get_project() key = self._get_cache_key() workspace = self._get_workspace() workspace.last_successful = key workspace.clear_running_files() - project.workspaces.save_config() + self._get_context().get_workspaces().save_config() # _assemble(): # @@ -1763,8 +1761,8 @@ class Element(Plugin): # (Workspace|None): A workspace associated with this element # def _get_workspace(self): - project = self._get_project() - return project.workspaces.get_workspace(self.name) + workspaces = self._get_context().get_workspaces() + return workspaces.get_workspace(self._get_full_name()) # _write_script(): # @@ -1932,7 +1930,7 @@ class Element(Plugin): 'execution-environment': self.__sandbox_config.get_unique_key(), 'environment': cache_env, 'sources': [s._get_unique_key(workspace is None) for s in self.__sources], - 'workspace': '' if workspace is None else workspace.get_key(), + 'workspace': '' if workspace is None else workspace.get_key(self._get_project()), 'public': self.__public, 'cache': type(self.__artifacts).__name__ } -- cgit v1.2.1 From acde3ba8fa09605775a6627398bc2af26658d69a Mon Sep 17 00:00:00 2001 From: Valentin David Date: Mon, 14 May 2018 18:48:32 +0200 Subject: Allow tracking dependencies within sub-projects. --track-cross-junctions now concerns crossing junctions rather than forbidding elements in sub-project to be tracked. Part of #359. --- buildstream/_pipeline.py | 16 +++++++++------- buildstream/_stream.py | 24 +++++++++++++++++++++--- 2 files changed, 30 insertions(+), 10 deletions(-) (limited to 'buildstream') diff --git a/buildstream/_pipeline.py b/buildstream/_pipeline.py index ba27ca6b6..a280c22ee 100644 --- a/buildstream/_pipeline.py +++ b/buildstream/_pipeline.py @@ -346,6 +346,8 @@ class Pipeline(): # lists targetted at tracking. # # Args: + # project (Project): Project used for cross_junction filtering. + # All elements are expected to belong to that project. # elements (list of Element): The list of elements to filter # cross_junction_requested (bool): Whether the user requested # cross junction tracking @@ -353,12 +355,11 @@ class Pipeline(): # Returns: # (list of Element): The filtered or asserted result # - def track_cross_junction_filter(self, elements, cross_junction_requested): + def track_cross_junction_filter(self, project, elements, cross_junction_requested): # Filter out cross junctioned elements - if cross_junction_requested: - self._assert_junction_tracking(elements) - else: - elements = self._filter_cross_junctions(elements) + if not cross_junction_requested: + elements = self._filter_cross_junctions(project, elements) + self._assert_junction_tracking(elements) return elements @@ -403,16 +404,17 @@ class Pipeline(): # Filters out cross junction elements from the elements # # Args: + # project (Project): The project on which elements are allowed # elements (list of Element): The list of elements to be tracked # # Returns: # (list): A filtered list of `elements` which does # not contain any cross junction elements. # - def _filter_cross_junctions(self, elements): + def _filter_cross_junctions(self, project, elements): return [ element for element in elements - if element._get_project() is self._project + if element._get_project() is project ] # _assert_junction_tracking() diff --git a/buildstream/_stream.py b/buildstream/_stream.py index c2fce58c0..0680f2a1f 100644 --- a/buildstream/_stream.py +++ b/buildstream/_stream.py @@ -831,12 +831,30 @@ class Stream(): # done before resolving element states. # assert track_selection != PipelineSelection.PLAN - track_selected = self._pipeline.get_selection(track_elements, track_selection) + + # Tracked elements are split by owner projects in order to + # filter cross junctions tracking dependencies on their + # respective project. + track_projects = {} + for element in track_elements: + project = element._get_project() + if project not in track_projects: + track_projects[project] = [element] + else: + track_projects[project].append(element) + + track_selected = [] + + for project, project_elements in track_projects.items(): + selected = self._pipeline.get_selection(project_elements, track_selection) + selected = self._pipeline.track_cross_junction_filter(project, + selected, + track_cross_junctions) + track_selected.extend(selected) + track_selected = self._pipeline.except_elements(track_elements, track_selected, track_except_elements) - track_selected = self._pipeline.track_cross_junction_filter(track_selected, - track_cross_junctions) for element in track_selected: element._schedule_tracking() -- cgit v1.2.1 From 055b77e8e9b7ab6ffdfcad8d01c0f20737dbd3dd Mon Sep 17 00:00:00 2001 From: Tristan Van Berkom Date: Sat, 9 Jun 2018 17:14:10 -0400 Subject: doc: Added plugins as ToC elements instead of orphaned links o Now the page titles are declared in plugins, allowing for a more descriptive ToC o Makefile and plugin.rsttemplate updated to not produce the title, to no longer use `:orphan:` for plugin pages, and to ignore any private modules in the plugin directories. o Interestingly, now the docs will fail to build if you add a new plugin and forget to add it to the documentation. --- buildstream/plugins/elements/autotools.py | 5 +++-- buildstream/plugins/elements/cmake.py | 5 +++-- buildstream/plugins/elements/compose.py | 5 +++-- buildstream/plugins/elements/distutils.py | 5 +++-- buildstream/plugins/elements/filter.py | 5 +++-- buildstream/plugins/elements/import.py | 5 +++-- buildstream/plugins/elements/junction.py | 5 +++-- buildstream/plugins/elements/make.py | 5 +++-- buildstream/plugins/elements/makemaker.py | 5 +++-- buildstream/plugins/elements/manual.py | 5 +++-- buildstream/plugins/elements/meson.py | 5 +++-- buildstream/plugins/elements/modulebuild.py | 5 +++-- buildstream/plugins/elements/pip.py | 5 +++-- buildstream/plugins/elements/qmake.py | 5 +++-- buildstream/plugins/elements/script.py | 5 +++-- buildstream/plugins/elements/stack.py | 5 +++-- buildstream/plugins/sources/bzr.py | 4 +++- buildstream/plugins/sources/deb.py | 4 +++- buildstream/plugins/sources/git.py | 4 +++- buildstream/plugins/sources/local.py | 4 +++- buildstream/plugins/sources/ostree.py | 4 +++- buildstream/plugins/sources/patch.py | 4 +++- buildstream/plugins/sources/tar.py | 5 ++++- buildstream/plugins/sources/zip.py | 4 +++- 24 files changed, 73 insertions(+), 40 deletions(-) (limited to 'buildstream') diff --git a/buildstream/plugins/elements/autotools.py b/buildstream/plugins/elements/autotools.py index 5f54c3953..93fc6be87 100644 --- a/buildstream/plugins/elements/autotools.py +++ b/buildstream/plugins/elements/autotools.py @@ -18,8 +18,9 @@ # Authors: # Tristan Van Berkom -"""Autotools build element - +""" +autotools - Autotools build element +=================================== This is a :mod:`BuildElement ` implementation for using Autotools build scripts (also known as the `GNU Build System `_). diff --git a/buildstream/plugins/elements/cmake.py b/buildstream/plugins/elements/cmake.py index 292785e81..0b3de0143 100644 --- a/buildstream/plugins/elements/cmake.py +++ b/buildstream/plugins/elements/cmake.py @@ -18,8 +18,9 @@ # Authors: # Tristan Van Berkom -"""CMake build element - +""" +cmake - CMake build element +=========================== This is a :mod:`BuildElement ` implementation for using the `CMake `_ build system. diff --git a/buildstream/plugins/elements/compose.py b/buildstream/plugins/elements/compose.py index 0e666c6e5..0769b8814 100644 --- a/buildstream/plugins/elements/compose.py +++ b/buildstream/plugins/elements/compose.py @@ -18,8 +18,9 @@ # Authors: # Tristan Van Berkom -"""Compose element - +""" +compose - Compose the output of multiple elements +================================================= This element creates a selective composition of its dependencies. This is normally used at near the end of a pipeline to prepare diff --git a/buildstream/plugins/elements/distutils.py b/buildstream/plugins/elements/distutils.py index 948e08b62..30ba6892c 100644 --- a/buildstream/plugins/elements/distutils.py +++ b/buildstream/plugins/elements/distutils.py @@ -18,8 +18,9 @@ # Authors: # Tristan Van Berkom -"""Python Distutils element - +""" +distutils - Python distutils element +==================================== A :mod:`BuildElement ` implementation for using python distutils diff --git a/buildstream/plugins/elements/filter.py b/buildstream/plugins/elements/filter.py index 8ce16ff9f..ee9143ba7 100644 --- a/buildstream/plugins/elements/filter.py +++ b/buildstream/plugins/elements/filter.py @@ -18,8 +18,9 @@ # Authors: # Jonathan Maw -"""Filter element - +""" +filter - Extract a subset of files from another element +======================================================= This filters another element by producing an output that is a subset of the filtered element. diff --git a/buildstream/plugins/elements/import.py b/buildstream/plugins/elements/import.py index 747455d70..c19a8bc36 100644 --- a/buildstream/plugins/elements/import.py +++ b/buildstream/plugins/elements/import.py @@ -18,8 +18,9 @@ # Authors: # Tristan Van Berkom -"""Import element - +""" +import - Import sources directly +================================ Import elements produce artifacts directly from its sources without any kind of processing. These are typically used to import an SDK to build on top of or to overlay your build with diff --git a/buildstream/plugins/elements/junction.py b/buildstream/plugins/elements/junction.py index 81fd57445..512c862ff 100644 --- a/buildstream/plugins/elements/junction.py +++ b/buildstream/plugins/elements/junction.py @@ -18,8 +18,9 @@ # Authors: # Jürg Billeter -"""Junction element - +""" +junction - Integrate subprojects +================================ This element is a link to another BuildStream project. It allows integration of multiple projects into a single pipeline. diff --git a/buildstream/plugins/elements/make.py b/buildstream/plugins/elements/make.py index 37024926c..d8dff4c41 100644 --- a/buildstream/plugins/elements/make.py +++ b/buildstream/plugins/elements/make.py @@ -18,8 +18,9 @@ # Authors: # Ed Baunton -"""Make build element - +""" +make - Make build element +========================= This is a :mod:`BuildElement ` implementation for using GNU make based build. diff --git a/buildstream/plugins/elements/makemaker.py b/buildstream/plugins/elements/makemaker.py index 94d459f93..eb4d96930 100644 --- a/buildstream/plugins/elements/makemaker.py +++ b/buildstream/plugins/elements/makemaker.py @@ -18,8 +18,9 @@ # Authors: # Tristan Van Berkom -"""Perl MakeMaker build element - +""" +makemaker - Perl MakeMaker build element +======================================== A :mod:`BuildElement ` implementation for using the Perl ExtUtil::MakeMaker build system diff --git a/buildstream/plugins/elements/manual.py b/buildstream/plugins/elements/manual.py index 998394b05..dbb799fb4 100644 --- a/buildstream/plugins/elements/manual.py +++ b/buildstream/plugins/elements/manual.py @@ -18,8 +18,9 @@ # Authors: # Tristan Van Berkom -"""Manual build element - +""" +manual - Manual build element +============================= The most basic build element does nothing but allows users to add custom build commands to the array understood by the :mod:`BuildElement ` diff --git a/buildstream/plugins/elements/meson.py b/buildstream/plugins/elements/meson.py index 2b7b7831a..228e90ad1 100644 --- a/buildstream/plugins/elements/meson.py +++ b/buildstream/plugins/elements/meson.py @@ -14,8 +14,9 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see . -"""Meson build element - +""" +meson - Meson build element +=========================== This is a :mod:`BuildElement ` implementation for using `Meson `_ build scripts. diff --git a/buildstream/plugins/elements/modulebuild.py b/buildstream/plugins/elements/modulebuild.py index c790cafb3..897430bae 100644 --- a/buildstream/plugins/elements/modulebuild.py +++ b/buildstream/plugins/elements/modulebuild.py @@ -18,8 +18,9 @@ # Authors: # Tristan Van Berkom -"""Perl Module::Build build element - +""" +modulebuild - Perl Module::Build build element +============================================== A :mod:`BuildElement ` implementation for using the Perl Module::Build build system diff --git a/buildstream/plugins/elements/pip.py b/buildstream/plugins/elements/pip.py index b979a6d21..e2f33b207 100644 --- a/buildstream/plugins/elements/pip.py +++ b/buildstream/plugins/elements/pip.py @@ -18,8 +18,9 @@ # Authors: # Mathieu Bridon -"""Pip build element - +""" +pip - Pip build element +======================= A :mod:`BuildElement ` implementation for installing Python modules with pip diff --git a/buildstream/plugins/elements/qmake.py b/buildstream/plugins/elements/qmake.py index ab5843d8b..b0cd3f178 100644 --- a/buildstream/plugins/elements/qmake.py +++ b/buildstream/plugins/elements/qmake.py @@ -18,8 +18,9 @@ # Authors: # Tristan Van Berkom -"""CMake build element - +""" +qmake - QMake build element +=========================== A :mod:`BuildElement ` implementation for using the qmake build system diff --git a/buildstream/plugins/elements/script.py b/buildstream/plugins/elements/script.py index 6778b3fac..fbc15bd78 100644 --- a/buildstream/plugins/elements/script.py +++ b/buildstream/plugins/elements/script.py @@ -19,8 +19,9 @@ # Tristan Van Berkom # Jonathan Maw -"""Script element - +""" +script - Run scripts to create output +===================================== This element allows one to run some commands to mutate the input and create some output. diff --git a/buildstream/plugins/elements/stack.py b/buildstream/plugins/elements/stack.py index 45c49c514..f6903c933 100644 --- a/buildstream/plugins/elements/stack.py +++ b/buildstream/plugins/elements/stack.py @@ -18,8 +18,9 @@ # Authors: # Tristan Van Berkom -"""Stack element - +""" +stack - Symbolic Element for dependency grouping +================================================ Stack elements are simply a symbolic element used for representing a logical group of elements. """ diff --git a/buildstream/plugins/sources/bzr.py b/buildstream/plugins/sources/bzr.py index 3732304fb..b499d49d3 100644 --- a/buildstream/plugins/sources/bzr.py +++ b/buildstream/plugins/sources/bzr.py @@ -17,7 +17,9 @@ # Authors: # Jonathan Maw -"""A source implementation for staging bazaar branches +""" +bzr - stage files from a bazaar repository +========================================== **Usage:** diff --git a/buildstream/plugins/sources/deb.py b/buildstream/plugins/sources/deb.py index daf6f94c7..4948d1933 100644 --- a/buildstream/plugins/sources/deb.py +++ b/buildstream/plugins/sources/deb.py @@ -19,7 +19,9 @@ # Jonathan Maw # Richard Maw -"""A source implementation for staging deb files +""" +deb - stage files from .deb packages +==================================== **Usage:** diff --git a/buildstream/plugins/sources/git.py b/buildstream/plugins/sources/git.py index f178656b0..44065ad8f 100644 --- a/buildstream/plugins/sources/git.py +++ b/buildstream/plugins/sources/git.py @@ -18,7 +18,9 @@ # Authors: # Tristan Van Berkom -"""A Source implementation for staging git checkouts +""" +git - stage files from a git repository +======================================= **Usage:** diff --git a/buildstream/plugins/sources/local.py b/buildstream/plugins/sources/local.py index 3193d101d..673add1bf 100644 --- a/buildstream/plugins/sources/local.py +++ b/buildstream/plugins/sources/local.py @@ -18,7 +18,9 @@ # Authors: # Tristan Van Berkom -"""A Source implementation for staging local project files +""" +local - stage local files and directories +========================================= **Usage:** diff --git a/buildstream/plugins/sources/ostree.py b/buildstream/plugins/sources/ostree.py index b311e24bf..c77b3a77f 100644 --- a/buildstream/plugins/sources/ostree.py +++ b/buildstream/plugins/sources/ostree.py @@ -18,7 +18,9 @@ # Authors: # Andrew Leeming -"""A Source implementation for importing/staging of OSTree checkouts. +""" +ostree - stage files from an OSTree repository +============================================== **Usage:** diff --git a/buildstream/plugins/sources/patch.py b/buildstream/plugins/sources/patch.py index c9e40b1e6..88fb2d59c 100644 --- a/buildstream/plugins/sources/patch.py +++ b/buildstream/plugins/sources/patch.py @@ -18,7 +18,9 @@ # Authors: # Chandan Singh -"""A Source implementation for applying local patches +""" +patch - apply locally stored patches +==================================== **Usage:** diff --git a/buildstream/plugins/sources/tar.py b/buildstream/plugins/sources/tar.py index e41824505..324006b21 100644 --- a/buildstream/plugins/sources/tar.py +++ b/buildstream/plugins/sources/tar.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# # Copyright (C) 2017 Codethink Limited # # This program is free software; you can redistribute it and/or @@ -17,7 +18,9 @@ # Authors: # Jonathan Maw -"""A source implementation for staging tar files +""" +tar - stage files from tar archives +=================================== **Usage:** diff --git a/buildstream/plugins/sources/zip.py b/buildstream/plugins/sources/zip.py index fdf8947ec..f167c3e63 100644 --- a/buildstream/plugins/sources/zip.py +++ b/buildstream/plugins/sources/zip.py @@ -18,7 +18,9 @@ # Authors: # Mathieu Bridon -"""A source implementation for staging zip files +""" +zip - stage files from zip archives +=================================== **Usage:** -- cgit v1.2.1 From 96c219d0f39be6a50badcd11abaf93d05ef0a072 Mon Sep 17 00:00:00 2001 From: Tristan Van Berkom Date: Sat, 9 Jun 2018 17:36:07 -0400 Subject: doc/source/core_framework.rst: Use a toctree instead of links --- buildstream/buildelement.py | 4 ++-- buildstream/element.py | 4 ++-- buildstream/plugin.py | 4 ++-- buildstream/sandbox/sandbox.py | 5 ++--- buildstream/scriptelement.py | 5 ++--- buildstream/source.py | 4 ++-- 6 files changed, 12 insertions(+), 14 deletions(-) (limited to 'buildstream') diff --git a/buildstream/buildelement.py b/buildstream/buildelement.py index 5f9a856d1..20c5e620f 100644 --- a/buildstream/buildelement.py +++ b/buildstream/buildelement.py @@ -18,8 +18,8 @@ # Authors: # Tristan Van Berkom """ -BuildElement -============ +BuildElement - Abstract class for build elements +================================================ The BuildElement class is a convenience element one can derive from for implementing the most common case of element. diff --git a/buildstream/element.py b/buildstream/element.py index 796d79d66..e876eb120 100644 --- a/buildstream/element.py +++ b/buildstream/element.py @@ -19,8 +19,8 @@ # Tristan Van Berkom """ -Element -======= +Element - Base element class +============================ .. _core_element_abstract_methods: diff --git a/buildstream/plugin.py b/buildstream/plugin.py index b208e2d06..0c216ed5d 100644 --- a/buildstream/plugin.py +++ b/buildstream/plugin.py @@ -18,8 +18,8 @@ # Authors: # Tristan Van Berkom """ -Plugin -====== +Plugin - Base plugin class +========================== BuildStream supports third party plugins to define additional kinds of :mod:`Elements ` and :mod:`Sources `. diff --git a/buildstream/sandbox/sandbox.py b/buildstream/sandbox/sandbox.py index 3ab75f1a8..c174c4adb 100644 --- a/buildstream/sandbox/sandbox.py +++ b/buildstream/sandbox/sandbox.py @@ -19,9 +19,8 @@ # Andrew Leeming # Tristan Van Berkom """ -Sandbox -======= - +Sandbox - The build sandbox +=========================== :class:`.Element` plugins which want to interface with the sandbox need only understand this interface, while it may be given a different sandbox implementation, any sandbox implementation it is given will diff --git a/buildstream/scriptelement.py b/buildstream/scriptelement.py index 95e6928ee..a0b12eef5 100644 --- a/buildstream/scriptelement.py +++ b/buildstream/scriptelement.py @@ -19,9 +19,8 @@ # Jonathan Maw """ -ScriptElement -============= - +ScriptElement - Abstract class for scripting elements +===================================================== The ScriptElement class is a convenience class one can derive for implementing elements that stage elements and run command-lines on them. diff --git a/buildstream/source.py b/buildstream/source.py index fa547d641..470a407ad 100644 --- a/buildstream/source.py +++ b/buildstream/source.py @@ -18,8 +18,8 @@ # Authors: # Tristan Van Berkom """ -Source -====== +Source - base source class +========================== .. _core_source_abstract_methods: -- cgit v1.2.1 From 9f77cb66cf4be2b09df48469c05735f3c5efce76 Mon Sep 17 00:00:00 2001 From: Tristan Maat Date: Fri, 8 Jun 2018 13:46:40 +0100 Subject: Handle missing tags in git repositories correctly Fixes issue #380 --- buildstream/utils.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'buildstream') diff --git a/buildstream/utils.py b/buildstream/utils.py index 70759dc12..af5bac3fb 100644 --- a/buildstream/utils.py +++ b/buildstream/utils.py @@ -472,6 +472,10 @@ def get_bst_version(): from . import __version__ versions = __version__.split('.')[:2] + if versions[0] == '0+untagged': + raise UtilError("Your git repository has no tags - BuildStream can't " + "determine its version. Please run `git fetch --tags`.") + return (int(versions[0]), int(versions[1])) -- cgit v1.2.1 From 9d7296eb84d604a02e5e9f989a09cb30994503c1 Mon Sep 17 00:00:00 2001 From: Tristan Van Berkom Date: Tue, 12 Jun 2018 23:20:14 -0400 Subject: doc: Overhaul of page names and titles o Giving main pages simple word titles This makes the main page: * About * Installing * Using * Reference * Contributing o Now named all rst files with their parent page name as a prefix. o Also changed some titles to make overall consistent titles. --- buildstream/source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'buildstream') diff --git a/buildstream/source.py b/buildstream/source.py index 470a407ad..387ed6a99 100644 --- a/buildstream/source.py +++ b/buildstream/source.py @@ -18,7 +18,7 @@ # Authors: # Tristan Van Berkom """ -Source - base source class +Source - Base source class ========================== -- cgit v1.2.1 From 4016bec1d747481ca8354c1c798fba0271d4696b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6k=C3=A7en=20Nurlu?= Date: Mon, 18 Jun 2018 16:27:35 +0100 Subject: Remove shebangs from python files Fixes #424 --- buildstream/__init__.py | 1 - buildstream/_artifactcache/__init__.py | 1 - buildstream/_artifactcache/artifactcache.py | 1 - buildstream/_artifactcache/ostreecache.py | 1 - buildstream/_artifactcache/tarcache.py | 1 - buildstream/_cachekey.py | 1 - buildstream/_context.py | 1 - buildstream/_elementfactory.py | 1 - buildstream/_exceptions.py | 1 - buildstream/_frontend/__init__.py | 1 - buildstream/_frontend/app.py | 1 - buildstream/_frontend/complete.py | 1 - buildstream/_frontend/linuxapp.py | 1 - buildstream/_frontend/profile.py | 1 - buildstream/_frontend/status.py | 1 - buildstream/_frontend/widget.py | 1 - buildstream/_fuse/__init__.py | 1 - buildstream/_fuse/hardlinks.py | 1 - buildstream/_fuse/mount.py | 1 - buildstream/_loader/__init__.py | 1 - buildstream/_loader/loadelement.py | 1 - buildstream/_loader/loader.py | 1 - buildstream/_loader/metaelement.py | 1 - buildstream/_loader/metasource.py | 1 - buildstream/_loader/types.py | 1 - buildstream/_message.py | 1 - buildstream/_options/__init__.py | 1 - buildstream/_options/option.py | 1 - buildstream/_options/optionarch.py | 1 - buildstream/_options/optionbool.py | 1 - buildstream/_options/optioneltmask.py | 1 - buildstream/_options/optionenum.py | 1 - buildstream/_options/optionflags.py | 1 - buildstream/_options/optionpool.py | 1 - buildstream/_ostree.py | 1 - buildstream/_pipeline.py | 1 - buildstream/_platform/__init__.py | 1 - buildstream/_platform/linux.py | 1 - buildstream/_platform/platform.py | 1 - buildstream/_platform/unix.py | 1 - buildstream/_plugincontext.py | 1 - buildstream/_profile.py | 1 - buildstream/_project.py | 1 - buildstream/_projectrefs.py | 1 - buildstream/_scheduler/__init__.py | 1 - buildstream/_scheduler/buildqueue.py | 1 - buildstream/_scheduler/fetchqueue.py | 1 - buildstream/_scheduler/job.py | 1 - buildstream/_scheduler/pullqueue.py | 1 - buildstream/_scheduler/pushqueue.py | 1 - buildstream/_scheduler/queue.py | 1 - buildstream/_scheduler/scheduler.py | 1 - buildstream/_scheduler/trackqueue.py | 1 - buildstream/_signals.py | 1 - buildstream/_site.py | 1 - buildstream/_sourcefactory.py | 1 - buildstream/_stream.py | 1 - buildstream/_variables.py | 1 - buildstream/_versions.py | 1 - buildstream/_workspaces.py | 1 - buildstream/_yaml.py | 1 - buildstream/buildelement.py | 1 - buildstream/element.py | 1 - buildstream/plugin.py | 1 - buildstream/plugins/elements/autotools.py | 1 - buildstream/plugins/elements/cmake.py | 1 - buildstream/plugins/elements/compose.py | 1 - buildstream/plugins/elements/distutils.py | 1 - buildstream/plugins/elements/filter.py | 1 - buildstream/plugins/elements/import.py | 1 - buildstream/plugins/elements/junction.py | 1 - buildstream/plugins/elements/make.py | 1 - buildstream/plugins/elements/makemaker.py | 1 - buildstream/plugins/elements/manual.py | 1 - buildstream/plugins/elements/modulebuild.py | 1 - buildstream/plugins/elements/pip.py | 1 - buildstream/plugins/elements/qmake.py | 1 - buildstream/plugins/elements/script.py | 1 - buildstream/plugins/elements/stack.py | 1 - buildstream/plugins/sources/bzr.py | 1 - buildstream/plugins/sources/deb.py | 1 - buildstream/plugins/sources/git.py | 1 - buildstream/plugins/sources/local.py | 1 - buildstream/plugins/sources/ostree.py | 1 - buildstream/plugins/sources/patch.py | 1 - buildstream/plugins/sources/tar.py | 1 - buildstream/plugins/sources/zip.py | 1 - buildstream/sandbox/__init__.py | 1 - buildstream/sandbox/_config.py | 1 - buildstream/sandbox/_mount.py | 1 - buildstream/sandbox/_mounter.py | 1 - buildstream/sandbox/_sandboxbwrap.py | 1 - buildstream/sandbox/_sandboxchroot.py | 1 - buildstream/sandbox/sandbox.py | 1 - buildstream/scriptelement.py | 1 - buildstream/source.py | 1 - buildstream/utils.py | 1 - 97 files changed, 97 deletions(-) (limited to 'buildstream') diff --git a/buildstream/__init__.py b/buildstream/__init__.py index 7c4b5e5e6..cf56ecfe1 100644 --- a/buildstream/__init__.py +++ b/buildstream/__init__.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016 Codethink Limited # diff --git a/buildstream/_artifactcache/__init__.py b/buildstream/_artifactcache/__init__.py index 66373fd75..07ed52b4b 100644 --- a/buildstream/_artifactcache/__init__.py +++ b/buildstream/_artifactcache/__init__.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017-2018 Codethink Limited # diff --git a/buildstream/_artifactcache/artifactcache.py b/buildstream/_artifactcache/artifactcache.py index 9cc281524..2d745f8c2 100644 --- a/buildstream/_artifactcache/artifactcache.py +++ b/buildstream/_artifactcache/artifactcache.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017-2018 Codethink Limited # diff --git a/buildstream/_artifactcache/ostreecache.py b/buildstream/_artifactcache/ostreecache.py index 9fc12f8d0..707a974eb 100644 --- a/buildstream/_artifactcache/ostreecache.py +++ b/buildstream/_artifactcache/ostreecache.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017-2018 Codethink Limited # diff --git a/buildstream/_artifactcache/tarcache.py b/buildstream/_artifactcache/tarcache.py index 10ae9d008..ab814abb0 100644 --- a/buildstream/_artifactcache/tarcache.py +++ b/buildstream/_artifactcache/tarcache.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Codethink Limited # diff --git a/buildstream/_cachekey.py b/buildstream/_cachekey.py index 3d0c19b44..fe407e96f 100644 --- a/buildstream/_cachekey.py +++ b/buildstream/_cachekey.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2018 Codethink Limited # diff --git a/buildstream/_context.py b/buildstream/_context.py index 114ac9e86..1a59af2b9 100644 --- a/buildstream/_context.py +++ b/buildstream/_context.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016-2018 Codethink Limited # diff --git a/buildstream/_elementfactory.py b/buildstream/_elementfactory.py index bc1d95082..9d5b258cb 100644 --- a/buildstream/_elementfactory.py +++ b/buildstream/_elementfactory.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016 Codethink Limited # diff --git a/buildstream/_exceptions.py b/buildstream/_exceptions.py index bcea65a8d..3aadd526d 100644 --- a/buildstream/_exceptions.py +++ b/buildstream/_exceptions.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016 Codethink Limited # diff --git a/buildstream/_frontend/__init__.py b/buildstream/_frontend/__init__.py index b2e41301e..febd4979d 100644 --- a/buildstream/_frontend/__init__.py +++ b/buildstream/_frontend/__init__.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Codethink Limited # diff --git a/buildstream/_frontend/app.py b/buildstream/_frontend/app.py index d30b59265..4675b0eb0 100644 --- a/buildstream/_frontend/app.py +++ b/buildstream/_frontend/app.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016-2018 Codethink Limited # diff --git a/buildstream/_frontend/complete.py b/buildstream/_frontend/complete.py index 4a367e62a..79bb92758 100644 --- a/buildstream/_frontend/complete.py +++ b/buildstream/_frontend/complete.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016 Codethink Limited # diff --git a/buildstream/_frontend/linuxapp.py b/buildstream/_frontend/linuxapp.py index 92586bc40..176c5d052 100644 --- a/buildstream/_frontend/linuxapp.py +++ b/buildstream/_frontend/linuxapp.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2018 Codethink Limited # diff --git a/buildstream/_frontend/profile.py b/buildstream/_frontend/profile.py index 00a5980d3..dda0f7ffe 100644 --- a/buildstream/_frontend/profile.py +++ b/buildstream/_frontend/profile.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2018 Codethink Limited # diff --git a/buildstream/_frontend/status.py b/buildstream/_frontend/status.py index 0e5855181..3f66e009a 100644 --- a/buildstream/_frontend/status.py +++ b/buildstream/_frontend/status.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2018 Codethink Limited # diff --git a/buildstream/_frontend/widget.py b/buildstream/_frontend/widget.py index fe7229e8a..dab8cab56 100644 --- a/buildstream/_frontend/widget.py +++ b/buildstream/_frontend/widget.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Codethink Limited # diff --git a/buildstream/_fuse/__init__.py b/buildstream/_fuse/__init__.py index 3ef9d631f..a5e882634 100644 --- a/buildstream/_fuse/__init__.py +++ b/buildstream/_fuse/__init__.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Codethink Limited # diff --git a/buildstream/_fuse/hardlinks.py b/buildstream/_fuse/hardlinks.py index d23f3fff6..4da51bb22 100644 --- a/buildstream/_fuse/hardlinks.py +++ b/buildstream/_fuse/hardlinks.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016 Stavros Korokithakis # Copyright (C) 2017 Codethink Limited diff --git a/buildstream/_fuse/mount.py b/buildstream/_fuse/mount.py index 3848ad305..0ab1ce715 100644 --- a/buildstream/_fuse/mount.py +++ b/buildstream/_fuse/mount.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Codethink Limited # diff --git a/buildstream/_loader/__init__.py b/buildstream/_loader/__init__.py index dbc89ba83..a2c31796e 100644 --- a/buildstream/_loader/__init__.py +++ b/buildstream/_loader/__init__.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2018 Codethink Limited # diff --git a/buildstream/_loader/loadelement.py b/buildstream/_loader/loadelement.py index b270fbef8..065364a87 100644 --- a/buildstream/_loader/loadelement.py +++ b/buildstream/_loader/loadelement.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016 Codethink Limited # diff --git a/buildstream/_loader/loader.py b/buildstream/_loader/loader.py index 9e4406bc6..42e7feef5 100644 --- a/buildstream/_loader/loader.py +++ b/buildstream/_loader/loader.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2018 Codethink Limited # diff --git a/buildstream/_loader/metaelement.py b/buildstream/_loader/metaelement.py index 7ba6ed0ed..16788e92b 100644 --- a/buildstream/_loader/metaelement.py +++ b/buildstream/_loader/metaelement.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016 Codethink Limited # diff --git a/buildstream/_loader/metasource.py b/buildstream/_loader/metasource.py index 75e191595..3bcc21ec6 100644 --- a/buildstream/_loader/metasource.py +++ b/buildstream/_loader/metasource.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016 Codethink Limited # diff --git a/buildstream/_loader/types.py b/buildstream/_loader/types.py index 9d96894c0..000925a6e 100644 --- a/buildstream/_loader/types.py +++ b/buildstream/_loader/types.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2018 Codethink Limited # diff --git a/buildstream/_message.py b/buildstream/_message.py index 9073e3803..32650450a 100644 --- a/buildstream/_message.py +++ b/buildstream/_message.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Codethink Limited # diff --git a/buildstream/_options/__init__.py b/buildstream/_options/__init__.py index 7b8f36553..70bbe35aa 100644 --- a/buildstream/_options/__init__.py +++ b/buildstream/_options/__init__.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Codethink Limited # diff --git a/buildstream/_options/option.py b/buildstream/_options/option.py index 9501a2bde..ffdb4d272 100644 --- a/buildstream/_options/option.py +++ b/buildstream/_options/option.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Codethink Limited # diff --git a/buildstream/_options/optionarch.py b/buildstream/_options/optionarch.py index 2ced60935..13a691643 100644 --- a/buildstream/_options/optionarch.py +++ b/buildstream/_options/optionarch.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Codethink Limited # diff --git a/buildstream/_options/optionbool.py b/buildstream/_options/optionbool.py index e0e1474d9..ffef55ca1 100644 --- a/buildstream/_options/optionbool.py +++ b/buildstream/_options/optionbool.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Codethink Limited # diff --git a/buildstream/_options/optioneltmask.py b/buildstream/_options/optioneltmask.py index 46c7fcd62..09c2ce8c2 100644 --- a/buildstream/_options/optioneltmask.py +++ b/buildstream/_options/optioneltmask.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Codethink Limited # diff --git a/buildstream/_options/optionenum.py b/buildstream/_options/optionenum.py index bc21bd81c..095b9c356 100644 --- a/buildstream/_options/optionenum.py +++ b/buildstream/_options/optionenum.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Codethink Limited # diff --git a/buildstream/_options/optionflags.py b/buildstream/_options/optionflags.py index 84ecc1360..0271208d9 100644 --- a/buildstream/_options/optionflags.py +++ b/buildstream/_options/optionflags.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Codethink Limited # diff --git a/buildstream/_options/optionpool.py b/buildstream/_options/optionpool.py index 70acd268e..f90fd820c 100644 --- a/buildstream/_options/optionpool.py +++ b/buildstream/_options/optionpool.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Codethink Limited # diff --git a/buildstream/_ostree.py b/buildstream/_ostree.py index e40df5fb5..238c6b4db 100644 --- a/buildstream/_ostree.py +++ b/buildstream/_ostree.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Codethink Limited # diff --git a/buildstream/_pipeline.py b/buildstream/_pipeline.py index a280c22ee..9f4504d3f 100644 --- a/buildstream/_pipeline.py +++ b/buildstream/_pipeline.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016-2018 Codethink Limited # diff --git a/buildstream/_platform/__init__.py b/buildstream/_platform/__init__.py index 49400c3f2..29a29894b 100644 --- a/buildstream/_platform/__init__.py +++ b/buildstream/_platform/__init__.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Codethink Limited # diff --git a/buildstream/_platform/linux.py b/buildstream/_platform/linux.py index 26dafb995..fec512b0a 100644 --- a/buildstream/_platform/linux.py +++ b/buildstream/_platform/linux.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Codethink Limited # diff --git a/buildstream/_platform/platform.py b/buildstream/_platform/platform.py index fc6a74bdd..29da33563 100644 --- a/buildstream/_platform/platform.py +++ b/buildstream/_platform/platform.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Codethink Limited # diff --git a/buildstream/_platform/unix.py b/buildstream/_platform/unix.py index 6d7b46374..8b1d2ece7 100644 --- a/buildstream/_platform/unix.py +++ b/buildstream/_platform/unix.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Codethink Limited # diff --git a/buildstream/_plugincontext.py b/buildstream/_plugincontext.py index 0be3de1b1..38d2231ba 100644 --- a/buildstream/_plugincontext.py +++ b/buildstream/_plugincontext.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016 Codethink Limited # diff --git a/buildstream/_profile.py b/buildstream/_profile.py index 4d39cfc5e..40cd2ab7e 100644 --- a/buildstream/_profile.py +++ b/buildstream/_profile.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Codethink Limited # diff --git a/buildstream/_project.py b/buildstream/_project.py index 9f42bf613..b568cf852 100644 --- a/buildstream/_project.py +++ b/buildstream/_project.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016-2018 Codethink Limited # diff --git a/buildstream/_projectrefs.py b/buildstream/_projectrefs.py index 83dd2619b..4009d7449 100644 --- a/buildstream/_projectrefs.py +++ b/buildstream/_projectrefs.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2018 Codethink Limited # diff --git a/buildstream/_scheduler/__init__.py b/buildstream/_scheduler/__init__.py index 14cdebf8e..80523db6f 100644 --- a/buildstream/_scheduler/__init__.py +++ b/buildstream/_scheduler/__init__.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Codethink Limited # diff --git a/buildstream/_scheduler/buildqueue.py b/buildstream/_scheduler/buildqueue.py index 24a124b32..50ba312ff 100644 --- a/buildstream/_scheduler/buildqueue.py +++ b/buildstream/_scheduler/buildqueue.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016 Codethink Limited # diff --git a/buildstream/_scheduler/fetchqueue.py b/buildstream/_scheduler/fetchqueue.py index 61055725d..24512bddb 100644 --- a/buildstream/_scheduler/fetchqueue.py +++ b/buildstream/_scheduler/fetchqueue.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016 Codethink Limited # diff --git a/buildstream/_scheduler/job.py b/buildstream/_scheduler/job.py index b8b81f2a9..8b9af9393 100644 --- a/buildstream/_scheduler/job.py +++ b/buildstream/_scheduler/job.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016 Codethink Limited # diff --git a/buildstream/_scheduler/pullqueue.py b/buildstream/_scheduler/pullqueue.py index f9928a342..b4f5b0d73 100644 --- a/buildstream/_scheduler/pullqueue.py +++ b/buildstream/_scheduler/pullqueue.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016 Codethink Limited # diff --git a/buildstream/_scheduler/pushqueue.py b/buildstream/_scheduler/pushqueue.py index 8a68d5953..624eefd1d 100644 --- a/buildstream/_scheduler/pushqueue.py +++ b/buildstream/_scheduler/pushqueue.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016 Codethink Limited # diff --git a/buildstream/_scheduler/queue.py b/buildstream/_scheduler/queue.py index 083822364..15caf8348 100644 --- a/buildstream/_scheduler/queue.py +++ b/buildstream/_scheduler/queue.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016 Codethink Limited # diff --git a/buildstream/_scheduler/scheduler.py b/buildstream/_scheduler/scheduler.py index 25e1e6790..f8a66ae92 100644 --- a/buildstream/_scheduler/scheduler.py +++ b/buildstream/_scheduler/scheduler.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016 Codethink Limited # diff --git a/buildstream/_scheduler/trackqueue.py b/buildstream/_scheduler/trackqueue.py index 2e7bc8b97..e48e1ae28 100644 --- a/buildstream/_scheduler/trackqueue.py +++ b/buildstream/_scheduler/trackqueue.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016 Codethink Limited # diff --git a/buildstream/_signals.py b/buildstream/_signals.py index 9749e0d1a..f1e520d37 100644 --- a/buildstream/_signals.py +++ b/buildstream/_signals.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Codethink Limited # diff --git a/buildstream/_site.py b/buildstream/_site.py index f4780ef3d..ff169180f 100644 --- a/buildstream/_site.py +++ b/buildstream/_site.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016 Codethink Limited # diff --git a/buildstream/_sourcefactory.py b/buildstream/_sourcefactory.py index dad3ddf68..88a130e10 100644 --- a/buildstream/_sourcefactory.py +++ b/buildstream/_sourcefactory.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016 Codethink Limited # diff --git a/buildstream/_stream.py b/buildstream/_stream.py index 0680f2a1f..5013daf3b 100644 --- a/buildstream/_stream.py +++ b/buildstream/_stream.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2018 Codethink Limited # diff --git a/buildstream/_variables.py b/buildstream/_variables.py index b4c920a71..8299f1c1e 100644 --- a/buildstream/_variables.py +++ b/buildstream/_variables.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016 Codethink Limited # diff --git a/buildstream/_versions.py b/buildstream/_versions.py index a101106aa..4b1bc7ec2 100644 --- a/buildstream/_versions.py +++ b/buildstream/_versions.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2018 Codethink Limited # diff --git a/buildstream/_workspaces.py b/buildstream/_workspaces.py index 41b4e9f05..3f474b8ca 100644 --- a/buildstream/_workspaces.py +++ b/buildstream/_workspaces.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2018 Codethink Limited # diff --git a/buildstream/_yaml.py b/buildstream/_yaml.py index 8954c7da9..bfd65865c 100644 --- a/buildstream/_yaml.py +++ b/buildstream/_yaml.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016 Codethink Limited # diff --git a/buildstream/buildelement.py b/buildstream/buildelement.py index 20c5e620f..ec05acbeb 100644 --- a/buildstream/buildelement.py +++ b/buildstream/buildelement.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016 Codethink Limited # diff --git a/buildstream/element.py b/buildstream/element.py index e876eb120..6fb826165 100644 --- a/buildstream/element.py +++ b/buildstream/element.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016-2018 Codethink Limited # diff --git a/buildstream/plugin.py b/buildstream/plugin.py index 0c216ed5d..29fe2cb11 100644 --- a/buildstream/plugin.py +++ b/buildstream/plugin.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Codethink Limited # diff --git a/buildstream/plugins/elements/autotools.py b/buildstream/plugins/elements/autotools.py index 93fc6be87..14d04d9a3 100644 --- a/buildstream/plugins/elements/autotools.py +++ b/buildstream/plugins/elements/autotools.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016, 2018 Codethink Limited # diff --git a/buildstream/plugins/elements/cmake.py b/buildstream/plugins/elements/cmake.py index 0b3de0143..8126a80ac 100644 --- a/buildstream/plugins/elements/cmake.py +++ b/buildstream/plugins/elements/cmake.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016, 2018 Codethink Limited # diff --git a/buildstream/plugins/elements/compose.py b/buildstream/plugins/elements/compose.py index 0769b8814..44a760215 100644 --- a/buildstream/plugins/elements/compose.py +++ b/buildstream/plugins/elements/compose.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Codethink Limited # diff --git a/buildstream/plugins/elements/distutils.py b/buildstream/plugins/elements/distutils.py index 30ba6892c..5201013c1 100644 --- a/buildstream/plugins/elements/distutils.py +++ b/buildstream/plugins/elements/distutils.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016 Codethink Limited # diff --git a/buildstream/plugins/elements/filter.py b/buildstream/plugins/elements/filter.py index ee9143ba7..22fddd14f 100644 --- a/buildstream/plugins/elements/filter.py +++ b/buildstream/plugins/elements/filter.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2018 Codethink Limited # diff --git a/buildstream/plugins/elements/import.py b/buildstream/plugins/elements/import.py index c19a8bc36..93594b623 100644 --- a/buildstream/plugins/elements/import.py +++ b/buildstream/plugins/elements/import.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016 Codethink Limited # diff --git a/buildstream/plugins/elements/junction.py b/buildstream/plugins/elements/junction.py index 512c862ff..ee5ed24d5 100644 --- a/buildstream/plugins/elements/junction.py +++ b/buildstream/plugins/elements/junction.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Codethink Limited # diff --git a/buildstream/plugins/elements/make.py b/buildstream/plugins/elements/make.py index d8dff4c41..1f37cb412 100644 --- a/buildstream/plugins/elements/make.py +++ b/buildstream/plugins/elements/make.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright Bloomberg Finance LP # diff --git a/buildstream/plugins/elements/makemaker.py b/buildstream/plugins/elements/makemaker.py index eb4d96930..fccfaadab 100644 --- a/buildstream/plugins/elements/makemaker.py +++ b/buildstream/plugins/elements/makemaker.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016 Codethink Limited # diff --git a/buildstream/plugins/elements/manual.py b/buildstream/plugins/elements/manual.py index dbb799fb4..c7bdba95f 100644 --- a/buildstream/plugins/elements/manual.py +++ b/buildstream/plugins/elements/manual.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016 Codethink Limited # diff --git a/buildstream/plugins/elements/modulebuild.py b/buildstream/plugins/elements/modulebuild.py index 897430bae..5189af1a6 100644 --- a/buildstream/plugins/elements/modulebuild.py +++ b/buildstream/plugins/elements/modulebuild.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016 Codethink Limited # diff --git a/buildstream/plugins/elements/pip.py b/buildstream/plugins/elements/pip.py index e2f33b207..e62f713a6 100644 --- a/buildstream/plugins/elements/pip.py +++ b/buildstream/plugins/elements/pip.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Mathieu Bridon # diff --git a/buildstream/plugins/elements/qmake.py b/buildstream/plugins/elements/qmake.py index b0cd3f178..7896692a6 100644 --- a/buildstream/plugins/elements/qmake.py +++ b/buildstream/plugins/elements/qmake.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016 Codethink Limited # diff --git a/buildstream/plugins/elements/script.py b/buildstream/plugins/elements/script.py index fbc15bd78..4e422c5db 100644 --- a/buildstream/plugins/elements/script.py +++ b/buildstream/plugins/elements/script.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Codethink Limited # diff --git a/buildstream/plugins/elements/stack.py b/buildstream/plugins/elements/stack.py index f6903c933..087d4dac0 100644 --- a/buildstream/plugins/elements/stack.py +++ b/buildstream/plugins/elements/stack.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016 Codethink Limited # diff --git a/buildstream/plugins/sources/bzr.py b/buildstream/plugins/sources/bzr.py index b499d49d3..ba96f7e3a 100644 --- a/buildstream/plugins/sources/bzr.py +++ b/buildstream/plugins/sources/bzr.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # Copyright (C) 2017 Codethink Limited # # This program is free software; you can redistribute it and/or diff --git a/buildstream/plugins/sources/deb.py b/buildstream/plugins/sources/deb.py index 4948d1933..b142b888d 100644 --- a/buildstream/plugins/sources/deb.py +++ b/buildstream/plugins/sources/deb.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # Copyright (C) 2017 Codethink Limited # # This program is free software; you can redistribute it and/or diff --git a/buildstream/plugins/sources/git.py b/buildstream/plugins/sources/git.py index 44065ad8f..67490b744 100644 --- a/buildstream/plugins/sources/git.py +++ b/buildstream/plugins/sources/git.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016 Codethink Limited # diff --git a/buildstream/plugins/sources/local.py b/buildstream/plugins/sources/local.py index 673add1bf..e3b019f1a 100644 --- a/buildstream/plugins/sources/local.py +++ b/buildstream/plugins/sources/local.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016 Codethink Limited # diff --git a/buildstream/plugins/sources/ostree.py b/buildstream/plugins/sources/ostree.py index c77b3a77f..94fe5093f 100644 --- a/buildstream/plugins/sources/ostree.py +++ b/buildstream/plugins/sources/ostree.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016 Codethink Limited # diff --git a/buildstream/plugins/sources/patch.py b/buildstream/plugins/sources/patch.py index 88fb2d59c..4640de166 100644 --- a/buildstream/plugins/sources/patch.py +++ b/buildstream/plugins/sources/patch.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright Bloomberg Finance LP # diff --git a/buildstream/plugins/sources/tar.py b/buildstream/plugins/sources/tar.py index 324006b21..3cd47cb8f 100644 --- a/buildstream/plugins/sources/tar.py +++ b/buildstream/plugins/sources/tar.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Codethink Limited # diff --git a/buildstream/plugins/sources/zip.py b/buildstream/plugins/sources/zip.py index f167c3e63..9b47d7f78 100644 --- a/buildstream/plugins/sources/zip.py +++ b/buildstream/plugins/sources/zip.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Mathieu Bridon # diff --git a/buildstream/sandbox/__init__.py b/buildstream/sandbox/__init__.py index 7ee871cab..53e170fbd 100644 --- a/buildstream/sandbox/__init__.py +++ b/buildstream/sandbox/__init__.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Codethink Limited # diff --git a/buildstream/sandbox/_config.py b/buildstream/sandbox/_config.py index 8893e3faa..5debe24b2 100644 --- a/buildstream/sandbox/_config.py +++ b/buildstream/sandbox/_config.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2018 Codethink Limited # diff --git a/buildstream/sandbox/_mount.py b/buildstream/sandbox/_mount.py index 84ab30ada..1540d9d4f 100644 --- a/buildstream/sandbox/_mount.py +++ b/buildstream/sandbox/_mount.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Codethink Limited # diff --git a/buildstream/sandbox/_mounter.py b/buildstream/sandbox/_mounter.py index c039b31df..921d06bb6 100644 --- a/buildstream/sandbox/_mounter.py +++ b/buildstream/sandbox/_mounter.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Codethink Limited # diff --git a/buildstream/sandbox/_sandboxbwrap.py b/buildstream/sandbox/_sandboxbwrap.py index 798bcb690..3a0645aae 100644 --- a/buildstream/sandbox/_sandboxbwrap.py +++ b/buildstream/sandbox/_sandboxbwrap.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016 Codethink Limited # diff --git a/buildstream/sandbox/_sandboxchroot.py b/buildstream/sandbox/_sandboxchroot.py index 7f27f50d0..22e69d549 100644 --- a/buildstream/sandbox/_sandboxchroot.py +++ b/buildstream/sandbox/_sandboxchroot.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Codethink Limited # diff --git a/buildstream/sandbox/sandbox.py b/buildstream/sandbox/sandbox.py index c174c4adb..595b9e778 100644 --- a/buildstream/sandbox/sandbox.py +++ b/buildstream/sandbox/sandbox.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Codethink Limited # diff --git a/buildstream/scriptelement.py b/buildstream/scriptelement.py index a0b12eef5..46afda807 100644 --- a/buildstream/scriptelement.py +++ b/buildstream/scriptelement.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2017 Codethink Limited # diff --git a/buildstream/source.py b/buildstream/source.py index 387ed6a99..ec38ae8f2 100644 --- a/buildstream/source.py +++ b/buildstream/source.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016 Codethink Limited # diff --git a/buildstream/utils.py b/buildstream/utils.py index af5bac3fb..b81a6c852 100644 --- a/buildstream/utils.py +++ b/buildstream/utils.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # # Copyright (C) 2016-2018 Codethink Limited # -- cgit v1.2.1 From b5d917940d79b859c7ebab0d25f512a805647b25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Sun, 10 Jun 2018 21:07:05 +0100 Subject: Source plugin tar depends on host's lzip Issue #353 --- buildstream/plugins/sources/tar.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'buildstream') diff --git a/buildstream/plugins/sources/tar.py b/buildstream/plugins/sources/tar.py index 3cd47cb8f..e32cc3dc8 100644 --- a/buildstream/plugins/sources/tar.py +++ b/buildstream/plugins/sources/tar.py @@ -21,6 +21,10 @@ tar - stage files from tar archives =================================== +**Host dependencies:** + + * lzip (for .tar.lz files) + **Usage:** .. code:: yaml -- cgit v1.2.1 From 7d97c6d3f3d59cf078df3bedd178e7977868d9d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Sun, 10 Jun 2018 21:09:46 +0100 Subject: Source plugin bzr depends on host's bzr Issue #353 --- buildstream/plugins/sources/bzr.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'buildstream') diff --git a/buildstream/plugins/sources/bzr.py b/buildstream/plugins/sources/bzr.py index ba96f7e3a..21121428c 100644 --- a/buildstream/plugins/sources/bzr.py +++ b/buildstream/plugins/sources/bzr.py @@ -20,6 +20,10 @@ bzr - stage files from a bazaar repository ========================================== +**Host dependencies:** + + * bzr + **Usage:** .. code:: yaml -- cgit v1.2.1 From 3b1e869bd8c64750a2e725a523acd55dc8e94360 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Sun, 10 Jun 2018 21:13:26 +0100 Subject: Source plugin git depends on host's git Issue #353 --- buildstream/plugins/sources/git.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'buildstream') diff --git a/buildstream/plugins/sources/git.py b/buildstream/plugins/sources/git.py index 67490b744..d079d8747 100644 --- a/buildstream/plugins/sources/git.py +++ b/buildstream/plugins/sources/git.py @@ -21,6 +21,10 @@ git - stage files from a git repository ======================================= +**Host dependencies:** + + * git + **Usage:** .. code:: yaml -- cgit v1.2.1 From be92cc8d7ff2dafee1a15b07a43802be01d5ad5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Sun, 10 Jun 2018 21:14:51 +0100 Subject: Source plugin patch depends on host's patch Issue #353 --- buildstream/plugins/sources/patch.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'buildstream') diff --git a/buildstream/plugins/sources/patch.py b/buildstream/plugins/sources/patch.py index 4640de166..11b66b3ea 100644 --- a/buildstream/plugins/sources/patch.py +++ b/buildstream/plugins/sources/patch.py @@ -21,6 +21,10 @@ patch - apply locally stored patches ==================================== +**Host dependencies:** + + * patch + **Usage:** .. code:: yaml -- cgit v1.2.1 From e90098e0e0decbd40f8e1980772d921e7843b8ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Sun, 10 Jun 2018 21:20:07 +0100 Subject: Source plugin deb depends on host's arpy python package Issue #353 --- buildstream/plugins/sources/deb.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'buildstream') diff --git a/buildstream/plugins/sources/deb.py b/buildstream/plugins/sources/deb.py index b142b888d..1cf8beb22 100644 --- a/buildstream/plugins/sources/deb.py +++ b/buildstream/plugins/sources/deb.py @@ -22,6 +22,10 @@ deb - stage files from .deb packages ==================================== +**Host dependencies:** + + * arpy (python package) + **Usage:** .. code:: yaml -- cgit v1.2.1 From 10d21ff040e8916a82741fd5527e0d29393c46d2 Mon Sep 17 00:00:00 2001 From: Valentin David Date: Tue, 12 Jun 2018 13:51:36 +0200 Subject: Fix element check for BST_FORBID_BDEPENDS --- buildstream/element.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'buildstream') diff --git a/buildstream/element.py b/buildstream/element.py index 6fb826165..fc21f80b6 100644 --- a/buildstream/element.py +++ b/buildstream/element.py @@ -1155,7 +1155,7 @@ class Element(Plugin): def _preflight(self): if self.BST_FORBID_RDEPENDS and self.BST_FORBID_BDEPENDS: - if any(self.dependencies(Scope.RUN, recurse=False)) or any(self.dependencies(Scope.RUN, recurse=False)): + if any(self.dependencies(Scope.RUN, recurse=False)) or any(self.dependencies(Scope.BUILD, recurse=False)): raise ElementError("{}: Dependencies are forbidden for '{}' elements" .format(self, self.get_kind()), reason="element-forbidden-depends") -- cgit v1.2.1 From 91d87e3c901dd9ac2cd3e7d148e5313cc5b915b0 Mon Sep 17 00:00:00 2001 From: Tristan Van Berkom Date: Tue, 26 Jun 2018 19:40:28 -0400 Subject: _scheduler/job.py: Added long comment This explains the nature of a complicated asyncio callback, which I've looked up on multiple occasions and is not available in python online docs (only in the source can you follow it). --- buildstream/_scheduler/job.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) (limited to 'buildstream') diff --git a/buildstream/_scheduler/job.py b/buildstream/_scheduler/job.py index 8b9af9393..cc350649e 100644 --- a/buildstream/_scheduler/job.py +++ b/buildstream/_scheduler/job.py @@ -137,7 +137,28 @@ class Job(): with _signals.blocked([signal.SIGINT, signal.SIGTSTP, signal.SIGTERM], ignore=False): self._process.start() - # Wait for it to complete + # Wait for the child task to complete. + # + # This is a tricky part of python which doesnt seem to + # make it to the online docs: + # + # o asyncio.get_child_watcher() will return a SafeChildWatcher() instance + # which is the default type of watcher, and the instance belongs to the + # "event loop policy" in use (so there is only one in the main process). + # + # o SafeChildWatcher() will register a SIGCHLD handler with the asyncio + # loop, and will selectively reap any child pids which have been + # terminated. + # + # o At registration time, the process will immediately be checked with + # `os.waitpid()` and will be reaped immediately, before add_child_handler() + # returns. + # + # The self._parent_child_completed callback passed here will normally + # be called after the child task has been reaped with `os.waitpid()`, in + # an event loop callback. Otherwise, if the job completes too fast, then + # the callback is called immediately. + # self._watcher = asyncio.get_child_watcher() self._watcher.add_child_handler(self._process.pid, self._parent_child_completed) -- cgit v1.2.1 From 463698ec3203190aeac1f41ed3885f08e95bceae Mon Sep 17 00:00:00 2001 From: Francisco Redondo Marchena Date: Thu, 28 Jun 2018 11:35:57 +0100 Subject: Add error message when running commands on directories Issue #446 --- buildstream/_exceptions.py | 3 +++ buildstream/_yaml.py | 4 ++++ 2 files changed, 7 insertions(+) (limited to 'buildstream') diff --git a/buildstream/_exceptions.py b/buildstream/_exceptions.py index 3aadd526d..377963244 100644 --- a/buildstream/_exceptions.py +++ b/buildstream/_exceptions.py @@ -197,6 +197,9 @@ class LoadErrorReason(Enum): # A project.conf file was missing MISSING_PROJECT_CONF = 17 + # Try to load a directory not a yaml file + LOADING_DIRECTORY = 18 + # LoadError # diff --git a/buildstream/_yaml.py b/buildstream/_yaml.py index bfd65865c..00336281c 100644 --- a/buildstream/_yaml.py +++ b/buildstream/_yaml.py @@ -184,6 +184,10 @@ def load(filename, shortname=None, copy_tree=False): except FileNotFoundError as e: raise LoadError(LoadErrorReason.MISSING_FILE, "Could not find file at {}".format(filename)) from e + except IsADirectoryError as e: + raise LoadError(LoadErrorReason.LOADING_DIRECTORY, + "{} is a directory. bst command is expecting a .bst file" + .format(filename)) from e # Like load(), but doesnt require the data to be in a file -- cgit v1.2.1 From 48848f4e69201b78524443dc34a77cf872e0ad6c Mon Sep 17 00:00:00 2001 From: James Ennis Date: Thu, 28 Jun 2018 18:34:47 +0100 Subject: loader.py: Check whether the dir specified is also a .bst file - Slight modification to the wording of existing error message. - Closes #446 --- buildstream/_loader/loader.py | 10 ++++++++++ buildstream/_yaml.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) (limited to 'buildstream') diff --git a/buildstream/_loader/loader.py b/buildstream/_loader/loader.py index 42e7feef5..07b0de996 100644 --- a/buildstream/_loader/loader.py +++ b/buildstream/_loader/loader.py @@ -229,6 +229,16 @@ class Loader(): detail = "Did you mean '{}'?".format(element_relpath) raise LoadError(LoadErrorReason.MISSING_FILE, message, detail=detail) from e + elif e.reason == LoadErrorReason.LOADING_DIRECTORY: + # If a .bst file exists in the element path, + # let's suggest this as a plausible alternative. + message = str(e) + detail = None + if os.path.exists(os.path.join(self._basedir, filename + '.bst')): + element_name = filename + '.bst' + detail = "Did you mean '{}'?\n".format(element_name) + raise LoadError(LoadErrorReason.LOADING_DIRECTORY, + message, detail=detail) from e else: raise self._options.process_node(node) diff --git a/buildstream/_yaml.py b/buildstream/_yaml.py index 00336281c..0e090e2e7 100644 --- a/buildstream/_yaml.py +++ b/buildstream/_yaml.py @@ -186,7 +186,7 @@ def load(filename, shortname=None, copy_tree=False): "Could not find file at {}".format(filename)) from e except IsADirectoryError as e: raise LoadError(LoadErrorReason.LOADING_DIRECTORY, - "{} is a directory. bst command is expecting a .bst file" + "{} is a directory. bst command expects a .bst file." .format(filename)) from e -- cgit v1.2.1 From aaf52d036c67a45356fc9f2eb7aa21542bf39782 Mon Sep 17 00:00:00 2001 From: Tristan Maat Date: Fri, 29 Jun 2018 16:56:20 +0100 Subject: cli.py: Make buildstream check element paths instead of click This is to avoid inconsistencies when dealing with paths inside an elements directory --- buildstream/_frontend/cli.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) (limited to 'buildstream') diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py index 465124557..e59b1baec 100644 --- a/buildstream/_frontend/cli.py +++ b/buildstream/_frontend/cli.py @@ -291,20 +291,20 @@ def init(app, project_name, format_version, element_path, force): @click.option('--all', 'all_', default=False, is_flag=True, help="Build elements that would not be needed for the current build plan") @click.option('--track', 'track_', multiple=True, - type=click.Path(dir_okay=False, readable=True), + type=click.Path(readable=False), help="Specify elements to track during the build. Can be used " "repeatedly to specify multiple elements") @click.option('--track-all', default=False, is_flag=True, help="Track all elements in the pipeline") @click.option('--track-except', multiple=True, - type=click.Path(dir_okay=False, readable=True), + type=click.Path(readable=False), help="Except certain dependencies from tracking") @click.option('--track-cross-junctions', '-J', default=False, is_flag=True, help="Allow tracking to cross junction boundaries") @click.option('--track-save', default=False, is_flag=True, help="Deprecated: This is ignored") @click.argument('elements', nargs=-1, - type=click.Path(dir_okay=False, readable=True)) + type=click.Path(readable=False)) @click.pass_obj def build(app, elements, all_, track_, track_save, track_all, track_except, track_cross_junctions): """Build elements in a pipeline""" @@ -333,7 +333,7 @@ def build(app, elements, all_, track_, track_save, track_all, track_except, trac ################################################################## @cli.command(short_help="Fetch sources in a pipeline") @click.option('--except', 'except_', multiple=True, - type=click.Path(dir_okay=False, readable=True), + type=click.Path(readable=False), help="Except certain dependencies from fetching") @click.option('--deps', '-d', default='plan', type=click.Choice(['none', 'plan', 'all']), @@ -343,7 +343,7 @@ def build(app, elements, all_, track_, track_save, track_all, track_except, trac @click.option('--track-cross-junctions', '-J', default=False, is_flag=True, help="Allow tracking to cross junction boundaries") @click.argument('elements', nargs=-1, - type=click.Path(dir_okay=False, readable=True)) + type=click.Path(readable=False)) @click.pass_obj def fetch(app, elements, deps, track_, except_, track_cross_junctions): """Fetch sources required to build the pipeline @@ -384,7 +384,7 @@ def fetch(app, elements, deps, track_, except_, track_cross_junctions): ################################################################## @cli.command(short_help="Track new source references") @click.option('--except', 'except_', multiple=True, - type=click.Path(dir_okay=False, readable=True), + type=click.Path(readable=False), help="Except certain dependencies from tracking") @click.option('--deps', '-d', default='none', type=click.Choice(['none', 'all']), @@ -392,7 +392,7 @@ def fetch(app, elements, deps, track_, except_, track_cross_junctions): @click.option('--cross-junctions', '-J', default=False, is_flag=True, help="Allow crossing junction boundaries") @click.argument('elements', nargs=-1, - type=click.Path(dir_okay=False, readable=True)) + type=click.Path(readable=False)) @click.pass_obj def track(app, elements, deps, except_, cross_junctions): """Consults the specified tracking branches for new versions available @@ -424,7 +424,7 @@ def track(app, elements, deps, except_, cross_junctions): @click.option('--remote', '-r', help="The URL of the remote cache (defaults to the first configured cache)") @click.argument('elements', nargs=-1, - type=click.Path(dir_okay=False, readable=True)) + type=click.Path(readable=False)) @click.pass_obj def pull(app, elements, deps, remote): """Pull a built artifact from the configured remote artifact cache. @@ -453,7 +453,7 @@ def pull(app, elements, deps, remote): @click.option('--remote', '-r', default=None, help="The URL of the remote cache (defaults to the first configured cache)") @click.argument('elements', nargs=-1, - type=click.Path(dir_okay=False, readable=True)) + type=click.Path(readable=False)) @click.pass_obj def push(app, elements, deps, remote): """Push a built artifact to a remote artifact cache. @@ -476,7 +476,7 @@ def push(app, elements, deps, remote): ################################################################## @cli.command(short_help="Show elements in the pipeline") @click.option('--except', 'except_', multiple=True, - type=click.Path(dir_okay=False, readable=True), + type=click.Path(readable=False), help="Except certain dependencies") @click.option('--deps', '-d', default='all', type=click.Choice(['none', 'plan', 'run', 'build', 'all']), @@ -488,7 +488,7 @@ def push(app, elements, deps, remote): type=click.STRING, help='Format string for each element') @click.argument('elements', nargs=-1, - type=click.Path(dir_okay=False, readable=True)) + type=click.Path(readable=False)) @click.pass_obj def show(app, elements, deps, except_, order, format_): """Show elements in the pipeline @@ -567,7 +567,7 @@ def show(app, elements, deps, except_, order, format_): @click.option('--isolate', is_flag=True, default=False, help='Create an isolated build sandbox') @click.argument('element', - type=click.Path(dir_okay=False, readable=True)) + type=click.Path(readable=False)) @click.argument('command', type=click.STRING, nargs=-1) @click.pass_obj def shell(app, element, sysroot, mount, isolate, build_, command): @@ -628,7 +628,7 @@ def shell(app, element, sysroot, mount, isolate, build_, command): @click.option('--hardlinks', default=False, is_flag=True, help="Checkout hardlinks instead of copies (handle with care)") @click.argument('element', - type=click.Path(dir_okay=False, readable=True)) + type=click.Path(readable=False)) @click.argument('directory', type=click.Path(file_okay=False)) @click.pass_obj def checkout(app, element, directory, force, integrate, hardlinks): @@ -662,7 +662,7 @@ def workspace(): @click.option('--track', 'track_', default=False, is_flag=True, help="Track and fetch new source references before checking out the workspace") @click.argument('element', - type=click.Path(dir_okay=False, readable=True)) + type=click.Path(readable=False)) @click.argument('directory', type=click.Path(file_okay=False)) @click.pass_obj def workspace_open(app, no_checkout, force, track_, element, directory): @@ -694,7 +694,7 @@ def workspace_open(app, no_checkout, force, track_, element, directory): @click.option('--all', '-a', 'all_', default=False, is_flag=True, help="Close all open workspaces") @click.argument('elements', nargs=-1, - type=click.Path(dir_okay=False, readable=True)) + type=click.Path(readable=False)) @click.pass_obj def workspace_close(app, remove_dir, all_, elements): """Close a workspace""" @@ -743,7 +743,7 @@ def workspace_close(app, remove_dir, all_, elements): @click.option('--all', '-a', 'all_', default=False, is_flag=True, help="Reset all open workspaces") @click.argument('elements', nargs=-1, - type=click.Path(dir_okay=False, readable=True)) + type=click.Path(readable=False)) @click.pass_obj def workspace_reset(app, soft, track_, all_, elements): """Reset a workspace to its original state""" @@ -785,7 +785,7 @@ def workspace_list(app): ################################################################## @cli.command(name="source-bundle", short_help="Produce a build bundle to be manually executed") @click.option('--except', 'except_', multiple=True, - type=click.Path(dir_okay=False, readable=True), + type=click.Path(readable=False), help="Elements to except from the tarball") @click.option('--compression', default='gz', type=click.Choice(['none', 'gz', 'bz2', 'xz']), @@ -797,7 +797,7 @@ def workspace_list(app): @click.option('--directory', default=os.getcwd(), help="The directory to write the tarball to") @click.argument('element', - type=click.Path(dir_okay=False, readable=True)) + type=click.Path(readable=False)) @click.pass_obj def source_bundle(app, element, force, directory, track_, compression, except_): -- cgit v1.2.1 From a85aaee6630c2e58c20f538a235179ec65b6b32e Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Thu, 28 Jun 2018 11:29:53 +0100 Subject: Fix documentation typo --- buildstream/element.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'buildstream') diff --git a/buildstream/element.py b/buildstream/element.py index fc21f80b6..3bdf9720a 100644 --- a/buildstream/element.py +++ b/buildstream/element.py @@ -65,7 +65,7 @@ Miscellaneous abstract methods also exist: * :func:`Element.generate_script() ` - For the purpose of ``bst source bundle``, an Element may optionally implmenent this. + For the purpose of ``bst source bundle``, an Element may optionally implement this. Class Reference -- cgit v1.2.1 From 9067e269a9f2866e659ef33a69aad72b01cb6633 Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Thu, 28 Jun 2018 11:31:09 +0100 Subject: Provide better error message on missing commands Before running a command in the sandbox, check its existence and fail early if it does not. This fixes issue #289. --- buildstream/sandbox/_sandboxbwrap.py | 6 ++++++ buildstream/sandbox/_sandboxchroot.py | 5 +++++ buildstream/sandbox/sandbox.py | 22 ++++++++++++++++++++++ 3 files changed, 33 insertions(+) (limited to 'buildstream') diff --git a/buildstream/sandbox/_sandboxbwrap.py b/buildstream/sandbox/_sandboxbwrap.py index 3a0645aae..9ed677620 100644 --- a/buildstream/sandbox/_sandboxbwrap.py +++ b/buildstream/sandbox/_sandboxbwrap.py @@ -28,6 +28,7 @@ from contextlib import ExitStack import psutil +from .._exceptions import SandboxError from .. import utils, _signals from ._mount import MountMap from . import Sandbox, SandboxFlags @@ -66,6 +67,11 @@ class SandboxBwrap(Sandbox): if env is None: env = self._get_environment() + if not self._has_command(command[0], env): + raise SandboxError("Staged artifacts do not provide command " + "'{}'".format(command[0]), + reason='missing-command') + # We want command args as a list of strings if isinstance(command, str): command = [command] diff --git a/buildstream/sandbox/_sandboxchroot.py b/buildstream/sandbox/_sandboxchroot.py index 22e69d549..8788c3031 100644 --- a/buildstream/sandbox/_sandboxchroot.py +++ b/buildstream/sandbox/_sandboxchroot.py @@ -58,6 +58,11 @@ class SandboxChroot(Sandbox): if env is None: env = self._get_environment() + if not self._has_command(command[0], env): + raise SandboxError("Staged artifacts do not provide command " + "'{}'".format(command[0]), + reason='missing-command') + # Command must be a list if isinstance(command, str): command = [command] diff --git a/buildstream/sandbox/sandbox.py b/buildstream/sandbox/sandbox.py index 595b9e778..7e1e32b65 100644 --- a/buildstream/sandbox/sandbox.py +++ b/buildstream/sandbox/sandbox.py @@ -279,3 +279,25 @@ class Sandbox(): # data passed in during construction. def _get_config(self): return self.__config + + # _has_command() + # + # Tests whether a command exists inside the sandbox + # + # Args: + # command (list): The command to test. + # env (dict): A dictionary of string key, value pairs to set as environment + # variables inside the sandbox environment. + # Returns: + # (bool): Whether a command exists inside the sandbox. + def _has_command(self, command, env=None): + if os.path.isabs(command): + return os.path.exists(os.path.join( + self.get_directory(), command.lstrip(os.sep))) + + for path in env.get('PATH').split(':'): + if os.path.exists(os.path.join( + self.get_directory(), path.lstrip(os.sep), command)): + return True + + return False -- cgit v1.2.1 From 93af52993b59d23b0100f6c1cdfaa652f4907c58 Mon Sep 17 00:00:00 2001 From: Jim MacArthur Date: Wed, 2 May 2018 18:24:33 +0100 Subject: Add BST_VIRTUAL_DIRECTORY flag for element plugins --- buildstream/element.py | 10 +++++++++- buildstream/sandbox/sandbox.py | 13 ++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) (limited to 'buildstream') diff --git a/buildstream/element.py b/buildstream/element.py index 3bdf9720a..19bf2717d 100644 --- a/buildstream/element.py +++ b/buildstream/element.py @@ -191,6 +191,13 @@ class Element(Plugin): *Since: 1.2* """ + BST_VIRTUAL_DIRECTORY = False + """Whether to raise exceptions if an element uses Sandbox.get_directory + instead of Sandbox.get_virtual_directory. + + *Since: 1.2* + """ + def __init__(self, context, project, artifacts, meta, plugin_conf): super().__init__(meta.name, context, project, meta.provenance, "element") @@ -2078,7 +2085,8 @@ class Element(Plugin): directory, stdout=stdout, stderr=stderr, - config=config) + config=config, + allow_real_directory=not self.BST_VIRTUAL_DIRECTORY) yield sandbox else: diff --git a/buildstream/sandbox/sandbox.py b/buildstream/sandbox/sandbox.py index 7e1e32b65..7e73e0d9a 100644 --- a/buildstream/sandbox/sandbox.py +++ b/buildstream/sandbox/sandbox.py @@ -29,7 +29,7 @@ See also: :ref:`sandboxing`. """ import os -from .._exceptions import ImplError +from .._exceptions import ImplError, BstError class SandboxFlags(): @@ -90,6 +90,8 @@ class Sandbox(): self.__cwd = None self.__env = None self.__mount_sources = {} + self.__allow_real_directory = kwargs['allow_real_directory'] + # Configuration from kwargs common to all subclasses self.__config = kwargs['config'] self.__stdout = kwargs['stdout'] @@ -106,12 +108,17 @@ class Sandbox(): """Fetches the sandbox root directory The root directory is where artifacts for the base - runtime environment should be staged. + runtime environment should be staged. Only works if + BST_VIRTUAL_DIRECTORY is not set. Returns: (str): The sandbox root directory + """ - return self.__root + if self.__allow_real_directory: + return self.__root + else: + raise BstError("You can't use get_directory") def set_environment(self, environment): """Sets the environment variables for the sandbox -- cgit v1.2.1 From ea353624da57c47d7da809acd428749e3ee03bed Mon Sep 17 00:00:00 2001 From: Jim MacArthur Date: Tue, 8 May 2018 16:08:12 +0100 Subject: Add the virtual directory class 'Directory' and one implementation, _filebaseddirectory.py. These are not used anywhere yet. buildstream/sandbox/Directory.py: New file. buildstream/sandbox/_filebaseddirectory.py: New file. buildstream/_exceptions.py: New VIRTUAL_FS exception source. --- buildstream/_exceptions.py | 1 + buildstream/sandbox/Directory.py | 120 ++++++++++++++++ buildstream/sandbox/_filebaseddirectory.py | 223 +++++++++++++++++++++++++++++ 3 files changed, 344 insertions(+) create mode 100644 buildstream/sandbox/Directory.py create mode 100644 buildstream/sandbox/_filebaseddirectory.py (limited to 'buildstream') diff --git a/buildstream/_exceptions.py b/buildstream/_exceptions.py index 377963244..34ce91081 100644 --- a/buildstream/_exceptions.py +++ b/buildstream/_exceptions.py @@ -88,6 +88,7 @@ class ErrorDomain(Enum): ELEMENT = 11 APP = 12 STREAM = 13 + VIRTUAL_FS = 14 # BstError is an internal base exception class for BuildSream diff --git a/buildstream/sandbox/Directory.py b/buildstream/sandbox/Directory.py new file mode 100644 index 000000000..07f22badc --- /dev/null +++ b/buildstream/sandbox/Directory.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2018 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . +# +# Authors: +# Jim MacArthur + +""" +Directory +========= + +Virtual Directory class to isolate the rest of BuildStream from the backing store implementation. +Sandboxes are allowed to read from and write to the underlying storage, but all others must use this +Directory class to access files and directories in the sandbox. + +See also: :ref:`sandboxing`. +""" + +from typing import List +from ..utils import FileListResult + + +class Directory(): + def __init__(self, external_directory=None): + raise NotImplementedError() + + def descend(self, subdirectory_spec: List[str]) -> 'Directory': + """ + Descend one or more levels of directory hierarchy and return a new + Directory object for that directory. + + Arguments: + subdirectory_spec (list of strings): A list of strings which are all directory + names. + create (boolean): If this is true, the directories will be created if + they don't already exist. + """ + raise NotImplementedError() + + # Import and export of files and links + def import_files(self, external_pathspec: any, files: List[str] = None, + report_written: bool = True, update_utimes: bool = False, + link_ok: bool = False) -> FileListResult: + """Imports some or all files from external_path into this directory. + + Keyword arguments: external_pathspec: Either a string + containing a pathname, or a Directory object, to use as the + source. + + files (list of strings): A list of all the files relative to + the external_pathspec to copy. If 'None' is supplied, all + files are copied. + + report_written (bool): Return the full list of files + written. Defaults to true. If false, only a list of + overwritten files is returned. + + update_utimes (bool): Update the access and modification time + of each file copied to the current time. + + can_link (bool): Whether it's OK to create a hard link to the + original content, meaning the stored copy will change when the + original files change. Setting this doesn't guarantee hard + links will be made. can_link will never be used if + update_utimes is set. + """ + + raise NotImplementedError() + + def export_files(self, to_directory: str, link_ok: bool = False) -> None: + """Copies everything from this into to_directory. + + Arguments: + + to_directory (string): a path outside this directory object + where the contents will be copied to. + + can_link (bool): Whether we can create hard links in to_directory + instead of copying. Setting this does not guarantee hard links will be used. + + """ + + raise NotImplementedError() + + # Convenience functions + def is_empty(self) -> bool: + raise NotImplementedError() + + def set_deterministic_mtime(self) -> None: + """ Sets a static modification time for all regular files in this directory. + The magic number for timestamps: 2011-11-11 11:11:11 + """ + raise NotImplementedError() + + def set_deterministic_user(self) -> None: + """ Sets all files in this directory to the current user's euid/egid. + """ + raise NotImplementedError() + + def list_relative_paths_with_mtimes(self) -> Dict[str, float]: + """Provide a list of relative paths with modification times for + each. Used to detect changed changed files during a Compose + operation. + + Return value: Dict(str->float) - dictionary with all paths and mtime in seconds. + """ + raise NotImplementedError() diff --git a/buildstream/sandbox/_filebaseddirectory.py b/buildstream/sandbox/_filebaseddirectory.py new file mode 100644 index 000000000..640e254b1 --- /dev/null +++ b/buildstream/sandbox/_filebaseddirectory.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2018 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . +# +# Authors: +# Jim MacArthur + +""" +FileBasedDirectory +========= + +Implementation of the Directory class which backs onto a normal POSIX filing system. + +See also: :ref:`sandboxing`. +""" + +from typing import List +from collections import OrderedDict + +import os +import time +from .._exceptions import BstError, ErrorDomain +from .Directory import Directory +from ..utils import link_files, copy_files, FileListResult, list_relative_paths +from ..utils import _set_deterministic_user, _set_deterministic_mtime + + +class VirtualDirectoryError(BstError): + """Raised by Directory functions when system calls fail. + This will be handled internally by the BuildStream core, + if you need to handle this error, then it should be reraised, + or either of the :class:`.ElementError` or :class:`.SourceError` + exceptions should be raised from this error. + """ + def __init__(self, message, reason=None): + super().__init__(message, domain=ErrorDomain.VIRTUAL_FS, reason=reason) + + +# Like os.path.getmtime(), but doesnt explode on symlinks +# Copy/pasted from compose.py +def getmtime(path): + stat = os.lstat(path) + return stat.st_mtime + +# FileBasedDirectory intentionally doesn't call its superclass constuctor, +# which is mean to be unimplemented. +# pylint: disable=super-init-not-called + + +class _FileObject(): + """A description of a file in a virtual directory. The contents of + this class are never used, but there needs to be something present + for files so is_empty() works correctly. + + """ + def __init__(self, virtual_directory: Directory, filename: str): + self.directory = virtual_directory + self.filename = filename + + +class FileBasedDirectory(Directory): + def __init__(self, external_directory=None): + self.external_directory = external_directory + self.index = OrderedDict() + self._directory_read = False + + def _populate_index(self) -> None: + if self._directory_read: + return + for entry in os.listdir(self.external_directory): + if os.path.isdir(os.path.join(self.external_directory, entry)): + self.index[entry] = FileBasedDirectory(os.path.join(self.external_directory, entry)) + else: + self.index[entry] = _FileObject(self, entry) + self._directory_read = True + + def descend(self, subdirectory_spec: List[str], create: bool = False) -> Directory: + """ Descend one or more levels of directory hierarchy and return a new + Directory object for that directory. + + Arguments: + * subdirectory_spec (list of strings): A list of strings which are all directory + names. + * create (boolean): If this is true, the directories will be created if + they don't already exist. + """ + + # It's very common to send a directory name instead of a list and this causes + # bizarre errors, so check for it here + if not isinstance(subdirectory_spec, list): + subdirectory_spec = [subdirectory_spec] + if not subdirectory_spec: + return self + + # Because of the way split works, it's common to get a list which begins with + # an empty string. Detect these and remove them, then start again. + if subdirectory_spec[0] == "": + return self.descend(subdirectory_spec[1:], create) + + self._populate_index() + if subdirectory_spec[0] in self.index: + entry = self.index[subdirectory_spec[0]] + if isinstance(entry, FileBasedDirectory): + new_path = os.path.join(self.external_directory, subdirectory_spec[0]) + return FileBasedDirectory(new_path).descend(subdirectory_spec[1:], create) + else: + error = "Cannot descend into {}, which is a '{}' in the directory {}" + raise VirtualDirectoryError(error.format(subdirectory_spec[0], + type(entry).__name__, + self.external_directory)) + else: + if create: + new_path = os.path.join(self.external_directory, subdirectory_spec[0]) + os.makedirs(new_path, exist_ok=True) + return FileBasedDirectory(new_path).descend(subdirectory_spec[1:], create) + else: + error = "No entry called '{}' found in the directory rooted at {}" + raise VirtualDirectoryError(error.format(subdirectory_spec[0], self.external_directory)) + return None + + def import_files(self, external_pathspec: any, files: List[str] = None, + report_written: bool = True, update_utimes: bool = False, + can_link: bool = False) -> FileListResult: + """Imports some or all files from external_path into this directory. + + Keyword arguments: external_pathspec: Either a string + containing a pathname, or a Directory object, to use as the + source. + + files (list of strings): A list of all the files relative to + the external_pathspec to copy. If 'None' is supplied, all + files are copied. + + report_written (bool): Return the full list of files + written. Defaults to true. If false, only a list of + overwritten files is returned. + + update_utimes (bool): Update the access and modification time + of each file copied to the current time. + + can_link (bool): Whether it's OK to create a hard link to the + original content, meaning the stored copy will change when the + original files change. Setting this doesn't guarantee hard + links will be made. can_link will never be used if + update_utimes is set. + """ + + if isinstance(external_pathspec, Directory): + source_directory = external_pathspec.external_directory + else: + source_directory = external_pathspec + + if can_link and not update_utimes: + import_result = link_files(source_directory, self.external_directory, files=files, + ignore_missing=False, report_written=report_written) + else: + import_result = copy_files(source_directory, self.external_directory, files=files, + ignore_missing=False, report_written=report_written) + if update_utimes: + cur_time = time.time() + + for f in import_result.files_written: + os.utime(os.path.join(self.external_directory, f), times=(cur_time, cur_time)) + return import_result + + def set_deterministic_mtime(self) -> None: + """ Sets a static modification time for all regular files in this directory. + The magic number for timestamps: 2011-11-11 11:11:11 + """ + _set_deterministic_mtime(self.external_directory) + + def set_deterministic_user(self) -> None: + """ Sets all files in this directory to the current user's euid/egid. + """ + _set_deterministic_user(self.external_directory) + + def export_files(self, to_directory: str, can_link: bool = False) -> None: + """Copies everything from this into to_directory. + + Arguments: + + to_directory (string): a path outside this directory object + where the contents will be copied to. + + can_link (bool): Whether we can create hard links in to_directory + instead of copying. + + """ + if can_link: + link_files(self.external_directory, to_directory) + else: + copy_files(self.external_directory, to_directory) + + def is_empty(self) -> bool: + """ Return true if this directory has no files, subdirectories or links in it. + """ + self._populate_index() + return len(self.index) == 0 + + def list_relative_paths_with_mtimes(self) -> Dict[str, float]: + return { + f: getmtime(os.path.join(self.external_directory, f)) + for f in list_relative_paths(self.external_directory) + } + + def __str__(self) -> str: + # This returns the whole path (since we don't know where the directory started) + # which exposes the sandbox directory; we will have to assume for the time being + # that people will not abuse __str__. + return self.external_directory -- cgit v1.2.1 From 341afb936a710a8cc95a4a84d39b90005ef4fe8c Mon Sep 17 00:00:00 2001 From: Jim MacArthur Date: Tue, 8 May 2018 16:14:37 +0100 Subject: Sandbox.py: Rename __root to _root to allow its use by subclasses. Since access to get_directories is now blocked for some plugins, and the subclasses of Sandbox do not have configuration defined by YAML files, they need another way to get at the root directory. NB Could this be done just with get_virtual_directory and .external_directory? --- buildstream/sandbox/_sandboxchroot.py | 2 +- buildstream/sandbox/sandbox.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) (limited to 'buildstream') diff --git a/buildstream/sandbox/_sandboxchroot.py b/buildstream/sandbox/_sandboxchroot.py index 8788c3031..de4eb46e2 100644 --- a/buildstream/sandbox/_sandboxchroot.py +++ b/buildstream/sandbox/_sandboxchroot.py @@ -90,7 +90,7 @@ class SandboxChroot(Sandbox): # Nonetheless a better solution could perhaps be found. rootfs = stack.enter_context(utils._tempdir(dir='/var/run/buildstream')) - stack.enter_context(self.create_devices(self.get_directory(), flags)) + stack.enter_context(self.create_devices(self._root, flags)) stack.enter_context(self.mount_dirs(rootfs, flags, stdout, stderr)) if flags & SandboxFlags.INTERACTIVE: diff --git a/buildstream/sandbox/sandbox.py b/buildstream/sandbox/sandbox.py index 7e73e0d9a..90b80af2a 100644 --- a/buildstream/sandbox/sandbox.py +++ b/buildstream/sandbox/sandbox.py @@ -97,11 +97,12 @@ class Sandbox(): self.__stdout = kwargs['stdout'] self.__stderr = kwargs['stderr'] - # Setup the directories + # Setup the directories. Root should be available to subclasses, hence + # being single-underscore. The others are private to this class. + self._root = os.path.join(directory, 'root') self.__directory = directory - self.__root = os.path.join(self.__directory, 'root') self.__scratch = os.path.join(self.__directory, 'scratch') - for directory_ in [self.__root, self.__scratch]: + for directory_ in [self._root, self.__scratch]: os.makedirs(directory_, exist_ok=True) def get_directory(self): @@ -116,7 +117,7 @@ class Sandbox(): """ if self.__allow_real_directory: - return self.__root + return self._root else: raise BstError("You can't use get_directory") -- cgit v1.2.1 From 3a435724a0529ccf19a66b083208a51be8c72d04 Mon Sep 17 00:00:00 2001 From: Jim MacArthur Date: Tue, 8 May 2018 16:17:50 +0100 Subject: Sandbox.py: Add get_virtual_directory and get_virtual_toplevel_directory --- buildstream/sandbox/sandbox.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) (limited to 'buildstream') diff --git a/buildstream/sandbox/sandbox.py b/buildstream/sandbox/sandbox.py index 90b80af2a..68e6b13b2 100644 --- a/buildstream/sandbox/sandbox.py +++ b/buildstream/sandbox/sandbox.py @@ -30,6 +30,7 @@ See also: :ref:`sandboxing`. import os from .._exceptions import ImplError, BstError +from ._filebaseddirectory import FileBasedDirectory class SandboxFlags(): @@ -121,6 +122,33 @@ class Sandbox(): else: raise BstError("You can't use get_directory") + def get_virtual_directory(self): + """Fetches the sandbox root directory + + The root directory is where artifacts for the base + runtime environment should be staged. Only works if + BST_VIRTUAL_DIRECTORY is not set. + + Returns: + (str): The sandbox root directory + + """ + # For now, just create a new Directory every time we're asked + return FileBasedDirectory(self._root) + + def get_virtual_toplevel_directory(self): + """Fetches the sandbox's toplevel directory + + The toplevel directory contains 'root', 'scratch' and later + 'artifact' where output is copied to. + + Returns: + (str): The sandbox toplevel directory + + """ + # For now, just create a new Directory every time we're asked + return FileBasedDirectory(self.__directory) + def set_environment(self, environment): """Sets the environment variables for the sandbox -- cgit v1.2.1 From 304db1c1e25e3b3631bdb04928667a48264516ce Mon Sep 17 00:00:00 2001 From: Jim MacArthur Date: Tue, 8 May 2018 16:18:36 +0100 Subject: _stream.py: Convert to virtual directories. --- buildstream/_stream.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) (limited to 'buildstream') diff --git a/buildstream/_stream.py b/buildstream/_stream.py index 5013daf3b..28a480982 100644 --- a/buildstream/_stream.py +++ b/buildstream/_stream.py @@ -387,13 +387,13 @@ class Stream(): with target._prepare_sandbox(Scope.RUN, None, integrate=integrate) as sandbox: # Copy or move the sandbox to the target directory - sandbox_root = sandbox.get_directory() + sandbox_vroot = sandbox.get_virtual_directory() with target.timed_activity("Checking out files in {}".format(directory)): try: if hardlinks: - self._checkout_hardlinks(sandbox_root, directory) + self._checkout_hardlinks(sandbox_vroot, directory) else: - utils.copy_files(sandbox_root, directory) + sandbox_vroot.export_files(directory) except OSError as e: raise StreamError("Failed to checkout files: {}".format(e)) from e except BstError as e: @@ -990,22 +990,19 @@ class Stream(): # Helper function for checkout() # - def _checkout_hardlinks(self, sandbox_root, directory): + def _checkout_hardlinks(self, sandbox_vroot, directory): try: removed = utils.safe_remove(directory) except OSError as e: raise StreamError("Failed to remove checkout directory: {}".format(e)) from e if removed: - # Try a simple rename of the sandbox root; if that - # doesnt cut it, then do the regular link files code path - try: - os.rename(sandbox_root, directory) - except OSError: - os.makedirs(directory, exist_ok=True) - utils.link_files(sandbox_root, directory) + # TODO: Direct rename is no longer possible with the new Virtual Directory interface. + # See what options there are to restore it. + os.makedirs(directory, exist_ok=True) + sandbox_vroot.export_files(directory, can_link=True) else: - utils.link_files(sandbox_root, directory) + sandbox_vroot.export_files(directory, can_link=True) # Write the element build script to the given directory def _write_element_script(self, directory, element): -- cgit v1.2.1 From cdfc72a4125d3e0b7d049375ceff1f55d288ce0b Mon Sep 17 00:00:00 2001 From: Jim MacArthur Date: Tue, 8 May 2018 16:19:22 +0100 Subject: element.py: Conversion to use virtual directories --- buildstream/element.py | 87 +++++++++++++++++++++++++------------------------- 1 file changed, 44 insertions(+), 43 deletions(-) (limited to 'buildstream') diff --git a/buildstream/element.py b/buildstream/element.py index 19bf2717d..8dea76758 100644 --- a/buildstream/element.py +++ b/buildstream/element.py @@ -80,7 +80,6 @@ from collections import Mapping, OrderedDict from contextlib import contextmanager from enum import Enum import tempfile -import time import shutil from . import _yaml @@ -97,6 +96,9 @@ from . import _site from ._platform import Platform from .sandbox._config import SandboxConfig +from .sandbox.Directory import Directory +from .sandbox._filebaseddirectory import FileBasedDirectory, VirtualDirectoryError + # _KeyStrength(): # @@ -626,10 +628,10 @@ class Element(Plugin): # Hard link it into the staging area # - basedir = sandbox.get_directory() - stagedir = basedir \ + vbasedir = sandbox.get_virtual_directory() + vstagedir = vbasedir \ if path is None \ - else os.path.join(basedir, path.lstrip(os.sep)) + else vbasedir.descend(path.lstrip(os.sep).split(os.sep)) files = list(self.__compute_splits(include, exclude, orphans)) @@ -641,15 +643,8 @@ class Element(Plugin): link_files = files copy_files = [] - link_result = utils.link_files(artifact, stagedir, files=link_files, - report_written=True) - copy_result = utils.copy_files(artifact, stagedir, files=copy_files, - report_written=True) - - cur_time = time.time() - - for f in copy_result.files_written: - os.utime(os.path.join(stagedir, f), times=(cur_time, cur_time)) + link_result = vstagedir.import_files(artifact, files=link_files, report_written=True, can_link=True) + copy_result = vstagedir.import_files(artifact, files=copy_files, report_written=True, update_utimes=True) return link_result.combine(copy_result) @@ -1293,8 +1288,8 @@ class Element(Plugin): sandbox._set_mount_source(directory, workspace.get_absolute_path()) # Stage all sources that need to be copied - sandbox_root = sandbox.get_directory() - host_directory = os.path.join(sandbox_root, directory.lstrip(os.sep)) + sandbox_vroot = sandbox.get_virtual_directory() + host_directory = sandbox_vroot.descend(directory.lstrip(os.sep).split(os.sep), create=True) self._stage_sources_at(host_directory, mount_workspaces=mount_workspaces) # _stage_sources_at(): @@ -1305,28 +1300,33 @@ class Element(Plugin): # directory (str): An absolute path to stage the sources at # mount_workspaces (bool): mount workspaces if True, copy otherwise # - def _stage_sources_at(self, directory, mount_workspaces=True): + def _stage_sources_at(self, vdirectory, mount_workspaces=True): with self.timed_activity("Staging sources", silent_nested=True): - if os.path.isdir(directory) and os.listdir(directory): - raise ElementError("Staging directory '{}' is not empty".format(directory)) - - workspace = self._get_workspace() - if workspace: - # If mount_workspaces is set and we're doing incremental builds, - # the workspace is already mounted into the sandbox. - if not (mount_workspaces and self.__can_build_incrementally()): - with self.timed_activity("Staging local files at {}".format(workspace.path)): - workspace.stage(directory) - else: - # No workspace, stage directly - for source in self.sources(): - source._stage(directory) - + if not isinstance(vdirectory, Directory): + vdirectory = FileBasedDirectory(vdirectory) + if not vdirectory.is_empty(): + raise ElementError("Staging directory '{}' is not empty".format(vdirectory)) + + with tempfile.TemporaryDirectory() as temp_staging_directory: + + workspace = self._get_workspace() + if workspace: + # If mount_workspaces is set and we're doing incremental builds, + # the workspace is already mounted into the sandbox. + if not (mount_workspaces and self.__can_build_incrementally()): + with self.timed_activity("Staging local files at {}".format(workspace.path)): + workspace.stage(temp_staging_directory) + else: + # No workspace, stage directly + for source in self.sources(): + source._stage(temp_staging_directory) + + vdirectory.import_files(temp_staging_directory, None) # Ensure deterministic mtime of sources at build time - utils._set_deterministic_mtime(directory) + vdirectory.set_deterministic_mtime() # Ensure deterministic owners of sources at build time - utils._set_deterministic_user(directory) + vdirectory.set_deterministic_user() # _set_required(): # @@ -1432,7 +1432,7 @@ class Element(Plugin): with _signals.terminator(cleanup_rootdir), \ self.__sandbox(rootdir, output_file, output_file, self.__sandbox_config) as sandbox: # nopep8 - sandbox_root = sandbox.get_directory() + sandbox_vroot = sandbox.get_virtual_directory() # By default, the dynamic public data is the same as the static public data. # The plugin's assemble() method may modify this, though. @@ -1462,23 +1462,24 @@ class Element(Plugin): # workspace = self._get_workspace() if workspace and self.__staged_sources_directory: - sandbox_root = sandbox.get_directory() - sandbox_path = os.path.join(sandbox_root, - self.__staged_sources_directory.lstrip(os.sep)) + sandbox_vroot = sandbox.get_virtual_directory() + path_components = self.__staged_sources_directory.lstrip(os.sep).split(os.sep) + sandbox_vpath = sandbox_vroot.descend(path_components) try: - utils.copy_files(workspace.path, sandbox_path) + sandbox_vpath.import_files(workspace.path) except UtilError as e: self.warn("Failed to preserve workspace state for failed build sysroot: {}" .format(e)) raise - collectdir = os.path.join(sandbox_root, collect.lstrip(os.sep)) - if not os.path.exists(collectdir): + try: + collectvdir = sandbox_vroot.descend(collect.lstrip(os.sep).split(os.sep)) + except VirtualDirectoryError: raise ElementError( - "Directory '{}' was not found inside the sandbox, " + "Subdirectory '{}' of '{}' does not exist following assembly, " "unable to collect artifact contents" - .format(collect)) + .format(collect, sandbox_vroot)) # At this point, we expect an exception was raised leading to # an error message, or we have good output to collect. @@ -1494,7 +1495,7 @@ class Element(Plugin): os.mkdir(metadir) # Hard link files from collect dir to files directory - utils.link_files(collectdir, filesdir) + collectvdir.export_files(filesdir, can_link=True) # Copy build log if self.__log_path: -- cgit v1.2.1 From 38dc6bfcbdab5afeb468b7951214c4fe831d4310 Mon Sep 17 00:00:00 2001 From: Jim MacArthur Date: Tue, 8 May 2018 16:20:41 +0100 Subject: plugins/elements/compose.py: Convert to virtual directories --- buildstream/plugins/elements/compose.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) (limited to 'buildstream') diff --git a/buildstream/plugins/elements/compose.py b/buildstream/plugins/elements/compose.py index 44a760215..3345d20ea 100644 --- a/buildstream/plugins/elements/compose.py +++ b/buildstream/plugins/elements/compose.py @@ -34,7 +34,6 @@ The default configuration and possible options are as such: """ import os -from buildstream import utils from buildstream import Element, Scope @@ -56,6 +55,9 @@ class ComposeElement(Element): # added, to reduce the potential for confusion BST_FORBID_SOURCES = True + # This plugin has been modified to avoid the use of Sandbox.get_directory + BST_VIRTUAL_DIRECTORY = True + def configure(self, node): self.node_validate(node, [ 'integrate', 'include', 'exclude', 'include-orphans' @@ -104,7 +106,8 @@ class ComposeElement(Element): orphans=self.include_orphans) manifest.update(files) - basedir = sandbox.get_directory() + # Make a snapshot of all the files. + vbasedir = sandbox.get_virtual_directory() modified_files = set() removed_files = set() added_files = set() @@ -116,23 +119,21 @@ class ComposeElement(Element): if require_split: # Make a snapshot of all the files before integration-commands are run. - snapshot = { - f: getmtime(os.path.join(basedir, f)) - for f in utils.list_relative_paths(basedir) - } + snapshot = vbasedir.list_relative_paths_with_mtimes() for dep in self.dependencies(Scope.BUILD): dep.integrate(sandbox) if require_split: - # Calculate added, modified and removed files - basedir_contents = set(utils.list_relative_paths(basedir)) + post_integration_snapshot = vbasedir.list_relative_paths_with_mtimes() + + basedir_contents = set(post_integration_snapshot.keys()) for path in manifest: if path in basedir_contents: if path in snapshot: preintegration_mtime = snapshot[path] - if preintegration_mtime != getmtime(os.path.join(basedir, path)): + if preintegration_mtime != post_integration_snapshot[path]: modified_files.add(path) else: # If the path appears in the manifest but not the initial snapshot, @@ -166,8 +167,7 @@ class ComposeElement(Element): # instead of into a subdir. The element assemble() method should # support this in some way. # - installdir = os.path.join(basedir, 'buildstream', 'install') - os.makedirs(installdir, exist_ok=True) + installdir = vbasedir.descend(['buildstream', 'install'], create=True) # We already saved the manifest for created files in the integration phase, # now collect the rest of the manifest. @@ -191,7 +191,7 @@ class ComposeElement(Element): with self.timed_activity("Creating composition", detail=detail, silent_nested=True): self.info("Composing {} files".format(len(manifest))) - utils.link_files(basedir, installdir, files=manifest) + installdir.import_files(vbasedir, files=manifest, can_link=True) # And we're done return os.path.join(os.sep, 'buildstream', 'install') -- cgit v1.2.1 From 08199733da98077e02eba03e303d1b2e5ffaa128 Mon Sep 17 00:00:00 2001 From: Jim MacArthur Date: Tue, 8 May 2018 16:20:55 +0100 Subject: plugins/elements/import.py: Convert to virtual directories --- buildstream/plugins/elements/import.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) (limited to 'buildstream') diff --git a/buildstream/plugins/elements/import.py b/buildstream/plugins/elements/import.py index 93594b623..11c3f9a6a 100644 --- a/buildstream/plugins/elements/import.py +++ b/buildstream/plugins/elements/import.py @@ -31,7 +31,6 @@ The empty configuration is as such: """ import os -import shutil from buildstream import Element, BuildElement, ElementError @@ -68,27 +67,22 @@ class ImportElement(BuildElement): # Do not mount workspaces as the files are copied from outside the sandbox self._stage_sources_in_sandbox(sandbox, 'input', mount_workspaces=False) - rootdir = sandbox.get_directory() - inputdir = os.path.join(rootdir, 'input') - outputdir = os.path.join(rootdir, 'output') + rootdir = sandbox.get_virtual_directory() + inputdir = rootdir.descend(['input']) + outputdir = rootdir.descend(['output'], create=True) # The directory to grab - inputdir = os.path.join(inputdir, self.source.lstrip(os.sep)) - inputdir = inputdir.rstrip(os.sep) + inputdir = inputdir.descend(self.source.strip(os.sep).split(os.sep)) # The output target directory - outputdir = os.path.join(outputdir, self.target.lstrip(os.sep)) - outputdir = outputdir.rstrip(os.sep) - - # Ensure target directory parent - os.makedirs(os.path.dirname(outputdir), exist_ok=True) + outputdir = outputdir.descend(self.target.strip(os.sep).split(os.sep), create=True) - if not os.path.exists(inputdir): + if inputdir.is_empty(): raise ElementError("{}: No files were found inside directory '{}'" .format(self, self.source)) # Move it over - shutil.move(inputdir, outputdir) + outputdir.import_files(inputdir) # And we're done return '/output' -- cgit v1.2.1 From 957c3d1c9292ae0a42a08a14220a3dd240fca17b Mon Sep 17 00:00:00 2001 From: Jim MacArthur Date: Tue, 8 May 2018 16:27:19 +0100 Subject: sandbox/_mount.py, sandbox/_sandboxbwrap.py: Remove instances of get_directory --- buildstream/sandbox/_mount.py | 3 ++- buildstream/sandbox/_sandboxbwrap.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) (limited to 'buildstream') diff --git a/buildstream/sandbox/_mount.py b/buildstream/sandbox/_mount.py index 1540d9d4f..0f96a92b7 100644 --- a/buildstream/sandbox/_mount.py +++ b/buildstream/sandbox/_mount.py @@ -32,7 +32,8 @@ from .._fuse import SafeHardlinks class Mount(): def __init__(self, sandbox, mount_point, safe_hardlinks): scratch_directory = sandbox._get_scratch_directory() - root_directory = sandbox.get_directory() + # Getting external_directory here is acceptable as we're part of the sandbox code. + root_directory = sandbox.get_virtual_directory().external_directory self.mount_point = mount_point self.safe_hardlinks = safe_hardlinks diff --git a/buildstream/sandbox/_sandboxbwrap.py b/buildstream/sandbox/_sandboxbwrap.py index 9ed677620..010e4791d 100644 --- a/buildstream/sandbox/_sandboxbwrap.py +++ b/buildstream/sandbox/_sandboxbwrap.py @@ -56,7 +56,9 @@ class SandboxBwrap(Sandbox): def run(self, command, flags, *, cwd=None, env=None): stdout, stderr = self._get_output() - root_directory = self.get_directory() + + # Allowable access to underlying storage as we're part of the sandbox + root_directory = self.get_virtual_directory().external_directory # Fallback to the sandbox default settings for # the cwd and env. -- cgit v1.2.1 From 3feeaefc80242fd20fc0cd4ef443e7df2080bd6f Mon Sep 17 00:00:00 2001 From: Jim MacArthur Date: Tue, 8 May 2018 16:27:45 +0100 Subject: plugins/elements/stack.py: Convert to virtual directories --- buildstream/plugins/elements/stack.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'buildstream') diff --git a/buildstream/plugins/elements/stack.py b/buildstream/plugins/elements/stack.py index 087d4dac0..d062b23bf 100644 --- a/buildstream/plugins/elements/stack.py +++ b/buildstream/plugins/elements/stack.py @@ -24,7 +24,6 @@ Stack elements are simply a symbolic element used for representing a logical group of elements. """ -import os from buildstream import Element @@ -52,7 +51,7 @@ class StackElement(Element): # Just create a dummy empty artifact, its existence is a statement # that all this stack's dependencies are built. - rootdir = sandbox.get_directory() + vrootdir = sandbox.get_virtual_directory() # XXX FIXME: This is currently needed because the artifact # cache wont let us commit an empty artifact. @@ -61,10 +60,7 @@ class StackElement(Element): # the actual artifact data in a subdirectory, then we # will be able to store some additional state in the # artifact cache, and we can also remove this hack. - outputdir = os.path.join(rootdir, 'output', 'bst') - - # Ensure target directory parent - os.makedirs(os.path.dirname(outputdir), exist_ok=True) + vrootdir.descend(['output', 'bst'], create=True) # And we're done return '/output' -- cgit v1.2.1 From 3e1389ac6987fca0bd3384c9f96cf1a5328303fd Mon Sep 17 00:00:00 2001 From: Jim MacArthur Date: Tue, 8 May 2018 16:22:21 +0100 Subject: scriptelement.py: Convert to virtual directories --- buildstream/scriptelement.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'buildstream') diff --git a/buildstream/scriptelement.py b/buildstream/scriptelement.py index 46afda807..645381a40 100644 --- a/buildstream/scriptelement.py +++ b/buildstream/scriptelement.py @@ -243,9 +243,8 @@ class ScriptElement(Element): with self.timed_activity("Staging {} at {}" .format(element.name, item['destination']), silent_nested=True): - real_dstdir = os.path.join(sandbox.get_directory(), - item['destination'].lstrip(os.sep)) - os.makedirs(os.path.dirname(real_dstdir), exist_ok=True) + virtual_dstdir = sandbox.get_virtual_directory() + virtual_dstdir.descend(item['destination'].lstrip(os.sep).split(os.sep), create=True) element.stage_dependency_artifacts(sandbox, Scope.RUN, path=item['destination']) for item in self.__layout: @@ -263,8 +262,8 @@ class ScriptElement(Element): for dep in element.dependencies(Scope.RUN): dep.integrate(sandbox) - os.makedirs(os.path.join(sandbox.get_directory(), self.__install_root.lstrip(os.sep)), - exist_ok=True) + install_root_path_components = self.__install_root.lstrip(os.sep).split(os.sep) + sandbox.get_virtual_directory().descend(install_root_path_components, create=True) def assemble(self, sandbox): -- cgit v1.2.1 From 42c43abe7ab5617ea3da11b4cf87fa3d3e1d41b5 Mon Sep 17 00:00:00 2001 From: Jim MacArthur Date: Tue, 8 May 2018 17:34:25 +0100 Subject: Directory.py: Change the mtime interface into mark_unmodified/list_modified_paths --- buildstream/plugins/elements/compose.py | 25 +++++++------------------ buildstream/sandbox/Directory.py | 21 ++++++++++++++++----- buildstream/sandbox/_filebaseddirectory.py | 29 ++++++++++++++++++++++++----- 3 files changed, 47 insertions(+), 28 deletions(-) (limited to 'buildstream') diff --git a/buildstream/plugins/elements/compose.py b/buildstream/plugins/elements/compose.py index 3345d20ea..44e846d0e 100644 --- a/buildstream/plugins/elements/compose.py +++ b/buildstream/plugins/elements/compose.py @@ -119,36 +119,25 @@ class ComposeElement(Element): if require_split: # Make a snapshot of all the files before integration-commands are run. - snapshot = vbasedir.list_relative_paths_with_mtimes() + snapshot = set(vbasedir.list_relative_paths()) + vbasedir.mark_unmodified() + for dep in self.dependencies(Scope.BUILD): dep.integrate(sandbox) if require_split: # Calculate added, modified and removed files - post_integration_snapshot = vbasedir.list_relative_paths_with_mtimes() - - basedir_contents = set(post_integration_snapshot.keys()) + post_integration_snapshot = vbasedir.list_relative_paths() + modified_files = set(vbasedir.list_modified_paths()) + basedir_contents = set(post_integration_snapshot) for path in manifest: - if path in basedir_contents: - if path in snapshot: - preintegration_mtime = snapshot[path] - if preintegration_mtime != post_integration_snapshot[path]: - modified_files.add(path) - else: - # If the path appears in the manifest but not the initial snapshot, - # it may be a file staged inside a directory symlink. In this case - # the path we got from the manifest won't show up in the snapshot - # because utils.list_relative_paths() doesn't recurse into symlink - # directories. - pass - elif path in snapshot: + if path in snapshot and not path in basedir_contents: removed_files.add(path) for path in basedir_contents: if path not in snapshot: added_files.add(path) - self.info("Integration modified {}, added {} and removed {} files" .format(len(modified_files), len(added_files), len(removed_files))) diff --git a/buildstream/sandbox/Directory.py b/buildstream/sandbox/Directory.py index 07f22badc..410749616 100644 --- a/buildstream/sandbox/Directory.py +++ b/buildstream/sandbox/Directory.py @@ -110,11 +110,22 @@ class Directory(): """ raise NotImplementedError() - def list_relative_paths_with_mtimes(self) -> Dict[str, float]: - """Provide a list of relative paths with modification times for - each. Used to detect changed changed files during a Compose - operation. + def mark_unmodified(self) -> None: + """ Marks all files in this directory (recursively) as unmodified. + """ + raise NotImplementedError() + + def list_modified_paths(self) -> List[str]: + """Provide a list of relative paths which have been modified since the + last call to mark_unmodified. + + Return value: List(str) - dictionary with all paths + """ + raise NotImplementedError() + + def list_relative_paths(self) -> List[str]: + """Provide a list of all relative paths in this directory. - Return value: Dict(str->float) - dictionary with all paths and mtime in seconds. + Return value: List(str) - dictionary with all paths """ raise NotImplementedError() diff --git a/buildstream/sandbox/_filebaseddirectory.py b/buildstream/sandbox/_filebaseddirectory.py index 640e254b1..975a4ccb0 100644 --- a/buildstream/sandbox/_filebaseddirectory.py +++ b/buildstream/sandbox/_filebaseddirectory.py @@ -30,6 +30,7 @@ See also: :ref:`sandboxing`. from typing import List from collections import OrderedDict +import calendar import os import time from .._exceptions import BstError, ErrorDomain @@ -210,11 +211,29 @@ class FileBasedDirectory(Directory): self._populate_index() return len(self.index) == 0 - def list_relative_paths_with_mtimes(self) -> Dict[str, float]: - return { - f: getmtime(os.path.join(self.external_directory, f)) - for f in list_relative_paths(self.external_directory) - } + def mark_unmodified(self) -> None: + """ Marks all files in this directory (recursively) as unmodified. + """ + _set_deterministic_mtime(self.external_directory) + + def list_modified_paths(self) -> List[str]: + """Provide a list of relative paths which have been modified since the + last call to mark_unmodified. + + Return value: List(str) - list of modified paths + """ + magic_timestamp = calendar.timegm([2011, 11, 11, 11, 11, 11]) + + return [f for f in list_relative_paths(self.external_directory) + if getmtime(os.path.join(self.external_directory, f)) != magic_timestamp] + + def list_relative_paths(self) -> List[str]: + """Provide a list of all relative paths. + + Return value: List(str) - list of all paths + """ + + return list_relative_paths(self.external_directory) def __str__(self) -> str: # This returns the whole path (since we don't know where the directory started) -- cgit v1.2.1 From 5eaaa7cb73640d659c98440ec6f6b27aa318fa2b Mon Sep 17 00:00:00 2001 From: Jim MacArthur Date: Thu, 17 May 2018 10:10:12 +0100 Subject: Fixup compose: pep8 --- buildstream/plugins/elements/compose.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'buildstream') diff --git a/buildstream/plugins/elements/compose.py b/buildstream/plugins/elements/compose.py index 44e846d0e..2a5979828 100644 --- a/buildstream/plugins/elements/compose.py +++ b/buildstream/plugins/elements/compose.py @@ -122,7 +122,6 @@ class ComposeElement(Element): snapshot = set(vbasedir.list_relative_paths()) vbasedir.mark_unmodified() - for dep in self.dependencies(Scope.BUILD): dep.integrate(sandbox) @@ -132,7 +131,7 @@ class ComposeElement(Element): modified_files = set(vbasedir.list_modified_paths()) basedir_contents = set(post_integration_snapshot) for path in manifest: - if path in snapshot and not path in basedir_contents: + if path in snapshot and path not in basedir_contents: removed_files.add(path) for path in basedir_contents: -- cgit v1.2.1 From e3f4f97a3a0a9a0cb44228c8f4a76130414e6986 Mon Sep 17 00:00:00 2001 From: Jim MacArthur Date: Wed, 9 May 2018 12:51:13 +0100 Subject: Directory.py: link_ok=>can_link (consistent with other use) --- buildstream/sandbox/Directory.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'buildstream') diff --git a/buildstream/sandbox/Directory.py b/buildstream/sandbox/Directory.py index 410749616..0a60fce0e 100644 --- a/buildstream/sandbox/Directory.py +++ b/buildstream/sandbox/Directory.py @@ -53,7 +53,7 @@ class Directory(): # Import and export of files and links def import_files(self, external_pathspec: any, files: List[str] = None, report_written: bool = True, update_utimes: bool = False, - link_ok: bool = False) -> FileListResult: + can_link: bool = False) -> FileListResult: """Imports some or all files from external_path into this directory. Keyword arguments: external_pathspec: Either a string @@ -80,7 +80,7 @@ class Directory(): raise NotImplementedError() - def export_files(self, to_directory: str, link_ok: bool = False) -> None: + def export_files(self, to_directory: str, can_link: bool = False) -> None: """Copies everything from this into to_directory. Arguments: -- cgit v1.2.1 From 27060bfe09f7bab2117fbe9284437703fe9a0618 Mon Sep 17 00:00:00 2001 From: Jim MacArthur Date: Wed, 9 May 2018 13:00:06 +0100 Subject: Implement can_destroy flag in _filebaseddirectory.py --- buildstream/sandbox/Directory.py | 5 ++++- buildstream/sandbox/_filebaseddirectory.py | 13 ++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) (limited to 'buildstream') diff --git a/buildstream/sandbox/Directory.py b/buildstream/sandbox/Directory.py index 0a60fce0e..f37fb98ad 100644 --- a/buildstream/sandbox/Directory.py +++ b/buildstream/sandbox/Directory.py @@ -80,7 +80,7 @@ class Directory(): raise NotImplementedError() - def export_files(self, to_directory: str, can_link: bool = False) -> None: + def export_files(self, to_directory: str, can_link: bool = False, can_destroy: bool = False) -> None: """Copies everything from this into to_directory. Arguments: @@ -91,6 +91,9 @@ class Directory(): can_link (bool): Whether we can create hard links in to_directory instead of copying. Setting this does not guarantee hard links will be used. + can_destroy (bool): Can we destroy the data already in this + directory when exporting? If set, this may allow data to be + moved rather than copied which will be quicker. """ raise NotImplementedError() diff --git a/buildstream/sandbox/_filebaseddirectory.py b/buildstream/sandbox/_filebaseddirectory.py index 975a4ccb0..50dca573b 100644 --- a/buildstream/sandbox/_filebaseddirectory.py +++ b/buildstream/sandbox/_filebaseddirectory.py @@ -188,7 +188,7 @@ class FileBasedDirectory(Directory): """ _set_deterministic_user(self.external_directory) - def export_files(self, to_directory: str, can_link: bool = False) -> None: + def export_files(self, to_directory: str, can_link: bool = False, can_destroy: bool = False) -> None: """Copies everything from this into to_directory. Arguments: @@ -200,6 +200,17 @@ class FileBasedDirectory(Directory): instead of copying. """ + + if can_destroy: + # Try a simple rename of the sandbox root; if that + # doesnt cut it, then do the regular link files code path + try: + os.rename(self.external_directory, to_directory) + return + except OSError: + # Proceed using normal link/copy + pass + if can_link: link_files(self.external_directory, to_directory) else: -- cgit v1.2.1 From 1e82e98676ef3a2f095cd7b627defc7663dfec02 Mon Sep 17 00:00:00 2001 From: Jim MacArthur Date: Wed, 9 May 2018 14:01:23 +0100 Subject: _stream.py: Alter _checkout_hardlinks to use can_destroy --- buildstream/_stream.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'buildstream') diff --git a/buildstream/_stream.py b/buildstream/_stream.py index 28a480982..5c0976bd7 100644 --- a/buildstream/_stream.py +++ b/buildstream/_stream.py @@ -997,12 +997,10 @@ class Stream(): raise StreamError("Failed to remove checkout directory: {}".format(e)) from e if removed: - # TODO: Direct rename is no longer possible with the new Virtual Directory interface. - # See what options there are to restore it. os.makedirs(directory, exist_ok=True) - sandbox_vroot.export_files(directory, can_link=True) + sandbox_vroot.export_files(directory, can_link=True, can_destroy=True) else: - sandbox_vroot.export_files(directory, can_link=True) + sandbox_vroot.export_files(directory, can_link=True, can_destroy=False) # Write the element build script to the given directory def _write_element_script(self, directory, element): -- cgit v1.2.1 From 6fee0923adc723d2bd51d1d4bc25b0e01533900d Mon Sep 17 00:00:00 2001 From: Jim MacArthur Date: Mon, 21 May 2018 18:03:50 +0100 Subject: Move virtual directory components to 'storage' module --- buildstream/element.py | 4 +- buildstream/sandbox/Directory.py | 134 --------------- buildstream/sandbox/_filebaseddirectory.py | 253 ----------------------------- buildstream/sandbox/sandbox.py | 2 +- buildstream/storage/_filebaseddirectory.py | 253 +++++++++++++++++++++++++++++ buildstream/storage/directory.py | 134 +++++++++++++++ 6 files changed, 390 insertions(+), 390 deletions(-) delete mode 100644 buildstream/sandbox/Directory.py delete mode 100644 buildstream/sandbox/_filebaseddirectory.py create mode 100644 buildstream/storage/_filebaseddirectory.py create mode 100644 buildstream/storage/directory.py (limited to 'buildstream') diff --git a/buildstream/element.py b/buildstream/element.py index 8dea76758..6f06cbb1c 100644 --- a/buildstream/element.py +++ b/buildstream/element.py @@ -96,8 +96,8 @@ from . import _site from ._platform import Platform from .sandbox._config import SandboxConfig -from .sandbox.Directory import Directory -from .sandbox._filebaseddirectory import FileBasedDirectory, VirtualDirectoryError +from .storage.directory import Directory +from .storage._filebaseddirectory import FileBasedDirectory, VirtualDirectoryError # _KeyStrength(): diff --git a/buildstream/sandbox/Directory.py b/buildstream/sandbox/Directory.py deleted file mode 100644 index f37fb98ad..000000000 --- a/buildstream/sandbox/Directory.py +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2018 Codethink Limited -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see . -# -# Authors: -# Jim MacArthur - -""" -Directory -========= - -Virtual Directory class to isolate the rest of BuildStream from the backing store implementation. -Sandboxes are allowed to read from and write to the underlying storage, but all others must use this -Directory class to access files and directories in the sandbox. - -See also: :ref:`sandboxing`. -""" - -from typing import List -from ..utils import FileListResult - - -class Directory(): - def __init__(self, external_directory=None): - raise NotImplementedError() - - def descend(self, subdirectory_spec: List[str]) -> 'Directory': - """ - Descend one or more levels of directory hierarchy and return a new - Directory object for that directory. - - Arguments: - subdirectory_spec (list of strings): A list of strings which are all directory - names. - create (boolean): If this is true, the directories will be created if - they don't already exist. - """ - raise NotImplementedError() - - # Import and export of files and links - def import_files(self, external_pathspec: any, files: List[str] = None, - report_written: bool = True, update_utimes: bool = False, - can_link: bool = False) -> FileListResult: - """Imports some or all files from external_path into this directory. - - Keyword arguments: external_pathspec: Either a string - containing a pathname, or a Directory object, to use as the - source. - - files (list of strings): A list of all the files relative to - the external_pathspec to copy. If 'None' is supplied, all - files are copied. - - report_written (bool): Return the full list of files - written. Defaults to true. If false, only a list of - overwritten files is returned. - - update_utimes (bool): Update the access and modification time - of each file copied to the current time. - - can_link (bool): Whether it's OK to create a hard link to the - original content, meaning the stored copy will change when the - original files change. Setting this doesn't guarantee hard - links will be made. can_link will never be used if - update_utimes is set. - """ - - raise NotImplementedError() - - def export_files(self, to_directory: str, can_link: bool = False, can_destroy: bool = False) -> None: - """Copies everything from this into to_directory. - - Arguments: - - to_directory (string): a path outside this directory object - where the contents will be copied to. - - can_link (bool): Whether we can create hard links in to_directory - instead of copying. Setting this does not guarantee hard links will be used. - - can_destroy (bool): Can we destroy the data already in this - directory when exporting? If set, this may allow data to be - moved rather than copied which will be quicker. - """ - - raise NotImplementedError() - - # Convenience functions - def is_empty(self) -> bool: - raise NotImplementedError() - - def set_deterministic_mtime(self) -> None: - """ Sets a static modification time for all regular files in this directory. - The magic number for timestamps: 2011-11-11 11:11:11 - """ - raise NotImplementedError() - - def set_deterministic_user(self) -> None: - """ Sets all files in this directory to the current user's euid/egid. - """ - raise NotImplementedError() - - def mark_unmodified(self) -> None: - """ Marks all files in this directory (recursively) as unmodified. - """ - raise NotImplementedError() - - def list_modified_paths(self) -> List[str]: - """Provide a list of relative paths which have been modified since the - last call to mark_unmodified. - - Return value: List(str) - dictionary with all paths - """ - raise NotImplementedError() - - def list_relative_paths(self) -> List[str]: - """Provide a list of all relative paths in this directory. - - Return value: List(str) - dictionary with all paths - """ - raise NotImplementedError() diff --git a/buildstream/sandbox/_filebaseddirectory.py b/buildstream/sandbox/_filebaseddirectory.py deleted file mode 100644 index 50dca573b..000000000 --- a/buildstream/sandbox/_filebaseddirectory.py +++ /dev/null @@ -1,253 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2018 Codethink Limited -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see . -# -# Authors: -# Jim MacArthur - -""" -FileBasedDirectory -========= - -Implementation of the Directory class which backs onto a normal POSIX filing system. - -See also: :ref:`sandboxing`. -""" - -from typing import List -from collections import OrderedDict - -import calendar -import os -import time -from .._exceptions import BstError, ErrorDomain -from .Directory import Directory -from ..utils import link_files, copy_files, FileListResult, list_relative_paths -from ..utils import _set_deterministic_user, _set_deterministic_mtime - - -class VirtualDirectoryError(BstError): - """Raised by Directory functions when system calls fail. - This will be handled internally by the BuildStream core, - if you need to handle this error, then it should be reraised, - or either of the :class:`.ElementError` or :class:`.SourceError` - exceptions should be raised from this error. - """ - def __init__(self, message, reason=None): - super().__init__(message, domain=ErrorDomain.VIRTUAL_FS, reason=reason) - - -# Like os.path.getmtime(), but doesnt explode on symlinks -# Copy/pasted from compose.py -def getmtime(path): - stat = os.lstat(path) - return stat.st_mtime - -# FileBasedDirectory intentionally doesn't call its superclass constuctor, -# which is mean to be unimplemented. -# pylint: disable=super-init-not-called - - -class _FileObject(): - """A description of a file in a virtual directory. The contents of - this class are never used, but there needs to be something present - for files so is_empty() works correctly. - - """ - def __init__(self, virtual_directory: Directory, filename: str): - self.directory = virtual_directory - self.filename = filename - - -class FileBasedDirectory(Directory): - def __init__(self, external_directory=None): - self.external_directory = external_directory - self.index = OrderedDict() - self._directory_read = False - - def _populate_index(self) -> None: - if self._directory_read: - return - for entry in os.listdir(self.external_directory): - if os.path.isdir(os.path.join(self.external_directory, entry)): - self.index[entry] = FileBasedDirectory(os.path.join(self.external_directory, entry)) - else: - self.index[entry] = _FileObject(self, entry) - self._directory_read = True - - def descend(self, subdirectory_spec: List[str], create: bool = False) -> Directory: - """ Descend one or more levels of directory hierarchy and return a new - Directory object for that directory. - - Arguments: - * subdirectory_spec (list of strings): A list of strings which are all directory - names. - * create (boolean): If this is true, the directories will be created if - they don't already exist. - """ - - # It's very common to send a directory name instead of a list and this causes - # bizarre errors, so check for it here - if not isinstance(subdirectory_spec, list): - subdirectory_spec = [subdirectory_spec] - if not subdirectory_spec: - return self - - # Because of the way split works, it's common to get a list which begins with - # an empty string. Detect these and remove them, then start again. - if subdirectory_spec[0] == "": - return self.descend(subdirectory_spec[1:], create) - - self._populate_index() - if subdirectory_spec[0] in self.index: - entry = self.index[subdirectory_spec[0]] - if isinstance(entry, FileBasedDirectory): - new_path = os.path.join(self.external_directory, subdirectory_spec[0]) - return FileBasedDirectory(new_path).descend(subdirectory_spec[1:], create) - else: - error = "Cannot descend into {}, which is a '{}' in the directory {}" - raise VirtualDirectoryError(error.format(subdirectory_spec[0], - type(entry).__name__, - self.external_directory)) - else: - if create: - new_path = os.path.join(self.external_directory, subdirectory_spec[0]) - os.makedirs(new_path, exist_ok=True) - return FileBasedDirectory(new_path).descend(subdirectory_spec[1:], create) - else: - error = "No entry called '{}' found in the directory rooted at {}" - raise VirtualDirectoryError(error.format(subdirectory_spec[0], self.external_directory)) - return None - - def import_files(self, external_pathspec: any, files: List[str] = None, - report_written: bool = True, update_utimes: bool = False, - can_link: bool = False) -> FileListResult: - """Imports some or all files from external_path into this directory. - - Keyword arguments: external_pathspec: Either a string - containing a pathname, or a Directory object, to use as the - source. - - files (list of strings): A list of all the files relative to - the external_pathspec to copy. If 'None' is supplied, all - files are copied. - - report_written (bool): Return the full list of files - written. Defaults to true. If false, only a list of - overwritten files is returned. - - update_utimes (bool): Update the access and modification time - of each file copied to the current time. - - can_link (bool): Whether it's OK to create a hard link to the - original content, meaning the stored copy will change when the - original files change. Setting this doesn't guarantee hard - links will be made. can_link will never be used if - update_utimes is set. - """ - - if isinstance(external_pathspec, Directory): - source_directory = external_pathspec.external_directory - else: - source_directory = external_pathspec - - if can_link and not update_utimes: - import_result = link_files(source_directory, self.external_directory, files=files, - ignore_missing=False, report_written=report_written) - else: - import_result = copy_files(source_directory, self.external_directory, files=files, - ignore_missing=False, report_written=report_written) - if update_utimes: - cur_time = time.time() - - for f in import_result.files_written: - os.utime(os.path.join(self.external_directory, f), times=(cur_time, cur_time)) - return import_result - - def set_deterministic_mtime(self) -> None: - """ Sets a static modification time for all regular files in this directory. - The magic number for timestamps: 2011-11-11 11:11:11 - """ - _set_deterministic_mtime(self.external_directory) - - def set_deterministic_user(self) -> None: - """ Sets all files in this directory to the current user's euid/egid. - """ - _set_deterministic_user(self.external_directory) - - def export_files(self, to_directory: str, can_link: bool = False, can_destroy: bool = False) -> None: - """Copies everything from this into to_directory. - - Arguments: - - to_directory (string): a path outside this directory object - where the contents will be copied to. - - can_link (bool): Whether we can create hard links in to_directory - instead of copying. - - """ - - if can_destroy: - # Try a simple rename of the sandbox root; if that - # doesnt cut it, then do the regular link files code path - try: - os.rename(self.external_directory, to_directory) - return - except OSError: - # Proceed using normal link/copy - pass - - if can_link: - link_files(self.external_directory, to_directory) - else: - copy_files(self.external_directory, to_directory) - - def is_empty(self) -> bool: - """ Return true if this directory has no files, subdirectories or links in it. - """ - self._populate_index() - return len(self.index) == 0 - - def mark_unmodified(self) -> None: - """ Marks all files in this directory (recursively) as unmodified. - """ - _set_deterministic_mtime(self.external_directory) - - def list_modified_paths(self) -> List[str]: - """Provide a list of relative paths which have been modified since the - last call to mark_unmodified. - - Return value: List(str) - list of modified paths - """ - magic_timestamp = calendar.timegm([2011, 11, 11, 11, 11, 11]) - - return [f for f in list_relative_paths(self.external_directory) - if getmtime(os.path.join(self.external_directory, f)) != magic_timestamp] - - def list_relative_paths(self) -> List[str]: - """Provide a list of all relative paths. - - Return value: List(str) - list of all paths - """ - - return list_relative_paths(self.external_directory) - - def __str__(self) -> str: - # This returns the whole path (since we don't know where the directory started) - # which exposes the sandbox directory; we will have to assume for the time being - # that people will not abuse __str__. - return self.external_directory diff --git a/buildstream/sandbox/sandbox.py b/buildstream/sandbox/sandbox.py index 68e6b13b2..f3cab41ec 100644 --- a/buildstream/sandbox/sandbox.py +++ b/buildstream/sandbox/sandbox.py @@ -30,7 +30,7 @@ See also: :ref:`sandboxing`. import os from .._exceptions import ImplError, BstError -from ._filebaseddirectory import FileBasedDirectory +from ..storage._filebaseddirectory import FileBasedDirectory class SandboxFlags(): diff --git a/buildstream/storage/_filebaseddirectory.py b/buildstream/storage/_filebaseddirectory.py new file mode 100644 index 000000000..60379eaed --- /dev/null +++ b/buildstream/storage/_filebaseddirectory.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2018 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . +# +# Authors: +# Jim MacArthur + +""" +FileBasedDirectory +========= + +Implementation of the Directory class which backs onto a normal POSIX filing system. + +See also: :ref:`sandboxing`. +""" + +from typing import List +from collections import OrderedDict + +import calendar +import os +import time +from .._exceptions import BstError, ErrorDomain +from .directory import Directory +from ..utils import link_files, copy_files, FileListResult, list_relative_paths +from ..utils import _set_deterministic_user, _set_deterministic_mtime + + +class VirtualDirectoryError(BstError): + """Raised by Directory functions when system calls fail. + This will be handled internally by the BuildStream core, + if you need to handle this error, then it should be reraised, + or either of the :class:`.ElementError` or :class:`.SourceError` + exceptions should be raised from this error. + """ + def __init__(self, message, reason=None): + super().__init__(message, domain=ErrorDomain.VIRTUAL_FS, reason=reason) + + +# Like os.path.getmtime(), but doesnt explode on symlinks +# Copy/pasted from compose.py +def getmtime(path): + stat = os.lstat(path) + return stat.st_mtime + +# FileBasedDirectory intentionally doesn't call its superclass constuctor, +# which is mean to be unimplemented. +# pylint: disable=super-init-not-called + + +class _FileObject(): + """A description of a file in a virtual directory. The contents of + this class are never used, but there needs to be something present + for files so is_empty() works correctly. + + """ + def __init__(self, virtual_directory: Directory, filename: str): + self.directory = virtual_directory + self.filename = filename + + +class FileBasedDirectory(Directory): + def __init__(self, external_directory=None): + self.external_directory = external_directory + self.index = OrderedDict() + self._directory_read = False + + def _populate_index(self) -> None: + if self._directory_read: + return + for entry in os.listdir(self.external_directory): + if os.path.isdir(os.path.join(self.external_directory, entry)): + self.index[entry] = FileBasedDirectory(os.path.join(self.external_directory, entry)) + else: + self.index[entry] = _FileObject(self, entry) + self._directory_read = True + + def descend(self, subdirectory_spec: List[str], create: bool = False) -> Directory: + """ Descend one or more levels of directory hierarchy and return a new + Directory object for that directory. + + Arguments: + * subdirectory_spec (list of strings): A list of strings which are all directory + names. + * create (boolean): If this is true, the directories will be created if + they don't already exist. + """ + + # It's very common to send a directory name instead of a list and this causes + # bizarre errors, so check for it here + if not isinstance(subdirectory_spec, list): + subdirectory_spec = [subdirectory_spec] + if not subdirectory_spec: + return self + + # Because of the way split works, it's common to get a list which begins with + # an empty string. Detect these and remove them, then start again. + if subdirectory_spec[0] == "": + return self.descend(subdirectory_spec[1:], create) + + self._populate_index() + if subdirectory_spec[0] in self.index: + entry = self.index[subdirectory_spec[0]] + if isinstance(entry, FileBasedDirectory): + new_path = os.path.join(self.external_directory, subdirectory_spec[0]) + return FileBasedDirectory(new_path).descend(subdirectory_spec[1:], create) + else: + error = "Cannot descend into {}, which is a '{}' in the directory {}" + raise VirtualDirectoryError(error.format(subdirectory_spec[0], + type(entry).__name__, + self.external_directory)) + else: + if create: + new_path = os.path.join(self.external_directory, subdirectory_spec[0]) + os.makedirs(new_path, exist_ok=True) + return FileBasedDirectory(new_path).descend(subdirectory_spec[1:], create) + else: + error = "No entry called '{}' found in the directory rooted at {}" + raise VirtualDirectoryError(error.format(subdirectory_spec[0], self.external_directory)) + return None + + def import_files(self, external_pathspec: any, files: List[str] = None, + report_written: bool = True, update_utimes: bool = False, + can_link: bool = False) -> FileListResult: + """Imports some or all files from external_path into this directory. + + Keyword arguments: external_pathspec: Either a string + containing a pathname, or a Directory object, to use as the + source. + + files (list of strings): A list of all the files relative to + the external_pathspec to copy. If 'None' is supplied, all + files are copied. + + report_written (bool): Return the full list of files + written. Defaults to true. If false, only a list of + overwritten files is returned. + + update_utimes (bool): Update the access and modification time + of each file copied to the current time. + + can_link (bool): Whether it's OK to create a hard link to the + original content, meaning the stored copy will change when the + original files change. Setting this doesn't guarantee hard + links will be made. can_link will never be used if + update_utimes is set. + """ + + if isinstance(external_pathspec, Directory): + source_directory = external_pathspec.external_directory + else: + source_directory = external_pathspec + + if can_link and not update_utimes: + import_result = link_files(source_directory, self.external_directory, files=files, + ignore_missing=False, report_written=report_written) + else: + import_result = copy_files(source_directory, self.external_directory, files=files, + ignore_missing=False, report_written=report_written) + if update_utimes: + cur_time = time.time() + + for f in import_result.files_written: + os.utime(os.path.join(self.external_directory, f), times=(cur_time, cur_time)) + return import_result + + def set_deterministic_mtime(self) -> None: + """ Sets a static modification time for all regular files in this directory. + The magic number for timestamps: 2011-11-11 11:11:11 + """ + _set_deterministic_mtime(self.external_directory) + + def set_deterministic_user(self) -> None: + """ Sets all files in this directory to the current user's euid/egid. + """ + _set_deterministic_user(self.external_directory) + + def export_files(self, to_directory: str, can_link: bool = False, can_destroy: bool = False) -> None: + """Copies everything from this into to_directory. + + Arguments: + + to_directory (string): a path outside this directory object + where the contents will be copied to. + + can_link (bool): Whether we can create hard links in to_directory + instead of copying. + + """ + + if can_destroy: + # Try a simple rename of the sandbox root; if that + # doesnt cut it, then do the regular link files code path + try: + os.rename(self.external_directory, to_directory) + return + except OSError: + # Proceed using normal link/copy + pass + + if can_link: + link_files(self.external_directory, to_directory) + else: + copy_files(self.external_directory, to_directory) + + def is_empty(self) -> bool: + """ Return true if this directory has no files, subdirectories or links in it. + """ + self._populate_index() + return len(self.index) == 0 + + def mark_unmodified(self) -> None: + """ Marks all files in this directory (recursively) as unmodified. + """ + _set_deterministic_mtime(self.external_directory) + + def list_modified_paths(self) -> List[str]: + """Provide a list of relative paths which have been modified since the + last call to mark_unmodified. + + Return value: List(str) - list of modified paths + """ + magic_timestamp = calendar.timegm([2011, 11, 11, 11, 11, 11]) + + return [f for f in list_relative_paths(self.external_directory) + if getmtime(os.path.join(self.external_directory, f)) != magic_timestamp] + + def list_relative_paths(self) -> List[str]: + """Provide a list of all relative paths. + + Return value: List(str) - list of all paths + """ + + return list_relative_paths(self.external_directory) + + def __str__(self) -> str: + # This returns the whole path (since we don't know where the directory started) + # which exposes the sandbox directory; we will have to assume for the time being + # that people will not abuse __str__. + return self.external_directory diff --git a/buildstream/storage/directory.py b/buildstream/storage/directory.py new file mode 100644 index 000000000..f37fb98ad --- /dev/null +++ b/buildstream/storage/directory.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2018 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . +# +# Authors: +# Jim MacArthur + +""" +Directory +========= + +Virtual Directory class to isolate the rest of BuildStream from the backing store implementation. +Sandboxes are allowed to read from and write to the underlying storage, but all others must use this +Directory class to access files and directories in the sandbox. + +See also: :ref:`sandboxing`. +""" + +from typing import List +from ..utils import FileListResult + + +class Directory(): + def __init__(self, external_directory=None): + raise NotImplementedError() + + def descend(self, subdirectory_spec: List[str]) -> 'Directory': + """ + Descend one or more levels of directory hierarchy and return a new + Directory object for that directory. + + Arguments: + subdirectory_spec (list of strings): A list of strings which are all directory + names. + create (boolean): If this is true, the directories will be created if + they don't already exist. + """ + raise NotImplementedError() + + # Import and export of files and links + def import_files(self, external_pathspec: any, files: List[str] = None, + report_written: bool = True, update_utimes: bool = False, + can_link: bool = False) -> FileListResult: + """Imports some or all files from external_path into this directory. + + Keyword arguments: external_pathspec: Either a string + containing a pathname, or a Directory object, to use as the + source. + + files (list of strings): A list of all the files relative to + the external_pathspec to copy. If 'None' is supplied, all + files are copied. + + report_written (bool): Return the full list of files + written. Defaults to true. If false, only a list of + overwritten files is returned. + + update_utimes (bool): Update the access and modification time + of each file copied to the current time. + + can_link (bool): Whether it's OK to create a hard link to the + original content, meaning the stored copy will change when the + original files change. Setting this doesn't guarantee hard + links will be made. can_link will never be used if + update_utimes is set. + """ + + raise NotImplementedError() + + def export_files(self, to_directory: str, can_link: bool = False, can_destroy: bool = False) -> None: + """Copies everything from this into to_directory. + + Arguments: + + to_directory (string): a path outside this directory object + where the contents will be copied to. + + can_link (bool): Whether we can create hard links in to_directory + instead of copying. Setting this does not guarantee hard links will be used. + + can_destroy (bool): Can we destroy the data already in this + directory when exporting? If set, this may allow data to be + moved rather than copied which will be quicker. + """ + + raise NotImplementedError() + + # Convenience functions + def is_empty(self) -> bool: + raise NotImplementedError() + + def set_deterministic_mtime(self) -> None: + """ Sets a static modification time for all regular files in this directory. + The magic number for timestamps: 2011-11-11 11:11:11 + """ + raise NotImplementedError() + + def set_deterministic_user(self) -> None: + """ Sets all files in this directory to the current user's euid/egid. + """ + raise NotImplementedError() + + def mark_unmodified(self) -> None: + """ Marks all files in this directory (recursively) as unmodified. + """ + raise NotImplementedError() + + def list_modified_paths(self) -> List[str]: + """Provide a list of relative paths which have been modified since the + last call to mark_unmodified. + + Return value: List(str) - dictionary with all paths + """ + raise NotImplementedError() + + def list_relative_paths(self) -> List[str]: + """Provide a list of all relative paths in this directory. + + Return value: List(str) - dictionary with all paths + """ + raise NotImplementedError() -- cgit v1.2.1 From af993bbb5319cc0568695b9f3ea26b738ef6f76f Mon Sep 17 00:00:00 2001 From: Jim MacArthur Date: Tue, 22 May 2018 15:09:07 +0100 Subject: storage: Add __init__.py --- buildstream/storage/__init__.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 buildstream/storage/__init__.py (limited to 'buildstream') diff --git a/buildstream/storage/__init__.py b/buildstream/storage/__init__.py new file mode 100644 index 000000000..49364bb86 --- /dev/null +++ b/buildstream/storage/__init__.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2017 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . +# +# Authors: +# Jim MacArthur + +from ._filebaseddirectory import FileBasedDirectory +from .directory import Directory -- cgit v1.2.1