diff options
Diffstat (limited to 'buildstream')
| -rw-r--r-- | buildstream/_frontend/cli.py | 24 | ||||
| -rw-r--r-- | buildstream/_stream.py | 114 |
2 files changed, 104 insertions, 34 deletions
diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py index 51744e806..bd2ce8a73 100644 --- a/buildstream/_frontend/cli.py +++ b/buildstream/_frontend/cli.py @@ -626,7 +626,7 @@ def shell(app, element, sysroot, mount, isolate, build_, command): ################################################################## @cli.command(short_help="Checkout a built artifact") @click.option('--force', '-f', default=False, is_flag=True, - help="Overwrite files existing in checkout directory") + help="Allow files to be overwritten") @click.option('--deps', '-d', default='run', type=click.Choice(['run', 'none']), help='The dependencies to checkout (default: run)') @@ -634,20 +634,30 @@ def shell(app, element, sysroot, mount, isolate, build_, command): help="Whether to run integration commands") @click.option('--hardlinks', default=False, is_flag=True, help="Checkout hardlinks instead of copies (handle with care)") +@click.option('--tar', default=False, is_flag=True, + help="Create a tarball from the artifact contents instead " + "of a file tree. If LOCATION is '-', the tarball " + "will be dumped to the standard output.") @click.argument('element', type=click.Path(readable=False)) -@click.argument('directory', type=click.Path(file_okay=False)) +@click.argument('location', type=click.Path()) @click.pass_obj -def checkout(app, element, directory, force, deps, integrate, hardlinks): - """Checkout a built artifact to the specified directory +def checkout(app, element, location, force, deps, integrate, hardlinks, tar): + """Checkout a built artifact to the specified location """ + + if hardlinks and tar: + click.echo("ERROR: options --hardlinks and --tar conflict", err=True) + sys.exit(-1) + with app.initialized(): app.stream.checkout(element, - directory=directory, - deps=deps, + location=location, force=force, + deps=deps, integrate=integrate, - hardlinks=hardlinks) + hardlinks=hardlinks, + tar=tar) ################################################################## diff --git a/buildstream/_stream.py b/buildstream/_stream.py index 002d37788..365709c46 100644 --- a/buildstream/_stream.py +++ b/buildstream/_stream.py @@ -20,6 +20,7 @@ # Tristan Maat <tristan.maat@codethink.co.uk> import os +import sys import stat import shlex import shutil @@ -350,56 +351,91 @@ class Stream(): # checkout() # - # Checkout the pipeline target artifact to the specified directory + # Checkout target artifact to the specified location # # Args: # target (str): Target to checkout - # directory (str): The directory to checkout the artifact to - # force (bool): Force overwrite files which exist in `directory` + # location (str): Location to checkout the artifact to + # force (bool): Whether files can be overwritten if necessary + # deps (str): The dependencies to checkout # integrate (bool): Whether to run integration commands # hardlinks (bool): Whether checking out files hardlinked to # their artifacts is acceptable + # tar (bool): If true, a tarball from the artifact contents will + # be created, otherwise the file tree of the artifact + # will be placed at the given location. If true and + # location is '-', the tarball will be dumped on the + # standard output. # def checkout(self, target, *, - deps='run', - directory=None, + location=None, force=False, + deps='run', integrate=True, - hardlinks=False): + hardlinks=False, + tar=False): # We only have one target in a checkout command elements, _ = self._load((target,), (), fetch_subprojects=True) target = elements[0] - try: - os.makedirs(directory, exist_ok=True) - except OSError as e: - raise StreamError("Failed to create checkout directory: {}".format(e)) from e - - if not os.access(directory, os.W_OK): - raise StreamError("Directory {} not writable".format(directory)) + if not tar: + try: + os.makedirs(location, exist_ok=True) + except OSError as e: + raise StreamError("Failed to create checkout directory: '{}'" + .format(e)) from e - if not force and os.listdir(directory): - raise StreamError("Checkout directory is not empty: {}" - .format(directory)) + if not tar: + if not os.access(location, os.W_OK): + raise StreamError("Checkout directory '{}' not writable" + .format(location)) + if not force and os.listdir(location): + raise StreamError("Checkout directory '{}' not empty" + .format(location)) + elif os.path.exists(location) and location != '-': + if not os.access(location, os.W_OK): + raise StreamError("Output file '{}' not writable" + .format(location)) + if not force and os.path.exists(location): + raise StreamError("Output file '{}' already exists" + .format(location)) # Stage deps into a temporary sandbox first try: - with target._prepare_sandbox(Scope.RUN, None, deps=deps, integrate=integrate) as sandbox: + with target._prepare_sandbox(Scope.RUN, None, deps=deps, + integrate=integrate) as sandbox: # Copy or move the sandbox to the target directory sandbox_root = sandbox.get_directory() - with target.timed_activity("Checking out files in {}".format(directory)): - try: - if hardlinks: - self._checkout_hardlinks(sandbox_root, directory) - else: - utils.copy_files(sandbox_root, directory) - except OSError as e: - raise StreamError("Failed to checkout files: {}".format(e)) from e + if not tar: + with target.timed_activity("Checking out files in '{}'" + .format(location)): + try: + if hardlinks: + self._checkout_hardlinks(sandbox_root, location) + else: + utils.copy_files(sandbox_root, location) + except OSError as e: + raise StreamError("Failed to checkout files: '{}'" + .format(e)) from e + else: + if location == '-': + with target.timed_activity("Creating tarball"): + with os.fdopen(sys.stdout.fileno(), 'wb') as fo: + with tarfile.open(fileobj=fo, mode="w|") as tf: + Stream._add_directory_to_tarfile( + tf, sandbox_root, '.') + else: + with target.timed_activity("Creating tarball '{}'" + .format(location)): + with tarfile.open(location, "w:") as tf: + Stream._add_directory_to_tarfile( + tf, sandbox_root, '.') + except BstError as e: - raise StreamError("Error while staging dependencies into a sandbox: {}".format(e), - reason=e.reason) from e + raise StreamError("Error while staging dependencies into a sandbox" + ": '{}'".format(e), reason=e.reason) from e # workspace_open # @@ -1027,6 +1063,30 @@ class Stream(): else: utils.link_files(sandbox_root, directory) + # Add a directory entry deterministically to a tar file + # + # This function takes extra steps to ensure the output is deterministic. + # First, it sorts the results of os.listdir() to ensure the ordering of + # the files in the archive is the same. Second, it sets a fixed + # timestamp for each entry. See also https://bugs.python.org/issue24465. + @staticmethod + def _add_directory_to_tarfile(tf, dir_name, dir_arcname, mtime=0): + for filename in sorted(os.listdir(dir_name)): + name = os.path.join(dir_name, filename) + arcname = os.path.join(dir_arcname, filename) + + tarinfo = tf.gettarinfo(name, arcname) + tarinfo.mtime = mtime + + if tarinfo.isreg(): + with open(name, "rb") as f: + tf.addfile(tarinfo, f) + elif tarinfo.isdir(): + tf.addfile(tarinfo) + Stream._add_directory_to_tarfile(tf, name, arcname, mtime) + else: + tf.addfile(tarinfo) + # Write the element build script to the given directory def _write_element_script(self, directory, element): try: |