diff options
| -rw-r--r-- | buildstream/_artifactcache/__init__.py | 5 | ||||
| -rw-r--r-- | buildstream/_artifactcache/tarcache.py | 347 | ||||
| -rwxr-xr-x | setup.py | 2 |
3 files changed, 350 insertions, 4 deletions
diff --git a/buildstream/_artifactcache/__init__.py b/buildstream/_artifactcache/__init__.py index 638575f5a..16e508d7f 100644 --- a/buildstream/_artifactcache/__init__.py +++ b/buildstream/_artifactcache/__init__.py @@ -19,6 +19,5 @@ # Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> from .artifactcache import ArtifactCache - -# Entry points for our local ostree-push implementation -from .pushreceive import receive_main +from .ostreecache import OSTreeCache +from .tarcache import TarCache diff --git a/buildstream/_artifactcache/tarcache.py b/buildstream/_artifactcache/tarcache.py new file mode 100644 index 000000000..56e3edd6c --- /dev/null +++ b/buildstream/_artifactcache/tarcache.py @@ -0,0 +1,347 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2017 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Maat <tristan.maat@codethink.co.uk> + +import os +import shutil +import tarfile +import subprocess + +from .. import utils +from ..element import _KeyStrength +from .._message import Message, MessageType +from ..exceptions import _ArtifactError, ProgramNotFoundError + +from . import ArtifactCache + + +def buildref(element, key): + project = element.get_project() + + # Normalize ostree ref unsupported chars + element_name = element.normal_name.replace('+', 'X') + + # assume project and element names are not allowed to contain slashes + return '{0}/{1}/{2}'.format(project.name, element_name, key) + + +def tarpath(element, key): + project = element.get_project() + return os.path.join(project.name, element.normal_name, key + '.tar.bz2') + + +# A helper class that contains tar archive/extract functions +class Tar(): + + # archive() + # + # Attempt to archive the given tarfile with the `tar` command, + # falling back to python's `tarfile` if this fails. + # + # Args: + # location (str): The path to the tar to create + # content (str): The path to the content to archvive + # cwd (str): The cwd + # + # This is done since AIX tar does not support 2G+ files. + # + @classmethod + def archive(cls, location, content, cwd=os.getcwd()): + + try: + cls._archive_with_tar(location, content, cwd) + return + except tarfile.TarError: + pass + except ProgramNotFoundError: + pass + + # If the former did not complete successfully, we try with + # python's tar implementation (since it's hard to detect + # specific issues with specific tar implementations - a + # fallback). + + try: + cls._archive_with_python(location, content, cwd) + except tarfile.TarError as e: + raise _ArtifactError("Failed to archive {}: {}" + .format(location, e)) from e + + # extract() + # + # Attempt to extract the given tarfile with the `tar` command, + # falling back to python's `tarfile` if this fails. + # + # Args: + # location (str): The path to the tar to extract + # dest (str): The destination path to extract to + # + # This is done since python tarfile extraction is horrendously + # slow (2 hrs+ for base images). + # + @classmethod + def extract(cls, location, dest): + + try: + cls._extract_with_tar(location, dest) + return + except tarfile.TarError: + pass + except ProgramNotFoundError: + pass + + try: + cls._extract_with_python(location, dest) + except tarfile.TarError as e: + raise _ArtifactError("Failed to extract {}: {}" + .format(location, e)) from e + + # _get_host_tar() + # + # Get the host tar command. + # + # Raises: + # ProgramNotFoundError: If the tar executable cannot be + # located + # + @classmethod + def _get_host_tar(cls): + tar_cmd = None + + for potential_tar_cmd in ['gtar', 'tar']: + try: + tar_cmd = utils.get_host_tool(potential_tar_cmd) + break + except ProgramNotFoundError: + continue + + # If we still couldn't find tar, raise the ProgramNotfounderror + if tar_cmd is None: + raise ProgramNotFoundError("Did not find tar in PATH: {}" + .format(os.environ.get('PATH'))) + + return tar_cmd + + # _archive_with_tar() + # + # Archive with an implementation of the `tar` command + # + # Args: + # location (str): The path to the tar to create + # content (str): The path to the content to archvive + # cwd (str): The cwd + # + # Raises: + # TarError: If an error occurs during extraction + # ProgramNotFoundError: If the tar executable cannot be + # located + # + @classmethod + def _archive_with_tar(cls, location, content, cwd): + tar_cmd = cls._get_host_tar() + + process = subprocess.Popen( + [tar_cmd, 'jcaf', location, content], + cwd=cwd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + + _, err = process.communicate() + if process.poll() != 0: + # Clean up in case the command failed in a broken state + try: + os.remove(location) + except FileNotFoundError: + pass + + raise tarfile.TarError("Failed to archive '{}': {}" + .format(content, err.decode('utf8'))) + + # _archive_with_python() + # + # Archive with the python `tarfile` module + # + # Args: + # location (str): The path to the tar to create + # content (str): The path to the content to archvive + # cwd (str): The cwd + # + # Raises: + # TarError: If an error occurs during extraction + # + @classmethod + def _archive_with_python(cls, location, content, cwd): + with tarfile.open(location, mode='w:bz2') as tar: + tar.add(os.path.join(cwd, content), arcname=content) + + # _extract_with_tar() + # + # Extract with an implementation of the `tar` command + # + # Args: + # location (str): The path to the tar to extract + # dest (str): The destination path to extract to + # + # Raises: + # TarError: If an error occurs during extraction + # + @classmethod + def _extract_with_tar(cls, location, dest): + tar_cmd = cls._get_host_tar() + + # Some tar implementations do not support '-C' + process = subprocess.Popen( + [tar_cmd, 'jxf', location], + cwd=dest, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + + _, err = process.communicate() + if process.poll() != 0: + raise tarfile.TarError("Failed to extract '{}': {}" + .format(location, err.decode('utf8'))) + + # _extract_with_python() + # + # Extract with the python `tarfile` module + # + # Args: + # location (str): The path to the tar to extract + # dest (str): The destination path to extract to + # + # Raises: + # TarError: If an error occurs during extraction + # + @classmethod + def _extract_with_python(cls, location, dest): + with tarfile.open(location) as tar: + tar.extractall(path=dest) + + +class TarCache(ArtifactCache): + + def __init__(self, context, project): + + super().__init__(context, project) + + self.tardir = os.path.join(context.artifactdir, 'tar') + os.makedirs(self.tardir, exist_ok=True) + + # contains() + # + # Implements artifactcache.contains(). + # + def contains(self, element, strength=None): + if strength is None: + strength = _KeyStrength.STRONG if self.context.strict_build_plan else _KeyStrength.WEAK + + key = element._get_cache_key(strength) + + if not key: + return False + + path = os.path.join(self.tardir, tarpath(element, key)) + return os.path.isfile(path) + + # remove() + # + # Implements artifactcache.remove(). + # + # FIXME: Untested... + # + def remove(self, element): + key = element._get_cache_key() + if not key: + return + + path = (os.path.join(self.tardir, tarpath(element, key))) + shutil.rmtree(path) + + # commit() + # + # Implements artifactcache.commit(). + # + def commit(self, element, content): + ref = tarpath(element, element._get_cache_key_for_build()) + weak_ref = tarpath(element, element._get_cache_key(strength=_KeyStrength.WEAK)) + + os.makedirs(os.path.join(self.tardir, element.get_project().name, element.normal_name), exist_ok=True) + + with utils._tempdir() as temp: + refdir = os.path.join(temp, element._get_cache_key_for_build()) + shutil.copytree(content, refdir, symlinks=True) + + if ref != weak_ref: + weak_refdir = os.path.join(temp, element._get_cache_key(strength=_KeyStrength.WEAK)) + shutil.copytree(content, weak_refdir, symlinks=True) + + Tar.archive(os.path.join(self.tardir, ref), + element._get_cache_key_for_build(), + temp) + + if ref != weak_ref: + Tar.archive(os.path.join(self.tardir, weak_ref), + element._get_cache_key(strength=_KeyStrength.WEAK), + temp) + + # extract() + # + # Implements artifactcache.extract(). + # + def extract(self, element): + + key = element._get_cache_key() + ref = buildref(element, key) + path = tarpath(element, key) + + if not os.path.isfile(os.path.join(self.tardir, path)): + key = element._get_cache_key(strength=_KeyStrength.WEAK) + ref = buildref(element, key) + path = tarpath(element, key) + + if not os.path.isfile(os.path.join(self.tardir, path)): + raise _ArtifactError("Artifact missing for {}".format(ref)) + + # If the destination already exists, the artifact has been extracted + dest = os.path.join(self.extractdir, ref) + if os.path.isdir(dest): + return dest + + os.makedirs(self.extractdir, exist_ok=True) + + with utils._tempdir(dir=self.extractdir) as tmpdir: + Tar.extract(os.path.join(self.tardir, path), tmpdir) + + os.makedirs(os.path.join(self.extractdir, element.get_project().name, element.normal_name), + exist_ok=True) + try: + os.rename(os.path.join(tmpdir, key), dest) + except OSError as e: + # With rename, it's possible to get either ENOTEMPTY or EEXIST + # in the case that the destination path is a not empty directory. + # + # If rename fails with these errors, another process beat + # us to it so just ignore. + if e.errno not in [os.errno.ENOTEMPTY, os.errno.EEXIST]: + raise _ArtifactError("Failed to extract artifact for ref '{}': {}" + .format(ref, e)) from e + + return dest @@ -136,7 +136,7 @@ setup(name='BuildStream', entry_points=''' [console_scripts] bst=buildstream._frontend:cli - bst-artifact-receive=buildstream._artifactcache:receive_main + bst-artifact-receive=buildstream._artifactcache.pushreceive:receive_main ''', setup_requires=['pytest-runner', 'setuptools_scm'], tests_require=['pep8', |