diff options
Diffstat (limited to 'buildstream/_artifactcache/tarcache.py')
| -rw-r--r-- | buildstream/_artifactcache/tarcache.py | 297 |
1 files changed, 0 insertions, 297 deletions
diff --git a/buildstream/_artifactcache/tarcache.py b/buildstream/_artifactcache/tarcache.py deleted file mode 100644 index ab814abb0..000000000 --- a/buildstream/_artifactcache/tarcache.py +++ /dev/null @@ -1,297 +0,0 @@ -# -# Copyright (C) 2017 Codethink Limited -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see <http://www.gnu.org/licenses/>. -# -# Authors: -# Tristan Maat <tristan.maat@codethink.co.uk> - -import os -import shutil -import tarfile -import subprocess - -from .. import utils, ProgramNotFoundError -from .._exceptions import ArtifactError - -from . import ArtifactCache - - -class TarCache(ArtifactCache): - - def __init__(self, context): - - super().__init__(context) - - self.tardir = os.path.join(context.artifactdir, 'tar') - os.makedirs(self.tardir, exist_ok=True) - - ################################################ - # Implementation of abstract methods # - ################################################ - def contains(self, element, key): - path = os.path.join(self.tardir, _tarpath(element, key)) - return os.path.isfile(path) - - def commit(self, element, content, keys): - os.makedirs(os.path.join(self.tardir, element._get_project().name, element.normal_name), exist_ok=True) - - with utils._tempdir() as temp: - for key in keys: - ref = _tarpath(element, key) - - refdir = os.path.join(temp, key) - shutil.copytree(content, refdir, symlinks=True) - - _Tar.archive(os.path.join(self.tardir, ref), key, temp) - - def extract(self, element, key): - - fullname = self.get_artifact_fullname(element, key) - path = _tarpath(element, key) - - # Extracting a nonexistent artifact is a bug - assert os.path.isfile(os.path.join(self.tardir, path)), "Artifact missing for {}".format(fullname) - - # If the destination already exists, the artifact has been extracted - dest = os.path.join(self.extractdir, fullname) - if os.path.isdir(dest): - return dest - - os.makedirs(self.extractdir, exist_ok=True) - - with utils._tempdir(dir=self.extractdir) as tmpdir: - _Tar.extract(os.path.join(self.tardir, path), tmpdir) - - os.makedirs(os.path.join(self.extractdir, element._get_project().name, element.normal_name), - exist_ok=True) - try: - os.rename(os.path.join(tmpdir, key), dest) - except OSError as e: - # With rename, it's possible to get either ENOTEMPTY or EEXIST - # in the case that the destination path is a not empty directory. - # - # If rename fails with these errors, another process beat - # us to it so just ignore. - if e.errno not in [os.errno.ENOTEMPTY, os.errno.EEXIST]: - raise ArtifactError("Failed to extract artifact '{}': {}" - .format(fullname, e)) from e - - return dest - - -# _tarpath() -# -# Generate a relative tarball path for a given element and it's cache key -# -# Args: -# element (Element): The Element object -# key (str): The element's cache key -# -# Returns: -# (str): The relative path to use for storing tarballs -# -def _tarpath(element, key): - project = element._get_project() - return os.path.join(project.name, element.normal_name, key + '.tar.bz2') - - -# A helper class that contains tar archive/extract functions -class _Tar(): - - # archive() - # - # Attempt to archive the given tarfile with the `tar` command, - # falling back to python's `tarfile` if this fails. - # - # Args: - # location (str): The path to the tar to create - # content (str): The path to the content to archvive - # cwd (str): The cwd - # - # This is done since AIX tar does not support 2G+ files. - # - @classmethod - def archive(cls, location, content, cwd=os.getcwd()): - - try: - cls._archive_with_tar(location, content, cwd) - return - except tarfile.TarError: - pass - except ProgramNotFoundError: - pass - - # If the former did not complete successfully, we try with - # python's tar implementation (since it's hard to detect - # specific issues with specific tar implementations - a - # fallback). - - try: - cls._archive_with_python(location, content, cwd) - except tarfile.TarError as e: - raise ArtifactError("Failed to archive {}: {}" - .format(location, e)) from e - - # extract() - # - # Attempt to extract the given tarfile with the `tar` command, - # falling back to python's `tarfile` if this fails. - # - # Args: - # location (str): The path to the tar to extract - # dest (str): The destination path to extract to - # - # This is done since python tarfile extraction is horrendously - # slow (2 hrs+ for base images). - # - @classmethod - def extract(cls, location, dest): - - try: - cls._extract_with_tar(location, dest) - return - except tarfile.TarError: - pass - except ProgramNotFoundError: - pass - - try: - cls._extract_with_python(location, dest) - except tarfile.TarError as e: - raise ArtifactError("Failed to extract {}: {}" - .format(location, e)) from e - - # _get_host_tar() - # - # Get the host tar command. - # - # Raises: - # ProgramNotFoundError: If the tar executable cannot be - # located - # - @classmethod - def _get_host_tar(cls): - tar_cmd = None - - for potential_tar_cmd in ['gtar', 'tar']: - try: - tar_cmd = utils.get_host_tool(potential_tar_cmd) - break - except ProgramNotFoundError: - continue - - # If we still couldn't find tar, raise the ProgramNotfounderror - if tar_cmd is None: - raise ProgramNotFoundError("Did not find tar in PATH: {}" - .format(os.environ.get('PATH'))) - - return tar_cmd - - # _archive_with_tar() - # - # Archive with an implementation of the `tar` command - # - # Args: - # location (str): The path to the tar to create - # content (str): The path to the content to archvive - # cwd (str): The cwd - # - # Raises: - # TarError: If an error occurs during extraction - # ProgramNotFoundError: If the tar executable cannot be - # located - # - @classmethod - def _archive_with_tar(cls, location, content, cwd): - tar_cmd = cls._get_host_tar() - - process = subprocess.Popen( - [tar_cmd, 'jcaf', location, content], - cwd=cwd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE - ) - - _, err = process.communicate() - if process.poll() != 0: - # Clean up in case the command failed in a broken state - try: - os.remove(location) - except FileNotFoundError: - pass - - raise tarfile.TarError("Failed to archive '{}': {}" - .format(content, err.decode('utf8'))) - - # _archive_with_python() - # - # Archive with the python `tarfile` module - # - # Args: - # location (str): The path to the tar to create - # content (str): The path to the content to archvive - # cwd (str): The cwd - # - # Raises: - # TarError: If an error occurs during extraction - # - @classmethod - def _archive_with_python(cls, location, content, cwd): - with tarfile.open(location, mode='w:bz2') as tar: - tar.add(os.path.join(cwd, content), arcname=content) - - # _extract_with_tar() - # - # Extract with an implementation of the `tar` command - # - # Args: - # location (str): The path to the tar to extract - # dest (str): The destination path to extract to - # - # Raises: - # TarError: If an error occurs during extraction - # - @classmethod - def _extract_with_tar(cls, location, dest): - tar_cmd = cls._get_host_tar() - - # Some tar implementations do not support '-C' - process = subprocess.Popen( - [tar_cmd, 'jxf', location], - cwd=dest, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE - ) - - _, err = process.communicate() - if process.poll() != 0: - raise tarfile.TarError("Failed to extract '{}': {}" - .format(location, err.decode('utf8'))) - - # _extract_with_python() - # - # Extract with the python `tarfile` module - # - # Args: - # location (str): The path to the tar to extract - # dest (str): The destination path to extract to - # - # Raises: - # TarError: If an error occurs during extraction - # - @classmethod - def _extract_with_python(cls, location, dest): - with tarfile.open(location) as tar: - tar.extractall(path=dest) |