From c57eb4ef59609f840691807c5d7083e45046ffcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrg=20Billeter?= Date: Fri, 11 May 2018 16:39:50 +0200 Subject: Remove tar artifact cache No longer used. --- buildstream/_artifactcache/tarcache.py | 298 --------------------------------- tests/artifactcache/tar.py | 82 --------- 2 files changed, 380 deletions(-) delete mode 100644 buildstream/_artifactcache/tarcache.py delete mode 100644 tests/artifactcache/tar.py diff --git a/buildstream/_artifactcache/tarcache.py b/buildstream/_artifactcache/tarcache.py deleted file mode 100644 index 10ae9d008..000000000 --- a/buildstream/_artifactcache/tarcache.py +++ /dev/null @@ -1,298 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2017 Codethink Limited -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see . -# -# Authors: -# Tristan Maat - -import os -import shutil -import tarfile -import subprocess - -from .. import utils, ProgramNotFoundError -from .._exceptions import ArtifactError - -from . import ArtifactCache - - -class TarCache(ArtifactCache): - - def __init__(self, context): - - super().__init__(context) - - self.tardir = os.path.join(context.artifactdir, 'tar') - os.makedirs(self.tardir, exist_ok=True) - - ################################################ - # Implementation of abstract methods # - ################################################ - def contains(self, element, key): - path = os.path.join(self.tardir, _tarpath(element, key)) - return os.path.isfile(path) - - def commit(self, element, content, keys): - os.makedirs(os.path.join(self.tardir, element._get_project().name, element.normal_name), exist_ok=True) - - with utils._tempdir() as temp: - for key in keys: - ref = _tarpath(element, key) - - refdir = os.path.join(temp, key) - shutil.copytree(content, refdir, symlinks=True) - - _Tar.archive(os.path.join(self.tardir, ref), key, temp) - - def extract(self, element, key): - - fullname = self.get_artifact_fullname(element, key) - path = _tarpath(element, key) - - # Extracting a nonexistent artifact is a bug - assert os.path.isfile(os.path.join(self.tardir, path)), "Artifact missing for {}".format(fullname) - - # If the destination already exists, the artifact has been extracted - dest = os.path.join(self.extractdir, fullname) - if os.path.isdir(dest): - return dest - - os.makedirs(self.extractdir, exist_ok=True) - - with utils._tempdir(dir=self.extractdir) as tmpdir: - _Tar.extract(os.path.join(self.tardir, path), tmpdir) - - os.makedirs(os.path.join(self.extractdir, element._get_project().name, element.normal_name), - exist_ok=True) - try: - os.rename(os.path.join(tmpdir, key), dest) - except OSError as e: - # With rename, it's possible to get either ENOTEMPTY or EEXIST - # in the case that the destination path is a not empty directory. - # - # If rename fails with these errors, another process beat - # us to it so just ignore. - if e.errno not in [os.errno.ENOTEMPTY, os.errno.EEXIST]: - raise ArtifactError("Failed to extract artifact '{}': {}" - .format(fullname, e)) from e - - return dest - - -# _tarpath() -# -# Generate a relative tarball path for a given element and it's cache key -# -# Args: -# element (Element): The Element object -# key (str): The element's cache key -# -# Returns: -# (str): The relative path to use for storing tarballs -# -def _tarpath(element, key): - project = element._get_project() - return os.path.join(project.name, element.normal_name, key + '.tar.bz2') - - -# A helper class that contains tar archive/extract functions -class _Tar(): - - # archive() - # - # Attempt to archive the given tarfile with the `tar` command, - # falling back to python's `tarfile` if this fails. - # - # Args: - # location (str): The path to the tar to create - # content (str): The path to the content to archvive - # cwd (str): The cwd - # - # This is done since AIX tar does not support 2G+ files. - # - @classmethod - def archive(cls, location, content, cwd=os.getcwd()): - - try: - cls._archive_with_tar(location, content, cwd) - return - except tarfile.TarError: - pass - except ProgramNotFoundError: - pass - - # If the former did not complete successfully, we try with - # python's tar implementation (since it's hard to detect - # specific issues with specific tar implementations - a - # fallback). - - try: - cls._archive_with_python(location, content, cwd) - except tarfile.TarError as e: - raise ArtifactError("Failed to archive {}: {}" - .format(location, e)) from e - - # extract() - # - # Attempt to extract the given tarfile with the `tar` command, - # falling back to python's `tarfile` if this fails. - # - # Args: - # location (str): The path to the tar to extract - # dest (str): The destination path to extract to - # - # This is done since python tarfile extraction is horrendously - # slow (2 hrs+ for base images). - # - @classmethod - def extract(cls, location, dest): - - try: - cls._extract_with_tar(location, dest) - return - except tarfile.TarError: - pass - except ProgramNotFoundError: - pass - - try: - cls._extract_with_python(location, dest) - except tarfile.TarError as e: - raise ArtifactError("Failed to extract {}: {}" - .format(location, e)) from e - - # _get_host_tar() - # - # Get the host tar command. - # - # Raises: - # ProgramNotFoundError: If the tar executable cannot be - # located - # - @classmethod - def _get_host_tar(cls): - tar_cmd = None - - for potential_tar_cmd in ['gtar', 'tar']: - try: - tar_cmd = utils.get_host_tool(potential_tar_cmd) - break - except ProgramNotFoundError: - continue - - # If we still couldn't find tar, raise the ProgramNotfounderror - if tar_cmd is None: - raise ProgramNotFoundError("Did not find tar in PATH: {}" - .format(os.environ.get('PATH'))) - - return tar_cmd - - # _archive_with_tar() - # - # Archive with an implementation of the `tar` command - # - # Args: - # location (str): The path to the tar to create - # content (str): The path to the content to archvive - # cwd (str): The cwd - # - # Raises: - # TarError: If an error occurs during extraction - # ProgramNotFoundError: If the tar executable cannot be - # located - # - @classmethod - def _archive_with_tar(cls, location, content, cwd): - tar_cmd = cls._get_host_tar() - - process = subprocess.Popen( - [tar_cmd, 'jcaf', location, content], - cwd=cwd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE - ) - - _, err = process.communicate() - if process.poll() != 0: - # Clean up in case the command failed in a broken state - try: - os.remove(location) - except FileNotFoundError: - pass - - raise tarfile.TarError("Failed to archive '{}': {}" - .format(content, err.decode('utf8'))) - - # _archive_with_python() - # - # Archive with the python `tarfile` module - # - # Args: - # location (str): The path to the tar to create - # content (str): The path to the content to archvive - # cwd (str): The cwd - # - # Raises: - # TarError: If an error occurs during extraction - # - @classmethod - def _archive_with_python(cls, location, content, cwd): - with tarfile.open(location, mode='w:bz2') as tar: - tar.add(os.path.join(cwd, content), arcname=content) - - # _extract_with_tar() - # - # Extract with an implementation of the `tar` command - # - # Args: - # location (str): The path to the tar to extract - # dest (str): The destination path to extract to - # - # Raises: - # TarError: If an error occurs during extraction - # - @classmethod - def _extract_with_tar(cls, location, dest): - tar_cmd = cls._get_host_tar() - - # Some tar implementations do not support '-C' - process = subprocess.Popen( - [tar_cmd, 'jxf', location], - cwd=dest, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE - ) - - _, err = process.communicate() - if process.poll() != 0: - raise tarfile.TarError("Failed to extract '{}': {}" - .format(location, err.decode('utf8'))) - - # _extract_with_python() - # - # Extract with the python `tarfile` module - # - # Args: - # location (str): The path to the tar to extract - # dest (str): The destination path to extract to - # - # Raises: - # TarError: If an error occurs during extraction - # - @classmethod - def _extract_with_python(cls, location, dest): - with tarfile.open(location) as tar: - tar.extractall(path=dest) diff --git a/tests/artifactcache/tar.py b/tests/artifactcache/tar.py deleted file mode 100644 index ef39be31c..000000000 --- a/tests/artifactcache/tar.py +++ /dev/null @@ -1,82 +0,0 @@ -import os -import tarfile -import tempfile -from contextlib import ExitStack - -import pytest - -from buildstream._artifactcache.tarcache import _Tar -from buildstream import utils, ProgramNotFoundError - - -# Test that it 'works' - this may be equivalent to test_archive_no_tar() -# on some systems. -def test_archive_default(): - with ExitStack() as stack: - src = stack.enter_context(tempfile.TemporaryDirectory()) - tar_dir = stack.enter_context(tempfile.TemporaryDirectory()) - scratch = stack.enter_context(tempfile.TemporaryDirectory()) - test_file = stack.enter_context(open(os.path.join(src, 'test'), 'a')) - test_file.write('Test') - - _Tar.archive(os.path.join(tar_dir, 'test.tar'), '.', src) - - with tarfile.open(os.path.join(tar_dir, 'test.tar')) as tar: - tar.extractall(path=scratch) - - assert os.listdir(scratch) == os.listdir(src) - - -def test_archive_no_tar(): - # Modify the path to exclude 'tar' - old_path = os.environ.get('PATH') - os.environ['PATH'] = '' - - # Ensure we can't find 'tar' or 'gtar' - try: - for tar in ['gtar', 'tar']: - with pytest.raises(ProgramNotFoundError): - utils.get_host_tool(tar) - - # Run the same test as before, this time 'tar' should not be available - test_archive_default() - - # Reset the environment - finally: - os.environ['PATH'] = old_path - - -# Same thing as test_archive_default() -def test_extract_default(): - with ExitStack() as stack: - src = stack.enter_context(tempfile.TemporaryDirectory()) - tar_dir = stack.enter_context(tempfile.TemporaryDirectory()) - scratch = stack.enter_context(tempfile.TemporaryDirectory()) - test_file = stack.enter_context(open(os.path.join(src, 'test'), 'a')) - test_file.write('Test') - - with tarfile.open(os.path.join(tar_dir, 'test.tar'), 'a:') as tar: - tar.add(src, 'contents') - - _Tar.extract(os.path.join(tar_dir, 'test.tar'), scratch) - - assert os.listdir(os.path.join(scratch, 'contents')) == os.listdir(src) - - -def test_extract_no_tar(): - # Modify the path to exclude 'tar' - old_path = os.environ.get('PATH') - os.environ['PATH'] = '' - - # Ensure we can't find 'tar' or 'gtar' - for tar in ['gtar', 'tar']: - with pytest.raises(ProgramNotFoundError): - utils.get_host_tool(tar) - - # Run the same test as before, this time 'tar' should not be available - try: - test_extract_default() - - # Reset the environment - finally: - os.environ['PATH'] = old_path -- cgit v1.2.1