diff options
Diffstat (limited to 'src/buildstream/plugins/sources')
-rw-r--r-- | src/buildstream/plugins/sources/_downloadablefilesource.py | 61 | ||||
-rw-r--r-- | src/buildstream/plugins/sources/bzr.py | 109 | ||||
-rw-r--r-- | src/buildstream/plugins/sources/deb.py | 6 | ||||
-rw-r--r-- | src/buildstream/plugins/sources/local.py | 8 | ||||
-rw-r--r-- | src/buildstream/plugins/sources/patch.py | 12 | ||||
-rw-r--r-- | src/buildstream/plugins/sources/pip.py | 106 | ||||
-rw-r--r-- | src/buildstream/plugins/sources/remote.py | 11 | ||||
-rw-r--r-- | src/buildstream/plugins/sources/tar.py | 45 | ||||
-rw-r--r-- | src/buildstream/plugins/sources/workspace.py | 12 | ||||
-rw-r--r-- | src/buildstream/plugins/sources/zip.py | 14 |
10 files changed, 205 insertions, 179 deletions
diff --git a/src/buildstream/plugins/sources/_downloadablefilesource.py b/src/buildstream/plugins/sources/_downloadablefilesource.py index 1e759b94f..4e43ee3e3 100644 --- a/src/buildstream/plugins/sources/_downloadablefilesource.py +++ b/src/buildstream/plugins/sources/_downloadablefilesource.py @@ -12,7 +12,6 @@ from buildstream import utils class _NetrcFTPOpener(urllib.request.FTPHandler): - def __init__(self, netrc_config): self.netrc = netrc_config @@ -28,11 +27,11 @@ class _NetrcFTPOpener(urllib.request.FTPHandler): def _unsplit(self, host, port, user, passwd): if port: - host = '{}:{}'.format(host, port) + host = "{}:{}".format(host, port) if user: if passwd: - user = '{}:{}'.format(user, passwd) - host = '{}@{}'.format(user, host) + user = "{}:{}".format(user, passwd) + host = "{}@{}".format(user, host) return host @@ -50,7 +49,6 @@ class _NetrcFTPOpener(urllib.request.FTPHandler): class _NetrcPasswordManager: - def __init__(self, netrc_config): self.netrc = netrc_config @@ -72,17 +70,16 @@ class _NetrcPasswordManager: class DownloadableFileSource(Source): # pylint: disable=attribute-defined-outside-init - COMMON_CONFIG_KEYS = Source.COMMON_CONFIG_KEYS + ['url', 'ref', 'etag'] + COMMON_CONFIG_KEYS = Source.COMMON_CONFIG_KEYS + ["url", "ref", "etag"] __urlopener = None __default_mirror_file = None def configure(self, node): - self.original_url = node.get_str('url') - self.ref = node.get_str('ref', None) + self.original_url = node.get_str("url") + self.ref = node.get_str("ref", None) self.url = self.translate_url(self.original_url) - self._mirror_dir = os.path.join(self.get_mirror_directory(), - utils.url_directory_name(self.original_url)) + self._mirror_dir = os.path.join(self.get_mirror_directory(), utils.url_directory_name(self.original_url)) self._warn_deprecated_etag(node) def preflight(self): @@ -102,28 +99,29 @@ class DownloadableFileSource(Source): return Consistency.RESOLVED def load_ref(self, node): - self.ref = node.get_str('ref', None) + self.ref = node.get_str("ref", None) self._warn_deprecated_etag(node) def get_ref(self): return self.ref def set_ref(self, ref, node): - node['ref'] = self.ref = ref + node["ref"] = self.ref = ref def track(self): # pylint: disable=arguments-differ # there is no 'track' field in the source to determine what/whether # or not to update refs, because tracking a ref is always a conscious # decision by the user. - with self.timed_activity("Tracking {}".format(self.url), - silent_nested=True): + with self.timed_activity("Tracking {}".format(self.url), silent_nested=True): new_ref = self._ensure_mirror() if self.ref and self.ref != new_ref: - detail = "When tracking, new ref differs from current ref:\n" \ - + " Tracked URL: {}\n".format(self.url) \ - + " Current ref: {}\n".format(self.ref) \ + detail = ( + "When tracking, new ref differs from current ref:\n" + + " Tracked URL: {}\n".format(self.url) + + " Current ref: {}\n".format(self.ref) + " New ref: {}\n".format(new_ref) + ) self.warn("Potential man-in-the-middle attack!", detail=detail) return new_ref @@ -142,25 +140,26 @@ class DownloadableFileSource(Source): with self.timed_activity("Fetching {}".format(self.url), silent_nested=True): sha256 = self._ensure_mirror() if sha256 != self.ref: - raise SourceError("File downloaded from {} has sha256sum '{}', not '{}'!" - .format(self.url, sha256, self.ref)) + raise SourceError( + "File downloaded from {} has sha256sum '{}', not '{}'!".format(self.url, sha256, self.ref) + ) def _warn_deprecated_etag(self, node): - etag = node.get_str('etag', None) + etag = node.get_str("etag", None) if etag: provenance = node.get_scalar(etag).get_provenance() self.warn('{} "etag" is deprecated and ignored.'.format(provenance)) def _get_etag(self, ref): - etagfilename = os.path.join(self._mirror_dir, '{}.etag'.format(ref)) + etagfilename = os.path.join(self._mirror_dir, "{}.etag".format(ref)) if os.path.exists(etagfilename): - with open(etagfilename, 'r') as etagfile: + with open(etagfilename, "r") as etagfile: return etagfile.read() return None def _store_etag(self, ref, etag): - etagfilename = os.path.join(self._mirror_dir, '{}.etag'.format(ref)) + etagfilename = os.path.join(self._mirror_dir, "{}.etag".format(ref)) with utils.save_file_atomic(etagfilename) as etagfile: etagfile.write(etag) @@ -170,7 +169,7 @@ class DownloadableFileSource(Source): with self.tempdir() as td: default_name = os.path.basename(self.url) request = urllib.request.Request(self.url) - request.add_header('Accept', '*/*') + request.add_header("Accept", "*/*") # We do not use etag in case what we have in cache is # not matching ref in order to be able to recover from @@ -180,18 +179,18 @@ class DownloadableFileSource(Source): # Do not re-download the file if the ETag matches. if etag and self.get_consistency() == Consistency.CACHED: - request.add_header('If-None-Match', etag) + request.add_header("If-None-Match", etag) opener = self.__get_urlopener() with contextlib.closing(opener.open(request)) as response: info = response.info() - etag = info['ETag'] if 'ETag' in info else None + etag = info["ETag"] if "ETag" in info else None filename = info.get_filename(default_name) filename = os.path.basename(filename) local_file = os.path.join(td, filename) - with open(local_file, 'wb') as dest: + with open(local_file, "wb") as dest: shutil.copyfileobj(response, dest) # Make sure url-specific mirror dir exists. @@ -214,14 +213,12 @@ class DownloadableFileSource(Source): # Because we use etag only for matching ref, currently specified ref is what # we would have downloaded. return self.ref - raise SourceError("{}: Error mirroring {}: {}" - .format(self, self.url, e), temporary=True) from e + raise SourceError("{}: Error mirroring {}: {}".format(self, self.url, e), temporary=True) from e except (urllib.error.URLError, urllib.error.ContentTooShortError, OSError, ValueError) as e: # Note that urllib.request.Request in the try block may throw a # ValueError for unknown url types, so we handle it here. - raise SourceError("{}: Error mirroring {}: {}" - .format(self, self.url, e), temporary=True) from e + raise SourceError("{}: Error mirroring {}: {}".format(self, self.url, e), temporary=True) from e def _get_mirror_file(self, sha=None): if sha is not None: @@ -245,7 +242,7 @@ class DownloadableFileSource(Source): # DownloadableFileSource.__urlopener = urllib.request.build_opener() except netrc.NetrcParseError as e: - self.warn('{}: While reading .netrc: {}'.format(self, e)) + self.warn("{}: While reading .netrc: {}".format(self, e)) return urllib.request.build_opener() else: netrc_pw_mgr = _NetrcPasswordManager(netrc_config) diff --git a/src/buildstream/plugins/sources/bzr.py b/src/buildstream/plugins/sources/bzr.py index 88dba7dc2..30ce55585 100644 --- a/src/buildstream/plugins/sources/bzr.py +++ b/src/buildstream/plugins/sources/bzr.py @@ -67,16 +67,16 @@ class BzrSource(Source): # pylint: disable=attribute-defined-outside-init def configure(self, node): - node.validate_keys(['url', 'track', 'ref', *Source.COMMON_CONFIG_KEYS]) + node.validate_keys(["url", "track", "ref", *Source.COMMON_CONFIG_KEYS]) - self.original_url = node.get_str('url') - self.tracking = node.get_str('track') - self.ref = node.get_str('ref', None) + self.original_url = node.get_str("url") + self.tracking = node.get_str("track") + self.ref = node.get_str("ref", None) self.url = self.translate_url(self.original_url) def preflight(self): # Check if bzr is installed, get the binary at the same time. - self.host_bzr = utils.get_host_tool('bzr') + self.host_bzr = utils.get_host_tool("bzr") def get_unique_key(self): return [self.original_url, self.tracking, self.ref] @@ -93,39 +93,44 @@ class BzrSource(Source): return Consistency.RESOLVED def load_ref(self, node): - self.ref = node.get_str('ref', None) + self.ref = node.get_str("ref", None) def get_ref(self): return self.ref def set_ref(self, ref, node): - node['ref'] = self.ref = ref + node["ref"] = self.ref = ref def track(self): # pylint: disable=arguments-differ - with self.timed_activity("Tracking {}".format(self.url), - silent_nested=True), self._locked(): + with self.timed_activity("Tracking {}".format(self.url), silent_nested=True), self._locked(): self._ensure_mirror(skip_ref_check=True) - ret, out = self.check_output([self.host_bzr, "version-info", - "--custom", "--template={revno}", - self._get_branch_dir()], - fail="Failed to read the revision number at '{}'" - .format(self._get_branch_dir())) + ret, out = self.check_output( + [self.host_bzr, "version-info", "--custom", "--template={revno}", self._get_branch_dir()], + fail="Failed to read the revision number at '{}'".format(self._get_branch_dir()), + ) if ret != 0: raise SourceError("{}: Failed to get ref for tracking {}".format(self, self.tracking)) return out def fetch(self): # pylint: disable=arguments-differ - with self.timed_activity("Fetching {}".format(self.url), - silent_nested=True), self._locked(): + with self.timed_activity("Fetching {}".format(self.url), silent_nested=True), self._locked(): self._ensure_mirror() def stage(self, directory): - self.call([self.host_bzr, "checkout", "--lightweight", - "--revision=revno:{}".format(self.ref), - self._get_branch_dir(), directory], - fail="Failed to checkout revision {} from branch {} to {}" - .format(self.ref, self._get_branch_dir(), directory)) + self.call( + [ + self.host_bzr, + "checkout", + "--lightweight", + "--revision=revno:{}".format(self.ref), + self._get_branch_dir(), + directory, + ], + fail="Failed to checkout revision {} from branch {} to {}".format( + self.ref, self._get_branch_dir(), directory + ), + ) # Remove .bzr dir shutil.rmtree(os.path.join(directory, ".bzr")) @@ -133,16 +138,24 @@ class BzrSource(Source): url = os.path.join(self.url, self.tracking) with self.timed_activity('Setting up workspace "{}"'.format(directory), silent_nested=True): # Checkout from the cache - self.call([self.host_bzr, "branch", - "--use-existing-dir", - "--revision=revno:{}".format(self.ref), - self._get_branch_dir(), directory], - fail="Failed to branch revision {} from branch {} to {}" - .format(self.ref, self._get_branch_dir(), directory)) + self.call( + [ + self.host_bzr, + "branch", + "--use-existing-dir", + "--revision=revno:{}".format(self.ref), + self._get_branch_dir(), + directory, + ], + fail="Failed to branch revision {} from branch {} to {}".format( + self.ref, self._get_branch_dir(), directory + ), + ) # Switch the parent branch to the source's origin - self.call([self.host_bzr, "switch", - "--directory={}".format(directory), url], - fail="Failed to switch workspace's parent branch to {}".format(url)) + self.call( + [self.host_bzr, "switch", "--directory={}".format(directory), url], + fail="Failed to switch workspace's parent branch to {}".format(url), + ) # _locked() # @@ -151,13 +164,10 @@ class BzrSource(Source): # @contextmanager def _locked(self): - lockdir = os.path.join(self.get_mirror_directory(), 'locks') - lockfile = os.path.join( - lockdir, - utils.url_directory_name(self.original_url) + '.lock' - ) + lockdir = os.path.join(self.get_mirror_directory(), "locks") + lockfile = os.path.join(lockdir, utils.url_directory_name(self.original_url) + ".lock") os.makedirs(lockdir, exist_ok=True) - with open(lockfile, 'w') as lock: + with open(lockfile, "w") as lock: fcntl.flock(lock, fcntl.LOCK_EX) try: yield @@ -169,41 +179,42 @@ class BzrSource(Source): if not os.path.exists(self._get_branch_dir()): return False - return self.call([self.host_bzr, "revno", - "--revision=revno:{}".format(self.ref), - self._get_branch_dir()]) == 0 + return self.call([self.host_bzr, "revno", "--revision=revno:{}".format(self.ref), self._get_branch_dir()]) == 0 def _get_branch_dir(self): return os.path.join(self._get_mirror_dir(), self.tracking) def _get_mirror_dir(self): - return os.path.join(self.get_mirror_directory(), - utils.url_directory_name(self.original_url)) + return os.path.join(self.get_mirror_directory(), utils.url_directory_name(self.original_url)) def _ensure_mirror(self, skip_ref_check=False): mirror_dir = self._get_mirror_dir() bzr_metadata_dir = os.path.join(mirror_dir, ".bzr") if not os.path.exists(bzr_metadata_dir): - self.call([self.host_bzr, "init-repo", "--no-trees", mirror_dir], - fail="Failed to initialize bzr repository") + self.call( + [self.host_bzr, "init-repo", "--no-trees", mirror_dir], fail="Failed to initialize bzr repository" + ) branch_dir = os.path.join(mirror_dir, self.tracking) branch_url = self.url + "/" + self.tracking if not os.path.exists(branch_dir): # `bzr branch` the branch if it doesn't exist # to get the upstream code - self.call([self.host_bzr, "branch", branch_url, branch_dir], - fail="Failed to branch from {} to {}".format(branch_url, branch_dir)) + self.call( + [self.host_bzr, "branch", branch_url, branch_dir], + fail="Failed to branch from {} to {}".format(branch_url, branch_dir), + ) else: # `bzr pull` the branch if it does exist # to get any changes to the upstream code - self.call([self.host_bzr, "pull", "--directory={}".format(branch_dir), branch_url], - fail="Failed to pull new changes for {}".format(branch_dir)) + self.call( + [self.host_bzr, "pull", "--directory={}".format(branch_dir), branch_url], + fail="Failed to pull new changes for {}".format(branch_dir), + ) if not skip_ref_check and not self._check_ref(): - raise SourceError("Failed to ensure ref '{}' was mirrored".format(self.ref), - reason="ref-not-mirrored") + raise SourceError("Failed to ensure ref '{}' was mirrored".format(self.ref), reason="ref-not-mirrored") def setup(): diff --git a/src/buildstream/plugins/sources/deb.py b/src/buildstream/plugins/sources/deb.py index cc88cf53c..a7437b150 100644 --- a/src/buildstream/plugins/sources/deb.py +++ b/src/buildstream/plugins/sources/deb.py @@ -50,7 +50,7 @@ details on common configuration options for sources. import tarfile from contextlib import contextmanager -import arpy # pylint: disable=import-error +import arpy # pylint: disable=import-error from .tar import TarSource @@ -61,14 +61,14 @@ class DebSource(TarSource): def configure(self, node): super().configure(node) - self.base_dir = node.get_str('base-dir', None) + self.base_dir = node.get_str("base-dir", None) def preflight(self): return @contextmanager def _get_tar(self): - with open(self._get_mirror_file(), 'rb') as deb_file: + with open(self._get_mirror_file(), "rb") as deb_file: arpy_archive = arpy.Archive(fileobj=deb_file) arpy_archive.read_all_headers() data_tar_arpy = [v for k, v in arpy_archive.archived_files.items() if b"data.tar" in k][0] diff --git a/src/buildstream/plugins/sources/local.py b/src/buildstream/plugins/sources/local.py index f40fd79c0..90d8a8f6f 100644 --- a/src/buildstream/plugins/sources/local.py +++ b/src/buildstream/plugins/sources/local.py @@ -54,8 +54,8 @@ class LocalSource(Source): self.__unique_key = None def configure(self, node): - node.validate_keys(['path', *Source.COMMON_CONFIG_KEYS]) - self.path = self.node_get_project_path(node.get_scalar('path')) + node.validate_keys(["path", *Source.COMMON_CONFIG_KEYS]) + self.path = self.node_get_project_path(node.get_scalar("path")) self.fullpath = os.path.join(self.get_project_directory(), self.path) def preflight(self): @@ -89,8 +89,8 @@ class LocalSource(Source): if result.overwritten or result.ignored: raise SourceError( - "Failed to stage source: files clash with existing directory", - reason='ensure-stage-dir-fail') + "Failed to stage source: files clash with existing directory", reason="ensure-stage-dir-fail" + ) def _get_local_path(self): return self.fullpath diff --git a/src/buildstream/plugins/sources/patch.py b/src/buildstream/plugins/sources/patch.py index 86811cb4d..082983023 100644 --- a/src/buildstream/plugins/sources/patch.py +++ b/src/buildstream/plugins/sources/patch.py @@ -56,8 +56,7 @@ class PatchSource(Source): def configure(self, node): node.validate_keys(["path", "strip-level", *Source.COMMON_CONFIG_KEYS]) - self.path = self.node_get_project_path(node.get_scalar('path'), - check_is_file=True) + self.path = self.node_get_project_path(node.get_scalar("path"), check_is_file=True) self.strip_level = node.get_int("strip-level", default=1) self.fullpath = os.path.join(self.get_project_directory(), self.path) @@ -89,12 +88,13 @@ class PatchSource(Source): # Bail out with a comprehensive message if the target directory is empty if not os.listdir(directory): - raise SourceError("Nothing to patch in directory '{}'".format(directory), - reason="patch-no-files") + raise SourceError("Nothing to patch in directory '{}'".format(directory), reason="patch-no-files") strip_level_option = "-p{}".format(self.strip_level) - self.call([self.host_patch, strip_level_option, "-i", self.fullpath, "-d", directory], - fail="Failed to apply patch {}".format(self.path)) + self.call( + [self.host_patch, strip_level_option, "-i", self.fullpath, "-d", directory], + fail="Failed to apply patch {}".format(self.path), + ) # Plugin entry point diff --git a/src/buildstream/plugins/sources/pip.py b/src/buildstream/plugins/sources/pip.py index 758ef665f..2c9773787 100644 --- a/src/buildstream/plugins/sources/pip.py +++ b/src/buildstream/plugins/sources/pip.py @@ -74,30 +74,28 @@ import re from buildstream import Consistency, Source, SourceError, utils -_OUTPUT_DIRNAME = '.bst_pip_downloads' -_PYPI_INDEX_URL = 'https://pypi.org/simple/' +_OUTPUT_DIRNAME = ".bst_pip_downloads" +_PYPI_INDEX_URL = "https://pypi.org/simple/" # Used only for finding pip command _PYTHON_VERSIONS = [ - 'python', # when running in a venv, we might not have the exact version - 'python2.7', - 'python3.0', - 'python3.1', - 'python3.2', - 'python3.3', - 'python3.4', - 'python3.5', - 'python3.6', - 'python3.7', + "python", # when running in a venv, we might not have the exact version + "python2.7", + "python3.0", + "python3.1", + "python3.2", + "python3.3", + "python3.4", + "python3.5", + "python3.6", + "python3.7", ] # List of allowed extensions taken from # https://docs.python.org/3/distutils/sourcedist.html. # Names of source distribution archives must be of the form # '%{package-name}-%{version}.%{extension}'. -_SDIST_RE = re.compile( - r'^([\w.-]+?)-((?:[\d.]+){2,})\.(?:tar|tar.bz2|tar.gz|tar.xz|tar.Z|zip)$', - re.IGNORECASE) +_SDIST_RE = re.compile(r"^([\w.-]+?)-((?:[\d.]+){2,})\.(?:tar|tar.bz2|tar.gz|tar.xz|tar.Z|zip)$", re.IGNORECASE) class PipSource(Source): @@ -109,16 +107,15 @@ class PipSource(Source): BST_REQUIRES_PREVIOUS_SOURCES_TRACK = True def configure(self, node): - node.validate_keys(['url', 'packages', 'ref', 'requirements-files'] + - Source.COMMON_CONFIG_KEYS) - self.ref = node.get_str('ref', None) - self.original_url = node.get_str('url', _PYPI_INDEX_URL) + node.validate_keys(["url", "packages", "ref", "requirements-files"] + Source.COMMON_CONFIG_KEYS) + self.ref = node.get_str("ref", None) + self.original_url = node.get_str("url", _PYPI_INDEX_URL) self.index_url = self.translate_url(self.original_url) - self.packages = node.get_str_list('packages', []) - self.requirements_files = node.get_str_list('requirements-files', []) + self.packages = node.get_str_list("packages", []) + self.requirements_files = node.get_str_list("requirements-files", []) if not (self.packages or self.requirements_files): - raise SourceError("{}: Either 'packages' or 'requirements-files' must be specified". format(self)) + raise SourceError("{}: Either 'packages' or 'requirements-files' must be specified".format(self)) def preflight(self): # Try to find a pip version that spports download command @@ -126,9 +123,9 @@ class PipSource(Source): for python in reversed(_PYTHON_VERSIONS): try: host_python = utils.get_host_tool(python) - rc = self.call([host_python, '-m', 'pip', 'download', '--help']) + rc = self.call([host_python, "-m", "pip", "download", "--help"]) if rc == 0: - self.host_pip = [host_python, '-m', 'pip'] + self.host_pip = [host_python, "-m", "pip"] break except utils.ProgramNotFoundError: pass @@ -150,10 +147,10 @@ class PipSource(Source): return self.ref def load_ref(self, node): - self.ref = node.get_str('ref', None) + self.ref = node.get_str("ref", None) def set_ref(self, ref, node): - node['ref'] = self.ref = ref + node["ref"] = self.ref = ref def track(self, previous_sources_dir): # pylint: disable=arguments-differ # XXX pip does not offer any public API other than the CLI tool so it @@ -163,32 +160,44 @@ class PipSource(Source): # for details. # As a result, we have to wastefully install the packages during track. with self.tempdir() as tmpdir: - install_args = self.host_pip + ['download', - '--no-binary', ':all:', - '--index-url', self.index_url, - '--dest', tmpdir] + install_args = self.host_pip + [ + "download", + "--no-binary", + ":all:", + "--index-url", + self.index_url, + "--dest", + tmpdir, + ] for requirement_file in self.requirements_files: fpath = os.path.join(previous_sources_dir, requirement_file) - install_args += ['-r', fpath] + install_args += ["-r", fpath] install_args += self.packages self.call(install_args, fail="Failed to install python packages") reqs = self._parse_sdist_names(tmpdir) - return '\n'.join(["{}=={}".format(pkg, ver) for pkg, ver in reqs]) + return "\n".join(["{}=={}".format(pkg, ver) for pkg, ver in reqs]) def fetch(self): # pylint: disable=arguments-differ with self.tempdir() as tmpdir: - packages = self.ref.strip().split('\n') - package_dir = os.path.join(tmpdir, 'packages') + packages = self.ref.strip().split("\n") + package_dir = os.path.join(tmpdir, "packages") os.makedirs(package_dir) - self.call([*self.host_pip, - 'download', - '--no-binary', ':all:', - '--index-url', self.index_url, - '--dest', package_dir, - *packages], - fail="Failed to install python packages: {}".format(packages)) + self.call( + [ + *self.host_pip, + "download", + "--no-binary", + ":all:", + "--index-url", + self.index_url, + "--dest", + package_dir, + *packages, + ], + fail="Failed to install python packages: {}".format(packages), + ) # If the mirror directory already exists, assume that some other # process has fetched the sources before us and ensure that we do @@ -200,8 +209,11 @@ class PipSource(Source): # before us. pass except OSError as e: - raise SourceError("{}: Failed to move downloaded pip packages from '{}' to '{}': {}" - .format(self, package_dir, self._mirror, e)) from e + raise SourceError( + "{}: Failed to move downloaded pip packages from '{}' to '{}': {}".format( + self, package_dir, self._mirror, e + ) + ) from e def stage(self, directory): with self.timed_activity("Staging Python packages", silent_nested=True): @@ -213,9 +225,11 @@ class PipSource(Source): def _mirror(self): if not self.ref: return None - return os.path.join(self.get_mirror_directory(), - utils.url_directory_name(self.original_url), - hashlib.sha256(self.ref.encode()).hexdigest()) + return os.path.join( + self.get_mirror_directory(), + utils.url_directory_name(self.original_url), + hashlib.sha256(self.ref.encode()).hexdigest(), + ) # Parse names of downloaded source distributions # diff --git a/src/buildstream/plugins/sources/remote.py b/src/buildstream/plugins/sources/remote.py index 68aa577fc..da1a1f964 100644 --- a/src/buildstream/plugins/sources/remote.py +++ b/src/buildstream/plugins/sources/remote.py @@ -62,13 +62,14 @@ class RemoteSource(DownloadableFileSource): def configure(self, node): super().configure(node) - self.filename = node.get_str('filename', os.path.basename(self.url)) - self.executable = node.get_bool('executable', default=False) + self.filename = node.get_str("filename", os.path.basename(self.url)) + self.executable = node.get_bool("executable", default=False) if os.sep in self.filename: - raise SourceError('{}: filename parameter cannot contain directories'.format(self), - reason="filename-contains-directory") - node.validate_keys(DownloadableFileSource.COMMON_CONFIG_KEYS + ['filename', 'executable']) + raise SourceError( + "{}: filename parameter cannot contain directories".format(self), reason="filename-contains-directory" + ) + node.validate_keys(DownloadableFileSource.COMMON_CONFIG_KEYS + ["filename", "executable"]) def get_unique_key(self): return super().get_unique_key() + [self.filename, self.executable] diff --git a/src/buildstream/plugins/sources/tar.py b/src/buildstream/plugins/sources/tar.py index 60d464457..658cc2735 100644 --- a/src/buildstream/plugins/sources/tar.py +++ b/src/buildstream/plugins/sources/tar.py @@ -73,6 +73,7 @@ class ReadableTarInfo(tarfile.TarInfo): `mode` attribute in `TarInfo`, the class that encapsulates the internal meta-data of the tarball, so that the owner-read bit is always set. """ + @property def mode(self): # ensure file is readable by owner @@ -89,13 +90,13 @@ class TarSource(DownloadableFileSource): def configure(self, node): super().configure(node) - self.base_dir = node.get_str('base-dir', '*') - node.validate_keys(DownloadableFileSource.COMMON_CONFIG_KEYS + ['base-dir']) + self.base_dir = node.get_str("base-dir", "*") + node.validate_keys(DownloadableFileSource.COMMON_CONFIG_KEYS + ["base-dir"]) def preflight(self): self.host_lzip = None - if self.url.endswith('.lz'): - self.host_lzip = utils.get_host_tool('lzip') + if self.url.endswith(".lz"): + self.host_lzip = utils.get_host_tool("lzip") def get_unique_key(self): return super().get_unique_key() + [self.base_dir] @@ -104,19 +105,17 @@ class TarSource(DownloadableFileSource): def _run_lzip(self): assert self.host_lzip with TemporaryFile() as lzip_stdout: - with open(self._get_mirror_file(), 'r') as lzip_file: - self.call([self.host_lzip, '-d'], - stdin=lzip_file, - stdout=lzip_stdout) + with open(self._get_mirror_file(), "r") as lzip_file: + self.call([self.host_lzip, "-d"], stdin=lzip_file, stdout=lzip_stdout) lzip_stdout.seek(0, 0) yield lzip_stdout @contextmanager def _get_tar(self): - if self.url.endswith('.lz'): + if self.url.endswith(".lz"): with self._run_lzip() as lzip_dec: - with tarfile.open(fileobj=lzip_dec, mode='r:', tarinfo=ReadableTarInfo) as tar: + with tarfile.open(fileobj=lzip_dec, mode="r:", tarinfo=ReadableTarInfo) as tar: yield tar else: with tarfile.open(self._get_mirror_file(), tarinfo=ReadableTarInfo) as tar: @@ -147,14 +146,18 @@ class TarSource(DownloadableFileSource): def assert_safe(member): final_path = os.path.abspath(os.path.join(target_dir, member.path)) if not final_path.startswith(target_dir): - raise SourceError("{}: Tarfile attempts to extract outside the staging area: " - "{} -> {}".format(self, member.path, final_path)) + raise SourceError( + "{}: Tarfile attempts to extract outside the staging area: " + "{} -> {}".format(self, member.path, final_path) + ) if member.islnk(): linked_path = os.path.abspath(os.path.join(target_dir, member.linkname)) if not linked_path.startswith(target_dir): - raise SourceError("{}: Tarfile attempts to hardlink outside the staging area: " - "{} -> {}".format(self, member.path, final_path)) + raise SourceError( + "{}: Tarfile attempts to hardlink outside the staging area: " + "{} -> {}".format(self, member.path, final_path) + ) # Don't need to worry about symlinks because they're just # files here and won't be able to do much harm once we are @@ -167,9 +170,9 @@ class TarSource(DownloadableFileSource): for member in tar.getmembers(): # First, ensure that a member never starts with `./` - if member.path.startswith('./'): + if member.path.startswith("./"): member.path = member.path[2:] - if member.islnk() and member.linkname.startswith('./'): + if member.islnk() and member.linkname.startswith("./"): member.linkname = member.linkname[2:] # Now extract only the paths which match the normalized path @@ -202,16 +205,16 @@ class TarSource(DownloadableFileSource): # Remove any possible leading './', offer more consistent behavior # across tarballs encoded with or without a leading '.' - member_name = member.name.lstrip('./') + member_name = member.name.lstrip("./") if not member.isdir(): # Loop over the components of a path, for a path of a/b/c/d # we will first visit 'a', then 'a/b' and then 'a/b/c', excluding # the final component - components = member_name.split('/') + components = member_name.split("/") for i in range(len(components) - 1): - dir_component = '/'.join([components[j] for j in range(i + 1)]) + dir_component = "/".join([components[j] for j in range(i + 1)]) if dir_component not in visited: visited.add(dir_component) try: @@ -219,7 +222,7 @@ class TarSource(DownloadableFileSource): # exist in the archive _ = tar.getmember(dir_component) except KeyError: - if dir_component != '.': + if dir_component != ".": yield dir_component continue @@ -227,7 +230,7 @@ class TarSource(DownloadableFileSource): # Avoid considering the '.' directory, if any is included in the archive # this is to avoid the default 'base-dir: *' value behaving differently # depending on whether the tarball was encoded with a leading '.' or not - elif member_name == '.': + elif member_name == ".": continue yield member_name diff --git a/src/buildstream/plugins/sources/workspace.py b/src/buildstream/plugins/sources/workspace.py index f40f5fae8..a845fd440 100644 --- a/src/buildstream/plugins/sources/workspace.py +++ b/src/buildstream/plugins/sources/workspace.py @@ -59,9 +59,9 @@ class WorkspaceSource(Source): return None def configure(self, node: MappingNode) -> None: - node.validate_keys(['path', 'ref', 'kind']) - self.path = node.get_str('path') - self.__digest = node.get_str('ref') + node.validate_keys(["path", "ref", "kind"]) + self.path = node.get_str("path") + self.__digest = node.get_str("ref") def preflight(self) -> None: pass # pragma: nocover @@ -79,7 +79,7 @@ class WorkspaceSource(Source): # # Raises AssertionError: existing workspaces should not be reinitialized def init_workspace(self, directory: Directory) -> None: - raise AssertionError('Attempting to re-open an existing workspace') + raise AssertionError("Attempting to re-open an existing workspace") def get_consistency(self): # always return cached state @@ -95,8 +95,8 @@ class WorkspaceSource(Source): if result.overwritten or result.ignored: raise SourceError( - "Failed to stage source: files clash with existing directory", - reason='ensure-stage-dir-fail') + "Failed to stage source: files clash with existing directory", reason="ensure-stage-dir-fail" + ) def _get_local_path(self) -> str: return self.path diff --git a/src/buildstream/plugins/sources/zip.py b/src/buildstream/plugins/sources/zip.py index 322be58d7..47933c8eb 100644 --- a/src/buildstream/plugins/sources/zip.py +++ b/src/buildstream/plugins/sources/zip.py @@ -72,8 +72,8 @@ class ZipSource(DownloadableFileSource): def configure(self, node): super().configure(node) - self.base_dir = node.get_str('base-dir', '*') - node.validate_keys(DownloadableFileSource.COMMON_CONFIG_KEYS + ['base-dir']) + self.base_dir = node.get_str("base-dir", "*") + node.validate_keys(DownloadableFileSource.COMMON_CONFIG_KEYS + ["base-dir"]) def get_unique_key(self): return super().get_unique_key() + [self.base_dir] @@ -139,14 +139,14 @@ class ZipSource(DownloadableFileSource): # ZipInfo.is_dir() is only available in python >= 3.6, but all # it does is check for a trailing '/' in the name # - if not member.filename.endswith('/'): + if not member.filename.endswith("/"): # Loop over the components of a path, for a path of a/b/c/d # we will first visit 'a', then 'a/b' and then 'a/b/c', excluding # the final component - components = member.filename.split('/') + components = member.filename.split("/") for i in range(len(components) - 1): - dir_component = '/'.join([components[j] for j in range(i + 1)]) + dir_component = "/".join([components[j] for j in range(i + 1)]) if dir_component not in visited: visited[dir_component] = True try: @@ -154,7 +154,7 @@ class ZipSource(DownloadableFileSource): # exist in the archive _ = archive.getinfo(dir_component) except KeyError: - if dir_component != '.': + if dir_component != ".": yield dir_component continue @@ -162,7 +162,7 @@ class ZipSource(DownloadableFileSource): # Avoid considering the '.' directory, if any is included in the archive # this is to avoid the default 'base-dir: *' value behaving differently # depending on whether the archive was encoded with a leading '.' or not - elif member.filename == '.' or member.filename == './': + elif member.filename == "." or member.filename == "./": continue yield member.filename |