diff options
| author | Chandan Singh <csingh43@bloomberg.net> | 2019-11-11 17:07:09 +0000 |
|---|---|---|
| committer | Chandan Singh <chandan@chandansingh.net> | 2019-11-14 21:21:06 +0000 |
| commit | 122177153b14664a0e4fed85aa4f22b87cfabf56 (patch) | |
| tree | 032c2e46825af91f6fe27f22b5b567eea2b7935d | |
| parent | a3ee349558f36a220f79665873b36c1b0f990c8e (diff) | |
| download | buildstream-122177153b14664a0e4fed85aa4f22b87cfabf56.tar.gz | |
Reformat code using Black
As discussed over the mailing list, reformat code using Black. This is a
one-off change to reformat all our codebase. Moving forward, we
shouldn't expect such blanket reformats. Rather, we expect each change
to already comply with the Black formatting style.
275 files changed, 12097 insertions, 13933 deletions
diff --git a/src/buildstream/__init__.py b/src/buildstream/__init__.py index cd8d0f1cf..c78fcbbf6 100644 --- a/src/buildstream/__init__.py +++ b/src/buildstream/__init__.py @@ -19,11 +19,13 @@ # Plugin author facing APIs import os + if "_BST_COMPLETION" not in os.environ: # Special sauce to get the version from versioneer from ._version import get_versions - __version__ = get_versions()['version'] + + __version__ = get_versions()["version"] del get_versions from .utils import UtilError, ProgramNotFoundError diff --git a/src/buildstream/__main__.py b/src/buildstream/__main__.py index 4b0fdabfe..556a0f67e 100644 --- a/src/buildstream/__main__.py +++ b/src/buildstream/__main__.py @@ -11,7 +11,8 @@ # This is used when we need to run BuildStream before installing, # like when we build documentation. # -if __name__ == '__main__': +if __name__ == "__main__": # pylint: disable=no-value-for-parameter from ._frontend.cli import cli + cli() diff --git a/src/buildstream/_artifact.py b/src/buildstream/_artifact.py index e5174eaea..feba3898b 100644 --- a/src/buildstream/_artifact.py +++ b/src/buildstream/_artifact.py @@ -47,7 +47,7 @@ from .storage._casbaseddirectory import CasBasedDirectory # strong_key (str): The elements strong cache key, dependent on context # weak_key (str): The elements weak cache key # -class Artifact(): +class Artifact: version = 0 @@ -61,11 +61,11 @@ class Artifact(): self._tmpdir = context.tmpdir self._proto = None - self._metadata_keys = None # Strong and weak key tuple extracted from the artifact - self._metadata_dependencies = None # Dictionary of dependency strong keys from the artifact - self._metadata_workspaced = None # Boolean of whether it's a workspaced artifact + self._metadata_keys = None # Strong and weak key tuple extracted from the artifact + self._metadata_dependencies = None # Dictionary of dependency strong keys from the artifact + self._metadata_workspaced = None # Boolean of whether it's a workspaced artifact self._metadata_workspaced_dependencies = None # List of which dependencies are workspaced from the artifact - self._cached = None # Boolean of whether the artifact is cached + self._cached = None # Boolean of whether the artifact is cached # get_files(): # @@ -193,12 +193,11 @@ class Artifact(): artifact.buildtree.CopyFrom(buildtreevdir._get_digest()) size += buildtreevdir.get_size() - os.makedirs(os.path.dirname(os.path.join( - self._artifactdir, element.get_artifact_name())), exist_ok=True) + os.makedirs(os.path.dirname(os.path.join(self._artifactdir, element.get_artifact_name())), exist_ok=True) keys = utils._deduplicate([self._cache_key, self._weak_cache_key]) for key in keys: path = os.path.join(self._artifactdir, element.get_artifact_name(key=key)) - with utils.save_file_atomic(path, mode='wb') as f: + with utils.save_file_atomic(path, mode="wb") as f: f.write(artifact.SerializeToString()) return size @@ -247,7 +246,7 @@ class Artifact(): # Load the public data from the artifact artifact = self._get_proto() meta_file = self._cas.objpath(artifact.public_data) - data = _yaml.load(meta_file, shortname='public.yaml') + data = _yaml.load(meta_file, shortname="public.yaml") return data @@ -263,9 +262,7 @@ class Artifact(): def load_build_result(self): artifact = self._get_proto() - build_result = (artifact.build_success, - artifact.build_error, - artifact.build_error_details) + build_result = (artifact.build_success, artifact.build_error, artifact.build_error_details) return build_result @@ -345,8 +342,9 @@ class Artifact(): # Extract proto artifact = self._get_proto() - self._metadata_workspaced_dependencies = [dep.element_name for dep in artifact.build_deps - if dep.was_workspaced] + self._metadata_workspaced_dependencies = [ + dep.element_name for dep in artifact.build_deps if dep.was_workspaced + ] return self._metadata_workspaced_dependencies @@ -419,12 +417,14 @@ class Artifact(): # Determine whether directories are required require_directories = context.require_artifact_directories # Determine whether file contents are required as well - require_files = (context.require_artifact_files or - self._element._artifact_files_required()) + require_files = context.require_artifact_files or self._element._artifact_files_required() # Check whether 'files' subdirectory is available, with or without file contents - if (require_directories and str(artifact.files) and - not self._cas.contains_directory(artifact.files, with_files=require_files)): + if ( + require_directories + and str(artifact.files) + and not self._cas.contains_directory(artifact.files, with_files=require_files) + ): self._cached = False return False @@ -471,11 +471,10 @@ class Artifact(): key = self.get_extract_key() - proto_path = os.path.join(self._artifactdir, - self._element.get_artifact_name(key=key)) + proto_path = os.path.join(self._artifactdir, self._element.get_artifact_name(key=key)) artifact = ArtifactProto() try: - with open(proto_path, mode='r+b') as f: + with open(proto_path, mode="r+b") as f: artifact.ParseFromString(f.read()) except FileNotFoundError: return None diff --git a/src/buildstream/_artifactcache.py b/src/buildstream/_artifactcache.py index 79d0dc50b..03c47b906 100644 --- a/src/buildstream/_artifactcache.py +++ b/src/buildstream/_artifactcache.py @@ -23,8 +23,7 @@ import grpc from ._basecache import BaseCache from ._exceptions import ArtifactError, CASError, CASCacheError, CASRemoteError, RemoteError -from ._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc, \ - artifact_pb2, artifact_pb2_grpc +from ._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc, artifact_pb2, artifact_pb2_grpc from ._remote import BaseRemote from .storage._casbaseddirectory import CasBasedDirectory @@ -38,7 +37,6 @@ from . import utils # artifact remotes. # class ArtifactRemote(BaseRemote): - def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.artifact_service = None @@ -78,8 +76,10 @@ class ArtifactRemote(BaseRemote): except grpc.RpcError as e: # Check if this remote has the artifact service if e.code() == grpc.StatusCode.UNIMPLEMENTED: - raise RemoteError("Configured remote does not have the BuildStream " - "capabilities service. Please check remote configuration.") + raise RemoteError( + "Configured remote does not have the BuildStream " + "capabilities service. Please check remote configuration." + ) # Else raise exception with details raise RemoteError("Remote initialisation failed: {}".format(e.details())) @@ -263,9 +263,11 @@ class ArtifactCache(BaseCache): if self._push_artifact_blobs(artifact, remote): element.info("Pushed data from artifact {} -> {}".format(display_key, remote)) else: - element.info("Remote ({}) already has all data of artifact {} cached".format( - remote, element._get_brief_display_key() - )) + element.info( + "Remote ({}) already has all data of artifact {} cached".format( + remote, element._get_brief_display_key() + ) + ) for remote in index_remotes: remote.init() @@ -275,9 +277,9 @@ class ArtifactCache(BaseCache): element.info("Pushed artifact {} -> {}".format(display_key, remote)) pushed = True else: - element.info("Remote ({}) already has artifact {} cached".format( - remote, element._get_brief_display_key() - )) + element.info( + "Remote ({}) already has artifact {} cached".format(remote, element._get_brief_display_key()) + ) return pushed @@ -295,7 +297,7 @@ class ArtifactCache(BaseCache): # def pull(self, element, key, *, pull_buildtrees=False): artifact = None - display_key = key[:self.context.log_key_length] + display_key = key[: self.context.log_key_length] project = element._get_project() errors = [] @@ -310,16 +312,15 @@ class ArtifactCache(BaseCache): element.info("Pulled artifact {} <- {}".format(display_key, remote)) break else: - element.info("Remote ({}) does not have artifact {} cached".format( - remote, display_key - )) + element.info("Remote ({}) does not have artifact {} cached".format(remote, display_key)) except CASError as e: element.warn("Could not pull from remote {}: {}".format(remote, e)) errors.append(e) if errors and not artifact: - raise ArtifactError("Failed to pull artifact {}".format(display_key), - detail="\n".join(str(e) for e in errors)) + raise ArtifactError( + "Failed to pull artifact {}".format(display_key), detail="\n".join(str(e) for e in errors) + ) # If we don't have an artifact, we can't exactly pull our # artifact @@ -337,16 +338,15 @@ class ArtifactCache(BaseCache): element.info("Pulled data for artifact {} <- {}".format(display_key, remote)) return True - element.info("Remote ({}) does not have artifact {} cached".format( - remote, display_key - )) + element.info("Remote ({}) does not have artifact {} cached".format(remote, display_key)) except CASError as e: element.warn("Could not pull from remote {}: {}".format(remote, e)) errors.append(e) if errors: - raise ArtifactError("Failed to pull artifact {}".format(display_key), - detail="\n".join(str(e) for e in errors)) + raise ArtifactError( + "Failed to pull artifact {}".format(display_key), detail="\n".join(str(e) for e in errors) + ) return False @@ -388,8 +388,9 @@ class ArtifactCache(BaseCache): push_remotes = [] if not push_remotes: - raise ArtifactError("push_message was called, but no remote artifact " + - "servers are configured as push remotes.") + raise ArtifactError( + "push_message was called, but no remote artifact " + "servers are configured as push remotes." + ) for remote in push_remotes: message_digest = remote.push_message(message) @@ -410,8 +411,7 @@ class ArtifactCache(BaseCache): newref = element.get_artifact_name(newkey) if not os.path.exists(os.path.join(self.artifactdir, newref)): - os.link(os.path.join(self.artifactdir, oldref), - os.path.join(self.artifactdir, newref)) + os.link(os.path.join(self.artifactdir, oldref), os.path.join(self.artifactdir, newref)) # get_artifact_logs(): # @@ -425,7 +425,7 @@ class ArtifactCache(BaseCache): # def get_artifact_logs(self, ref): cache_id = self.cas.resolve_ref(ref, update_mtime=True) - vdir = CasBasedDirectory(self.cas, digest=cache_id).descend('logs') + vdir = CasBasedDirectory(self.cas, digest=cache_id).descend("logs") return vdir # fetch_missing_blobs(): @@ -517,7 +517,7 @@ class ArtifactCache(BaseCache): for root, _, files in os.walk(self.artifactdir): for artifact_file in files: artifact = artifact_pb2.Artifact() - with open(os.path.join(root, artifact_file), 'r+b') as f: + with open(os.path.join(root, artifact_file), "r+b") as f: artifact.ParseFromString(f.read()) if str(artifact.files): @@ -535,7 +535,7 @@ class ArtifactCache(BaseCache): for root, _, files in os.walk(self.artifactdir): for artifact_file in files: artifact = artifact_pb2.Artifact() - with open(os.path.join(root, artifact_file), 'r+b') as f: + with open(os.path.join(root, artifact_file), "r+b") as f: artifact.ParseFromString(f.read()) if str(artifact.public_data): @@ -620,8 +620,7 @@ class ArtifactCache(BaseCache): remote.get_artifact(element.get_artifact_name(key=key)) except grpc.RpcError as e: if e.code() != grpc.StatusCode.NOT_FOUND: - raise ArtifactError("Error checking artifact cache: {}" - .format(e.details())) + raise ArtifactError("Error checking artifact cache: {}".format(e.details())) else: return False @@ -710,7 +709,7 @@ class ArtifactCache(BaseCache): # Write the artifact proto to cache artifact_path = os.path.join(self.artifactdir, artifact_name) os.makedirs(os.path.dirname(artifact_path), exist_ok=True) - with utils.save_file_atomic(artifact_path, mode='wb') as f: + with utils.save_file_atomic(artifact_path, mode="wb") as f: f.write(artifact.SerializeToString()) return artifact diff --git a/src/buildstream/_artifactelement.py b/src/buildstream/_artifactelement.py index 48c3d1769..1c1c5db46 100644 --- a/src/buildstream/_artifactelement.py +++ b/src/buildstream/_artifactelement.py @@ -40,7 +40,7 @@ if TYPE_CHECKING: class ArtifactElement(Element): # A hash of ArtifactElement by ref - __instantiated_artifacts = {} # type: Dict[str, ArtifactElement] + __instantiated_artifacts = {} # type: Dict[str, ArtifactElement] # ArtifactElement's require this as the sandbox will use a normal # directory when we checkout @@ -138,7 +138,7 @@ class ArtifactElement(Element): # sandbox (Sandbox) # def configure_sandbox(self, sandbox): - install_root = self.get_variable('install-root') + install_root = self.get_variable("install-root") # Tell the sandbox to mount the build root and install root sandbox.mark_directory(install_root) @@ -173,7 +173,7 @@ class ArtifactElement(Element): # def verify_artifact_ref(ref): try: - project, element, key = ref.split('/', 2) # This will raise a Value error if unable to split + project, element, key = ref.split("/", 2) # This will raise a Value error if unable to split # Explicitly raise a ValueError if the key length is not as expected if not _cachekey.is_key(key): raise ValueError diff --git a/src/buildstream/_basecache.py b/src/buildstream/_basecache.py index fc2e92456..516119cd1 100644 --- a/src/buildstream/_basecache.py +++ b/src/buildstream/_basecache.py @@ -37,21 +37,21 @@ if TYPE_CHECKING: # Base Cache for Caches to derive from # -class BaseCache(): +class BaseCache: # None of these should ever be called in the base class, but this appeases # pylint to some degree - spec_name = None # type: str - spec_error = None # type: Type[BstError] - config_node_name = None # type: str - index_remote_class = None # type: Type[BaseRemote] + spec_name = None # type: str + spec_error = None # type: Type[BstError] + config_node_name = None # type: str + index_remote_class = None # type: Type[BaseRemote] storage_remote_class = CASRemote # type: Type[BaseRemote] def __init__(self, context): self.context = context self.cas = context.get_cascache() - self._remotes_setup = False # Check to prevent double-setup of remotes + self._remotes_setup = False # Check to prevent double-setup of remotes # Per-project list of Remote instances. self._storage_remotes = {} self._index_remotes = {} @@ -116,8 +116,12 @@ class BaseCache(): artifacts = config_node.get_sequence(cls.config_node_name, default=[]) except LoadError: provenance = config_node.get_node(cls.config_node_name).get_provenance() - raise _yaml.LoadError("{}: '{}' must be a single remote mapping, or a list of mappings" - .format(provenance, cls.config_node_name), _yaml.LoadErrorReason.INVALID_DATA) + raise _yaml.LoadError( + "{}: '{}' must be a single remote mapping, or a list of mappings".format( + provenance, cls.config_node_name + ), + _yaml.LoadErrorReason.INVALID_DATA, + ) for spec_node in artifacts: cache_specs.append(RemoteSpec.new_from_config_node(spec_node)) @@ -144,8 +148,7 @@ class BaseCache(): project_specs = getattr(project, cls.spec_name) context_specs = getattr(context, cls.spec_name) - return list(utils._deduplicate( - project_extra_specs + project_specs + context_specs)) + return list(utils._deduplicate(project_extra_specs + project_specs + context_specs)) # setup_remotes(): # @@ -266,8 +269,9 @@ class BaseCache(): # Check whether the specified element's project has push remotes index_remotes = self._index_remotes[plugin._get_project()] storage_remotes = self._storage_remotes[plugin._get_project()] - return (any(remote.spec.push for remote in index_remotes) and - any(remote.spec.push for remote in storage_remotes)) + return any(remote.spec.push for remote in index_remotes) and any( + remote.spec.push for remote in storage_remotes + ) ################################################ # Local Private Methods # @@ -323,8 +327,9 @@ class BaseCache(): storage_remotes[remote_spec] = storage self._has_fetch_remotes = storage_remotes and index_remotes - self._has_push_remotes = (any(spec.push for spec in storage_remotes) and - any(spec.push for spec in index_remotes)) + self._has_push_remotes = any(spec.push for spec in storage_remotes) and any( + spec.push for spec in index_remotes + ) return index_remotes, storage_remotes @@ -366,8 +371,7 @@ class BaseCache(): # def _message(self, message_type, message, **kwargs): args = dict(kwargs) - self.context.messenger.message( - Message(message_type, message, **args)) + self.context.messenger.message(Message(message_type, message, **args)) # _set_remotes(): # diff --git a/src/buildstream/_cachekey.py b/src/buildstream/_cachekey.py index 89d47671e..dd9207516 100644 --- a/src/buildstream/_cachekey.py +++ b/src/buildstream/_cachekey.py @@ -62,5 +62,5 @@ def is_key(key): # (str): An sha256 hex digest of the given value # def generate_key(value): - ustring = ujson.dumps(value, sort_keys=True, escape_forward_slashes=False).encode('utf-8') + ustring = ujson.dumps(value, sort_keys=True, escape_forward_slashes=False).encode("utf-8") return hashlib.sha256(ustring).hexdigest() diff --git a/src/buildstream/_cas/cascache.py b/src/buildstream/_cas/cascache.py index 022730445..c1f2b30b0 100644 --- a/src/buildstream/_cas/cascache.py +++ b/src/buildstream/_cas/cascache.py @@ -68,15 +68,14 @@ class CASLogLevel(FastEnum): # protect_session_blobs (bool): Disable expiry for blobs used in the current session # log_level (LogLevel): Log level to give to buildbox-casd for logging # -class CASCache(): - +class CASCache: def __init__( - self, path, *, casd=True, cache_quota=None, protect_session_blobs=True, log_level=CASLogLevel.WARNING + self, path, *, casd=True, cache_quota=None, protect_session_blobs=True, log_level=CASLogLevel.WARNING ): - self.casdir = os.path.join(path, 'cas') - self.tmpdir = os.path.join(path, 'tmp') - os.makedirs(os.path.join(self.casdir, 'refs', 'heads'), exist_ok=True) - os.makedirs(os.path.join(self.casdir, 'objects'), exist_ok=True) + self.casdir = os.path.join(path, "cas") + self.tmpdir = os.path.join(path, "tmp") + os.makedirs(os.path.join(self.casdir, "refs", "heads"), exist_ok=True) + os.makedirs(os.path.join(self.casdir, "objects"), exist_ok=True) os.makedirs(self.tmpdir, exist_ok=True) self._casd_channel = None @@ -88,19 +87,19 @@ class CASCache(): if casd: # Place socket in global/user temporary directory to avoid hitting # the socket path length limit. - self._casd_socket_tempdir = tempfile.mkdtemp(prefix='buildstream') - self._casd_socket_path = os.path.join(self._casd_socket_tempdir, 'casd.sock') + self._casd_socket_tempdir = tempfile.mkdtemp(prefix="buildstream") + self._casd_socket_path = os.path.join(self._casd_socket_tempdir, "casd.sock") - casd_args = [utils.get_host_tool('buildbox-casd')] - casd_args.append('--bind=unix:' + self._casd_socket_path) - casd_args.append('--log-level=' + log_level.value) + casd_args = [utils.get_host_tool("buildbox-casd")] + casd_args.append("--bind=unix:" + self._casd_socket_path) + casd_args.append("--log-level=" + log_level.value) if cache_quota is not None: - casd_args.append('--quota-high={}'.format(int(cache_quota))) - casd_args.append('--quota-low={}'.format(int(cache_quota / 2))) + casd_args.append("--quota-high={}".format(int(cache_quota))) + casd_args.append("--quota-low={}".format(int(cache_quota / 2))) if protect_session_blobs: - casd_args.append('--protect-session-blobs') + casd_args.append("--protect-session-blobs") casd_args.append(path) @@ -112,7 +111,8 @@ class CASCache(): # The frontend will take care of it if needed with _signals.blocked([signal.SIGINT], ignore=False): self._casd_process = subprocess.Popen( - casd_args, cwd=path, stdout=logfile_fp, stderr=subprocess.STDOUT) + casd_args, cwd=path, stdout=logfile_fp, stderr=subprocess.STDOUT + ) self._cache_usage_monitor = _CASCacheUsageMonitor(self) else: @@ -123,16 +123,16 @@ class CASCache(): # Popen objects are not pickle-able, however, child processes only # need the information whether a casd subprocess was started or not. - assert '_casd_process' in state - state['_casd_process'] = bool(self._casd_process) + assert "_casd_process" in state + state["_casd_process"] = bool(self._casd_process) # The usage monitor is not pickle-able, but we also don't need it in # child processes currently. Make sure that if this changes, we get a # bug report, by setting _cache_usage_monitor_forbidden. - assert '_cache_usage_monitor' in state - assert '_cache_usage_monitor_forbidden' in state - state['_cache_usage_monitor'] = None - state['_cache_usage_monitor_forbidden'] = True + assert "_cache_usage_monitor" in state + assert "_cache_usage_monitor_forbidden" in state + state["_cache_usage_monitor"] = None + state["_cache_usage_monitor_forbidden"] = True return state @@ -148,7 +148,7 @@ class CASCache(): time.sleep(0.01) - self._casd_channel = grpc.insecure_channel('unix:' + self._casd_socket_path) + self._casd_channel = grpc.insecure_channel("unix:" + self._casd_socket_path) self._casd_cas = remote_execution_pb2_grpc.ContentAddressableStorageStub(self._casd_channel) self._local_cas = local_cas_pb2_grpc.LocalContentAddressableStorageStub(self._casd_channel) @@ -179,8 +179,8 @@ class CASCache(): # Preflight check. # def preflight(self): - headdir = os.path.join(self.casdir, 'refs', 'heads') - objdir = os.path.join(self.casdir, 'objects') + headdir = os.path.join(self.casdir, "refs", "heads") + objdir = os.path.join(self.casdir, "objects") if not (os.path.isdir(headdir) and os.path.isdir(objdir)): raise CASCacheError("CAS repository check failed for '{}'".format(self.casdir)) @@ -285,7 +285,7 @@ class CASCache(): directory = remote_execution_pb2.Directory() - with open(self.objpath(tree), 'rb') as f: + with open(self.objpath(tree), "rb") as f: directory.ParseFromString(f.read()) for filenode in directory.files: @@ -297,8 +297,16 @@ class CASCache(): utils.safe_copy(self.objpath(filenode.digest), fullpath) if filenode.is_executable: - os.chmod(fullpath, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | - stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) + os.chmod( + fullpath, + stat.S_IRUSR + | stat.S_IWUSR + | stat.S_IXUSR + | stat.S_IRGRP + | stat.S_IXGRP + | stat.S_IROTH + | stat.S_IXOTH, + ) for dirnode in directory.directories: fullpath = os.path.join(dest, dirnode.name) @@ -365,7 +373,7 @@ class CASCache(): # (str): The path of the object # def objpath(self, digest): - return os.path.join(self.casdir, 'objects', digest.hash[:2], digest.hash[2:]) + return os.path.join(self.casdir, "objects", digest.hash[:2], digest.hash[2:]) # add_object(): # @@ -450,7 +458,7 @@ class CASCache(): treepath = self.objpath(tree_response.tree_digest) tree = remote_execution_pb2.Tree() - with open(treepath, 'rb') as f: + with open(treepath, "rb") as f: tree.ParseFromString(f.read()) root_directory = tree.root.SerializeToString() @@ -467,7 +475,7 @@ class CASCache(): def set_ref(self, ref, tree): refpath = self._refpath(ref) os.makedirs(os.path.dirname(refpath), exist_ok=True) - with utils.save_file_atomic(refpath, 'wb', tempdir=self.tmpdir) as f: + with utils.save_file_atomic(refpath, "wb", tempdir=self.tmpdir) as f: f.write(tree.SerializeToString()) # resolve_ref(): @@ -485,7 +493,7 @@ class CASCache(): refpath = self._refpath(ref) try: - with open(refpath, 'rb') as f: + with open(refpath, "rb") as f: if update_mtime: os.utime(refpath) @@ -521,7 +529,7 @@ class CASCache(): def remove(self, ref, *, basedir=None): if basedir is None: - basedir = os.path.join(self.casdir, 'refs', 'heads') + basedir = os.path.join(self.casdir, "refs", "heads") # Remove cache ref self._remove_ref(ref, basedir) @@ -611,7 +619,7 @@ class CASCache(): directory = remote_execution_pb2.Directory() - with open(self.objpath(directory_digest), 'rb') as f: + with open(self.objpath(directory_digest), "rb") as f: directory.ParseFromString(f.read()) for filenode in directory.files: @@ -626,21 +634,19 @@ class CASCache(): dir_b = remote_execution_pb2.Directory() if tree_a: - with open(self.objpath(tree_a), 'rb') as f: + with open(self.objpath(tree_a), "rb") as f: dir_a.ParseFromString(f.read()) if tree_b: - with open(self.objpath(tree_b), 'rb') as f: + with open(self.objpath(tree_b), "rb") as f: dir_b.ParseFromString(f.read()) a = 0 b = 0 while a < len(dir_a.files) or b < len(dir_b.files): - if b < len(dir_b.files) and (a >= len(dir_a.files) or - dir_a.files[a].name > dir_b.files[b].name): + if b < len(dir_b.files) and (a >= len(dir_a.files) or dir_a.files[a].name > dir_b.files[b].name): added.append(os.path.join(path, dir_b.files[b].name)) b += 1 - elif a < len(dir_a.files) and (b >= len(dir_b.files) or - dir_b.files[b].name > dir_a.files[a].name): + elif a < len(dir_a.files) and (b >= len(dir_b.files) or dir_b.files[b].name > dir_a.files[a].name): removed.append(os.path.join(path, dir_a.files[a].name)) a += 1 else: @@ -653,24 +659,41 @@ class CASCache(): a = 0 b = 0 while a < len(dir_a.directories) or b < len(dir_b.directories): - if b < len(dir_b.directories) and (a >= len(dir_a.directories) or - dir_a.directories[a].name > dir_b.directories[b].name): - self.diff_trees(None, dir_b.directories[b].digest, - added=added, removed=removed, modified=modified, - path=os.path.join(path, dir_b.directories[b].name)) + if b < len(dir_b.directories) and ( + a >= len(dir_a.directories) or dir_a.directories[a].name > dir_b.directories[b].name + ): + self.diff_trees( + None, + dir_b.directories[b].digest, + added=added, + removed=removed, + modified=modified, + path=os.path.join(path, dir_b.directories[b].name), + ) b += 1 - elif a < len(dir_a.directories) and (b >= len(dir_b.directories) or - dir_b.directories[b].name > dir_a.directories[a].name): - self.diff_trees(dir_a.directories[a].digest, None, - added=added, removed=removed, modified=modified, - path=os.path.join(path, dir_a.directories[a].name)) + elif a < len(dir_a.directories) and ( + b >= len(dir_b.directories) or dir_b.directories[b].name > dir_a.directories[a].name + ): + self.diff_trees( + dir_a.directories[a].digest, + None, + added=added, + removed=removed, + modified=modified, + path=os.path.join(path, dir_a.directories[a].name), + ) a += 1 else: # Subdirectory exists in both directories if dir_a.directories[a].digest.hash != dir_b.directories[b].digest.hash: - self.diff_trees(dir_a.directories[a].digest, dir_b.directories[b].digest, - added=added, removed=removed, modified=modified, - path=os.path.join(path, dir_a.directories[a].name)) + self.diff_trees( + dir_a.directories[a].digest, + dir_b.directories[b].digest, + added=added, + removed=removed, + modified=modified, + path=os.path.join(path, dir_a.directories[a].name), + ) a += 1 b += 1 @@ -703,7 +726,7 @@ class CASCache(): return os.path.join(log_dir, str(self._casd_start_time) + ".log") def _refpath(self, ref): - return os.path.join(self.casdir, 'refs', 'heads', ref) + return os.path.join(self.casdir, "refs", "heads", ref) # _remove_ref() # @@ -763,7 +786,7 @@ class CASCache(): directory = remote_execution_pb2.Directory() - with open(self.objpath(tree), 'rb') as f: + with open(self.objpath(tree), "rb") as f: directory.ParseFromString(f.read()) for dirnode in directory.directories: @@ -783,7 +806,7 @@ class CASCache(): directory = remote_execution_pb2.Directory() - with open(self.objpath(tree), 'rb') as f: + with open(self.objpath(tree), "rb") as f: directory.ParseFromString(f.read()) except FileNotFoundError: @@ -813,8 +836,7 @@ class CASCache(): @contextlib.contextmanager def _temporary_object(self): with utils._tempnamedfile(dir=self.tmpdir) as f: - os.chmod(f.name, - stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH) + os.chmod(f.name, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH) yield f # _ensure_blob(): @@ -898,12 +920,13 @@ class CASCache(): objpath = self._ensure_blob(remote, dir_digest) directory = remote_execution_pb2.Directory() - with open(objpath, 'rb') as f: + with open(objpath, "rb") as f: directory.ParseFromString(f.read()) for dirnode in directory.directories: - batch = self._fetch_directory_node(remote, dirnode.digest, batch, - fetch_queue, fetch_next_queue, recursive=True) + batch = self._fetch_directory_node( + remote, dirnode.digest, batch, fetch_queue, fetch_next_queue, recursive=True + ) # Fetch final batch self._fetch_directory_batch(remote, batch, fetch_queue, fetch_next_queue) @@ -913,7 +936,7 @@ class CASCache(): tree = remote_execution_pb2.Tree() - with open(objpath, 'rb') as f: + with open(objpath, "rb") as f: tree.ParseFromString(f.read()) tree.children.extend([tree.root]) @@ -1062,8 +1085,7 @@ class CASCache(): # used_size (int): Total size used by the local cache, in bytes. # quota_size (int): Disk quota for the local cache, in bytes. # -class _CASCacheUsage(): - +class _CASCacheUsage: def __init__(self, used_size, quota_size): self.used_size = used_size self.quota_size = quota_size @@ -1080,10 +1102,11 @@ class _CASCacheUsage(): elif self.quota_size is None: return utils._pretty_size(self.used_size, dec_places=1) else: - return "{} / {} ({}%)" \ - .format(utils._pretty_size(self.used_size, dec_places=1), - utils._pretty_size(self.quota_size, dec_places=1), - self.used_percent) + return "{} / {} ({}%)".format( + utils._pretty_size(self.used_size, dec_places=1), + utils._pretty_size(self.quota_size, dec_places=1), + self.used_percent, + ) # _CASCacheUsageMonitor diff --git a/src/buildstream/_cas/casremote.py b/src/buildstream/_cas/casremote.py index a054b288a..ee6f4679c 100644 --- a/src/buildstream/_cas/casremote.py +++ b/src/buildstream/_cas/casremote.py @@ -32,7 +32,6 @@ _MAX_DIGESTS = _MAX_PAYLOAD_BYTES / 80 class BlobNotFound(CASRemoteError): - def __init__(self, blob, msg): self.blob = blob super().__init__(msg) @@ -41,7 +40,6 @@ class BlobNotFound(CASRemoteError): # Represents a single remote CAS cache. # class CASRemote(BaseRemote): - def __init__(self, spec, cascache, **kwargs): super().__init__(spec, **kwargs) @@ -90,7 +88,7 @@ class CASRemote(BaseRemote): # Represents a batch of blobs queued for fetching. # -class _CASBatchRead(): +class _CASBatchRead: def __init__(self, remote): self._remote = remote self._requests = [] @@ -123,22 +121,28 @@ class _CASBatchRead(): for response in batch_response.responses: if response.status.code == code_pb2.NOT_FOUND: if missing_blobs is None: - raise BlobNotFound(response.digest.hash, "Failed to download blob {}: {}".format( - response.digest.hash, response.status.code)) + raise BlobNotFound( + response.digest.hash, + "Failed to download blob {}: {}".format(response.digest.hash, response.status.code), + ) missing_blobs.append(response.digest) if response.status.code != code_pb2.OK: - raise CASRemoteError("Failed to download blob {}: {}".format( - response.digest.hash, response.status.code)) + raise CASRemoteError( + "Failed to download blob {}: {}".format(response.digest.hash, response.status.code) + ) if response.digest.size_bytes != len(response.data): - raise CASRemoteError("Failed to download blob {}: expected {} bytes, received {} bytes".format( - response.digest.hash, response.digest.size_bytes, len(response.data))) + raise CASRemoteError( + "Failed to download blob {}: expected {} bytes, received {} bytes".format( + response.digest.hash, response.digest.size_bytes, len(response.data) + ) + ) # Represents a batch of blobs queued for upload. # -class _CASBatchUpdate(): +class _CASBatchUpdate: def __init__(self, remote): self._remote = remote self._requests = [] @@ -175,5 +179,7 @@ class _CASBatchUpdate(): else: reason = None - raise CASRemoteError("Failed to upload blob {}: {}".format( - response.digest.hash, response.status.code), reason=reason) + raise CASRemoteError( + "Failed to upload blob {}: {}".format(response.digest.hash, response.status.code), + reason=reason, + ) diff --git a/src/buildstream/_cas/casserver.py b/src/buildstream/_cas/casserver.py index d4241435a..a2110d8a2 100644 --- a/src/buildstream/_cas/casserver.py +++ b/src/buildstream/_cas/casserver.py @@ -33,8 +33,14 @@ import click from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc from .._protos.google.bytestream import bytestream_pb2, bytestream_pb2_grpc from .._protos.google.rpc import code_pb2 -from .._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc, \ - artifact_pb2, artifact_pb2_grpc, source_pb2, source_pb2_grpc +from .._protos.buildstream.v2 import ( + buildstream_pb2, + buildstream_pb2_grpc, + artifact_pb2, + artifact_pb2_grpc, + source_pb2, + source_pb2_grpc, +) from .. import utils from .._exceptions import CASError, CASCacheError @@ -61,8 +67,8 @@ def create_server(repo, *, enable_push, quota, index_only): cas = CASCache(os.path.abspath(repo), cache_quota=quota, protect_session_blobs=False) try: - artifactdir = os.path.join(os.path.abspath(repo), 'artifacts', 'refs') - sourcedir = os.path.join(os.path.abspath(repo), 'source_protos') + artifactdir = os.path.join(os.path.abspath(repo), "artifacts", "refs") + sourcedir = os.path.join(os.path.abspath(repo), "source_protos") # Use max_workers default from Python 3.5+ max_workers = (os.cpu_count() or 1) * 5 @@ -70,31 +76,31 @@ def create_server(repo, *, enable_push, quota, index_only): if not index_only: bytestream_pb2_grpc.add_ByteStreamServicer_to_server( - _ByteStreamServicer(cas, enable_push=enable_push), server) + _ByteStreamServicer(cas, enable_push=enable_push), server + ) remote_execution_pb2_grpc.add_ContentAddressableStorageServicer_to_server( - _ContentAddressableStorageServicer(cas, enable_push=enable_push), server) + _ContentAddressableStorageServicer(cas, enable_push=enable_push), server + ) - remote_execution_pb2_grpc.add_CapabilitiesServicer_to_server( - _CapabilitiesServicer(), server) + remote_execution_pb2_grpc.add_CapabilitiesServicer_to_server(_CapabilitiesServicer(), server) buildstream_pb2_grpc.add_ReferenceStorageServicer_to_server( - _ReferenceStorageServicer(cas, enable_push=enable_push), server) + _ReferenceStorageServicer(cas, enable_push=enable_push), server + ) artifact_pb2_grpc.add_ArtifactServiceServicer_to_server( - _ArtifactServicer(cas, artifactdir, update_cas=not index_only), server) + _ArtifactServicer(cas, artifactdir, update_cas=not index_only), server + ) - source_pb2_grpc.add_SourceServiceServicer_to_server( - _SourceServicer(sourcedir), server) + source_pb2_grpc.add_SourceServiceServicer_to_server(_SourceServicer(sourcedir), server) # Create up reference storage and artifact capabilities - artifact_capabilities = buildstream_pb2.ArtifactCapabilities( - allow_updates=enable_push) - source_capabilities = buildstream_pb2.SourceCapabilities( - allow_updates=enable_push) + artifact_capabilities = buildstream_pb2.ArtifactCapabilities(allow_updates=enable_push) + source_capabilities = buildstream_pb2.SourceCapabilities(allow_updates=enable_push) buildstream_pb2_grpc.add_CapabilitiesServicer_to_server( - _BuildStreamCapabilitiesServicer(artifact_capabilities, source_capabilities), - server) + _BuildStreamCapabilitiesServicer(artifact_capabilities, source_capabilities), server + ) yield server @@ -103,28 +109,25 @@ def create_server(repo, *, enable_push, quota, index_only): @click.command(short_help="CAS Artifact Server") -@click.option('--port', '-p', type=click.INT, required=True, help="Port number") -@click.option('--server-key', help="Private server key for TLS (PEM-encoded)") -@click.option('--server-cert', help="Public server certificate for TLS (PEM-encoded)") -@click.option('--client-certs', help="Public client certificates for TLS (PEM-encoded)") -@click.option('--enable-push', is_flag=True, - help="Allow clients to upload blobs and update artifact cache") -@click.option('--quota', type=click.INT, default=10e9, show_default=True, - help="Maximum disk usage in bytes") -@click.option('--index-only', is_flag=True, - help="Only provide the BuildStream artifact and source services (\"index\"), not the CAS (\"storage\")") -@click.argument('repo') -def server_main(repo, port, server_key, server_cert, client_certs, enable_push, - quota, index_only): +@click.option("--port", "-p", type=click.INT, required=True, help="Port number") +@click.option("--server-key", help="Private server key for TLS (PEM-encoded)") +@click.option("--server-cert", help="Public server certificate for TLS (PEM-encoded)") +@click.option("--client-certs", help="Public client certificates for TLS (PEM-encoded)") +@click.option("--enable-push", is_flag=True, help="Allow clients to upload blobs and update artifact cache") +@click.option("--quota", type=click.INT, default=10e9, show_default=True, help="Maximum disk usage in bytes") +@click.option( + "--index-only", + is_flag=True, + help='Only provide the BuildStream artifact and source services ("index"), not the CAS ("storage")', +) +@click.argument("repo") +def server_main(repo, port, server_key, server_cert, client_certs, enable_push, quota, index_only): # Handle SIGTERM by calling sys.exit(0), which will raise a SystemExit exception, # properly executing cleanup code in `finally` clauses and context managers. # This is required to terminate buildbox-casd on SIGTERM. signal.signal(signal.SIGTERM, lambda signalnum, frame: sys.exit(0)) - with create_server(repo, - quota=quota, - enable_push=enable_push, - index_only=index_only) as server: + with create_server(repo, quota=quota, enable_push=enable_push, index_only=index_only) as server: use_tls = bool(server_key) @@ -138,23 +141,25 @@ def server_main(repo, port, server_key, server_cert, client_certs, enable_push, if use_tls: # Read public/private key pair - with open(server_key, 'rb') as f: + with open(server_key, "rb") as f: server_key_bytes = f.read() - with open(server_cert, 'rb') as f: + with open(server_cert, "rb") as f: server_cert_bytes = f.read() if client_certs: - with open(client_certs, 'rb') as f: + with open(client_certs, "rb") as f: client_certs_bytes = f.read() else: client_certs_bytes = None - credentials = grpc.ssl_server_credentials([(server_key_bytes, server_cert_bytes)], - root_certificates=client_certs_bytes, - require_client_auth=bool(client_certs)) - server.add_secure_port('[::]:{}'.format(port), credentials) + credentials = grpc.ssl_server_credentials( + [(server_key_bytes, server_cert_bytes)], + root_certificates=client_certs_bytes, + require_client_auth=bool(client_certs), + ) + server.add_secure_port("[::]:{}".format(port), credentials) else: - server.add_insecure_port('[::]:{}'.format(port)) + server.add_insecure_port("[::]:{}".format(port)) # Run artifact server server.start() @@ -183,7 +188,7 @@ class _ByteStreamServicer(bytestream_pb2_grpc.ByteStreamServicer): return try: - with open(self.cas.objpath(client_digest), 'rb') as f: + with open(self.cas.objpath(client_digest), "rb") as f: if os.fstat(f.fileno()).st_size != client_digest.size_bytes: context.set_code(grpc.StatusCode.NOT_FOUND) return @@ -317,7 +322,7 @@ class _ContentAddressableStorageServicer(remote_execution_pb2_grpc.ContentAddres blob_response.digest.size_bytes = digest.size_bytes try: objpath = self.cas.objpath(digest) - with open(objpath, 'rb') as f: + with open(objpath, "rb") as f: if os.fstat(f.fileno()).st_size != digest.size_bytes: blob_response.status.code = code_pb2.NOT_FOUND continue @@ -437,7 +442,6 @@ class _ReferenceStorageServicer(buildstream_pb2_grpc.ReferenceStorageServicer): class _ArtifactServicer(artifact_pb2_grpc.ArtifactServiceServicer): - def __init__(self, cas, artifactdir, *, update_cas=True): super().__init__() self.cas = cas @@ -451,7 +455,7 @@ class _ArtifactServicer(artifact_pb2_grpc.ArtifactServiceServicer): context.abort(grpc.StatusCode.NOT_FOUND, "Artifact proto not found") artifact = artifact_pb2.Artifact() - with open(artifact_path, 'rb') as f: + with open(artifact_path, "rb") as f: artifact.ParseFromString(f.read()) # Artifact-only servers will not have blobs on their system, @@ -489,11 +493,9 @@ class _ArtifactServicer(artifact_pb2_grpc.ArtifactServiceServicer): except FileNotFoundError: os.unlink(artifact_path) - context.abort(grpc.StatusCode.NOT_FOUND, - "Artifact files incomplete") + context.abort(grpc.StatusCode.NOT_FOUND, "Artifact files incomplete") except DecodeError: - context.abort(grpc.StatusCode.NOT_FOUND, - "Artifact files not valid") + context.abort(grpc.StatusCode.NOT_FOUND, "Artifact files not valid") return artifact @@ -516,7 +518,7 @@ class _ArtifactServicer(artifact_pb2_grpc.ArtifactServiceServicer): # Add the artifact proto to the cas artifact_path = os.path.join(self.artifactdir, request.cache_key) os.makedirs(os.path.dirname(artifact_path), exist_ok=True) - with utils.save_file_atomic(artifact_path, mode='wb') as f: + with utils.save_file_atomic(artifact_path, mode="wb") as f: f.write(artifact.SerializeToString()) return artifact @@ -527,19 +529,18 @@ class _ArtifactServicer(artifact_pb2_grpc.ArtifactServiceServicer): def _check_directory(self, name, digest, context): try: directory = remote_execution_pb2.Directory() - with open(self.cas.objpath(digest), 'rb') as f: + with open(self.cas.objpath(digest), "rb") as f: directory.ParseFromString(f.read()) except FileNotFoundError: - context.abort(grpc.StatusCode.FAILED_PRECONDITION, - "Artifact {} specified but no files found".format(name)) + context.abort(grpc.StatusCode.FAILED_PRECONDITION, "Artifact {} specified but no files found".format(name)) except DecodeError: - context.abort(grpc.StatusCode.FAILED_PRECONDITION, - "Artifact {} specified but directory not found".format(name)) + context.abort( + grpc.StatusCode.FAILED_PRECONDITION, "Artifact {} specified but directory not found".format(name) + ) def _check_file(self, name, digest, context): if not os.path.exists(self.cas.objpath(digest)): - context.abort(grpc.StatusCode.FAILED_PRECONDITION, - "Artifact {} specified but not found".format(name)) + context.abort(grpc.StatusCode.FAILED_PRECONDITION, "Artifact {} specified but not found".format(name)) class _BuildStreamCapabilitiesServicer(buildstream_pb2_grpc.CapabilitiesServicer): @@ -564,8 +565,7 @@ class _SourceServicer(source_pb2_grpc.SourceServiceServicer): except FileNotFoundError: context.abort(grpc.StatusCode.NOT_FOUND, "Source not found") except DecodeError: - context.abort(grpc.StatusCode.NOT_FOUND, - "Sources gives invalid directory") + context.abort(grpc.StatusCode.NOT_FOUND, "Sources gives invalid directory") return source_proto @@ -576,7 +576,7 @@ class _SourceServicer(source_pb2_grpc.SourceServiceServicer): def _get_source(self, cache_key): path = os.path.join(self.sourcedir, cache_key) source_proto = source_pb2.Source() - with open(path, 'r+b') as f: + with open(path, "r+b") as f: source_proto.ParseFromString(f.read()) os.utime(path) return source_proto @@ -584,18 +584,18 @@ class _SourceServicer(source_pb2_grpc.SourceServiceServicer): def _set_source(self, cache_key, source_proto): path = os.path.join(self.sourcedir, cache_key) os.makedirs(os.path.dirname(path), exist_ok=True) - with utils.save_file_atomic(path, 'w+b') as f: + with utils.save_file_atomic(path, "w+b") as f: f.write(source_proto.SerializeToString()) def _digest_from_download_resource_name(resource_name): - parts = resource_name.split('/') + parts = resource_name.split("/") # Accept requests from non-conforming BuildStream 1.1.x clients if len(parts) == 2: - parts.insert(0, 'blobs') + parts.insert(0, "blobs") - if len(parts) != 3 or parts[0] != 'blobs': + if len(parts) != 3 or parts[0] != "blobs": return None try: @@ -608,15 +608,15 @@ def _digest_from_download_resource_name(resource_name): def _digest_from_upload_resource_name(resource_name): - parts = resource_name.split('/') + parts = resource_name.split("/") # Accept requests from non-conforming BuildStream 1.1.x clients if len(parts) == 2: - parts.insert(0, 'uploads') + parts.insert(0, "uploads") parts.insert(1, str(uuid.uuid4())) - parts.insert(2, 'blobs') + parts.insert(2, "blobs") - if len(parts) < 5 or parts[0] != 'uploads' or parts[2] != 'blobs': + if len(parts) < 5 or parts[0] != "uploads" or parts[2] != "blobs": return None try: diff --git a/src/buildstream/_context.py b/src/buildstream/_context.py index 4e1007e28..f426f4bb0 100644 --- a/src/buildstream/_context.py +++ b/src/buildstream/_context.py @@ -46,13 +46,12 @@ from .sandbox import SandboxRemote # verbosity levels and basically anything pertaining to the context # in which BuildStream was invoked. # -class Context(): - +class Context: def __init__(self, *, use_casd=True): # Whether we are running as part of a test suite. This is only relevant # for developing BuildStream itself. - self.is_running_in_test_suite = 'BST_TEST_SUITE' in os.environ + self.is_running_in_test_suite = "BST_TEST_SUITE" in os.environ # Filename indicating which configuration file was used, or None for the defaults self.config_origin = None @@ -216,8 +215,7 @@ class Context(): # a $XDG_CONFIG_HOME/buildstream.conf file # if not config: - default_config = os.path.join(os.environ['XDG_CONFIG_HOME'], - 'buildstream.conf') + default_config = os.path.join(os.environ["XDG_CONFIG_HOME"], "buildstream.conf") if os.path.exists(default_config): config = default_config @@ -231,19 +229,32 @@ class Context(): user_config._composite(defaults) # Give obsoletion warnings - if 'builddir' in defaults: + if "builddir" in defaults: raise LoadError("builddir is obsolete, use cachedir", LoadErrorReason.INVALID_DATA) - if 'artifactdir' in defaults: + if "artifactdir" in defaults: raise LoadError("artifactdir is obsolete", LoadErrorReason.INVALID_DATA) - defaults.validate_keys([ - 'cachedir', 'sourcedir', 'builddir', 'logdir', 'scheduler', 'build', - 'artifacts', 'source-caches', 'logging', 'projects', 'cache', 'prompt', - 'workspacedir', 'remote-execution', - ]) - - for directory in ['cachedir', 'sourcedir', 'logdir', 'workspacedir']: + defaults.validate_keys( + [ + "cachedir", + "sourcedir", + "builddir", + "logdir", + "scheduler", + "build", + "artifacts", + "source-caches", + "logging", + "projects", + "cache", + "prompt", + "workspacedir", + "remote-execution", + ] + ) + + for directory in ["cachedir", "sourcedir", "logdir", "workspacedir"]: # Allow the ~ tilde expansion and any environment variables in # path specification in the config files. # @@ -256,25 +267,23 @@ class Context(): # Relative paths don't make sense in user configuration. The exception is # workspacedir where `.` is useful as it will be combined with the name # specified on the command line. - if not os.path.isabs(path) and not (directory == 'workspacedir' and path == '.'): + if not os.path.isabs(path) and not (directory == "workspacedir" and path == "."): raise LoadError("{} must be an absolute path".format(directory), LoadErrorReason.INVALID_DATA) # add directories not set by users - self.tmpdir = os.path.join(self.cachedir, 'tmp') - self.casdir = os.path.join(self.cachedir, 'cas') - self.builddir = os.path.join(self.cachedir, 'build') - self.artifactdir = os.path.join(self.cachedir, 'artifacts', 'refs') + self.tmpdir = os.path.join(self.cachedir, "tmp") + self.casdir = os.path.join(self.cachedir, "cas") + self.builddir = os.path.join(self.cachedir, "build") + self.artifactdir = os.path.join(self.cachedir, "artifacts", "refs") # Move old artifact cas to cas if it exists and create symlink - old_casdir = os.path.join(self.cachedir, 'artifacts', 'cas') - if (os.path.exists(old_casdir) and not os.path.islink(old_casdir) and - not os.path.exists(self.casdir)): + old_casdir = os.path.join(self.cachedir, "artifacts", "cas") + if os.path.exists(old_casdir) and not os.path.islink(old_casdir) and not os.path.exists(self.casdir): os.rename(old_casdir, self.casdir) os.symlink(self.casdir, old_casdir) # Cleanup old extract directories - old_extractdirs = [os.path.join(self.cachedir, 'artifacts', 'extract'), - os.path.join(self.cachedir, 'extract')] + old_extractdirs = [os.path.join(self.cachedir, "artifacts", "extract"), os.path.join(self.cachedir, "extract")] for old_extractdir in old_extractdirs: if os.path.isdir(old_extractdir): shutil.rmtree(old_extractdir, ignore_errors=True) @@ -282,21 +291,22 @@ class Context(): # Load quota configuration # We need to find the first existing directory in the path of our # casdir - the casdir may not have been created yet. - cache = defaults.get_mapping('cache') - cache.validate_keys(['quota', 'pull-buildtrees', 'cache-buildtrees']) + cache = defaults.get_mapping("cache") + cache.validate_keys(["quota", "pull-buildtrees", "cache-buildtrees"]) cas_volume = self.casdir while not os.path.exists(cas_volume): cas_volume = os.path.dirname(cas_volume) - self.config_cache_quota_string = cache.get_str('quota') + self.config_cache_quota_string = cache.get_str("quota") try: - self.config_cache_quota = utils._parse_size(self.config_cache_quota_string, - cas_volume) + self.config_cache_quota = utils._parse_size(self.config_cache_quota_string, cas_volume) except utils.UtilError as e: - raise LoadError("{}\nPlease specify the value in bytes or as a % of full disk space.\n" - "\nValid values are, for example: 800M 10G 1T 50%\n" - .format(str(e)), LoadErrorReason.INVALID_DATA) from e + raise LoadError( + "{}\nPlease specify the value in bytes or as a % of full disk space.\n" + "\nValid values are, for example: 800M 10G 1T 50%\n".format(str(e)), + LoadErrorReason.INVALID_DATA, + ) from e # Load artifact share configuration self.artifact_cache_specs = ArtifactCache.specs_from_config_node(defaults) @@ -305,73 +315,70 @@ class Context(): self.source_cache_specs = SourceCache.specs_from_config_node(defaults) # Load remote execution config getting pull-artifact-files from it - remote_execution = defaults.get_mapping('remote-execution', default=None) + remote_execution = defaults.get_mapping("remote-execution", default=None) if remote_execution: - self.pull_artifact_files = remote_execution.get_bool('pull-artifact-files', default=True) + self.pull_artifact_files = remote_execution.get_bool("pull-artifact-files", default=True) # This stops it being used in the remote service set up - remote_execution.safe_del('pull-artifact-files') + remote_execution.safe_del("pull-artifact-files") # Don't pass the remote execution settings if that was the only option if remote_execution.keys() == []: - del defaults['remote-execution'] + del defaults["remote-execution"] else: self.pull_artifact_files = True self.remote_execution_specs = SandboxRemote.specs_from_config_node(defaults) # Load pull build trees configuration - self.pull_buildtrees = cache.get_bool('pull-buildtrees') + self.pull_buildtrees = cache.get_bool("pull-buildtrees") # Load cache build trees configuration - self.cache_buildtrees = cache.get_enum('cache-buildtrees', _CacheBuildTrees) + self.cache_buildtrees = cache.get_enum("cache-buildtrees", _CacheBuildTrees) # Load logging config - logging = defaults.get_mapping('logging') - logging.validate_keys([ - 'key-length', 'verbose', - 'error-lines', 'message-lines', - 'debug', 'element-format', 'message-format' - ]) - self.log_key_length = logging.get_int('key-length') - self.log_debug = logging.get_bool('debug') - self.log_verbose = logging.get_bool('verbose') - self.log_error_lines = logging.get_int('error-lines') - self.log_message_lines = logging.get_int('message-lines') - self.log_element_format = logging.get_str('element-format') - self.log_message_format = logging.get_str('message-format') + logging = defaults.get_mapping("logging") + logging.validate_keys( + ["key-length", "verbose", "error-lines", "message-lines", "debug", "element-format", "message-format"] + ) + self.log_key_length = logging.get_int("key-length") + self.log_debug = logging.get_bool("debug") + self.log_verbose = logging.get_bool("verbose") + self.log_error_lines = logging.get_int("error-lines") + self.log_message_lines = logging.get_int("message-lines") + self.log_element_format = logging.get_str("element-format") + self.log_message_format = logging.get_str("message-format") # Load scheduler config - scheduler = defaults.get_mapping('scheduler') - scheduler.validate_keys([ - 'on-error', 'fetchers', 'builders', - 'pushers', 'network-retries' - ]) - self.sched_error_action = scheduler.get_enum('on-error', _SchedulerErrorAction) - self.sched_fetchers = scheduler.get_int('fetchers') - self.sched_builders = scheduler.get_int('builders') - self.sched_pushers = scheduler.get_int('pushers') - self.sched_network_retries = scheduler.get_int('network-retries') + scheduler = defaults.get_mapping("scheduler") + scheduler.validate_keys(["on-error", "fetchers", "builders", "pushers", "network-retries"]) + self.sched_error_action = scheduler.get_enum("on-error", _SchedulerErrorAction) + self.sched_fetchers = scheduler.get_int("fetchers") + self.sched_builders = scheduler.get_int("builders") + self.sched_pushers = scheduler.get_int("pushers") + self.sched_network_retries = scheduler.get_int("network-retries") # Load build config - build = defaults.get_mapping('build') - build.validate_keys(['max-jobs', 'dependencies']) - self.build_max_jobs = build.get_int('max-jobs') - - self.build_dependencies = build.get_str('dependencies') - if self.build_dependencies not in ['plan', 'all']: - provenance = build.get_scalar('dependencies').get_provenance() - raise LoadError("{}: Invalid value for 'dependencies'. Choose 'plan' or 'all'." - .format(provenance), LoadErrorReason.INVALID_DATA) + build = defaults.get_mapping("build") + build.validate_keys(["max-jobs", "dependencies"]) + self.build_max_jobs = build.get_int("max-jobs") + + self.build_dependencies = build.get_str("dependencies") + if self.build_dependencies not in ["plan", "all"]: + provenance = build.get_scalar("dependencies").get_provenance() + raise LoadError( + "{}: Invalid value for 'dependencies'. Choose 'plan' or 'all'.".format(provenance), + LoadErrorReason.INVALID_DATA, + ) # Load per-projects overrides - self._project_overrides = defaults.get_mapping('projects', default={}) + self._project_overrides = defaults.get_mapping("projects", default={}) # Shallow validation of overrides, parts of buildstream which rely # on the overrides are expected to validate elsewhere. for overrides_project in self._project_overrides.keys(): overrides = self._project_overrides.get_mapping(overrides_project) - overrides.validate_keys(['artifacts', 'source-caches', 'options', - 'strict', 'default-mirror', - 'remote-execution']) + overrides.validate_keys( + ["artifacts", "source-caches", "options", "strict", "default-mirror", "remote-execution"] + ) @property def platform(self): @@ -474,7 +481,7 @@ class Context(): # so work out if we should be strict, and then cache the result toplevel = self.get_toplevel_project() overrides = self.get_overrides(toplevel.name) - self._strict_build_plan = overrides.get_bool('strict', default=True) + self._strict_build_plan = overrides.get_bool("strict", default=True) # If it was set by the CLI, it overrides any config # Ditto if we've already computed this, then we return the computed @@ -505,12 +512,12 @@ class Context(): # preferred locations of things from user configuration # files. def _init_xdg(self): - if not os.environ.get('XDG_CACHE_HOME'): - os.environ['XDG_CACHE_HOME'] = os.path.expanduser('~/.cache') - if not os.environ.get('XDG_CONFIG_HOME'): - os.environ['XDG_CONFIG_HOME'] = os.path.expanduser('~/.config') - if not os.environ.get('XDG_DATA_HOME'): - os.environ['XDG_DATA_HOME'] = os.path.expanduser('~/.local/share') + if not os.environ.get("XDG_CACHE_HOME"): + os.environ["XDG_CACHE_HOME"] = os.path.expanduser("~/.cache") + if not os.environ.get("XDG_CONFIG_HOME"): + os.environ["XDG_CONFIG_HOME"] = os.path.expanduser("~/.config") + if not os.environ.get("XDG_DATA_HOME"): + os.environ["XDG_DATA_HOME"] = os.path.expanduser("~/.local/share") def get_cascache(self): if self._cascache is None: @@ -521,10 +528,9 @@ class Context(): else: log_level = CASLogLevel.WARNING - self._cascache = CASCache(self.cachedir, - casd=self.use_casd, - cache_quota=self.config_cache_quota, - log_level=log_level) + self._cascache = CASCache( + self.cachedir, casd=self.use_casd, cache_quota=self.config_cache_quota, log_level=log_level + ) return self._cascache # prepare_fork(): diff --git a/src/buildstream/_elementfactory.py b/src/buildstream/_elementfactory.py index d6591bf4c..5d219c627 100644 --- a/src/buildstream/_elementfactory.py +++ b/src/buildstream/_elementfactory.py @@ -30,14 +30,15 @@ from .element import Element # plugin_origins (list): Data used to search for external Element plugins # class ElementFactory(PluginContext): + def __init__(self, plugin_base, *, format_versions={}, plugin_origins=None): - def __init__(self, plugin_base, *, - format_versions={}, - plugin_origins=None): - - super().__init__(plugin_base, Element, [_site.element_plugins], - plugin_origins=plugin_origins, - format_versions=format_versions) + super().__init__( + plugin_base, + Element, + [_site.element_plugins], + plugin_origins=plugin_origins, + format_versions=format_versions, + ) # create(): # diff --git a/src/buildstream/_exceptions.py b/src/buildstream/_exceptions.py index 46de90796..ca17577f7 100644 --- a/src/buildstream/_exceptions.py +++ b/src/buildstream/_exceptions.py @@ -51,7 +51,7 @@ def get_last_exception(): # Used by regression tests # def get_last_task_error(): - if 'BST_TEST_SUITE' not in os.environ: + if "BST_TEST_SUITE" not in os.environ: raise BstError("Getting the last task error is only supported when running tests") global _last_task_error_domain @@ -71,7 +71,7 @@ def get_last_task_error(): # tests about how things failed in a machine readable way # def set_last_task_error(domain, reason): - if 'BST_TEST_SUITE' in os.environ: + if "BST_TEST_SUITE" in os.environ: global _last_task_error_domain global _last_task_error_reason @@ -108,7 +108,6 @@ class ErrorDomain(Enum): # context can then be communicated back to the main process. # class BstError(Exception): - def __init__(self, message, *, detail=None, domain=None, reason=None, temporary=False): global _last_exception @@ -133,7 +132,7 @@ class BstError(Exception): self.reason = reason # Hold on to the last raised exception for testing purposes - if 'BST_TEST_SUITE' in os.environ: + if "BST_TEST_SUITE" in os.environ: _last_exception = self @@ -330,7 +329,6 @@ class CASCacheError(CASError): # Raised from pipeline operations # class PipelineError(BstError): - def __init__(self, message, *, detail=None, reason=None): super().__init__(message, detail=detail, domain=ErrorDomain.PIPELINE, reason=reason) @@ -340,7 +338,6 @@ class PipelineError(BstError): # Raised when a stream operation fails # class StreamError(BstError): - def __init__(self, message=None, *, detail=None, reason=None, terminated=False): # The empty string should never appear to a user, diff --git a/src/buildstream/_frontend/app.py b/src/buildstream/_frontend/app.py index 99e164358..09610851f 100644 --- a/src/buildstream/_frontend/app.py +++ b/src/buildstream/_frontend/app.py @@ -56,19 +56,18 @@ INDENT = 4 # main_options (dict): The main CLI options of the `bst` # command, before any subcommand # -class App(): - +class App: def __init__(self, main_options): # # Public members # - self.context = None # The Context object - self.stream = None # The Stream object - self.project = None # The toplevel Project object - self.logger = None # The LogLine object - self.interactive = None # Whether we are running in interactive mode - self.colors = None # Whether to use colors in logging + self.context = None # The Context object + self.stream = None # The Stream object + self.project = None # The toplevel Project object + self.logger = None # The LogLine object + self.interactive = None # Whether we are running in interactive mode + self.colors = None # Whether to use colors in logging # # Private members @@ -76,18 +75,18 @@ class App(): self._session_start = datetime.datetime.now() self._session_name = None self._main_options = main_options # Main CLI options, before any command - self._status = None # The Status object - self._fail_messages = {} # Failure messages by unique plugin id + self._status = None # The Status object + self._fail_messages = {} # Failure messages by unique plugin id self._interactive_failures = None # Whether to handle failures interactively - self._started = False # Whether a session has started - self._set_project_dir = False # Whether -C option was used - self._state = None # Frontend reads this and registers callbacks + self._started = False # Whether a session has started + self._set_project_dir = False # Whether -C option was used + self._state = None # Frontend reads this and registers callbacks # UI Colors Profiles - self._content_profile = Profile(fg='yellow') - self._format_profile = Profile(fg='cyan', dim=True) - self._success_profile = Profile(fg='green') - self._error_profile = Profile(fg='red', dim=True) + self._content_profile = Profile(fg="yellow") + self._format_profile = Profile(fg="cyan", dim=True) + self._success_profile = Profile(fg="green") + self._error_profile = Profile(fg="red", dim=True) self._detail_profile = Profile(dim=True) # @@ -96,31 +95,31 @@ class App(): is_a_tty = sys.stdout.isatty() and sys.stderr.isatty() # Enable interactive mode if we're attached to a tty - if main_options['no_interactive']: + if main_options["no_interactive"]: self.interactive = False else: self.interactive = is_a_tty # Handle errors interactively if we're in interactive mode # and --on-error was not specified on the command line - if main_options.get('on_error') is not None: + if main_options.get("on_error") is not None: self._interactive_failures = False else: self._interactive_failures = self.interactive # Use color output if we're attached to a tty, unless # otherwise specified on the command line - if main_options['colors'] is None: + if main_options["colors"] is None: self.colors = is_a_tty - elif main_options['colors']: + elif main_options["colors"]: self.colors = True else: self.colors = False - if main_options['directory']: + if main_options["directory"]: self._set_project_dir = True else: - main_options['directory'] = os.getcwd() + main_options["directory"] = os.getcwd() # create() # @@ -133,9 +132,10 @@ class App(): # @classmethod def create(cls, *args, **kwargs): - if sys.platform.startswith('linux'): + if sys.platform.startswith("linux"): # Use an App with linux specific features from .linuxapp import LinuxApp # pylint: disable=cyclic-import + return LinuxApp(*args, **kwargs) else: # The base App() class is default @@ -163,8 +163,8 @@ class App(): # @contextmanager def initialized(self, *, session_name=None): - directory = self._main_options['directory'] - config = self._main_options['config'] + directory = self._main_options["directory"] + config = self._main_options["config"] self._session_name = session_name @@ -184,19 +184,19 @@ class App(): # the command line when used, trumps the config files. # override_map = { - 'strict': '_strict_build_plan', - 'debug': 'log_debug', - 'verbose': 'log_verbose', - 'error_lines': 'log_error_lines', - 'message_lines': 'log_message_lines', - 'on_error': 'sched_error_action', - 'fetchers': 'sched_fetchers', - 'builders': 'sched_builders', - 'pushers': 'sched_pushers', - 'max_jobs': 'build_max_jobs', - 'network_retries': 'sched_network_retries', - 'pull_buildtrees': 'pull_buildtrees', - 'cache_buildtrees': 'cache_buildtrees' + "strict": "_strict_build_plan", + "debug": "log_debug", + "verbose": "log_verbose", + "error_lines": "log_error_lines", + "message_lines": "log_message_lines", + "on_error": "sched_error_action", + "fetchers": "sched_fetchers", + "builders": "sched_builders", + "pushers": "sched_pushers", + "max_jobs": "build_max_jobs", + "network_retries": "sched_network_retries", + "pull_buildtrees": "pull_buildtrees", + "cache_buildtrees": "cache_buildtrees", } for cli_option, context_attr in override_map.items(): option_value = self._main_options.get(cli_option) @@ -208,10 +208,13 @@ class App(): self._error_exit(e, "Error instantiating platform") # Create the stream right away, we'll need to pass it around. - self.stream = Stream(self.context, self._session_start, - session_start_callback=self.session_start_cb, - interrupt_callback=self._interrupt_handler, - ticker_callback=self._tick) + self.stream = Stream( + self.context, + self._session_start, + session_start_callback=self.session_start_cb, + interrupt_callback=self._interrupt_handler, + ticker_callback=self._tick, + ) self._state = self.stream.get_state() @@ -219,13 +222,16 @@ class App(): self._state.register_task_failed_callback(self._job_failed) # Create the logger right before setting the message handler - self.logger = LogLine(self.context, self._state, - self._content_profile, - self._format_profile, - self._success_profile, - self._error_profile, - self._detail_profile, - indent=INDENT) + self.logger = LogLine( + self.context, + self._state, + self._content_profile, + self._format_profile, + self._success_profile, + self._error_profile, + self._detail_profile, + indent=INDENT, + ) # Propagate pipeline feedback to the user self.context.messenger.set_message_handler(self._message_handler) @@ -248,10 +254,15 @@ class App(): self.stream.init() # Create our status printer, only available in interactive - self._status = Status(self.context, self._state, - self._content_profile, self._format_profile, - self._success_profile, self._error_profile, - self.stream) + self._status = Status( + self.context, + self._state, + self._content_profile, + self._format_profile, + self._success_profile, + self._error_profile, + self.stream, + ) # Mark the beginning of the session if session_name: @@ -261,9 +272,13 @@ class App(): # Load the Project # try: - self.project = Project(directory, self.context, cli_options=self._main_options['option'], - default_mirror=self._main_options.get('default_mirror'), - fetch_subprojects=self.stream.fetch_subprojects) + self.project = Project( + directory, + self.context, + cli_options=self._main_options["option"], + default_mirror=self._main_options.get("default_mirror"), + fetch_subprojects=self.stream.fetch_subprojects, + ) self.stream.set_project(self.project) except LoadError as e: @@ -291,7 +306,7 @@ class App(): elapsed = self.stream.elapsed_time if isinstance(e, StreamError) and e.terminated: # pylint: disable=no-member - self._message(MessageType.WARN, session_name + ' Terminated', elapsed=elapsed) + self._message(MessageType.WARN, session_name + " Terminated", elapsed=elapsed) else: self._message(MessageType.FAIL, session_name, elapsed=elapsed) @@ -304,8 +319,9 @@ class App(): # Exit with the error self._error_exit(e) except RecursionError: - click.echo("RecursionError: Dependency depth is too large. Maximum recursion depth exceeded.", - err=True) + click.echo( + "RecursionError: Dependency depth is too large. Maximum recursion depth exceeded.", err=True + ) sys.exit(-1) else: @@ -331,41 +347,51 @@ class App(): # force (bool): Allow overwriting an existing project.conf # target_directory (str): The target directory the project should be initialized in # - def init_project(self, project_name, format_version=BST_FORMAT_VERSION, element_path='elements', - force=False, target_directory=None): + def init_project( + self, + project_name, + format_version=BST_FORMAT_VERSION, + element_path="elements", + force=False, + target_directory=None, + ): if target_directory: directory = os.path.abspath(target_directory) else: - directory = self._main_options['directory'] + directory = self._main_options["directory"] directory = os.path.abspath(directory) - project_path = os.path.join(directory, 'project.conf') + project_path = os.path.join(directory, "project.conf") try: if self._set_project_dir: - raise AppError("Attempted to use -C or --directory with init.", - reason='init-with-set-directory', - detail="Please use 'bst init {}' instead.".format(directory)) + raise AppError( + "Attempted to use -C or --directory with init.", + reason="init-with-set-directory", + detail="Please use 'bst init {}' instead.".format(directory), + ) # Abort if the project.conf already exists, unless `--force` was specified in `bst init` if not force and os.path.exists(project_path): - raise AppError("A project.conf already exists at: {}".format(project_path), - reason='project-exists') + raise AppError("A project.conf already exists at: {}".format(project_path), reason="project-exists") if project_name: # If project name was specified, user interaction is not desired, just # perform some validation and write the project.conf - node._assert_symbol_name(project_name, 'project name') + node._assert_symbol_name(project_name, "project name") self._assert_format_version(format_version) self._assert_element_path(element_path) elif not self.interactive: - raise AppError("Cannot initialize a new project without specifying the project name", - reason='unspecified-project-name') + raise AppError( + "Cannot initialize a new project without specifying the project name", + reason="unspecified-project-name", + ) else: # Collect the parameters using an interactive session - project_name, format_version, element_path = \ - self._init_project_interactive(project_name, format_version, element_path) + project_name, format_version, element_path = self._init_project_interactive( + project_name, format_version, element_path + ) # Create the directory if it doesnt exist try: @@ -378,20 +404,21 @@ class App(): try: os.makedirs(elements_path, exist_ok=True) except IOError as e: - raise AppError("Error creating elements sub-directory {}: {}" - .format(elements_path, e)) from e + raise AppError("Error creating elements sub-directory {}: {}".format(elements_path, e)) from e # Dont use ruamel.yaml here, because it doesnt let # us programatically insert comments or whitespace at # the toplevel. try: - with open(project_path, 'w') as f: - f.write("# Unique project name\n" + - "name: {}\n\n".format(project_name) + - "# Required BuildStream format version\n" + - "format-version: {}\n\n".format(format_version) + - "# Subdirectory where elements are stored\n" + - "element-path: {}\n".format(element_path)) + with open(project_path, "w") as f: + f.write( + "# Unique project name\n" + + "name: {}\n\n".format(project_name) + + "# Required BuildStream format version\n" + + "format-version: {}\n\n".format(format_version) + + "# Subdirectory where elements are stored\n" + + "element-path: {}\n".format(element_path) + ) except IOError as e: raise AppError("Error writing {}: {}".format(project_path, e)) from e @@ -419,15 +446,18 @@ class App(): _, key, dim = element_key if self.colors: - prompt = self._format_profile.fmt('[') + \ - self._content_profile.fmt(key, dim=dim) + \ - self._format_profile.fmt('@') + \ - self._content_profile.fmt(element_name) + \ - self._format_profile.fmt(':') + \ - self._content_profile.fmt('$PWD') + \ - self._format_profile.fmt(']$') + ' ' + prompt = ( + self._format_profile.fmt("[") + + self._content_profile.fmt(key, dim=dim) + + self._format_profile.fmt("@") + + self._content_profile.fmt(element_name) + + self._format_profile.fmt(":") + + self._content_profile.fmt("$PWD") + + self._format_profile.fmt("]$") + + " " + ) else: - prompt = '[{}@{}:${{PWD}}]$ '.format(key, element_name) + prompt = "[{}@{}:${{PWD}}]$ ".format(key, element_name) return prompt @@ -473,8 +503,7 @@ class App(): # def _message(self, message_type, message, **kwargs): args = dict(kwargs) - self.context.messenger.message( - Message(message_type, message, **args)) + self.context.messenger.message(Message(message_type, message, **args)) # Exception handler # @@ -482,8 +511,7 @@ class App(): # Print the regular BUG message formatted = "".join(traceback.format_exception(etype, value, tb)) - self._message(MessageType.BUG, str(value), - detail=formatted) + self._message(MessageType.BUG, str(value), detail=formatted) # If the scheduler has started, try to terminate all jobs gracefully, # otherwise exit immediately. @@ -498,8 +526,7 @@ class App(): def _maybe_render_status(self): # If we're suspended or terminating, then dont render the status area - if self._status and self.stream and \ - not (self.stream.suspended or self.stream.terminated): + if self._status and self.stream and not (self.stream.suspended or self.stream.terminated): self._status.render() # @@ -518,36 +545,40 @@ class App(): # the currently ongoing tasks. We can also print something more # intelligent, like how many tasks remain to complete overall. with self._interrupted(): - click.echo("\nUser interrupted with ^C\n" + - "\n" - "Choose one of the following options:\n" + - " (c)ontinue - Continue queueing jobs as much as possible\n" + - " (q)uit - Exit after all ongoing jobs complete\n" + - " (t)erminate - Terminate any ongoing jobs and exit\n" + - "\n" + - "Pressing ^C again will terminate jobs and exit\n", - err=True) + click.echo( + "\nUser interrupted with ^C\n" + "\n" + "Choose one of the following options:\n" + + " (c)ontinue - Continue queueing jobs as much as possible\n" + + " (q)uit - Exit after all ongoing jobs complete\n" + + " (t)erminate - Terminate any ongoing jobs and exit\n" + + "\n" + + "Pressing ^C again will terminate jobs and exit\n", + err=True, + ) try: - choice = click.prompt("Choice:", - value_proc=_prefix_choice_value_proc(['continue', 'quit', 'terminate']), - default='continue', err=True) + choice = click.prompt( + "Choice:", + value_proc=_prefix_choice_value_proc(["continue", "quit", "terminate"]), + default="continue", + err=True, + ) except (click.Abort, SystemError): # In some cases, the readline buffer underlying the prompt gets corrupted on the second CTRL+C # This throws a SystemError, which doesn't seem to be problematic for the rest of the program # Ensure a newline after automatically printed '^C' click.echo("", err=True) - choice = 'terminate' + choice = "terminate" - if choice == 'terminate': + if choice == "terminate": click.echo("\nTerminating all jobs at user request\n", err=True) self.stream.terminate() else: - if choice == 'quit': + if choice == "quit": click.echo("\nCompleting ongoing tasks before quitting\n", err=True) self.stream.quit() - elif choice == 'continue': + elif choice == "continue": click.echo("\nContinuing\n", err=True) def _tick(self): @@ -577,9 +608,11 @@ class App(): # the failure message reaches us ?? if not failure: self._status.clear() - click.echo("\n\n\nBUG: Message handling out of sync, " + - "unable to retrieve failure message for element {}\n\n\n\n\n" - .format(full_name), err=True) + click.echo( + "\n\n\nBUG: Message handling out of sync, " + + "unable to retrieve failure message for element {}\n\n\n\n\n".format(full_name), + err=True, + ) else: self._handle_failure(element, action_name, failure, full_name) @@ -604,69 +637,72 @@ class App(): # Interactive mode for element failures with self._interrupted(): - summary = ("\n{} failure on element: {}\n".format(failure.action_name, full_name) + - "\n" + - "Choose one of the following options:\n" + - " (c)ontinue - Continue queueing jobs as much as possible\n" + - " (q)uit - Exit after all ongoing jobs complete\n" + - " (t)erminate - Terminate any ongoing jobs and exit\n" + - " (r)etry - Retry this job\n") + summary = ( + "\n{} failure on element: {}\n".format(failure.action_name, full_name) + + "\n" + + "Choose one of the following options:\n" + + " (c)ontinue - Continue queueing jobs as much as possible\n" + + " (q)uit - Exit after all ongoing jobs complete\n" + + " (t)erminate - Terminate any ongoing jobs and exit\n" + + " (r)etry - Retry this job\n" + ) if failure.logfile: summary += " (l)og - View the full log file\n" if failure.sandbox: summary += " (s)hell - Drop into a shell in the failed build sandbox\n" summary += "\nPressing ^C will terminate jobs and exit\n" - choices = ['continue', 'quit', 'terminate', 'retry'] + choices = ["continue", "quit", "terminate", "retry"] if failure.logfile: - choices += ['log'] + choices += ["log"] if failure.sandbox: - choices += ['shell'] + choices += ["shell"] - choice = '' - while choice not in ['continue', 'quit', 'terminate', 'retry']: + choice = "" + while choice not in ["continue", "quit", "terminate", "retry"]: click.echo(summary, err=True) - self._notify("BuildStream failure", "{} on element {}" - .format(failure.action_name, full_name)) + self._notify("BuildStream failure", "{} on element {}".format(failure.action_name, full_name)) try: - choice = click.prompt("Choice:", default='continue', err=True, - value_proc=_prefix_choice_value_proc(choices)) + choice = click.prompt( + "Choice:", default="continue", err=True, value_proc=_prefix_choice_value_proc(choices) + ) except (click.Abort, SystemError): # In some cases, the readline buffer underlying the prompt gets corrupted on the second CTRL+C # This throws a SystemError, which doesn't seem to be problematic for the rest of the program # Ensure a newline after automatically printed '^C' click.echo("", err=True) - choice = 'terminate' + choice = "terminate" # Handle choices which you can come back from # - if choice == 'shell': + if choice == "shell": click.echo("\nDropping into an interactive shell in the failed build sandbox\n", err=True) try: unique_id, element_key = element prompt = self.shell_prompt(full_name, element_key) - self.stream.shell(None, Scope.BUILD, prompt, isolate=True, - usebuildtree='always', unique_id=unique_id) + self.stream.shell( + None, Scope.BUILD, prompt, isolate=True, usebuildtree="always", unique_id=unique_id + ) except BstError as e: click.echo("Error while attempting to create interactive shell: {}".format(e), err=True) - elif choice == 'log': - with open(failure.logfile, 'r') as logfile: + elif choice == "log": + with open(failure.logfile, "r") as logfile: content = logfile.read() click.echo_via_pager(content) - if choice == 'terminate': + if choice == "terminate": click.echo("\nTerminating all jobs\n", err=True) self.stream.terminate() else: - if choice == 'quit': + if choice == "quit": click.echo("\nCompleting ongoing tasks before quitting\n", err=True) self.stream.quit() - elif choice == 'continue': + elif choice == "continue": click.echo("\nContinuing with other non failing elements\n", err=True) - elif choice == 'retry': + elif choice == "retry": click.echo("\nRetrying failed job\n", err=True) unique_id = element[0] self.stream._failure_retry(action_name, unique_id) @@ -678,17 +714,14 @@ class App(): def session_start_cb(self): self._started = True if self._session_name: - self.logger.print_heading(self.project, - self.stream, - log_file=self._main_options['log_file']) + self.logger.print_heading(self.project, self.stream, log_file=self._main_options["log_file"]) # # Print a summary of the queues # def _print_summary(self): click.echo("", err=True) - self.logger.print_summary(self.stream, - self._main_options['log_file']) + self.logger.print_summary(self.stream, self._main_options["log_file"]) # _error_exit() # @@ -720,7 +753,7 @@ class App(): click.echo(main_error, err=True) if error.detail: indent = " " * INDENT - detail = '\n' + indent + indent.join(error.detail.splitlines(True)) + detail = "\n" + indent + indent.join(error.detail.splitlines(True)) click.echo(detail, err=True) sys.exit(-1) @@ -753,8 +786,8 @@ class App(): self._maybe_render_status() # Additionally log to a file - if self._main_options['log_file']: - click.echo(text, file=self._main_options['log_file'], color=False, nl=False) + if self._main_options["log_file"]: + click.echo(text, file=self._main_options["log_file"], color=False, nl=False) @contextmanager def _interrupted(self): @@ -768,25 +801,26 @@ class App(): # Some validation routines for project initialization # def _assert_format_version(self, format_version): - message = "The version must be supported by this " + \ - "version of buildstream (0 - {})\n".format(BST_FORMAT_VERSION) + message = "The version must be supported by this " + "version of buildstream (0 - {})\n".format( + BST_FORMAT_VERSION + ) # Validate that it is an integer try: number = int(format_version) except ValueError as e: - raise AppError(message, reason='invalid-format-version') from e + raise AppError(message, reason="invalid-format-version") from e # Validate that the specified version is supported if number < 0 or number > BST_FORMAT_VERSION: - raise AppError(message, reason='invalid-format-version') + raise AppError(message, reason="invalid-format-version") def _assert_element_path(self, element_path): message = "The element path cannot be an absolute path or contain any '..' components\n" # Validate the path is not absolute if os.path.isabs(element_path): - raise AppError(message, reason='invalid-element-path') + raise AppError(message, reason="invalid-element-path") # Validate that the path does not contain any '..' components path = element_path @@ -794,8 +828,8 @@ class App(): split = os.path.split(path) path = split[0] basename = split[1] - if basename == '..': - raise AppError(message, reason='invalid-element-path') + if basename == "..": + raise AppError(message, reason="invalid-element-path") # _init_project_interactive() # @@ -811,11 +845,10 @@ class App(): # format_version (int): The user selected format version # element_path (str): The user selected element path # - def _init_project_interactive(self, project_name, format_version=BST_FORMAT_VERSION, element_path='elements'): - + def _init_project_interactive(self, project_name, format_version=BST_FORMAT_VERSION, element_path="elements"): def project_name_proc(user_input): try: - node._assert_symbol_name(user_input, 'project name') + node._assert_symbol_name(user_input, "project name") except LoadError as e: message = "{}\n\n{}\n".format(e, e.detail) raise UsageError(message) from e @@ -835,63 +868,101 @@ class App(): raise UsageError(str(e)) from e return user_input - w = TextWrapper(initial_indent=' ', subsequent_indent=' ', width=79) + w = TextWrapper(initial_indent=" ", subsequent_indent=" ", width=79) # Collect project name click.echo("", err=True) click.echo(self._content_profile.fmt("Choose a unique name for your project"), err=True) click.echo(self._format_profile.fmt("-------------------------------------"), err=True) click.echo("", err=True) - click.echo(self._detail_profile.fmt( - w.fill("The project name is a unique symbol for your project and will be used " - "to distinguish your project from others in user preferences, namspaceing " - "of your project's artifacts in shared artifact caches, and in any case where " - "BuildStream needs to distinguish between multiple projects.")), err=True) + click.echo( + self._detail_profile.fmt( + w.fill( + "The project name is a unique symbol for your project and will be used " + "to distinguish your project from others in user preferences, namspaceing " + "of your project's artifacts in shared artifact caches, and in any case where " + "BuildStream needs to distinguish between multiple projects." + ) + ), + err=True, + ) click.echo("", err=True) - click.echo(self._detail_profile.fmt( - w.fill("The project name must contain only alphanumeric characters, " - "may not start with a digit, and may contain dashes or underscores.")), err=True) + click.echo( + self._detail_profile.fmt( + w.fill( + "The project name must contain only alphanumeric characters, " + "may not start with a digit, and may contain dashes or underscores." + ) + ), + err=True, + ) click.echo("", err=True) - project_name = click.prompt(self._content_profile.fmt("Project name"), - value_proc=project_name_proc, err=True) + project_name = click.prompt(self._content_profile.fmt("Project name"), value_proc=project_name_proc, err=True) click.echo("", err=True) # Collect format version click.echo(self._content_profile.fmt("Select the minimum required format version for your project"), err=True) click.echo(self._format_profile.fmt("-----------------------------------------------------------"), err=True) click.echo("", err=True) - click.echo(self._detail_profile.fmt( - w.fill("The format version is used to provide users who build your project " - "with a helpful error message in the case that they do not have a recent " - "enough version of BuildStream supporting all the features which your " - "project might use.")), err=True) + click.echo( + self._detail_profile.fmt( + w.fill( + "The format version is used to provide users who build your project " + "with a helpful error message in the case that they do not have a recent " + "enough version of BuildStream supporting all the features which your " + "project might use." + ) + ), + err=True, + ) click.echo("", err=True) - click.echo(self._detail_profile.fmt( - w.fill("The lowest version allowed is 0, the currently installed version of BuildStream " - "supports up to format version {}.".format(BST_FORMAT_VERSION))), err=True) + click.echo( + self._detail_profile.fmt( + w.fill( + "The lowest version allowed is 0, the currently installed version of BuildStream " + "supports up to format version {}.".format(BST_FORMAT_VERSION) + ) + ), + err=True, + ) click.echo("", err=True) - format_version = click.prompt(self._content_profile.fmt("Format version"), - value_proc=format_version_proc, - default=format_version, err=True) + format_version = click.prompt( + self._content_profile.fmt("Format version"), + value_proc=format_version_proc, + default=format_version, + err=True, + ) click.echo("", err=True) # Collect element path click.echo(self._content_profile.fmt("Select the element path"), err=True) click.echo(self._format_profile.fmt("-----------------------"), err=True) click.echo("", err=True) - click.echo(self._detail_profile.fmt( - w.fill("The element path is a project subdirectory where element .bst files are stored " - "within your project.")), err=True) + click.echo( + self._detail_profile.fmt( + w.fill( + "The element path is a project subdirectory where element .bst files are stored " + "within your project." + ) + ), + err=True, + ) click.echo("", err=True) - click.echo(self._detail_profile.fmt( - w.fill("Elements will be displayed in logs as filenames relative to " - "the element path, and similarly, dependencies must be expressed as filenames " - "relative to the element path.")), err=True) + click.echo( + self._detail_profile.fmt( + w.fill( + "Elements will be displayed in logs as filenames relative to " + "the element path, and similarly, dependencies must be expressed as filenames " + "relative to the element path." + ) + ), + err=True, + ) click.echo("", err=True) - element_path = click.prompt(self._content_profile.fmt("Element path"), - value_proc=element_path_proc, - default=element_path, err=True) + element_path = click.prompt( + self._content_profile.fmt("Element path"), value_proc=element_path_proc, default=element_path, err=True + ) return (project_name, format_version, element_path) @@ -909,7 +980,6 @@ class App(): # ask for a new input. # def _prefix_choice_value_proc(choices): - def value_proc(user_input): remaining_candidate = [choice for choice in choices if choice.startswith(user_input)] diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index 5c0293589..935a492d9 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -17,8 +17,8 @@ from ..utils import _get_compression, UtilError # Helper classes and methods for Click # ################################################################## -class FastEnumType(click.Choice): +class FastEnumType(click.Choice): def __init__(self, enum): self._enum = enum super().__init__(enum.values()) @@ -45,7 +45,7 @@ class FastEnumType(click.Choice): # def search_command(args, *, context=None): if context is None: - context = cli.make_context('bst', args, resilient_parsing=True) + context = cli.make_context("bst", args, resilient_parsing=True) # Loop into the deepest command command = cli @@ -54,9 +54,7 @@ def search_command(args, *, context=None): command = command_ctx.command.get_command(command_ctx, cmd) if command is None: return None - command_ctx = command.make_context(command.name, [command.name], - parent=command_ctx, - resilient_parsing=True) + command_ctx = command.make_context(command.name, [command.name], parent=command_ctx, resilient_parsing=True) return command_ctx @@ -65,8 +63,11 @@ def search_command(args, *, context=None): def complete_commands(cmd, args, incomplete): command_ctx = search_command(args[1:]) if command_ctx and command_ctx.command and isinstance(command_ctx.command, click.MultiCommand): - return [subcommand + " " for subcommand in command_ctx.command.list_commands(command_ctx) - if not command_ctx.command.get_command(command_ctx, subcommand).hidden] + return [ + subcommand + " " + for subcommand in command_ctx.command.list_commands(command_ctx) + if not command_ctx.command.get_command(command_ctx, subcommand).hidden + ] return [] @@ -80,18 +81,19 @@ def complete_target(args, incomplete): """ from .. import utils - project_conf = 'project.conf' + + project_conf = "project.conf" # First resolve the directory, in case there is an # active --directory/-C option # - base_directory = '.' + base_directory = "." idx = -1 try: - idx = args.index('-C') + idx = args.index("-C") except ValueError: try: - idx = args.index('--directory') + idx = args.index("--directory") except ValueError: pass @@ -116,7 +118,7 @@ def complete_target(args, incomplete): return [] # The project is not required to have an element-path - element_directory = project.get_str('element-path', default='') + element_directory = project.get_str("element-path", default="") # If a project was loaded, use its element-path to # adjust our completion's base directory @@ -132,19 +134,20 @@ def complete_target(args, incomplete): def complete_artifact(orig_args, args, incomplete): from .._context import Context + with Context(use_casd=False) as ctx: config = None if orig_args: for i, arg in enumerate(orig_args): - if arg in ('-c', '--config'): + if arg in ("-c", "--config"): try: config = orig_args[i + 1] except IndexError: pass if args: for i, arg in enumerate(args): - if arg in ('-c', '--config'): + if arg in ("-c", "--config"): try: config = args[i + 1] except IndexError: @@ -167,38 +170,40 @@ def override_completions(orig_args, cmd, cmd_param, args, incomplete): :return: all the possible user-specified completions for the param """ - if cmd.name == 'help': + if cmd.name == "help": return complete_commands(cmd, args, incomplete) # We can't easily extend click's data structures without # modifying click itself, so just do some weak special casing # right here and select which parameters we want to handle specially. if isinstance(cmd_param.type, click.Path): - if (cmd_param.name == 'elements' or - cmd_param.name == 'element' or - cmd_param.name == 'except_' or - cmd_param.opts == ['--track'] or - cmd_param.opts == ['--track-except']): + if ( + cmd_param.name == "elements" + or cmd_param.name == "element" + or cmd_param.name == "except_" + or cmd_param.opts == ["--track"] + or cmd_param.opts == ["--track-except"] + ): return complete_target(args, incomplete) - if cmd_param.name == 'artifacts' or cmd_param.name == 'target': + if cmd_param.name == "artifacts" or cmd_param.name == "target": return complete_artifact(orig_args, args, incomplete) raise CompleteUnhandled() def validate_output_streams(): - if sys.platform == 'win32': + if sys.platform == "win32": # Windows does not support 'fcntl', the module is unavailable there as # of Python 3.7, therefore early-out here. return import fcntl + for stream in (sys.stdout, sys.stderr): fileno = stream.fileno() flags = fcntl.fcntl(fileno, fcntl.F_GETFL) if flags & os.O_NONBLOCK: - click.echo("{} is currently set to O_NONBLOCK, try opening a new shell" - .format(stream.name), err=True) + click.echo("{} is currently set to O_NONBLOCK, try opening a new shell".format(stream.name), err=True) sys.exit(-1) @@ -237,8 +242,7 @@ def handle_bst_force_start_method_env(): sys.exit(-1) -def override_main(self, args=None, prog_name=None, complete_var=None, - standalone_mode=True, **extra): +def override_main(self, args=None, prog_name=None, complete_var=None, standalone_mode=True, **extra): # Hook for the Bash completion. This only activates if the Bash # completion is actually enabled, otherwise this is quite a fast @@ -250,7 +254,7 @@ def override_main(self, args=None, prog_name=None, complete_var=None, # # The below is a quicker exit path for the sake # of making completions respond faster. - if 'BST_TEST_SUITE' not in os.environ: + if "BST_TEST_SUITE" not in os.environ: sys.stdout.flush() sys.stderr.flush() os._exit(0) @@ -269,14 +273,13 @@ def override_main(self, args=None, prog_name=None, complete_var=None, # case of testing, our tests preceed our entrypoint, so we do our best. handle_bst_force_start_method_env() - original_main(self, args=args, prog_name=prog_name, complete_var=None, - standalone_mode=standalone_mode, **extra) + original_main(self, args=args, prog_name=prog_name, complete_var=None, standalone_mode=standalone_mode, **extra) original_main = click.BaseCommand.main # Disable type checking since mypy doesn't support assigning to a method. # See https://github.com/python/mypy/issues/2427. -click.BaseCommand.main = override_main # type: ignore +click.BaseCommand.main = override_main # type: ignore ################################################################## @@ -287,58 +290,78 @@ def print_version(ctx, param, value): return from .. import __version__ + click.echo(__version__) ctx.exit() -@click.group(context_settings=dict(help_option_names=['-h', '--help'])) -@click.option('--version', is_flag=True, callback=print_version, - expose_value=False, is_eager=True) -@click.option('--config', '-c', - type=click.Path(exists=True, dir_okay=False, readable=True), - help="Configuration file to use") -@click.option('--directory', '-C', default=None, # Set to os.getcwd() later. - type=click.Path(file_okay=False, readable=True), - help="Project directory (default: current directory)") -@click.option('--on-error', default=None, - type=FastEnumType(_SchedulerErrorAction), - help="What to do when an error is encountered") -@click.option('--fetchers', type=click.INT, default=None, - help="Maximum simultaneous download tasks") -@click.option('--builders', type=click.INT, default=None, - help="Maximum simultaneous build tasks") -@click.option('--pushers', type=click.INT, default=None, - help="Maximum simultaneous upload tasks") -@click.option('--max-jobs', type=click.INT, default=None, - help="Number of parallel jobs allowed for a given build task") -@click.option('--network-retries', type=click.INT, default=None, - help="Maximum retries for network tasks") -@click.option('--no-interactive', is_flag=True, - help="Force non interactive mode, otherwise this is automatically decided") -@click.option('--verbose/--no-verbose', default=None, - help="Be extra verbose") -@click.option('--debug/--no-debug', default=None, - help="Print debugging output") -@click.option('--error-lines', type=click.INT, default=None, - help="Maximum number of lines to show from a task log") -@click.option('--message-lines', type=click.INT, default=None, - help="Maximum number of lines to show in a detailed message") -@click.option('--log-file', - type=click.File(mode='w', encoding='UTF-8'), - help="A file to store the main log (allows storing the main log while in interactive mode)") -@click.option('--colors/--no-colors', default=None, - help="Force enable/disable ANSI color codes in output") -@click.option('--strict/--no-strict', default=None, is_flag=True, - help="Elements must be rebuilt when their dependencies have changed") -@click.option('--option', '-o', type=click.Tuple([str, str]), multiple=True, metavar='OPTION VALUE', - help="Specify a project option") -@click.option('--default-mirror', default=None, - help="The mirror to fetch from first, before attempting other mirrors") -@click.option('--pull-buildtrees', is_flag=True, default=None, - help="Include an element's build tree when pulling remote element artifacts") -@click.option('--cache-buildtrees', default=None, - type=FastEnumType(_CacheBuildTrees), - help="Cache artifact build tree content on creation") +@click.group(context_settings=dict(help_option_names=["-h", "--help"])) +@click.option("--version", is_flag=True, callback=print_version, expose_value=False, is_eager=True) +@click.option( + "--config", "-c", type=click.Path(exists=True, dir_okay=False, readable=True), help="Configuration file to use" +) +@click.option( + "--directory", + "-C", + default=None, # Set to os.getcwd() later. + type=click.Path(file_okay=False, readable=True), + help="Project directory (default: current directory)", +) +@click.option( + "--on-error", + default=None, + type=FastEnumType(_SchedulerErrorAction), + help="What to do when an error is encountered", +) +@click.option("--fetchers", type=click.INT, default=None, help="Maximum simultaneous download tasks") +@click.option("--builders", type=click.INT, default=None, help="Maximum simultaneous build tasks") +@click.option("--pushers", type=click.INT, default=None, help="Maximum simultaneous upload tasks") +@click.option( + "--max-jobs", type=click.INT, default=None, help="Number of parallel jobs allowed for a given build task" +) +@click.option("--network-retries", type=click.INT, default=None, help="Maximum retries for network tasks") +@click.option( + "--no-interactive", is_flag=True, help="Force non interactive mode, otherwise this is automatically decided" +) +@click.option("--verbose/--no-verbose", default=None, help="Be extra verbose") +@click.option("--debug/--no-debug", default=None, help="Print debugging output") +@click.option("--error-lines", type=click.INT, default=None, help="Maximum number of lines to show from a task log") +@click.option( + "--message-lines", type=click.INT, default=None, help="Maximum number of lines to show in a detailed message" +) +@click.option( + "--log-file", + type=click.File(mode="w", encoding="UTF-8"), + help="A file to store the main log (allows storing the main log while in interactive mode)", +) +@click.option("--colors/--no-colors", default=None, help="Force enable/disable ANSI color codes in output") +@click.option( + "--strict/--no-strict", + default=None, + is_flag=True, + help="Elements must be rebuilt when their dependencies have changed", +) +@click.option( + "--option", + "-o", + type=click.Tuple([str, str]), + multiple=True, + metavar="OPTION VALUE", + help="Specify a project option", +) +@click.option("--default-mirror", default=None, help="The mirror to fetch from first, before attempting other mirrors") +@click.option( + "--pull-buildtrees", + is_flag=True, + default=None, + help="Include an element's build tree when pulling remote element artifacts", +) +@click.option( + "--cache-buildtrees", + default=None, + type=FastEnumType(_CacheBuildTrees), + help="Cache artifact build tree content on creation", +) @click.pass_context def cli(context, **kwargs): """Build and manipulate BuildStream projects @@ -360,17 +383,15 @@ def cli(context, **kwargs): ################################################################## # Help Command # ################################################################## -@cli.command(name="help", short_help="Print usage information", - context_settings={"help_option_names": []}) -@click.argument("command", nargs=-1, metavar='COMMAND') +@cli.command(name="help", short_help="Print usage information", context_settings={"help_option_names": []}) +@click.argument("command", nargs=-1, metavar="COMMAND") @click.pass_context def help_command(ctx, command): """Print usage information about a given command """ command_ctx = search_command(command, context=ctx.parent) if not command_ctx: - click.echo("Not a valid command: '{} {}'" - .format(ctx.parent.info_name, " ".join(command)), err=True) + click.echo("Not a valid command: '{} {}'".format(ctx.parent.info_name, " ".join(command)), err=True) sys.exit(-1) click.echo(command_ctx.command.get_help(command_ctx), err=True) @@ -380,24 +401,32 @@ def help_command(ctx, command): detail = " " if command: detail = " {} ".format(" ".join(command)) - click.echo("\nFor usage on a specific command: {} help{}COMMAND" - .format(ctx.parent.info_name, detail), err=True) + click.echo( + "\nFor usage on a specific command: {} help{}COMMAND".format(ctx.parent.info_name, detail), err=True + ) ################################################################## # Init Command # ################################################################## @cli.command(short_help="Initialize a new BuildStream project") -@click.option('--project-name', type=click.STRING, - help="The project name to use") -@click.option('--format-version', type=click.INT, default=BST_FORMAT_VERSION, show_default=True, - help="The required format version") -@click.option('--element-path', type=click.Path(), default="elements", show_default=True, - help="The subdirectory to store elements in") -@click.option('--force', '-f', is_flag=True, - help="Allow overwriting an existing project.conf") -@click.argument('target-directory', nargs=1, required=False, - type=click.Path(file_okay=False, writable=True)) +@click.option("--project-name", type=click.STRING, help="The project name to use") +@click.option( + "--format-version", + type=click.INT, + default=BST_FORMAT_VERSION, + show_default=True, + help="The required format version", +) +@click.option( + "--element-path", + type=click.Path(), + default="elements", + show_default=True, + help="The subdirectory to store elements in", +) +@click.option("--force", "-f", is_flag=True, help="Allow overwriting an existing project.conf") +@click.argument("target-directory", nargs=1, required=False, type=click.Path(file_okay=False, writable=True)) @click.pass_obj def init(app, project_name, format_version, element_path, force, target_directory): """Initialize a new BuildStream project @@ -415,13 +444,11 @@ def init(app, project_name, format_version, element_path, force, target_director # Build Command # ################################################################## @cli.command(short_help="Build elements in a pipeline") -@click.option('--deps', '-d', default=None, - type=click.Choice(['plan', 'all']), - help='The dependencies to build') -@click.option('--remote', '-r', default=None, - help="The URL of the remote cache (defaults to the first configured cache)") -@click.argument('elements', nargs=-1, - type=click.Path(readable=False)) +@click.option("--deps", "-d", default=None, type=click.Choice(["plan", "all"]), help="The dependencies to build") +@click.option( + "--remote", "-r", default=None, help="The URL of the remote cache (defaults to the first configured cache)" +) +@click.argument("elements", nargs=-1, type=click.Path(readable=False)) @click.pass_obj def build(app, elements, deps, remote): """Build elements in a pipeline @@ -450,30 +477,41 @@ def build(app, elements, deps, remote): # Junction elements cannot be built, exclude them from default targets ignore_junction_targets = True - app.stream.build(elements, - selection=deps, - ignore_junction_targets=ignore_junction_targets, - remote=remote) + app.stream.build(elements, selection=deps, ignore_junction_targets=ignore_junction_targets, remote=remote) ################################################################## # Show Command # ################################################################## @cli.command(short_help="Show elements in the pipeline") -@click.option('--except', 'except_', multiple=True, - type=click.Path(readable=False), - help="Except certain dependencies") -@click.option('--deps', '-d', default='all', show_default=True, - type=click.Choice(['none', 'plan', 'run', 'build', 'all']), - help='The dependencies to show') -@click.option('--order', default="stage", show_default=True, - type=click.Choice(['stage', 'alpha']), - help='Staging or alphabetic ordering of dependencies') -@click.option('--format', '-f', 'format_', metavar='FORMAT', default=None, - type=click.STRING, - help='Format string for each element') -@click.argument('elements', nargs=-1, - type=click.Path(readable=False)) +@click.option( + "--except", "except_", multiple=True, type=click.Path(readable=False), help="Except certain dependencies" +) +@click.option( + "--deps", + "-d", + default="all", + show_default=True, + type=click.Choice(["none", "plan", "run", "build", "all"]), + help="The dependencies to show", +) +@click.option( + "--order", + default="stage", + show_default=True, + type=click.Choice(["stage", "alpha"]), + help="Staging or alphabetic ordering of dependencies", +) +@click.option( + "--format", + "-f", + "format_", + metavar="FORMAT", + default=None, + type=click.STRING, + help="Format string for each element", +) +@click.argument("elements", nargs=-1, type=click.Path(readable=False)) @click.pass_obj def show(app, elements, deps, except_, order, format_): """Show elements in the pipeline @@ -536,9 +574,7 @@ def show(app, elements, deps, except_, order, format_): if not elements: elements = app.project.get_default_targets() - dependencies = app.stream.load_selection(elements, - selection=deps, - except_targets=except_) + dependencies = app.stream.load_selection(elements, selection=deps, except_targets=except_) if order == "alpha": dependencies = sorted(dependencies) @@ -554,25 +590,34 @@ def show(app, elements, deps, except_, order, format_): # Shell Command # ################################################################## @cli.command(short_help="Shell into an element's sandbox environment") -@click.option('--build', '-b', 'build_', is_flag=True, - help='Stage dependencies and sources to build') -@click.option('--sysroot', '-s', default=None, - type=click.Path(exists=True, file_okay=False, readable=True), - help="An existing sysroot") -@click.option('--mount', type=click.Tuple([click.Path(exists=True), str]), multiple=True, - metavar='HOSTPATH PATH', - help="Mount a file or directory into the sandbox") -@click.option('--isolate', is_flag=True, - help='Create an isolated build sandbox') -@click.option('--use-buildtree', '-t', 'cli_buildtree', type=click.Choice(['ask', 'try', 'always', 'never']), - default='ask', show_default=True, - help=('Use a buildtree. If `always` is set, will always fail to ' - 'build if a buildtree is not available.')) -@click.option('--pull', 'pull_', is_flag=True, - help='Attempt to pull missing or incomplete artifacts') -@click.argument('element', required=False, - type=click.Path(readable=False)) -@click.argument('command', type=click.STRING, nargs=-1) +@click.option("--build", "-b", "build_", is_flag=True, help="Stage dependencies and sources to build") +@click.option( + "--sysroot", + "-s", + default=None, + type=click.Path(exists=True, file_okay=False, readable=True), + help="An existing sysroot", +) +@click.option( + "--mount", + type=click.Tuple([click.Path(exists=True), str]), + multiple=True, + metavar="HOSTPATH PATH", + help="Mount a file or directory into the sandbox", +) +@click.option("--isolate", is_flag=True, help="Create an isolated build sandbox") +@click.option( + "--use-buildtree", + "-t", + "cli_buildtree", + type=click.Choice(["ask", "try", "always", "never"]), + default="ask", + show_default=True, + help=("Use a buildtree. If `always` is set, will always fail to " "build if a buildtree is not available."), +) +@click.option("--pull", "pull_", is_flag=True, help="Attempt to pull missing or incomplete artifacts") +@click.argument("element", required=False, type=click.Path(readable=False)) +@click.argument("command", type=click.STRING, nargs=-1) @click.pass_obj def shell(app, element, sysroot, mount, isolate, build_, cli_buildtree, pull_, command): """Run a command in the target element's sandbox environment @@ -616,8 +661,7 @@ def shell(app, element, sysroot, mount, isolate, build_, cli_buildtree, pull_, c if not element: raise AppError('Missing argument "ELEMENT".') - elements = app.stream.load_selection((element,), selection=selection, - use_artifact_config=True) + elements = app.stream.load_selection((element,), selection=selection, use_artifact_config=True) # last one will be the element we want to stage, previous ones are # elements to try and pull @@ -628,10 +672,7 @@ def shell(app, element, sysroot, mount, isolate, build_, cli_buildtree, pull_, c element_key = element._get_display_key() prompt = app.shell_prompt(element_name, element_key) - mounts = [ - HostMount(path, host_path) - for host_path, path in mount - ] + mounts = [HostMount(path, host_path) for host_path, path in mount] cached = element._cached_buildtree() buildtree_exists = element._buildtree_exists() @@ -640,27 +681,31 @@ def shell(app, element, sysroot, mount, isolate, build_, cli_buildtree, pull_, c if buildtree_exists or pull_: use_buildtree = cli_buildtree if not cached and use_buildtree == "always": - click.echo("WARNING: buildtree is not cached locally, will attempt to pull from available remotes", - err=True) + click.echo( + "WARNING: buildtree is not cached locally, will attempt to pull from available remotes", + err=True, + ) else: if cli_buildtree == "always": # Exit early if it won't be possible to even fetch a buildtree with always option raise AppError("Artifact was created without buildtree, unable to launch shell with it") - click.echo("WARNING: Artifact created without buildtree, shell will be loaded without it", - err=True) + click.echo("WARNING: Artifact created without buildtree, shell will be loaded without it", err=True) else: # If the value has defaulted to ask and in non interactive mode, don't consider the buildtree, this # being the default behaviour of the command if app.interactive and cli_buildtree == "ask": - if cached and bool(click.confirm('Do you want to use the cached buildtree?')): + if cached and bool(click.confirm("Do you want to use the cached buildtree?")): use_buildtree = "always" elif buildtree_exists: try: - choice = click.prompt("Do you want to pull & use a cached buildtree?", - type=click.Choice(['try', 'always', 'never']), - err=True, show_choices=True) + choice = click.prompt( + "Do you want to pull & use a cached buildtree?", + type=click.Choice(["try", "always", "never"]), + err=True, + show_choices=True, + ) except click.Abort: - click.echo('Aborting', err=True) + click.echo("Aborting", err=True) sys.exit(-1) if choice != "never": @@ -671,13 +716,17 @@ def shell(app, element, sysroot, mount, isolate, build_, cli_buildtree, pull_, c click.echo("WARNING: using a buildtree from a failed build.", err=True) try: - exitcode = app.stream.shell(element, scope, prompt, - directory=sysroot, - mounts=mounts, - isolate=isolate, - command=command, - usebuildtree=use_buildtree, - pull_dependencies=pull_dependencies) + exitcode = app.stream.shell( + element, + scope, + prompt, + directory=sysroot, + mounts=mounts, + isolate=isolate, + command=command, + usebuildtree=use_buildtree, + pull_dependencies=pull_dependencies, + ) except BstError as e: raise AppError("Error launching shell: {}".format(e), detail=e.detail) from e @@ -697,20 +746,27 @@ def source(): # Source Fetch Command # ################################################################## @source.command(name="fetch", short_help="Fetch sources in a pipeline") -@click.option('--except', 'except_', multiple=True, - type=click.Path(readable=False), - help="Except certain dependencies from fetching") -@click.option('--deps', '-d', default='plan', show_default=True, - type=click.Choice(['none', 'plan', 'all']), - help='The dependencies to fetch') -@click.option('--track', 'track_', is_flag=True, - help="Track new source references before fetching") -@click.option('--track-cross-junctions', '-J', is_flag=True, - help="Allow tracking to cross junction boundaries") -@click.option('--remote', '-r', default=None, - help="The URL of the remote source cache (defaults to the first configured cache)") -@click.argument('elements', nargs=-1, - type=click.Path(readable=False)) +@click.option( + "--except", + "except_", + multiple=True, + type=click.Path(readable=False), + help="Except certain dependencies from fetching", +) +@click.option( + "--deps", + "-d", + default="plan", + show_default=True, + type=click.Choice(["none", "plan", "all"]), + help="The dependencies to fetch", +) +@click.option("--track", "track_", is_flag=True, help="Track new source references before fetching") +@click.option("--track-cross-junctions", "-J", is_flag=True, help="Allow tracking to cross junction boundaries") +@click.option( + "--remote", "-r", default=None, help="The URL of the remote source cache (defaults to the first configured cache)" +) +@click.argument("elements", nargs=-1, type=click.Path(readable=False)) @click.pass_obj def source_fetch(app, elements, deps, track_, except_, track_cross_junctions, remote): """Fetch sources required to build the pipeline @@ -741,36 +797,48 @@ def source_fetch(app, elements, deps, track_, except_, track_cross_junctions, re sys.exit(-1) if track_ and deps == PipelineSelection.PLAN: - click.echo("WARNING: --track specified for tracking of a build plan\n\n" - "Since tracking modifies the build plan, all elements will be tracked.", err=True) + click.echo( + "WARNING: --track specified for tracking of a build plan\n\n" + "Since tracking modifies the build plan, all elements will be tracked.", + err=True, + ) deps = PipelineSelection.ALL with app.initialized(session_name="Fetch"): if not elements: elements = app.project.get_default_targets() - app.stream.fetch(elements, - selection=deps, - except_targets=except_, - track_targets=track_, - track_cross_junctions=track_cross_junctions, - remote=remote) + app.stream.fetch( + elements, + selection=deps, + except_targets=except_, + track_targets=track_, + track_cross_junctions=track_cross_junctions, + remote=remote, + ) ################################################################## # Source Track Command # ################################################################## @source.command(name="track", short_help="Track new source references") -@click.option('--except', 'except_', multiple=True, - type=click.Path(readable=False), - help="Except certain dependencies from tracking") -@click.option('--deps', '-d', default='none', show_default=True, - type=click.Choice(['none', 'all']), - help='The dependencies to track') -@click.option('--cross-junctions', '-J', is_flag=True, - help="Allow crossing junction boundaries") -@click.argument('elements', nargs=-1, - type=click.Path(readable=False)) +@click.option( + "--except", + "except_", + multiple=True, + type=click.Path(readable=False), + help="Except certain dependencies from tracking", +) +@click.option( + "--deps", + "-d", + default="none", + show_default=True, + type=click.Choice(["none", "all"]), + help="The dependencies to track", +) +@click.option("--cross-junctions", "-J", is_flag=True, help="Allow crossing junction boundaries") +@click.argument("elements", nargs=-1, type=click.Path(readable=False)) @click.pass_obj def source_track(app, elements, deps, except_, cross_junctions): """Consults the specified tracking branches for new versions available @@ -800,41 +868,50 @@ def source_track(app, elements, deps, except_, cross_junctions): # Substitute 'none' for 'redirect' so that element redirections # will be done - if deps == 'none': - deps = 'redirect' - app.stream.track(elements, - selection=deps, - except_targets=except_, - cross_junctions=cross_junctions) + if deps == "none": + deps = "redirect" + app.stream.track(elements, selection=deps, except_targets=except_, cross_junctions=cross_junctions) ################################################################## # Source Checkout Command # ################################################################## -@source.command(name='checkout', short_help='Checkout sources of an element') -@click.option('--force', '-f', is_flag=True, - help="Allow files to be overwritten") -@click.option('--except', 'except_', multiple=True, - type=click.Path(readable=False), - help="Except certain dependencies") -@click.option('--deps', '-d', default='none', show_default=True, - type=click.Choice(['build', 'none', 'run', 'all']), - help='The dependencies whose sources to checkout') -@click.option('--tar', default=None, metavar='LOCATION', - type=click.Path(), - help="Create a tarball containing the sources instead " - "of a file tree.") -@click.option('--compression', default=None, - type=click.Choice(['gz', 'xz', 'bz2']), - help="The compression option of the tarball created.") -@click.option('--include-build-scripts', 'build_scripts', is_flag=True) -@click.option('--directory', default='source-checkout', - type=click.Path(file_okay=False), - help="The directory to checkout the sources to") -@click.argument('element', required=False, type=click.Path(readable=False)) +@source.command(name="checkout", short_help="Checkout sources of an element") +@click.option("--force", "-f", is_flag=True, help="Allow files to be overwritten") +@click.option( + "--except", "except_", multiple=True, type=click.Path(readable=False), help="Except certain dependencies" +) +@click.option( + "--deps", + "-d", + default="none", + show_default=True, + type=click.Choice(["build", "none", "run", "all"]), + help="The dependencies whose sources to checkout", +) +@click.option( + "--tar", + default=None, + metavar="LOCATION", + type=click.Path(), + help="Create a tarball containing the sources instead " "of a file tree.", +) +@click.option( + "--compression", + default=None, + type=click.Choice(["gz", "xz", "bz2"]), + help="The compression option of the tarball created.", +) +@click.option("--include-build-scripts", "build_scripts", is_flag=True) +@click.option( + "--directory", + default="source-checkout", + type=click.Path(file_okay=False), + help="The directory to checkout the sources to", +) +@click.argument("element", required=False, type=click.Path(readable=False)) @click.pass_obj -def source_checkout(app, element, directory, force, deps, except_, - tar, compression, build_scripts): +def source_checkout(app, element, directory, force, deps, except_, tar, compression, build_scripts): """Checkout sources of an element to the specified location When this command is executed from a workspace directory, the default @@ -859,14 +936,16 @@ def source_checkout(app, element, directory, force, deps, except_, if not element: raise AppError('Missing argument "ELEMENT".') - app.stream.source_checkout(element, - location=location, - force=force, - deps=deps, - except_targets=except_, - tar=bool(tar), - compression=compression, - include_build_scripts=build_scripts) + app.stream.source_checkout( + element, + location=location, + force=force, + deps=deps, + except_targets=except_, + tar=bool(tar), + compression=compression, + include_build_scripts=build_scripts, + ) ################################################################## @@ -880,39 +959,42 @@ def workspace(): ################################################################## # Workspace Open Command # ################################################################## -@workspace.command(name='open', short_help="Open a new workspace") -@click.option('--no-checkout', is_flag=True, - help="Do not checkout the source, only link to the given directory") -@click.option('--force', '-f', is_flag=True, - help="The workspace will be created even if the directory in which it will be created is not empty " + - "or if a workspace for that element already exists") -@click.option('--track', 'track_', is_flag=True, - help="Track and fetch new source references before checking out the workspace") -@click.option('--directory', type=click.Path(file_okay=False), default=None, - help="Only for use when a single Element is given: Set the directory to use to create the workspace") -@click.argument('elements', nargs=-1, type=click.Path(readable=False), required=True) +@workspace.command(name="open", short_help="Open a new workspace") +@click.option("--no-checkout", is_flag=True, help="Do not checkout the source, only link to the given directory") +@click.option( + "--force", + "-f", + is_flag=True, + help="The workspace will be created even if the directory in which it will be created is not empty " + + "or if a workspace for that element already exists", +) +@click.option( + "--track", "track_", is_flag=True, help="Track and fetch new source references before checking out the workspace" +) +@click.option( + "--directory", + type=click.Path(file_okay=False), + default=None, + help="Only for use when a single Element is given: Set the directory to use to create the workspace", +) +@click.argument("elements", nargs=-1, type=click.Path(readable=False), required=True) @click.pass_obj def workspace_open(app, no_checkout, force, track_, directory, elements): """Open a workspace for manual source modification""" with app.initialized(): - app.stream.workspace_open(elements, - no_checkout=no_checkout, - track_first=track_, - force=force, - custom_dir=directory) + app.stream.workspace_open( + elements, no_checkout=no_checkout, track_first=track_, force=force, custom_dir=directory + ) ################################################################## # Workspace Close Command # ################################################################## -@workspace.command(name='close', short_help="Close workspaces") -@click.option('--remove-dir', is_flag=True, - help="Remove the path that contains the closed workspace") -@click.option('--all', '-a', 'all_', is_flag=True, - help="Close all open workspaces") -@click.argument('elements', nargs=-1, - type=click.Path(readable=False)) +@workspace.command(name="close", short_help="Close workspaces") +@click.option("--remove-dir", is_flag=True, help="Remove the path that contains the closed workspace") +@click.option("--all", "-a", "all_", is_flag=True, help="Close all open workspaces") +@click.argument("elements", nargs=-1, type=click.Path(readable=False)) @click.pass_obj def workspace_close(app, remove_dir, all_, elements): """Close a workspace""" @@ -927,11 +1009,11 @@ def workspace_close(app, remove_dir, all_, elements): if element: elements = (element,) else: - raise AppError('No elements specified') + raise AppError("No elements specified") # Early exit if we specified `all` and there are no workspaces if all_ and not app.stream.workspace_exists(): - click.echo('No open workspaces to close', err=True) + click.echo("No open workspaces to close", err=True) sys.exit(0) if all_: @@ -958,21 +1040,19 @@ def workspace_close(app, remove_dir, all_, elements): if removed_required_element: click.echo( "Removed '{}', therefore you can no longer run BuildStream " - "commands from the current directory.".format(element_name), err=True) + "commands from the current directory.".format(element_name), + err=True, + ) ################################################################## # Workspace Reset Command # ################################################################## -@workspace.command(name='reset', short_help="Reset a workspace to its original state") -@click.option('--soft', is_flag=True, - help="Reset workspace state without affecting its contents") -@click.option('--track', 'track_', is_flag=True, - help="Track and fetch the latest source before resetting") -@click.option('--all', '-a', 'all_', is_flag=True, - help="Reset all open workspaces") -@click.argument('elements', nargs=-1, - type=click.Path(readable=False)) +@workspace.command(name="reset", short_help="Reset a workspace to its original state") +@click.option("--soft", is_flag=True, help="Reset workspace state without affecting its contents") +@click.option("--track", "track_", is_flag=True, help="Track and fetch the latest source before resetting") +@click.option("--all", "-a", "all_", is_flag=True, help="Reset all open workspaces") +@click.argument("elements", nargs=-1, type=click.Path(readable=False)) @click.pass_obj def workspace_reset(app, soft, track_, all_, elements): """Reset a workspace to its original state""" @@ -985,7 +1065,7 @@ def workspace_reset(app, soft, track_, all_, elements): if element: elements = (element,) else: - raise AppError('No elements specified to reset') + raise AppError("No elements specified to reset") if all_ and not app.stream.workspace_exists(): raise AppError("No open workspaces to reset") @@ -999,7 +1079,7 @@ def workspace_reset(app, soft, track_, all_, elements): ################################################################## # Workspace List Command # ################################################################## -@workspace.command(name='list', short_help="List open workspaces") +@workspace.command(name="list", short_help="List open workspaces") @click.pass_obj def workspace_list(app): """List open workspaces""" @@ -1044,11 +1124,16 @@ def artifact(): ############################################################# # Artifact show Command # ############################################################# -@artifact.command(name='show', short_help="Show the cached state of artifacts") -@click.option('--deps', '-d', default='none', show_default=True, - type=click.Choice(['build', 'run', 'all', 'none']), - help='The dependencies we also want to show') -@click.argument('artifacts', type=click.Path(), nargs=-1) +@artifact.command(name="show", short_help="Show the cached state of artifacts") +@click.option( + "--deps", + "-d", + default="none", + show_default=True, + type=click.Choice(["build", "run", "all", "none"]), + help="The dependencies we also want to show", +) +@click.argument("artifacts", type=click.Path(), nargs=-1) @click.pass_obj def artifact_show(app, deps, artifacts): """show the cached state of artifacts""" @@ -1061,31 +1146,38 @@ def artifact_show(app, deps, artifacts): ##################################################################### # Artifact Checkout Command # ##################################################################### -@artifact.command(name='checkout', short_help="Checkout contents of an artifact") -@click.option('--force', '-f', is_flag=True, - help="Allow files to be overwritten") -@click.option('--deps', '-d', default='run', show_default=True, - type=click.Choice(['run', 'build', 'none', 'all']), - help='The dependencies to checkout') -@click.option('--integrate/--no-integrate', default=None, is_flag=True, - help="Whether to run integration commands") -@click.option('--hardlinks', is_flag=True, - help="Checkout hardlinks instead of copying if possible") -@click.option('--tar', default=None, metavar='LOCATION', - type=click.Path(), - help="Create a tarball from the artifact contents instead " - "of a file tree. If LOCATION is '-', the tarball " - "will be dumped to the standard output.") -@click.option('--compression', default=None, - type=click.Choice(['gz', 'xz', 'bz2']), - help="The compression option of the tarball created.") -@click.option('--pull', 'pull_', is_flag=True, - help="Pull the artifact if it's missing or incomplete.") -@click.option('--directory', default=None, - type=click.Path(file_okay=False), - help="The directory to checkout the artifact to") -@click.argument('target', required=False, - type=click.Path(readable=False)) +@artifact.command(name="checkout", short_help="Checkout contents of an artifact") +@click.option("--force", "-f", is_flag=True, help="Allow files to be overwritten") +@click.option( + "--deps", + "-d", + default="run", + show_default=True, + type=click.Choice(["run", "build", "none", "all"]), + help="The dependencies to checkout", +) +@click.option("--integrate/--no-integrate", default=None, is_flag=True, help="Whether to run integration commands") +@click.option("--hardlinks", is_flag=True, help="Checkout hardlinks instead of copying if possible") +@click.option( + "--tar", + default=None, + metavar="LOCATION", + type=click.Path(), + help="Create a tarball from the artifact contents instead " + "of a file tree. If LOCATION is '-', the tarball " + "will be dumped to the standard output.", +) +@click.option( + "--compression", + default=None, + type=click.Choice(["gz", "xz", "bz2"]), + help="The compression option of the tarball created.", +) +@click.option("--pull", "pull_", is_flag=True, help="Pull the artifact if it's missing or incomplete.") +@click.option( + "--directory", default=None, type=click.Path(file_okay=False), help="The directory to checkout the artifact to" +) +@click.argument("target", required=False, type=click.Path(readable=False)) @click.pass_obj def artifact_checkout(app, force, deps, integrate, hardlinks, tar, compression, pull_, directory, target): """Checkout contents of an artifact @@ -1110,7 +1202,7 @@ def artifact_checkout(app, force, deps, integrate, hardlinks, tar, compression, location = os.path.abspath(os.path.join(os.getcwd(), target)) else: location = directory - if location[-4:] == '.bst': + if location[-4:] == ".bst": location = location[:-4] tar = False else: @@ -1120,9 +1212,12 @@ def artifact_checkout(app, force, deps, integrate, hardlinks, tar, compression, except UtilError as e: click.echo("ERROR: Invalid file extension given with '--tar': {}".format(e), err=True) sys.exit(-1) - if compression and inferred_compression != '' and inferred_compression != compression: - click.echo("WARNING: File extension and compression differ." - "File extension has been overridden by --compression", err=True) + if compression and inferred_compression != "" and inferred_compression != compression: + click.echo( + "WARNING: File extension and compression differ." + "File extension has been overridden by --compression", + err=True, + ) if not compression: compression = inferred_compression @@ -1132,28 +1227,35 @@ def artifact_checkout(app, force, deps, integrate, hardlinks, tar, compression, if not target: raise AppError('Missing argument "ELEMENT".') - app.stream.checkout(target, - location=location, - force=force, - selection=deps, - integrate=True if integrate is None else integrate, - hardlinks=hardlinks, - pull=pull_, - compression=compression, - tar=bool(tar)) + app.stream.checkout( + target, + location=location, + force=force, + selection=deps, + integrate=True if integrate is None else integrate, + hardlinks=hardlinks, + pull=pull_, + compression=compression, + tar=bool(tar), + ) ################################################################ # Artifact Pull Command # ################################################################ @artifact.command(name="pull", short_help="Pull a built artifact") -@click.option('--deps', '-d', default='none', show_default=True, - type=click.Choice(['none', 'all']), - help='The dependency artifacts to pull') -@click.option('--remote', '-r', default=None, - help="The URL of the remote cache (defaults to the first configured cache)") -@click.argument('artifacts', nargs=-1, - type=click.Path(readable=False)) +@click.option( + "--deps", + "-d", + default="none", + show_default=True, + type=click.Choice(["none", "all"]), + help="The dependency artifacts to pull", +) +@click.option( + "--remote", "-r", default=None, help="The URL of the remote cache (defaults to the first configured cache)" +) +@click.argument("artifacts", nargs=-1, type=click.Path(readable=False)) @click.pass_obj def artifact_pull(app, artifacts, deps, remote): """Pull a built artifact from the configured remote artifact cache. @@ -1184,21 +1286,25 @@ def artifact_pull(app, artifacts, deps, remote): # Junction elements cannot be pulled, exclude them from default targets ignore_junction_targets = True - app.stream.pull(artifacts, selection=deps, remote=remote, - ignore_junction_targets=ignore_junction_targets) + app.stream.pull(artifacts, selection=deps, remote=remote, ignore_junction_targets=ignore_junction_targets) ################################################################## # Artifact Push Command # ################################################################## @artifact.command(name="push", short_help="Push a built artifact") -@click.option('--deps', '-d', default='none', show_default=True, - type=click.Choice(['none', 'all']), - help='The dependencies to push') -@click.option('--remote', '-r', default=None, - help="The URL of the remote cache (defaults to the first configured cache)") -@click.argument('artifacts', nargs=-1, - type=click.Path(readable=False)) +@click.option( + "--deps", + "-d", + default="none", + show_default=True, + type=click.Choice(["none", "all"]), + help="The dependencies to push", +) +@click.option( + "--remote", "-r", default=None, help="The URL of the remote cache (defaults to the first configured cache)" +) +@click.argument("artifacts", nargs=-1, type=click.Path(readable=False)) @click.pass_obj def artifact_push(app, artifacts, deps, remote): """Push a built artifact to a remote artifact cache. @@ -1231,18 +1337,19 @@ def artifact_push(app, artifacts, deps, remote): # Junction elements cannot be pushed, exclude them from default targets ignore_junction_targets = True - app.stream.push(artifacts, selection=deps, remote=remote, - ignore_junction_targets=ignore_junction_targets) + app.stream.push(artifacts, selection=deps, remote=remote, ignore_junction_targets=ignore_junction_targets) ################################################################ # Artifact Log Command # ################################################################ -@artifact.command(name='log', short_help="Show logs of artifacts") -@click.option('--out', - type=click.Path(file_okay=True, writable=True), - help="Output logs to individual files in the specified path. If absent, logs are written to stdout.") -@click.argument('artifacts', type=click.Path(), nargs=-1) +@artifact.command(name="log", short_help="Show logs of artifacts") +@click.option( + "--out", + type=click.Path(file_okay=True, writable=True), + help="Output logs to individual files in the specified path. If absent, logs are written to stdout.", +) +@click.argument("artifacts", type=click.Path(), nargs=-1) @click.pass_obj def artifact_log(app, artifacts, out): """Show build logs of artifacts""" @@ -1252,7 +1359,7 @@ def artifact_log(app, artifacts, out): if not out: try: for log in list(artifact_logs.values()): - with open(log[0], 'r') as f: + with open(log[0], "r") as f: data = f.read() click.echo_via_pager(data) except (OSError, FileNotFoundError): @@ -1274,7 +1381,7 @@ def artifact_log(app, artifacts, out): shutil.copy(log, dest) # make a dir and write in log files else: - log_name = os.path.splitext(name)[0] + '.log' + log_name = os.path.splitext(name)[0] + ".log" dest = os.path.join(out, log_name) shutil.copy(log_files[0], dest) # write a log file @@ -1283,10 +1390,11 @@ def artifact_log(app, artifacts, out): ################################################################ # Artifact List-Contents Command # ################################################################ -@artifact.command(name='list-contents', short_help="List the contents of an artifact") -@click.option('--long', '-l', 'long_', is_flag=True, - help="Provide more information about the contents of the artifact.") -@click.argument('artifacts', type=click.Path(), nargs=-1) +@artifact.command(name="list-contents", short_help="List the contents of an artifact") +@click.option( + "--long", "-l", "long_", is_flag=True, help="Provide more information about the contents of the artifact." +) +@click.argument("artifacts", type=click.Path(), nargs=-1) @click.pass_obj def artifact_list_contents(app, artifacts, long_): """List the contents of an artifact. @@ -1308,11 +1416,16 @@ def artifact_list_contents(app, artifacts, long_): ################################################################### # Artifact Delete Command # ################################################################### -@artifact.command(name='delete', short_help="Remove artifacts from the local cache") -@click.option('--deps', '-d', default='none', show_default=True, - type=click.Choice(['none', 'run', 'build', 'all']), - help="The dependencies to delete") -@click.argument('artifacts', type=click.Path(), nargs=-1) +@artifact.command(name="delete", short_help="Remove artifacts from the local cache") +@click.option( + "--deps", + "-d", + default="none", + show_default=True, + type=click.Choice(["none", "run", "build", "all"]), + help="The dependencies to delete", +) +@click.argument("artifacts", type=click.Path(), nargs=-1) @click.pass_obj def artifact_delete(app, artifacts, deps): """Remove artifacts from the local cache""" @@ -1333,18 +1446,24 @@ def artifact_delete(app, artifacts, deps): # Fetch Command # ################################################################## @cli.command(short_help="COMMAND OBSOLETE - Fetch sources in a pipeline", hidden=True) -@click.option('--except', 'except_', multiple=True, - type=click.Path(readable=False), - help="Except certain dependencies from fetching") -@click.option('--deps', '-d', default='plan', show_default=True, - type=click.Choice(['none', 'plan', 'all']), - help='The dependencies to fetch') -@click.option('--track', 'track_', is_flag=True, - help="Track new source references before fetching") -@click.option('--track-cross-junctions', '-J', is_flag=True, - help="Allow tracking to cross junction boundaries") -@click.argument('elements', nargs=-1, - type=click.Path(readable=False)) +@click.option( + "--except", + "except_", + multiple=True, + type=click.Path(readable=False), + help="Except certain dependencies from fetching", +) +@click.option( + "--deps", + "-d", + default="plan", + show_default=True, + type=click.Choice(["none", "plan", "all"]), + help="The dependencies to fetch", +) +@click.option("--track", "track_", is_flag=True, help="Track new source references before fetching") +@click.option("--track-cross-junctions", "-J", is_flag=True, help="Allow tracking to cross junction boundaries") +@click.argument("elements", nargs=-1, type=click.Path(readable=False)) @click.pass_obj def fetch(app, elements, deps, track_, except_, track_cross_junctions): click.echo("This command is now obsolete. Use `bst source fetch` instead.", err=True) @@ -1355,16 +1474,23 @@ def fetch(app, elements, deps, track_, except_, track_cross_junctions): # Track Command # ################################################################## @cli.command(short_help="COMMAND OBSOLETE - Track new source references", hidden=True) -@click.option('--except', 'except_', multiple=True, - type=click.Path(readable=False), - help="Except certain dependencies from tracking") -@click.option('--deps', '-d', default='none', show_default=True, - type=click.Choice(['none', 'all']), - help='The dependencies to track') -@click.option('--cross-junctions', '-J', is_flag=True, - help="Allow crossing junction boundaries") -@click.argument('elements', nargs=-1, - type=click.Path(readable=False)) +@click.option( + "--except", + "except_", + multiple=True, + type=click.Path(readable=False), + help="Except certain dependencies from tracking", +) +@click.option( + "--deps", + "-d", + default="none", + show_default=True, + type=click.Choice(["none", "all"]), + help="The dependencies to track", +) +@click.option("--cross-junctions", "-J", is_flag=True, help="Allow crossing junction boundaries") +@click.argument("elements", nargs=-1, type=click.Path(readable=False)) @click.pass_obj def track(app, elements, deps, except_, cross_junctions): click.echo("This command is now obsolete. Use `bst source track` instead.", err=True) @@ -1375,26 +1501,33 @@ def track(app, elements, deps, except_, cross_junctions): # Checkout Command # ################################################################## @cli.command(short_help="COMMAND OBSOLETE - Checkout a built artifact", hidden=True) -@click.option('--force', '-f', is_flag=True, - help="Allow files to be overwritten") -@click.option('--deps', '-d', default='run', show_default=True, - type=click.Choice(['run', 'build', 'none']), - help='The dependencies to checkout') -@click.option('--integrate/--no-integrate', default=True, - help="Run integration commands (default is to run commands)") -@click.option('--hardlinks', is_flag=True, - help="Checkout hardlinks instead of copies (handle with care)") -@click.option('--tar', is_flag=True, - help="Create a tarball from the artifact contents instead " - "of a file tree. If LOCATION is '-', the tarball " - "will be dumped to the standard output.") -@click.argument('element', required=False, - type=click.Path(readable=False)) -@click.argument('location', type=click.Path(), required=False) +@click.option("--force", "-f", is_flag=True, help="Allow files to be overwritten") +@click.option( + "--deps", + "-d", + default="run", + show_default=True, + type=click.Choice(["run", "build", "none"]), + help="The dependencies to checkout", +) +@click.option("--integrate/--no-integrate", default=True, help="Run integration commands (default is to run commands)") +@click.option("--hardlinks", is_flag=True, help="Checkout hardlinks instead of copies (handle with care)") +@click.option( + "--tar", + is_flag=True, + help="Create a tarball from the artifact contents instead " + "of a file tree. If LOCATION is '-', the tarball " + "will be dumped to the standard output.", +) +@click.argument("element", required=False, type=click.Path(readable=False)) +@click.argument("location", type=click.Path(), required=False) @click.pass_obj def checkout(app, element, location, force, deps, integrate, hardlinks, tar): - click.echo("This command is now obsolete. Use `bst artifact checkout` instead " + - "and use the --directory option to specify LOCATION", err=True) + click.echo( + "This command is now obsolete. Use `bst artifact checkout` instead " + + "and use the --directory option to specify LOCATION", + err=True, + ) sys.exit(1) @@ -1402,13 +1535,16 @@ def checkout(app, element, location, force, deps, integrate, hardlinks, tar): # Pull Command # ################################################################ @cli.command(short_help="COMMAND OBSOLETE - Pull a built artifact", hidden=True) -@click.option('--deps', '-d', default='none', show_default=True, - type=click.Choice(['none', 'all']), - help='The dependency artifacts to pull') -@click.option('--remote', '-r', - help="The URL of the remote cache (defaults to the first configured cache)") -@click.argument('elements', nargs=-1, - type=click.Path(readable=False)) +@click.option( + "--deps", + "-d", + default="none", + show_default=True, + type=click.Choice(["none", "all"]), + help="The dependency artifacts to pull", +) +@click.option("--remote", "-r", help="The URL of the remote cache (defaults to the first configured cache)") +@click.argument("elements", nargs=-1, type=click.Path(readable=False)) @click.pass_obj def pull(app, elements, deps, remote): click.echo("This command is now obsolete. Use `bst artifact pull` instead.", err=True) @@ -1419,13 +1555,18 @@ def pull(app, elements, deps, remote): # Push Command # ################################################################## @cli.command(short_help="COMMAND OBSOLETE - Push a built artifact", hidden=True) -@click.option('--deps', '-d', default='none', show_default=True, - type=click.Choice(['none', 'all']), - help='The dependencies to push') -@click.option('--remote', '-r', default=None, - help="The URL of the remote cache (defaults to the first configured cache)") -@click.argument('elements', nargs=-1, - type=click.Path(readable=False)) +@click.option( + "--deps", + "-d", + default="none", + show_default=True, + type=click.Choice(["none", "all"]), + help="The dependencies to push", +) +@click.option( + "--remote", "-r", default=None, help="The URL of the remote cache (defaults to the first configured cache)" +) +@click.argument("elements", nargs=-1, type=click.Path(readable=False)) @click.pass_obj def push(app, elements, deps, remote): click.echo("This command is now obsolete. Use `bst artifact push` instead.", err=True) diff --git a/src/buildstream/_frontend/complete.py b/src/buildstream/_frontend/complete.py index 06067f6cc..45e857e3e 100644 --- a/src/buildstream/_frontend/complete.py +++ b/src/buildstream/_frontend/complete.py @@ -39,9 +39,9 @@ import click from click.core import MultiCommand, Option, Argument from click.parser import split_arg_string -WORDBREAK = '=' +WORDBREAK = "=" -COMPLETION_SCRIPT = ''' +COMPLETION_SCRIPT = """ %(complete_func)s() { local IFS=$'\n' COMPREPLY=( $( env COMP_WORDS="${COMP_WORDS[*]}" \\ @@ -51,7 +51,7 @@ COMPLETION_SCRIPT = ''' } complete -F %(complete_func)s -o nospace %(script_names)s -''' +""" # An exception for our custom completion handler to @@ -62,7 +62,7 @@ class CompleteUnhandled(Exception): pass -def complete_path(path_type, incomplete, base_directory='.'): +def complete_path(path_type, incomplete, base_directory="."): """Helper method for implementing the completions() method for File and Path parameter types. """ @@ -71,7 +71,7 @@ def complete_path(path_type, incomplete, base_directory='.'): # specified in `incomplete` minus the last path component, # otherwise list files starting from the current working directory. entries = [] - base_path = '' + base_path = "" # This is getting a bit messy listed_base_directory = False @@ -128,11 +128,11 @@ def complete_path(path_type, incomplete, base_directory='.'): return [ # Return an appropriate path for each entry - fix_path(e) for e in sorted(entries) - + fix_path(e) + for e in sorted(entries) # Filter out non directory elements when searching for a directory, # the opposite is fine, however. - if not (path_type == 'Directory' and not entry_is_dir(e)) + if not (path_type == "Directory" and not entry_is_dir(e)) ] @@ -183,7 +183,7 @@ def start_of_option(param_str): :param param_str: param_str to check :return: whether or not this is the start of an option declaration (i.e. starts "-" or "--") """ - return param_str and param_str[:1] == '-' + return param_str and param_str[:1] == "-" def is_incomplete_option(all_args, cmd_param): @@ -218,8 +218,11 @@ def is_incomplete_argument(current_params, cmd_param): return True if cmd_param.nargs == -1: return True - if isinstance(current_param_values, collections.abc.Iterable) \ - and cmd_param.nargs > 1 and len(current_param_values) < cmd_param.nargs: + if ( + isinstance(current_param_values, collections.abc.Iterable) + and cmd_param.nargs > 1 + and len(current_param_values) < cmd_param.nargs + ): return True return False @@ -237,10 +240,7 @@ def get_user_autocompletions(args, incomplete, cmd, cmd_param, override): # Use the type specific default completions unless it was overridden try: - return override(cmd=cmd, - cmd_param=cmd_param, - args=args, - incomplete=incomplete) + return override(cmd=cmd, cmd_param=cmd_param, args=args, incomplete=incomplete) except CompleteUnhandled: return get_param_type_completion(cmd_param.type, incomplete) or [] @@ -269,7 +269,7 @@ def get_choices(cli, prog_name, args, incomplete, override): all_args.append(partition_incomplete[0]) incomplete = partition_incomplete[2] elif incomplete == WORDBREAK: - incomplete = '' + incomplete = "" choices = [] found_param = False @@ -277,8 +277,13 @@ def get_choices(cli, prog_name, args, incomplete, override): # completions for options for param in ctx.command.params: if isinstance(param, Option): - choices.extend([param_opt + " " for param_opt in param.opts + param.secondary_opts - if param_opt not in all_args or param.multiple]) + choices.extend( + [ + param_opt + " " + for param_opt in param.opts + param.secondary_opts + if param_opt not in all_args or param.multiple + ] + ) found_param = True if not found_param: # completion for option values by choices @@ -297,14 +302,22 @@ def get_choices(cli, prog_name, args, incomplete, override): if not found_param and isinstance(ctx.command, MultiCommand): # completion for any subcommands - choices.extend([cmd + " " for cmd in ctx.command.list_commands(ctx) - if not ctx.command.get_command(ctx, cmd).hidden]) - - if not start_of_option(incomplete) and ctx.parent is not None \ - and isinstance(ctx.parent.command, MultiCommand) and ctx.parent.command.chain: + choices.extend( + [cmd + " " for cmd in ctx.command.list_commands(ctx) if not ctx.command.get_command(ctx, cmd).hidden] + ) + + if ( + not start_of_option(incomplete) + and ctx.parent is not None + and isinstance(ctx.parent.command, MultiCommand) + and ctx.parent.command.chain + ): # completion for chained commands - visible_commands = [cmd for cmd in ctx.parent.command.list_commands(ctx.parent) - if not ctx.parent.command.get_command(ctx.parent, cmd).hidden] + visible_commands = [ + cmd + for cmd in ctx.parent.command.list_commands(ctx.parent) + if not ctx.parent.command.get_command(ctx.parent, cmd).hidden + ] remaining_commands = set(visible_commands) - set(ctx.parent.protected_args) choices.extend([cmd + " " for cmd in remaining_commands]) @@ -314,13 +327,13 @@ def get_choices(cli, prog_name, args, incomplete, override): def do_complete(cli, prog_name, override): - cwords = split_arg_string(os.environ['COMP_WORDS']) - cword = int(os.environ['COMP_CWORD']) + cwords = split_arg_string(os.environ["COMP_WORDS"]) + cword = int(os.environ["COMP_CWORD"]) args = cwords[1:cword] try: incomplete = cwords[cword] except IndexError: - incomplete = '' + incomplete = "" for item in get_choices(cli, prog_name, args, incomplete, override): click.echo(item) @@ -331,7 +344,7 @@ def do_complete(cli, prog_name, override): def main_bashcomplete(cmd, prog_name, override): """Internal handler for the bash completion support.""" - if '_BST_COMPLETION' in os.environ: + if "_BST_COMPLETION" in os.environ: do_complete(cmd, prog_name, override) return True diff --git a/src/buildstream/_frontend/linuxapp.py b/src/buildstream/_frontend/linuxapp.py index 0444dc7b4..987b023bd 100644 --- a/src/buildstream/_frontend/linuxapp.py +++ b/src/buildstream/_frontend/linuxapp.py @@ -28,9 +28,9 @@ from .app import App # def _osc_777_supported(): - term = os.environ.get('TERM') + term = os.environ.get("TERM") - if term and (term.startswith('xterm') or term.startswith('vte')): + if term and (term.startswith("xterm") or term.startswith("vte")): # Since vte version 4600, upstream silently ignores # the OSC 777 without printing garbage to the terminal. @@ -39,7 +39,7 @@ def _osc_777_supported(): # will trigger a desktop notification and bring attention # to the terminal. # - vte_version = os.environ.get('VTE_VERSION') + vte_version = os.environ.get("VTE_VERSION") try: vte_version_int = int(vte_version) except (ValueError, TypeError): @@ -54,7 +54,6 @@ def _osc_777_supported(): # A linux specific App implementation # class LinuxApp(App): - def notify(self, title, text): # Currently we only try this notification method diff --git a/src/buildstream/_frontend/profile.py b/src/buildstream/_frontend/profile.py index dda0f7ffe..f49be5b0a 100644 --- a/src/buildstream/_frontend/profile.py +++ b/src/buildstream/_frontend/profile.py @@ -28,7 +28,7 @@ import click # Kwargs: # The same keyword arguments which can be used with click.style() # -class Profile(): +class Profile: def __init__(self, **kwargs): self._kwargs = dict(kwargs) @@ -64,7 +64,6 @@ class Profile(): # arguments # def fmt_subst(self, text, varname, value, **kwargs): - def subst_callback(match): # Extract and format the "{(varname)...}" portion of the match inner_token = match.group(1) diff --git a/src/buildstream/_frontend/status.py b/src/buildstream/_frontend/status.py index 85fdf7451..a3f0d8aa7 100644 --- a/src/buildstream/_frontend/status.py +++ b/src/buildstream/_frontend/status.py @@ -43,19 +43,12 @@ from .widget import TimeCode # error_profile (Profile): Formatting profile for error text # stream (Stream): The Stream # -class Status(): +class Status: # Table of the terminal capabilities we require and use - _TERM_CAPABILITIES = { - 'move_up': 'cuu1', - 'move_x': 'hpa', - 'clear_eol': 'el' - } + _TERM_CAPABILITIES = {"move_up": "cuu1", "move_x": "hpa", "clear_eol": "el"} - def __init__(self, context, state, - content_profile, format_profile, - success_profile, error_profile, - stream): + def __init__(self, context, state, content_profile, format_profile, success_profile, error_profile, stream): self._context = context self._state = state @@ -67,10 +60,9 @@ class Status(): self._jobs = OrderedDict() self._last_lines = 0 # Number of status lines we last printed to console self._spacing = 1 - self._header = _StatusHeader(context, state, - content_profile, format_profile, - success_profile, error_profile, - stream) + self._header = _StatusHeader( + context, state, content_profile, format_profile, success_profile, error_profile, stream + ) self._term_width, _ = click.get_terminal_size() self._alloc_lines = 0 @@ -131,7 +123,7 @@ class Status(): # feeds for the amount of lines we intend to print first, and # move cursor position back to the first line for _ in range(self._alloc_lines + self._header.lines): - click.echo('', err=True) + click.echo("", err=True) for _ in range(self._alloc_lines + self._header.lines): self._move_up() @@ -143,14 +135,14 @@ class Status(): # alignment of each column n_columns = len(self._alloc_columns) for line in self._job_lines(n_columns): - text = '' + text = "" for job in line: column = line.index(job) text += job.render(self._alloc_columns[column] - job.size, elapsed) # Add spacing between columns if column < (n_columns - 1): - text += ' ' * self._spacing + text += " " * self._spacing # Print the line click.echo(text, err=True) @@ -196,7 +188,7 @@ class Status(): # Initialized terminal, curses might decide it doesnt # support this terminal try: - curses.setupterm(os.environ.get('TERM', 'dumb')) + curses.setupterm(os.environ.get("TERM", "dumb")) except curses.error: return None @@ -221,7 +213,7 @@ class Status(): # as well, and should provide better compatibility with most # terminals. # - term_caps[capname] = code.decode('latin1') + term_caps[capname] = code.decode("latin1") return term_caps @@ -236,19 +228,19 @@ class Status(): # Explicitly move to beginning of line, fixes things up # when there was a ^C or ^Z printed to the terminal. - move_x = curses.tparm(self._term_caps['move_x'].encode('latin1'), 0) - move_x = move_x.decode('latin1') + move_x = curses.tparm(self._term_caps["move_x"].encode("latin1"), 0) + move_x = move_x.decode("latin1") - move_up = curses.tparm(self._term_caps['move_up'].encode('latin1')) - move_up = move_up.decode('latin1') + move_up = curses.tparm(self._term_caps["move_up"].encode("latin1")) + move_up = move_up.decode("latin1") click.echo(move_x + move_up, nl=False, err=True) def _clear_line(self): assert self._term_caps is not None - clear_eol = curses.tparm(self._term_caps['clear_eol'].encode('latin1')) - clear_eol = clear_eol.decode('latin1') + clear_eol = curses.tparm(self._term_caps["clear_eol"].encode("latin1")) + clear_eol = clear_eol.decode("latin1") click.echo(clear_eol, nl=False, err=True) def _allocate(self): @@ -277,7 +269,7 @@ class Status(): def _job_lines(self, columns): jobs_list = list(self._jobs.values()) for i in range(0, len(self._jobs), columns): - yield jobs_list[i:i + columns] + yield jobs_list[i : i + columns] # Returns an array of integers representing the maximum # length in characters for each column, given the current @@ -307,9 +299,7 @@ class Status(): def _add_job(self, action_name, full_name): task = self._state.tasks[(action_name, full_name)] elapsed = task.elapsed_offset - job = _StatusJob(self._context, action_name, full_name, - self._content_profile, self._format_profile, - elapsed) + job = _StatusJob(self._context, action_name, full_name, self._content_profile, self._format_profile, elapsed) self._jobs[(action_name, full_name)] = job self._need_alloc = True @@ -338,12 +328,8 @@ class Status(): # error_profile (Profile): Formatting profile for error text # stream (Stream): The Stream # -class _StatusHeader(): - - def __init__(self, context, state, - content_profile, format_profile, - success_profile, error_profile, - stream): +class _StatusHeader: + def __init__(self, context, state, content_profile, format_profile, success_profile, error_profile, stream): # # Public members @@ -375,19 +361,22 @@ class _StatusHeader(): total = str(len(self._stream.total_elements)) size = 0 - text = '' + text = "" size += len(total) + len(session) + 4 # Size for (N/N) with a leading space size += 8 # Size of time code size += len(project.name) + 1 text += self._time_code.render_time(elapsed) - text += ' ' + self._content_profile.fmt(project.name) - text += ' ' + self._format_profile.fmt('(') + \ - self._content_profile.fmt(session) + \ - self._format_profile.fmt('/') + \ - self._content_profile.fmt(total) + \ - self._format_profile.fmt(')') - - line1 = self._centered(text, size, line_length, '=') + text += " " + self._content_profile.fmt(project.name) + text += ( + " " + + self._format_profile.fmt("(") + + self._content_profile.fmt(session) + + self._format_profile.fmt("/") + + self._content_profile.fmt(total) + + self._format_profile.fmt(")") + ) + + line1 = self._centered(text, size, line_length, "=") # # Line 2: Dynamic list of queue status reports @@ -395,7 +384,7 @@ class _StatusHeader(): # (Sources Fetched:0 117 0)→ (Built:4 0 0) # size = 0 - text = '' + text = "" # Format and calculate size for each queue progress for index, task_group in enumerate(self._state.task_groups.values()): @@ -403,13 +392,13 @@ class _StatusHeader(): # Add spacing if index > 0: size += 2 - text += self._format_profile.fmt('→ ') + text += self._format_profile.fmt("→ ") group_text, group_size = self._render_task_group(task_group) size += group_size text += group_text - line2 = self._centered(text, size, line_length, ' ') + line2 = self._centered(text, size, line_length, " ") # # Line 3: Cache usage percentage report @@ -423,7 +412,7 @@ class _StatusHeader(): if usage.used_size is None: # Cache usage is unknown size = 0 - text = '' + text = "" else: size = 21 size += len(usage_string) @@ -434,15 +423,17 @@ class _StatusHeader(): else: formatted_usage = self._success_profile.fmt(usage_string) - text = self._format_profile.fmt("~~~~~~ ") + \ - self._content_profile.fmt('cache') + \ - self._format_profile.fmt(': ') + \ - formatted_usage + \ - self._format_profile.fmt(' ~~~~~~') + text = ( + self._format_profile.fmt("~~~~~~ ") + + self._content_profile.fmt("cache") + + self._format_profile.fmt(": ") + + formatted_usage + + self._format_profile.fmt(" ~~~~~~") + ) - line3 = self._centered(text, size, line_length, ' ') + line3 = self._centered(text, size, line_length, " ") - return line1 + '\n' + line2 + '\n' + line3 + return line1 + "\n" + line2 + "\n" + line3 ################################################### # Private Methods # @@ -455,13 +446,17 @@ class _StatusHeader(): size = 5 # Space for the formatting '[', ':', ' ', ' ' and ']' size += len(group.complete_name) size += len(processed) + len(skipped) + len(failed) - text = self._format_profile.fmt("(") + \ - self._content_profile.fmt(group.complete_name) + \ - self._format_profile.fmt(":") + \ - self._success_profile.fmt(processed) + ' ' + \ - self._content_profile.fmt(skipped) + ' ' + \ - self._error_profile.fmt(failed) + \ - self._format_profile.fmt(")") + text = ( + self._format_profile.fmt("(") + + self._content_profile.fmt(group.complete_name) + + self._format_profile.fmt(":") + + self._success_profile.fmt(processed) + + " " + + self._content_profile.fmt(skipped) + + " " + + self._error_profile.fmt(failed) + + self._format_profile.fmt(")") + ) return (text, size) @@ -469,9 +464,9 @@ class _StatusHeader(): remaining = line_length - size remaining -= 2 - final_text = self._format_profile.fmt(fill * (remaining // 2)) + ' ' + final_text = self._format_profile.fmt(fill * (remaining // 2)) + " " final_text += text - final_text += ' ' + self._format_profile.fmt(fill * (remaining // 2)) + final_text += " " + self._format_profile.fmt(fill * (remaining // 2)) return final_text @@ -488,14 +483,13 @@ class _StatusHeader(): # format_profile (Profile): Formatting profile for formatting text # elapsed (datetime): The offset into the session when this job is created # -class _StatusJob(): - +class _StatusJob: def __init__(self, context, action_name, full_name, content_profile, format_profile, elapsed): # # Public members # - self.action_name = action_name # The action name - self.size = None # The number of characters required to render + self.action_name = action_name # The action name + self.size = None # The number of characters required to render self.full_name = full_name # @@ -568,24 +562,26 @@ class _StatusJob(): # elapsed (datetime): The session elapsed time offset # def render(self, padding, elapsed): - text = self._format_profile.fmt('[') + \ - self._time_code.render_time(elapsed - self._offset) + \ - self._format_profile.fmt(']') - - text += self._format_profile.fmt('[') + \ - self._content_profile.fmt(self.action_name) + \ - self._format_profile.fmt(':') + \ - self._content_profile.fmt(self.full_name) + text = ( + self._format_profile.fmt("[") + + self._time_code.render_time(elapsed - self._offset) + + self._format_profile.fmt("]") + ) + + text += ( + self._format_profile.fmt("[") + + self._content_profile.fmt(self.action_name) + + self._format_profile.fmt(":") + + self._content_profile.fmt(self.full_name) + ) if self._current_progress is not None: - text += self._format_profile.fmt(':') + \ - self._content_profile.fmt(str(self._current_progress)) + text += self._format_profile.fmt(":") + self._content_profile.fmt(str(self._current_progress)) if self._maximum_progress is not None: - text += self._format_profile.fmt('/') + \ - self._content_profile.fmt(str(self._maximum_progress)) + text += self._format_profile.fmt("/") + self._content_profile.fmt(str(self._maximum_progress)) # Add padding before terminating ']' - terminator = (' ' * padding) + ']' + terminator = (" " * padding) + "]" text += self._format_profile.fmt(terminator) return text diff --git a/src/buildstream/_frontend/widget.py b/src/buildstream/_frontend/widget.py index 0a268b717..63fbfbb7d 100644 --- a/src/buildstream/_frontend/widget.py +++ b/src/buildstream/_frontend/widget.py @@ -45,8 +45,7 @@ ERROR_MESSAGES = [MessageType.FAIL, MessageType.ERROR, MessageType.BUG] # # An abstract class for printing output columns in our text UI. # -class Widget(): - +class Widget: def __init__(self, context, content_profile, format_profile): # The context @@ -74,7 +73,6 @@ class Widget(): # Used to add fixed text between columns class FixedText(Widget): - def __init__(self, context, text, content_profile, format_profile): super().__init__(context, content_profile, format_profile) self.text = text @@ -91,15 +89,13 @@ class WallclockTime(Widget): def render(self, message): - fields = [self.content_profile.fmt("{:02d}".format(x)) for x in - [message.creation_time.hour, - message.creation_time.minute, - message.creation_time.second, - ] - ] + fields = [ + self.content_profile.fmt("{:02d}".format(x)) + for x in [message.creation_time.hour, message.creation_time.minute, message.creation_time.second,] + ] text = self.format_profile.fmt(":").join(fields) - if self._output_format == 'us': + if self._output_format == "us": text += self.content_profile.fmt(".{:06d}".format(message.creation_time.microsecond)) return text @@ -107,11 +103,10 @@ class WallclockTime(Widget): # A widget for rendering the debugging column class Debug(Widget): - def render(self, message): element_name = "n/a" if message.element_name is None else message.element_name - text = self.format_profile.fmt('pid:') + text = self.format_profile.fmt("pid:") text += self.content_profile.fmt("{: <5}".format(message.pid)) text += self.format_profile.fmt("element name:") text += self.content_profile.fmt("{: <30}".format(element_name)) @@ -130,19 +125,13 @@ class TimeCode(Widget): def render_time(self, elapsed): if elapsed is None: - fields = [ - self.content_profile.fmt('--') - for i in range(3) - ] + fields = [self.content_profile.fmt("--") for i in range(3)] else: hours, remainder = divmod(int(elapsed.total_seconds()), 60 * 60) minutes, seconds = divmod(remainder, 60) - fields = [ - self.content_profile.fmt("{0:02d}".format(field)) - for field in [hours, minutes, seconds] - ] + fields = [self.content_profile.fmt("{0:02d}".format(field)) for field in [hours, minutes, seconds]] - text = self.format_profile.fmt(':').join(fields) + text = self.format_profile.fmt(":").join(fields) if self._microseconds: if elapsed is not None: @@ -169,41 +158,43 @@ class TypeName(Widget): } def render(self, message): - return self.content_profile.fmt("{: <7}" - .format(message.message_type.upper()), - bold=True, dim=True, - fg=self._action_colors[message.message_type]) + return self.content_profile.fmt( + "{: <7}".format(message.message_type.upper()), + bold=True, + dim=True, + fg=self._action_colors[message.message_type], + ) # A widget for displaying the Element name class ElementName(Widget): - def render(self, message): action_name = message.action_name element_name = message.element_name if element_name is not None: - name = '{: <30}'.format(element_name) + name = "{: <30}".format(element_name) else: - name = 'core activity' - name = '{: <30}'.format(name) + name = "core activity" + name = "{: <30}".format(name) if not action_name: action_name = "Main" - return self.content_profile.fmt("{: >8}".format(action_name.lower())) + \ - self.format_profile.fmt(':') + self.content_profile.fmt(name) + return ( + self.content_profile.fmt("{: >8}".format(action_name.lower())) + + self.format_profile.fmt(":") + + self.content_profile.fmt(name) + ) # A widget for displaying the primary message text class MessageText(Widget): - def render(self, message): return message.message # A widget for formatting the element cache key class CacheKey(Widget): - def __init__(self, context, content_profile, format_profile, err_profile): super().__init__(context, content_profile, format_profile) @@ -216,10 +207,10 @@ class CacheKey(Widget): return "" if message.element_name is None: - return ' ' * self._key_length + return " " * self._key_length missing = False - key = ' ' * self._key_length + key = " " * self._key_length if message.element_key: _, key, missing = message.element_key @@ -233,7 +224,6 @@ class CacheKey(Widget): # A widget for formatting the log file class LogFile(Widget): - def __init__(self, context, content_profile, format_profile, err_profile): super().__init__(context, content_profile, format_profile) @@ -248,7 +238,7 @@ class LogFile(Widget): logfile = message.logfile if abbrev and self._logdir != "" and logfile.startswith(self._logdir): - logfile = logfile[len(self._logdir):] + logfile = logfile[len(self._logdir) :] logfile = logfile.lstrip(os.sep) if message.message_type in ERROR_MESSAGES: @@ -256,7 +246,7 @@ class LogFile(Widget): else: text = self.content_profile.fmt(logfile, dim=True) else: - text = '' + text = "" return text @@ -273,8 +263,7 @@ class MessageOrLogFile(Widget): def render(self, message): # Show the log file only in the main start/success messages - if message.logfile and message.scheduler and \ - message.message_type in [MessageType.START, MessageType.SUCCESS]: + if message.logfile and message.scheduler and message.message_type in [MessageType.START, MessageType.SUCCESS]: text = self._logfile_widget.render(message) else: text = self._message_widget.render(message) @@ -296,14 +285,9 @@ class MessageOrLogFile(Widget): # indent (int): Number of spaces to use for general indentation # class LogLine(Widget): - - def __init__(self, context, state, - content_profile, - format_profile, - success_profile, - err_profile, - detail_profile, - indent=4): + def __init__( + self, context, state, content_profile, format_profile, success_profile, err_profile, detail_profile, indent=4 + ): super().__init__(context, content_profile, format_profile) self._columns = [] @@ -311,7 +295,7 @@ class LogLine(Widget): self._success_profile = success_profile self._err_profile = err_profile self._detail_profile = detail_profile - self._indent = ' ' * indent + self._indent = " " * indent self._log_lines = context.log_error_lines self._message_lines = context.log_message_lines self._resolved_keys = None @@ -320,19 +304,17 @@ class LogLine(Widget): self._logfile_widget = LogFile(context, content_profile, format_profile, err_profile) if context.log_debug: - self._columns.extend([ - Debug(context, content_profile, format_profile) - ]) + self._columns.extend([Debug(context, content_profile, format_profile)]) self.logfile_variable_names = { "elapsed": TimeCode(context, content_profile, format_profile, microseconds=False), "elapsed-us": TimeCode(context, content_profile, format_profile, microseconds=True), "wallclock": WallclockTime(context, content_profile, format_profile), - "wallclock-us": WallclockTime(context, content_profile, format_profile, output_format='us'), + "wallclock-us": WallclockTime(context, content_profile, format_profile, output_format="us"), "key": CacheKey(context, content_profile, format_profile, err_profile), "element": ElementName(context, content_profile, format_profile), "action": TypeName(context, content_profile, format_profile), - "message": MessageOrLogFile(context, content_profile, format_profile, err_profile) + "message": MessageOrLogFile(context, content_profile, format_profile, err_profile), } logfile_tokens = self._parse_logfile_format(context.log_message_format, content_profile, format_profile) self._columns.extend(logfile_tokens) @@ -352,7 +334,7 @@ class LogLine(Widget): # (str): The formatted list of elements # def show_pipeline(self, dependencies, format_): - report = '' + report = "" p = Profile() for element in dependencies: @@ -360,57 +342,57 @@ class LogLine(Widget): full_key, cache_key, dim_keys = element._get_display_key() - line = p.fmt_subst(line, 'name', element._get_full_name(), fg='blue', bold=True) - line = p.fmt_subst(line, 'key', cache_key, fg='yellow', dim=dim_keys) - line = p.fmt_subst(line, 'full-key', full_key, fg='yellow', dim=dim_keys) + line = p.fmt_subst(line, "name", element._get_full_name(), fg="blue", bold=True) + line = p.fmt_subst(line, "key", cache_key, fg="yellow", dim=dim_keys) + line = p.fmt_subst(line, "full-key", full_key, fg="yellow", dim=dim_keys) consistency = element._get_consistency() if consistency == Consistency.INCONSISTENT: - line = p.fmt_subst(line, 'state', "no reference", fg='red') + line = p.fmt_subst(line, "state", "no reference", fg="red") else: if element._cached_failure(): - line = p.fmt_subst(line, 'state', "failed", fg='red') + line = p.fmt_subst(line, "state", "failed", fg="red") elif element._cached_success(): - line = p.fmt_subst(line, 'state', "cached", fg='magenta') + line = p.fmt_subst(line, "state", "cached", fg="magenta") elif consistency == Consistency.RESOLVED and not element._source_cached(): - line = p.fmt_subst(line, 'state', "fetch needed", fg='red') + line = p.fmt_subst(line, "state", "fetch needed", fg="red") elif element._buildable(): - line = p.fmt_subst(line, 'state', "buildable", fg='green') + line = p.fmt_subst(line, "state", "buildable", fg="green") else: - line = p.fmt_subst(line, 'state', "waiting", fg='blue') + line = p.fmt_subst(line, "state", "waiting", fg="blue") # Element configuration if "%{config" in format_: line = p.fmt_subst( - line, 'config', - yaml.round_trip_dump(element._Element__config, default_flow_style=False, allow_unicode=True)) + line, + "config", + yaml.round_trip_dump(element._Element__config, default_flow_style=False, allow_unicode=True), + ) # Variables if "%{vars" in format_: variables = element._Element__variables.flat line = p.fmt_subst( - line, 'vars', - yaml.round_trip_dump(variables, default_flow_style=False, allow_unicode=True)) + line, "vars", yaml.round_trip_dump(variables, default_flow_style=False, allow_unicode=True) + ) # Environment if "%{env" in format_: environment = element._Element__environment line = p.fmt_subst( - line, 'env', - yaml.round_trip_dump(environment, default_flow_style=False, allow_unicode=True)) + line, "env", yaml.round_trip_dump(environment, default_flow_style=False, allow_unicode=True) + ) # Public if "%{public" in format_: environment = element._Element__public line = p.fmt_subst( - line, 'public', - yaml.round_trip_dump(environment, default_flow_style=False, allow_unicode=True)) + line, "public", yaml.round_trip_dump(environment, default_flow_style=False, allow_unicode=True) + ) # Workspaced if "%{workspaced" in format_: - line = p.fmt_subst( - line, 'workspaced', - '(workspaced)' if element._get_workspace() else '', fg='yellow') + line = p.fmt_subst(line, "workspaced", "(workspaced)" if element._get_workspace() else "", fg="yellow") # Workspace-dirs if "%{workspace-dirs" in format_: @@ -418,36 +400,31 @@ class LogLine(Widget): if workspace is not None: path = workspace.get_absolute_path() if path.startswith("~/"): - path = os.path.join(os.getenv('HOME', '/root'), path[2:]) - line = p.fmt_subst(line, 'workspace-dirs', "Workspace: {}".format(path)) + path = os.path.join(os.getenv("HOME", "/root"), path[2:]) + line = p.fmt_subst(line, "workspace-dirs", "Workspace: {}".format(path)) else: - line = p.fmt_subst( - line, 'workspace-dirs', '') + line = p.fmt_subst(line, "workspace-dirs", "") # Dependencies if "%{deps" in format_: deps = [e.name for e in element.dependencies(Scope.ALL, recurse=False)] - line = p.fmt_subst( - line, 'deps', - yaml.safe_dump(deps, default_style=None).rstrip('\n')) + line = p.fmt_subst(line, "deps", yaml.safe_dump(deps, default_style=None).rstrip("\n")) # Build Dependencies if "%{build-deps" in format_: build_deps = [e.name for e in element.dependencies(Scope.BUILD, recurse=False)] - line = p.fmt_subst( - line, 'build-deps', - yaml.safe_dump(build_deps, default_style=False).rstrip('\n')) + line = p.fmt_subst(line, "build-deps", yaml.safe_dump(build_deps, default_style=False).rstrip("\n")) # Runtime Dependencies if "%{runtime-deps" in format_: runtime_deps = [e.name for e in element.dependencies(Scope.RUN, recurse=False)] line = p.fmt_subst( - line, 'runtime-deps', - yaml.safe_dump(runtime_deps, default_style=False).rstrip('\n')) + line, "runtime-deps", yaml.safe_dump(runtime_deps, default_style=False).rstrip("\n") + ) - report += line + '\n' + report += line + "\n" - return report.rstrip('\n') + return report.rstrip("\n") # print_heading() # @@ -463,25 +440,24 @@ class LogLine(Widget): def print_heading(self, project, stream, *, log_file): context = self.context starttime = datetime.datetime.now() - text = '' + text = "" self._resolved_keys = {element: element._get_cache_key() for element in stream.session_elements} # Main invocation context - text += '\n' + text += "\n" text += self.content_profile.fmt("BuildStream Version {}\n".format(bst_version), bold=True) values = OrderedDict() - values["Session Start"] = starttime.strftime('%A, %d-%m-%Y at %H:%M:%S') + values["Session Start"] = starttime.strftime("%A, %d-%m-%Y at %H:%M:%S") values["Project"] = "{} ({})".format(project.name, project.directory) values["Targets"] = ", ".join([t.name for t in stream.targets]) text += self._format_values(values) # User configurations - text += '\n' + text += "\n" text += self.content_profile.fmt("User Configuration\n", bold=True) values = OrderedDict() - values["Configuration File"] = \ - "Default Configuration" if not context.config_origin else context.config_origin + values["Configuration File"] = "Default Configuration" if not context.config_origin else context.config_origin values["Cache Directory"] = context.cachedir values["Log Files"] = context.logdir values["Source Mirrors"] = context.sourcedir @@ -492,7 +468,7 @@ class LogLine(Widget): values["Maximum Push Tasks"] = context.sched_pushers values["Maximum Network Retries"] = context.sched_network_retries text += self._format_values(values) - text += '\n' + text += "\n" # Project Options values = OrderedDict() @@ -500,22 +476,25 @@ class LogLine(Widget): if values: text += self.content_profile.fmt("Project Options\n", bold=True) text += self._format_values(values) - text += '\n' + text += "\n" # Plugins - text += self._format_plugins(project.first_pass_config.element_factory.loaded_dependencies, - project.first_pass_config.source_factory.loaded_dependencies) + text += self._format_plugins( + project.first_pass_config.element_factory.loaded_dependencies, + project.first_pass_config.source_factory.loaded_dependencies, + ) if project.config.element_factory and project.config.source_factory: - text += self._format_plugins(project.config.element_factory.loaded_dependencies, - project.config.source_factory.loaded_dependencies) + text += self._format_plugins( + project.config.element_factory.loaded_dependencies, project.config.source_factory.loaded_dependencies + ) # Pipeline state text += self.content_profile.fmt("Pipeline\n", bold=True) text += self.show_pipeline(stream.total_elements, context.log_element_format) - text += '\n' + text += "\n" # Separator line before following output - text += self.format_profile.fmt("=" * 79 + '\n') + text += self.format_profile.fmt("=" * 79 + "\n") click.echo(text, nl=False, err=True) if log_file: @@ -537,7 +516,7 @@ class LogLine(Widget): if not self._state.task_groups: return - text = '' + text = "" assert self._resolved_keys is not None elements = sorted(e for (e, k) in self._resolved_keys.items() if k != e._get_cache_key()) @@ -554,7 +533,7 @@ class LogLine(Widget): # Exclude the failure messages if the job didn't ultimately fail # (e.g. succeeded on retry) if element_name in group.failed_tasks: - values[element_name] = ''.join(self._render(v) for v in messages) + values[element_name] = "".join(self._render(v) for v in messages) if values: text += self.content_profile.fmt("Failure Summary\n", bold=True) @@ -563,8 +542,8 @@ class LogLine(Widget): text += self.content_profile.fmt("Pipeline Summary\n", bold=True) values = OrderedDict() - values['Total'] = self.content_profile.fmt(str(len(stream.total_elements))) - values['Session'] = self.content_profile.fmt(str(len(stream.session_elements))) + values["Total"] = self.content_profile.fmt(str(len(stream.total_elements))) + values["Session"] = self.content_profile.fmt(str(len(stream.session_elements))) processed_maxlen = 1 skipped_maxlen = 1 @@ -579,20 +558,25 @@ class LogLine(Widget): skipped = str(group.skipped_tasks) failed = str(len(group.failed_tasks)) - processed_align = ' ' * (processed_maxlen - len(processed)) - skipped_align = ' ' * (skipped_maxlen - len(skipped)) - failed_align = ' ' * (failed_maxlen - len(failed)) - - status_text = self.content_profile.fmt("processed ") + \ - self._success_profile.fmt(processed) + \ - self.format_profile.fmt(', ') + processed_align - - status_text += self.content_profile.fmt("skipped ") + \ - self.content_profile.fmt(skipped) + \ - self.format_profile.fmt(', ') + skipped_align - - status_text += self.content_profile.fmt("failed ") + \ - self._err_profile.fmt(failed) + ' ' + failed_align + processed_align = " " * (processed_maxlen - len(processed)) + skipped_align = " " * (skipped_maxlen - len(skipped)) + failed_align = " " * (failed_maxlen - len(failed)) + + status_text = ( + self.content_profile.fmt("processed ") + + self._success_profile.fmt(processed) + + self.format_profile.fmt(", ") + + processed_align + ) + + status_text += ( + self.content_profile.fmt("skipped ") + + self.content_profile.fmt(skipped) + + self.format_profile.fmt(", ") + + skipped_align + ) + + status_text += self.content_profile.fmt("failed ") + self._err_profile.fmt(failed) + " " + failed_align values["{} Queue".format(group.name)] = status_text text += self._format_values(values, style_value=False) @@ -627,7 +611,7 @@ class LogLine(Widget): m = re.search(r"^%\{([^\}]+)\}", format_string) if m is not None: variable = m.group(1) - format_string = format_string[m.end(0):] + format_string = format_string[m.end(0) :] if variable not in self.logfile_variable_names: raise Exception("'{0}' is not a valid log variable name.".format(variable)) logfile_tokens.append(self.logfile_variable_names[variable]) @@ -635,7 +619,7 @@ class LogLine(Widget): m = re.search("^[^%]+", format_string) if m is not None: text = FixedText(self.context, m.group(0), content_profile, format_profile) - format_string = format_string[m.end(0):] + format_string = format_string[m.end(0) :] logfile_tokens.append(text) else: # No idea what to do now @@ -645,11 +629,11 @@ class LogLine(Widget): def _render(self, message): # Render the column widgets first - text = '' + text = "" for widget in self._columns: text += widget.render(message) - text += '\n' + text += "\n" extra_nl = False @@ -664,51 +648,53 @@ class LogLine(Widget): n_lines = len(lines) abbrev = False - if message.message_type not in ERROR_MESSAGES \ - and not frontend_message and n_lines > self._message_lines: - lines = lines[0:self._message_lines] + if message.message_type not in ERROR_MESSAGES and not frontend_message and n_lines > self._message_lines: + lines = lines[0 : self._message_lines] if self._message_lines > 0: abbrev = True else: - lines[n_lines - 1] = lines[n_lines - 1].rstrip('\n') + lines[n_lines - 1] = lines[n_lines - 1].rstrip("\n") detail = self._indent + self._indent.join(lines) - text += '\n' + text += "\n" if message.message_type in ERROR_MESSAGES: text += self._err_profile.fmt(detail, bold=True) else: text += self._detail_profile.fmt(detail) if abbrev: - text += self._indent + \ - self.content_profile.fmt('Message contains {} additional lines' - .format(n_lines - self._message_lines), dim=True) - text += '\n' + text += self._indent + self.content_profile.fmt( + "Message contains {} additional lines".format(n_lines - self._message_lines), dim=True + ) + text += "\n" extra_nl = True if message.scheduler and message.message_type == MessageType.FAIL: - text += '\n' + text += "\n" if self.context is not None and not self.context.log_verbose: text += self._indent + self._err_profile.fmt("Log file: ") - text += self._indent + self._logfile_widget.render(message) + '\n' + text += self._indent + self._logfile_widget.render(message) + "\n" elif self._log_lines > 0: - text += self._indent + self._err_profile.fmt("Printing the last {} lines from log file:" - .format(self._log_lines)) + '\n' - text += self._indent + self._logfile_widget.render_abbrev(message, abbrev=False) + '\n' - text += self._indent + self._err_profile.fmt("=" * 70) + '\n' + text += ( + self._indent + + self._err_profile.fmt("Printing the last {} lines from log file:".format(self._log_lines)) + + "\n" + ) + text += self._indent + self._logfile_widget.render_abbrev(message, abbrev=False) + "\n" + text += self._indent + self._err_profile.fmt("=" * 70) + "\n" log_content = self._read_last_lines(message.logfile) log_content = textwrap.indent(log_content, self._indent) text += self._detail_profile.fmt(log_content) - text += '\n' - text += self._indent + self._err_profile.fmt("=" * 70) + '\n' + text += "\n" + text += self._indent + self._err_profile.fmt("=" * 70) + "\n" extra_nl = True if extra_nl: - text += '\n' + text += "\n" return text @@ -716,14 +702,14 @@ class LogLine(Widget): with ExitStack() as stack: # mmap handles low-level memory details, allowing for # faster searches - f = stack.enter_context(open(logfile, 'r+')) + f = stack.enter_context(open(logfile, "r+")) log = stack.enter_context(mmap(f.fileno(), os.path.getsize(f.name))) count = 0 end = log.size() - 1 while count < self._log_lines and end >= 0: - location = log.rfind(b'\n', 0, end) + location = log.rfind(b"\n", 0, end) count += 1 # If location is -1 (none found), this will print the @@ -735,8 +721,8 @@ class LogLine(Widget): # then we get the first characther. If end is a newline position, # we discard it and only want to print the beginning of the next # line. - lines = log[(end + 1):].splitlines() - return '\n'.join([line.decode('utf-8') for line in lines]).rstrip() + lines = log[(end + 1) :].splitlines() + return "\n".join([line.decode("utf-8") for line in lines]).rstrip() def _format_plugins(self, element_plugins, source_plugins): text = "" @@ -756,7 +742,7 @@ class LogLine(Widget): for plugin in source_plugins: text += self.content_profile.fmt(" - {}\n".format(plugin)) - text += '\n' + text += "\n" return text @@ -773,23 +759,23 @@ class LogLine(Widget): # (str): The formatted values # def _format_values(self, values, style_value=True): - text = '' + text = "" max_key_len = 0 for key, value in values.items(): max_key_len = max(len(key), max_key_len) for key, value in values.items(): - if isinstance(value, str) and '\n' in value: + if isinstance(value, str) and "\n" in value: text += self.format_profile.fmt(" {}:\n".format(key)) text += textwrap.indent(value, self._indent) continue - text += self.format_profile.fmt(" {}: {}".format(key, ' ' * (max_key_len - len(key)))) + text += self.format_profile.fmt(" {}: {}".format(key, " " * (max_key_len - len(key)))) if style_value: text += self.content_profile.fmt(str(value)) else: text += str(value) - text += '\n' + text += "\n" return text @@ -806,20 +792,20 @@ class LogLine(Widget): # (str): The formatted values # def _pretty_print_dictionary(self, values, long_=False, style_value=True): - text = '' + text = "" max_key_len = 0 try: max_key_len = max(len(key) for key in values.keys()) except ValueError: - text = '' + text = "" for key, value in values.items(): - if isinstance(value, str) and '\n' in value: + if isinstance(value, str) and "\n" in value: text += self.format_profile.fmt(" {}:".format(key)) text += textwrap.indent(value, self._indent) continue - text += self.format_profile.fmt(" {}:{}".format(key, ' ' * (max_key_len - len(key)))) + text += self.format_profile.fmt(" {}:{}".format(key, " " * (max_key_len - len(key)))) value_list = "\n\t" + "\n\t".join((self._get_filestats(v, list_long=long_) for v in value)) if value == []: @@ -832,7 +818,7 @@ class LogLine(Widget): text += self.content_profile.fmt(value_list) else: text += value_list - text += '\n' + text += "\n" return text @@ -854,22 +840,22 @@ class LogLine(Widget): # cached status of # def show_state_of_artifacts(self, targets): - report = '' + report = "" p = Profile() for element in targets: - line = '%{state: >12} %{name}' - line = p.fmt_subst(line, 'name', element.name, fg='yellow') + line = "%{state: >12} %{name}" + line = p.fmt_subst(line, "name", element.name, fg="yellow") if element._cached_success(): - line = p.fmt_subst(line, 'state', "cached", fg='magenta') + line = p.fmt_subst(line, "state", "cached", fg="magenta") elif element._cached(): - line = p.fmt_subst(line, 'state', "failed", fg='red') + line = p.fmt_subst(line, "state", "failed", fg="red") elif element._cached_remotely(): - line = p.fmt_subst(line, 'state', "available", fg='green') + line = p.fmt_subst(line, "state", "available", fg="green") else: - line = p.fmt_subst(line, 'state', "not cached", fg='bright_red') + line = p.fmt_subst(line, "state", "not cached", fg="bright_red") - report += line + '\n' + report += line + "\n" return report @@ -890,15 +876,27 @@ class LogLine(Widget): # Support files up to 99G, meaning maximum characters is 11 max_v_len = 11 if entry["type"] == _FileType.DIRECTORY: - return "drwxr-xr-x dir {}".format(entry["size"]) +\ - "{} ".format(' ' * (max_v_len - len(size))) + "{}".format(entry["name"]) + return ( + "drwxr-xr-x dir {}".format(entry["size"]) + + "{} ".format(" " * (max_v_len - len(size))) + + "{}".format(entry["name"]) + ) elif entry["type"] == _FileType.SYMLINK: - return "lrwxrwxrwx link {}".format(entry["size"]) +\ - "{} ".format(' ' * (max_v_len - len(size))) + "{} -> {}".format(entry["name"], entry["target"]) + return ( + "lrwxrwxrwx link {}".format(entry["size"]) + + "{} ".format(" " * (max_v_len - len(size))) + + "{} -> {}".format(entry["name"], entry["target"]) + ) elif entry["executable"]: - return "-rwxr-xr-x exe {}".format(entry["size"]) +\ - "{} ".format(' ' * (max_v_len - len(size))) + "{}".format(entry["name"]) + return ( + "-rwxr-xr-x exe {}".format(entry["size"]) + + "{} ".format(" " * (max_v_len - len(size))) + + "{}".format(entry["name"]) + ) else: - return "-rw-r--r-- reg {}".format(entry["size"]) +\ - "{} ".format(' ' * (max_v_len - len(size))) + "{}".format(entry["name"]) + return ( + "-rw-r--r-- reg {}".format(entry["size"]) + + "{} ".format(" " * (max_v_len - len(size))) + + "{}".format(entry["name"]) + ) return entry["name"] diff --git a/src/buildstream/_gitsourcebase.py b/src/buildstream/_gitsourcebase.py index 120d8c72a..4e9e59161 100644 --- a/src/buildstream/_gitsourcebase.py +++ b/src/buildstream/_gitsourcebase.py @@ -35,7 +35,7 @@ from . import utils from .types import FastEnum from .utils import move_atomic, DirectoryExistsError -GIT_MODULES = '.gitmodules' +GIT_MODULES = ".gitmodules" # Warnings WARN_INCONSISTENT_SUBMODULE = "inconsistent-submodule" @@ -53,7 +53,6 @@ class _RefFormat(FastEnum): # might have at a given time # class _GitMirror(SourceFetcher): - def __init__(self, source, path, url, ref, *, primary=False, tags=[]): super().__init__() @@ -80,59 +79,64 @@ class _GitMirror(SourceFetcher): # system configured tmpdir is not on the same partition. # with self.source.tempdir() as tmpdir: - url = self.source.translate_url(self.url, alias_override=alias_override, - primary=self.primary) - self.source.call([self.source.host_git, 'clone', '--mirror', '-n', url, tmpdir], - fail="Failed to clone git repository {}".format(url), - fail_temporarily=True) + url = self.source.translate_url(self.url, alias_override=alias_override, primary=self.primary) + self.source.call( + [self.source.host_git, "clone", "--mirror", "-n", url, tmpdir], + fail="Failed to clone git repository {}".format(url), + fail_temporarily=True, + ) try: move_atomic(tmpdir, self.mirror) except DirectoryExistsError: # Another process was quicker to download this repository. # Let's discard our own - self.source.status("{}: Discarding duplicate clone of {}" - .format(self.source, url)) + self.source.status("{}: Discarding duplicate clone of {}".format(self.source, url)) except OSError as e: - raise SourceError("{}: Failed to move cloned git repository {} from '{}' to '{}': {}" - .format(self.source, url, tmpdir, self.mirror, e)) from e + raise SourceError( + "{}: Failed to move cloned git repository {} from '{}' to '{}': {}".format( + self.source, url, tmpdir, self.mirror, e + ) + ) from e def _fetch(self, alias_override=None): - url = self.source.translate_url(self.url, - alias_override=alias_override, - primary=self.primary) + url = self.source.translate_url(self.url, alias_override=alias_override, primary=self.primary) if alias_override: remote_name = utils.url_directory_name(alias_override) _, remotes = self.source.check_output( - [self.source.host_git, 'remote'], + [self.source.host_git, "remote"], fail="Failed to retrieve list of remotes in {}".format(self.mirror), - cwd=self.mirror + cwd=self.mirror, ) if remote_name not in remotes: self.source.call( - [self.source.host_git, 'remote', 'add', remote_name, url], + [self.source.host_git, "remote", "add", remote_name, url], fail="Failed to add remote {} with url {}".format(remote_name, url), - cwd=self.mirror + cwd=self.mirror, ) else: remote_name = "origin" - self.source.call([self.source.host_git, 'fetch', remote_name, '--prune', - '+refs/heads/*:refs/heads/*', '+refs/tags/*:refs/tags/*'], - fail="Failed to fetch from remote git repository: {}".format(url), - fail_temporarily=True, - cwd=self.mirror) + self.source.call( + [ + self.source.host_git, + "fetch", + remote_name, + "--prune", + "+refs/heads/*:refs/heads/*", + "+refs/tags/*:refs/tags/*", + ], + fail="Failed to fetch from remote git repository: {}".format(url), + fail_temporarily=True, + cwd=self.mirror, + ) def fetch(self, alias_override=None): # pylint: disable=arguments-differ # Resolve the URL for the message - resolved_url = self.source.translate_url(self.url, - alias_override=alias_override, - primary=self.primary) + resolved_url = self.source.translate_url(self.url, alias_override=alias_override, primary=self.primary) - with self.source.timed_activity("Fetching from {}" - .format(resolved_url), - silent_nested=True): + with self.source.timed_activity("Fetching from {}".format(resolved_url), silent_nested=True): self.ensure(alias_override) if not self.has_ref(): self._fetch(alias_override) @@ -147,48 +151,49 @@ class _GitMirror(SourceFetcher): return False # Check if the ref is really there - rc = self.source.call([self.source.host_git, 'cat-file', '-t', self.ref], cwd=self.mirror) + rc = self.source.call([self.source.host_git, "cat-file", "-t", self.ref], cwd=self.mirror) return rc == 0 def assert_ref(self): if not self.has_ref(): - raise SourceError("{}: expected ref '{}' was not found in git repository: '{}'" - .format(self.source, self.ref, self.url)) + raise SourceError( + "{}: expected ref '{}' was not found in git repository: '{}'".format(self.source, self.ref, self.url) + ) def latest_commit_with_tags(self, tracking, track_tags=False): _, output = self.source.check_output( - [self.source.host_git, 'rev-parse', tracking], + [self.source.host_git, "rev-parse", tracking], fail="Unable to find commit for specified branch name '{}'".format(tracking), - cwd=self.mirror) - ref = output.rstrip('\n') + cwd=self.mirror, + ) + ref = output.rstrip("\n") if self.source.ref_format == _RefFormat.GIT_DESCRIBE: # Prefix the ref with the closest tag, if available, # to make the ref human readable exit_code, output = self.source.check_output( - [self.source.host_git, 'describe', '--tags', '--abbrev=40', '--long', ref], - cwd=self.mirror) + [self.source.host_git, "describe", "--tags", "--abbrev=40", "--long", ref], cwd=self.mirror + ) if exit_code == 0: - ref = output.rstrip('\n') + ref = output.rstrip("\n") if not track_tags: return ref, [] tags = set() - for options in [[], ['--first-parent'], ['--tags'], ['--tags', '--first-parent']]: + for options in [[], ["--first-parent"], ["--tags"], ["--tags", "--first-parent"]]: exit_code, output = self.source.check_output( - [self.source.host_git, 'describe', '--abbrev=0', ref, *options], - cwd=self.mirror) + [self.source.host_git, "describe", "--abbrev=0", ref, *options], cwd=self.mirror + ) if exit_code == 0: tag = output.strip() _, commit_ref = self.source.check_output( - [self.source.host_git, 'rev-parse', tag + '^{commit}'], + [self.source.host_git, "rev-parse", tag + "^{commit}"], fail="Unable to resolve tag '{}'".format(tag), - cwd=self.mirror) - exit_code = self.source.call( - [self.source.host_git, 'cat-file', 'tag', tag], - cwd=self.mirror) - annotated = (exit_code == 0) + cwd=self.mirror, + ) + exit_code = self.source.call([self.source.host_git, "cat-file", "tag", tag], cwd=self.mirror) + annotated = exit_code == 0 tags.add((tag, commit_ref.strip(), annotated)) @@ -200,13 +205,17 @@ class _GitMirror(SourceFetcher): # Using --shared here avoids copying the objects into the checkout, in any # case we're just checking out a specific commit and then removing the .git/ # directory. - self.source.call([self.source.host_git, 'clone', '--no-checkout', '--shared', self.mirror, fullpath], - fail="Failed to create git mirror {} in directory: {}".format(self.mirror, fullpath), - fail_temporarily=True) - - self.source.call([self.source.host_git, 'checkout', '--force', self.ref], - fail="Failed to checkout git ref {}".format(self.ref), - cwd=fullpath) + self.source.call( + [self.source.host_git, "clone", "--no-checkout", "--shared", self.mirror, fullpath], + fail="Failed to create git mirror {} in directory: {}".format(self.mirror, fullpath), + fail_temporarily=True, + ) + + self.source.call( + [self.source.host_git, "checkout", "--force", self.ref], + fail="Failed to checkout git ref {}".format(self.ref), + cwd=fullpath, + ) # Remove .git dir shutil.rmtree(os.path.join(fullpath, ".git")) @@ -217,34 +226,37 @@ class _GitMirror(SourceFetcher): fullpath = os.path.join(directory, self.path) url = self.source.translate_url(self.url) - self.source.call([self.source.host_git, 'clone', '--no-checkout', self.mirror, fullpath], - fail="Failed to clone git mirror {} in directory: {}".format(self.mirror, fullpath), - fail_temporarily=True) + self.source.call( + [self.source.host_git, "clone", "--no-checkout", self.mirror, fullpath], + fail="Failed to clone git mirror {} in directory: {}".format(self.mirror, fullpath), + fail_temporarily=True, + ) - self.source.call([self.source.host_git, 'remote', 'set-url', 'origin', url], - fail='Failed to add remote origin "{}"'.format(url), - cwd=fullpath) + self.source.call( + [self.source.host_git, "remote", "set-url", "origin", url], + fail='Failed to add remote origin "{}"'.format(url), + cwd=fullpath, + ) - self.source.call([self.source.host_git, 'checkout', '--force', self.ref], - fail="Failed to checkout git ref {}".format(self.ref), - cwd=fullpath) + self.source.call( + [self.source.host_git, "checkout", "--force", self.ref], + fail="Failed to checkout git ref {}".format(self.ref), + cwd=fullpath, + ) # List the submodules (path/url tuples) present at the given ref of this repo def submodule_list(self): modules = "{}:{}".format(self.ref, GIT_MODULES) - exit_code, output = self.source.check_output( - [self.source.host_git, 'show', modules], cwd=self.mirror) + exit_code, output = self.source.check_output([self.source.host_git, "show", modules], cwd=self.mirror) # If git show reports error code 128 here, we take it to mean there is # no .gitmodules file to display for the given revision. if exit_code == 128: return elif exit_code != 0: - raise SourceError( - "{plugin}: Failed to show gitmodules at ref {ref}".format( - plugin=self, ref=self.ref)) + raise SourceError("{plugin}: Failed to show gitmodules at ref {ref}".format(plugin=self, ref=self.ref)) - content = '\n'.join([l.strip() for l in output.splitlines()]) + content = "\n".join([l.strip() for l in output.splitlines()]) io = StringIO(content) parser = RawConfigParser() @@ -253,8 +265,8 @@ class _GitMirror(SourceFetcher): for section in parser.sections(): # validate section name against the 'submodule "foo"' pattern if re.match(r'submodule "(.*)"', section): - path = parser.get(section, 'path') - url = parser.get(section, 'url') + path = parser.get(section, "path") + url = parser.get(section, "url") yield (path, url) @@ -266,31 +278,37 @@ class _GitMirror(SourceFetcher): # list objects in the parent repo tree to find the commit # object that corresponds to the submodule - _, output = self.source.check_output([self.source.host_git, 'ls-tree', ref, submodule], - fail="ls-tree failed for commit {} and submodule: {}".format( - ref, submodule), - cwd=self.mirror) + _, output = self.source.check_output( + [self.source.host_git, "ls-tree", ref, submodule], + fail="ls-tree failed for commit {} and submodule: {}".format(ref, submodule), + cwd=self.mirror, + ) # read the commit hash from the output fields = output.split() - if len(fields) >= 2 and fields[1] == 'commit': + if len(fields) >= 2 and fields[1] == "commit": submodule_commit = output.split()[2] # fail if the commit hash is invalid if len(submodule_commit) != 40: - raise SourceError("{}: Error reading commit information for submodule '{}'" - .format(self.source, submodule)) + raise SourceError( + "{}: Error reading commit information for submodule '{}'".format(self.source, submodule) + ) return submodule_commit else: - detail = "The submodule '{}' is defined either in the BuildStream source\n".format(submodule) + \ - "definition, or in a .gitmodules file. But the submodule was never added to the\n" + \ - "underlying git repository with `git submodule add`." + detail = ( + "The submodule '{}' is defined either in the BuildStream source\n".format(submodule) + + "definition, or in a .gitmodules file. But the submodule was never added to the\n" + + "underlying git repository with `git submodule add`." + ) - self.source.warn("{}: Ignoring inconsistent submodule '{}'" - .format(self.source, submodule), detail=detail, - warning_token=WARN_INCONSISTENT_SUBMODULE) + self.source.warn( + "{}: Ignoring inconsistent submodule '{}'".format(self.source, submodule), + detail=detail, + warning_token=WARN_INCONSISTENT_SUBMODULE, + ) return None @@ -307,17 +325,24 @@ class _GitMirror(SourceFetcher): # rev-list does not work in case of same rev shallow.add(self.ref) else: - _, out = self.source.check_output([self.source.host_git, 'rev-list', - '--ancestry-path', '--boundary', - '{}..{}'.format(commit_ref, self.ref)], - fail="Failed to get git history {}..{} in directory: {}" - .format(commit_ref, self.ref, fullpath), - fail_temporarily=True, - cwd=self.mirror) + _, out = self.source.check_output( + [ + self.source.host_git, + "rev-list", + "--ancestry-path", + "--boundary", + "{}..{}".format(commit_ref, self.ref), + ], + fail="Failed to get git history {}..{} in directory: {}".format( + commit_ref, self.ref, fullpath + ), + fail_temporarily=True, + cwd=self.mirror, + ) self.source.warn("refs {}..{}: {}".format(commit_ref, self.ref, out.splitlines())) for line in out.splitlines(): - rev = line.lstrip('-') - if line[0] == '-': + rev = line.lstrip("-") + if line[0] == "-": shallow.add(rev) else: included.add(rev) @@ -325,52 +350,64 @@ class _GitMirror(SourceFetcher): shallow -= included included |= shallow - self.source.call([self.source.host_git, 'init'], - fail="Cannot initialize git repository: {}".format(fullpath), - cwd=fullpath) + self.source.call( + [self.source.host_git, "init"], + fail="Cannot initialize git repository: {}".format(fullpath), + cwd=fullpath, + ) for rev in included: with TemporaryFile(dir=tmpdir) as commit_file: - self.source.call([self.source.host_git, 'cat-file', 'commit', rev], - stdout=commit_file, - fail="Failed to get commit {}".format(rev), - cwd=self.mirror) + self.source.call( + [self.source.host_git, "cat-file", "commit", rev], + stdout=commit_file, + fail="Failed to get commit {}".format(rev), + cwd=self.mirror, + ) commit_file.seek(0, 0) - self.source.call([self.source.host_git, 'hash-object', '-w', '-t', 'commit', '--stdin'], - stdin=commit_file, - fail="Failed to add commit object {}".format(rev), - cwd=fullpath) - - with open(os.path.join(fullpath, '.git', 'shallow'), 'w') as shallow_file: + self.source.call( + [self.source.host_git, "hash-object", "-w", "-t", "commit", "--stdin"], + stdin=commit_file, + fail="Failed to add commit object {}".format(rev), + cwd=fullpath, + ) + + with open(os.path.join(fullpath, ".git", "shallow"), "w") as shallow_file: for rev in shallow: - shallow_file.write('{}\n'.format(rev)) + shallow_file.write("{}\n".format(rev)) for tag, commit_ref, annotated in self.tags: if annotated: with TemporaryFile(dir=tmpdir) as tag_file: - tag_data = 'object {}\ntype commit\ntag {}\n'.format(commit_ref, tag) - tag_file.write(tag_data.encode('ascii')) + tag_data = "object {}\ntype commit\ntag {}\n".format(commit_ref, tag) + tag_file.write(tag_data.encode("ascii")) tag_file.seek(0, 0) _, tag_ref = self.source.check_output( - [self.source.host_git, 'hash-object', '-w', '-t', - 'tag', '--stdin'], + [self.source.host_git, "hash-object", "-w", "-t", "tag", "--stdin"], stdin=tag_file, fail="Failed to add tag object {}".format(tag), - cwd=fullpath) - - self.source.call([self.source.host_git, 'tag', tag, tag_ref.strip()], - fail="Failed to tag: {}".format(tag), - cwd=fullpath) + cwd=fullpath, + ) + + self.source.call( + [self.source.host_git, "tag", tag, tag_ref.strip()], + fail="Failed to tag: {}".format(tag), + cwd=fullpath, + ) else: - self.source.call([self.source.host_git, 'tag', tag, commit_ref], - fail="Failed to tag: {}".format(tag), - cwd=fullpath) + self.source.call( + [self.source.host_git, "tag", tag, commit_ref], + fail="Failed to tag: {}".format(tag), + cwd=fullpath, + ) - with open(os.path.join(fullpath, '.git', 'HEAD'), 'w') as head: - self.source.call([self.source.host_git, 'rev-parse', self.ref], - stdout=head, - fail="Failed to parse commit {}".format(self.ref), - cwd=self.mirror) + with open(os.path.join(fullpath, ".git", "HEAD"), "w") as head: + self.source.call( + [self.source.host_git, "rev-parse", self.ref], + stdout=head, + fail="Failed to parse commit {}".format(self.ref), + cwd=self.mirror, + ) class _GitSourceBase(Source): @@ -382,58 +419,57 @@ class _GitSourceBase(Source): BST_MIRROR_CLASS = _GitMirror def configure(self, node): - ref = node.get_str('ref', None) + ref = node.get_str("ref", None) - config_keys = ['url', 'track', 'ref', 'submodules', - 'checkout-submodules', 'ref-format', - 'track-tags', 'tags'] + config_keys = ["url", "track", "ref", "submodules", "checkout-submodules", "ref-format", "track-tags", "tags"] node.validate_keys(config_keys + Source.COMMON_CONFIG_KEYS) - tags_node = node.get_sequence('tags', []) + tags_node = node.get_sequence("tags", []) for tag_node in tags_node: - tag_node.validate_keys(['tag', 'commit', 'annotated']) + tag_node.validate_keys(["tag", "commit", "annotated"]) tags = self._load_tags(node) - self.track_tags = node.get_bool('track-tags', default=False) + self.track_tags = node.get_bool("track-tags", default=False) - self.original_url = node.get_str('url') - self.mirror = self.BST_MIRROR_CLASS(self, '', self.original_url, ref, tags=tags, primary=True) - self.tracking = node.get_str('track', None) + self.original_url = node.get_str("url") + self.mirror = self.BST_MIRROR_CLASS(self, "", self.original_url, ref, tags=tags, primary=True) + self.tracking = node.get_str("track", None) - self.ref_format = node.get_enum('ref-format', _RefFormat, _RefFormat.SHA1) + self.ref_format = node.get_enum("ref-format", _RefFormat, _RefFormat.SHA1) # At this point we now know if the source has a ref and/or a track. # If it is missing both then we will be unable to track or build. if self.mirror.ref is None and self.tracking is None: - raise SourceError("{}: Git sources require a ref and/or track".format(self), - reason="missing-track-and-ref") + raise SourceError( + "{}: Git sources require a ref and/or track".format(self), reason="missing-track-and-ref" + ) - self.checkout_submodules = node.get_bool('checkout-submodules', default=True) + self.checkout_submodules = node.get_bool("checkout-submodules", default=True) self.submodules = [] # Parse a dict of submodule overrides, stored in the submodule_overrides # and submodule_checkout_overrides dictionaries. self.submodule_overrides = {} self.submodule_checkout_overrides = {} - modules = node.get_mapping('submodules', {}) + modules = node.get_mapping("submodules", {}) for path in modules.keys(): submodule = modules.get_mapping(path) - url = submodule.get_str('url', None) + url = submodule.get_str("url", None) # Make sure to mark all URLs that are specified in the configuration if url: self.mark_download_url(url, primary=False) self.submodule_overrides[path] = url - if 'checkout' in submodule: - checkout = submodule.get_bool('checkout') + if "checkout" in submodule: + checkout = submodule.get_bool("checkout") self.submodule_checkout_overrides[path] = checkout self.mark_download_url(self.original_url) def preflight(self): # Check if git is installed, get the binary at the same time - self.host_git = utils.get_host_tool('git') + self.host_git = utils.get_host_tool("git") def get_unique_key(self): # Here we want to encode the local name of the repository and @@ -442,7 +478,7 @@ class _GitSourceBase(Source): key = [self.original_url, self.mirror.ref] if self.mirror.tags: tags = {tag: (commit, annotated) for tag, commit, annotated in self.mirror.tags} - key.append({'tags': tags}) + key.append({"tags": tags}) # Only modify the cache key with checkout_submodules if it's something # other than the default behaviour. @@ -467,7 +503,7 @@ class _GitSourceBase(Source): return Consistency.INCONSISTENT def load_ref(self, node): - self.mirror.ref = node.get_str('ref', None) + self.mirror.ref = node.get_str("ref", None) self.mirror.tags = self._load_tags(node) def get_ref(self): @@ -478,25 +514,23 @@ class _GitSourceBase(Source): def set_ref(self, ref, node): if not ref: self.mirror.ref = None - if 'ref' in node: - del node['ref'] + if "ref" in node: + del node["ref"] self.mirror.tags = [] - if 'tags' in node: - del node['tags'] + if "tags" in node: + del node["tags"] else: actual_ref, tags = ref - node['ref'] = self.mirror.ref = actual_ref + node["ref"] = self.mirror.ref = actual_ref self.mirror.tags = tags if tags: - node['tags'] = [] + node["tags"] = [] for tag, commit_ref, annotated in tags: - data = {'tag': tag, - 'commit': commit_ref, - 'annotated': annotated} - node['tags'].append(data) + data = {"tag": tag, "commit": commit_ref, "annotated": annotated} + node["tags"].append(data) else: - if 'tags' in node: - del node['tags'] + if "tags" in node: + del node["tags"] def track(self): # pylint: disable=arguments-differ @@ -504,17 +538,13 @@ class _GitSourceBase(Source): if not self.tracking: # Is there a better way to check if a ref is given. if self.mirror.ref is None: - detail = 'Without a tracking branch ref can not be updated. Please ' + \ - 'provide a ref or a track.' - raise SourceError("{}: No track or ref".format(self), - detail=detail, reason="track-attempt-no-track") + detail = "Without a tracking branch ref can not be updated. Please " + "provide a ref or a track." + raise SourceError("{}: No track or ref".format(self), detail=detail, reason="track-attempt-no-track") return None # Resolve the URL for the message resolved_url = self.translate_url(self.mirror.url) - with self.timed_activity("Tracking {} from {}" - .format(self.tracking, resolved_url), - silent_nested=True): + with self.timed_activity("Tracking {} from {}".format(self.tracking, resolved_url), silent_nested=True): self.mirror.ensure() self.mirror._fetch() @@ -578,11 +608,12 @@ class _GitSourceBase(Source): for path, url in invalid_submodules: detail.append(" Submodule URL '{}' at path '{}'".format(url, path)) - self.warn("{}: Invalid submodules specified".format(self), - warning_token=WARN_INVALID_SUBMODULE, - detail="The following submodules are specified in the source " - "description but do not exist according to the repository\n\n" + - "\n".join(detail)) + self.warn( + "{}: Invalid submodules specified".format(self), + warning_token=WARN_INVALID_SUBMODULE, + detail="The following submodules are specified in the source " + "description but do not exist according to the repository\n\n" + "\n".join(detail), + ) # Warn about submodules which exist but have not been explicitly configured if unlisted_submodules: @@ -590,37 +621,47 @@ class _GitSourceBase(Source): for path, url in unlisted_submodules: detail.append(" Submodule URL '{}' at path '{}'".format(url, path)) - self.warn("{}: Unlisted submodules exist".format(self), - warning_token=WARN_UNLISTED_SUBMODULE, - detail="The following submodules exist but are not specified " + - "in the source description\n\n" + - "\n".join(detail)) + self.warn( + "{}: Unlisted submodules exist".format(self), + warning_token=WARN_UNLISTED_SUBMODULE, + detail="The following submodules exist but are not specified " + + "in the source description\n\n" + + "\n".join(detail), + ) # Assert that the ref exists in the track tag/branch, if track has been specified. ref_in_track = False if self.tracking: - _, branch = self.check_output([self.host_git, 'branch', '--list', self.tracking, - '--contains', self.mirror.ref], - cwd=self.mirror.mirror) + _, branch = self.check_output( + [self.host_git, "branch", "--list", self.tracking, "--contains", self.mirror.ref], + cwd=self.mirror.mirror, + ) if branch: ref_in_track = True else: - _, tag = self.check_output([self.host_git, 'tag', '--list', self.tracking, - '--contains', self.mirror.ref], - cwd=self.mirror.mirror) + _, tag = self.check_output( + [self.host_git, "tag", "--list", self.tracking, "--contains", self.mirror.ref], + cwd=self.mirror.mirror, + ) if tag: ref_in_track = True if not ref_in_track: - detail = "The ref provided for the element does not exist locally " + \ - "in the provided track branch / tag '{}'.\n".format(self.tracking) + \ - "You may wish to track the element to update the ref from '{}' ".format(self.tracking) + \ - "with `bst source track`,\n" + \ - "or examine the upstream at '{}' for the specific ref.".format(self.mirror.url) + detail = ( + "The ref provided for the element does not exist locally " + + "in the provided track branch / tag '{}'.\n".format(self.tracking) + + "You may wish to track the element to update the ref from '{}' ".format(self.tracking) + + "with `bst source track`,\n" + + "or examine the upstream at '{}' for the specific ref.".format(self.mirror.url) + ) - self.warn("{}: expected ref '{}' was not found in given track '{}' for staged repository: '{}'\n" - .format(self, self.mirror.ref, self.tracking, self.mirror.url), - detail=detail, warning_token=CoreWarnings.REF_NOT_IN_TRACK) + self.warn( + "{}: expected ref '{}' was not found in given track '{}' for staged repository: '{}'\n".format( + self, self.mirror.ref, self.tracking, self.mirror.url + ), + detail=detail, + warning_token=CoreWarnings.REF_NOT_IN_TRACK, + ) ########################################################### # Local Functions # @@ -668,11 +709,11 @@ class _GitSourceBase(Source): def _load_tags(self, node): tags = [] - tags_node = node.get_sequence('tags', []) + tags_node = node.get_sequence("tags", []) for tag_node in tags_node: - tag = tag_node.get_str('tag') - commit_ref = tag_node.get_str('commit') - annotated = tag_node.get_bool('annotated') + tag = tag_node.get_str("tag") + commit_ref = tag_node.get_str("commit") + annotated = tag_node.get_bool("annotated") tags.append((tag, commit_ref, annotated)) return tags diff --git a/src/buildstream/_includes.py b/src/buildstream/_includes.py index c04601b91..bc0d7718b 100644 --- a/src/buildstream/_includes.py +++ b/src/buildstream/_includes.py @@ -14,7 +14,6 @@ from ._exceptions import LoadError, LoadErrorReason # provenance. Should be true if intended to be # serialized. class Includes: - def __init__(self, loader, *, copy_tree=False): self._loader = loader self._loaded = {} @@ -29,14 +28,11 @@ class Includes: # included (set): Fail for recursion if trying to load any files in this set # current_loader (Loader): Use alternative loader (for junction files) # only_local (bool): Whether to ignore junction files - def process(self, node, *, - included=set(), - current_loader=None, - only_local=False): + def process(self, node, *, included=set(), current_loader=None, only_local=False): if current_loader is None: current_loader = self._loader - includes_node = node.get_node('(@)', allowed_types=[ScalarNode, SequenceNode], allow_none=True) + includes_node = node.get_node("(@)", allowed_types=[ScalarNode, SequenceNode], allow_none=True) if includes_node: if type(includes_node) is ScalarNode: # pylint: disable=unidiomatic-typecheck @@ -44,23 +40,24 @@ class Includes: else: includes = includes_node.as_str_list() - del node['(@)'] + del node["(@)"] for include in reversed(includes): - if only_local and ':' in include: + if only_local and ":" in include: continue try: - include_node, file_path, sub_loader = self._include_file(include, - current_loader) + include_node, file_path, sub_loader = self._include_file(include, current_loader) except LoadError as e: include_provenance = includes_node.get_provenance() if e.reason == LoadErrorReason.MISSING_FILE: message = "{}: Include block references a file that could not be found: '{}'.".format( - include_provenance, include) + include_provenance, include + ) raise LoadError(message, LoadErrorReason.MISSING_FILE) from e if e.reason == LoadErrorReason.LOADING_DIRECTORY: message = "{}: Include block references a directory instead of a file: '{}'.".format( - include_provenance, include) + include_provenance, include + ) raise LoadError(message, LoadErrorReason.LOADING_DIRECTORY) from e # Otherwise, we don't know the reason, so just raise @@ -68,8 +65,10 @@ class Includes: if file_path in included: include_provenance = includes_node.get_provenance() - raise LoadError("{}: trying to recursively include {}". format(include_provenance, file_path), - LoadErrorReason.RECURSIVE_INCLUDE) + raise LoadError( + "{}: trying to recursively include {}".format(include_provenance, file_path), + LoadErrorReason.RECURSIVE_INCLUDE, + ) # Because the included node will be modified, we need # to copy it so that we do not modify the toplevel # node of the provenance. @@ -77,19 +76,14 @@ class Includes: try: included.add(file_path) - self.process(include_node, included=included, - current_loader=sub_loader, - only_local=only_local) + self.process(include_node, included=included, current_loader=sub_loader, only_local=only_local) finally: included.remove(file_path) include_node._composite_under(node) for value in node.values(): - self._process_value(value, - included=included, - current_loader=current_loader, - only_local=only_local) + self._process_value(value, included=included, current_loader=current_loader, only_local=only_local) # _include_file() # @@ -101,8 +95,8 @@ class Includes: # loader (Loader): Loader for the current project. def _include_file(self, include, loader): shortname = include - if ':' in include: - junction, include = include.split(':', 1) + if ":" in include: + junction, include = include.split(":", 1) junction_loader = loader._get_loader(junction) current_loader = junction_loader else: @@ -112,10 +106,7 @@ class Includes: file_path = os.path.join(directory, include) key = (current_loader, file_path) if key not in self._loaded: - self._loaded[key] = _yaml.load(file_path, - shortname=shortname, - project=project, - copy_tree=self._copy_tree) + self._loaded[key] = _yaml.load(file_path, shortname=shortname, project=project, copy_tree=self._copy_tree) return self._loaded[key], file_path, current_loader # _process_value() @@ -127,20 +118,11 @@ class Includes: # included (set): Fail for recursion if trying to load any files in this set # current_loader (Loader): Use alternative loader (for junction files) # only_local (bool): Whether to ignore junction files - def _process_value(self, value, *, - included=set(), - current_loader=None, - only_local=False): + def _process_value(self, value, *, included=set(), current_loader=None, only_local=False): value_type = type(value) if value_type is MappingNode: - self.process(value, - included=included, - current_loader=current_loader, - only_local=only_local) + self.process(value, included=included, current_loader=current_loader, only_local=only_local) elif value_type is SequenceNode: for v in value: - self._process_value(v, - included=included, - current_loader=current_loader, - only_local=only_local) + self._process_value(v, included=included, current_loader=current_loader, only_local=only_local) diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py index e5859e9e8..da0c0fb29 100644 --- a/src/buildstream/_loader/loader.py +++ b/src/buildstream/_loader/loader.py @@ -54,8 +54,7 @@ _NO_PROGRESS = object() # fetch_subprojects (callable): A function to fetch subprojects # parent (Loader): A parent Loader object, in the case this is a junctioned Loader # -class Loader(): - +class Loader: def __init__(self, context, project, *, fetch_subprojects, parent=None): # Ensure we have an absolute path for the base directory @@ -66,22 +65,22 @@ class Loader(): # # Public members # - self.project = project # The associated Project - self.loaded = None # The number of loaded Elements + self.project = project # The associated Project + self.loaded = None # The number of loaded Elements # # Private members # self._context = context - self._options = project.options # Project options (OptionPool) - self._basedir = basedir # Base project directory + self._options = project.options # Project options (OptionPool) + self._basedir = basedir # Base project directory self._first_pass_options = project.first_pass_config.options # Project options (OptionPool) - self._parent = parent # The parent loader + self._parent = parent # The parent loader self._fetch_subprojects = fetch_subprojects self._meta_elements = {} # Dict of resolved meta elements by name - self._elements = {} # Dict of elements - self._loaders = {} # Dict of junction loaders + self._elements = {} # Dict of elements + self._loaders = {} # Dict of junction loaders self._includes = Includes(self, copy_tree=True) @@ -105,9 +104,11 @@ class Loader(): if os.path.isabs(filename): # XXX Should this just be an assertion ? # Expect that the caller gives us the right thing at least ? - raise LoadError("Target '{}' was not specified as a relative " - "path to the base project directory: {}" - .format(filename, self._basedir), LoadErrorReason.INVALID_DATA) + raise LoadError( + "Target '{}' was not specified as a relative " + "path to the base project directory: {}".format(filename, self._basedir), + LoadErrorReason.INVALID_DATA, + ) self._warn_invalid_elements(targets) @@ -130,8 +131,7 @@ class Loader(): dummy_target = LoadElement(Node.from_dict({}), "", self) # Pylint is not very happy with Cython and can't understand 'dependencies' is a list dummy_target.dependencies.extend( # pylint: disable=no-member - Dependency(element, Symbol.RUNTIME, False) - for element in target_elements + Dependency(element, Symbol.RUNTIME, False) for element in target_elements ) with PROFILER.profile(Topics.CIRCULAR_CHECK, "_".join(targets)): @@ -180,12 +180,12 @@ class Loader(): # too late. The only time that seems just right is here, when preparing # the child process' copy of the Loader. # - del state['_fetch_subprojects'] + del state["_fetch_subprojects"] # Also there's no gain in pickling over the caches, and they might # contain things which are unpleasantly large or unable to pickle. - del state['_elements'] - del state['_meta_elements'] + del state["_elements"] + del state["_meta_elements"] return state @@ -230,14 +230,14 @@ class Loader(): # Load the data and process any conditional statements therein fullpath = os.path.join(self._basedir, filename) try: - node = _yaml.load(fullpath, shortname=filename, copy_tree=rewritable, - project=self.project) + node = _yaml.load(fullpath, shortname=filename, copy_tree=rewritable, project=self.project) except LoadError as e: if e.reason == LoadErrorReason.MISSING_FILE: if self.project.junction: - message = "Could not find element '{}' in project referred to by junction element '{}'" \ - .format(filename, self.project.junction.name) + message = "Could not find element '{}' in project referred to by junction element '{}'".format( + filename, self.project.junction.name + ) else: message = "Could not find element '{}' in elements directory '{}'".format(filename, self._basedir) @@ -262,8 +262,8 @@ class Loader(): if provenance: message = "{}: {}".format(provenance, message) detail = None - if os.path.exists(os.path.join(self._basedir, filename + '.bst')): - element_name = filename + '.bst' + if os.path.exists(os.path.join(self._basedir, filename + ".bst")): + element_name = filename + ".bst" detail = "Did you mean '{}'?\n".format(element_name) raise LoadError(message, LoadErrorReason.LOADING_DIRECTORY, detail=detail) from e @@ -333,10 +333,9 @@ class Loader(): if dep.junction: self._load_file(dep.junction, rewritable, ticker, dep.provenance) - loader = self._get_loader(dep.junction, - rewritable=rewritable, - ticker=ticker, - provenance=dep.provenance) + loader = self._get_loader( + dep.junction, rewritable=rewritable, ticker=ticker, provenance=dep.provenance + ) dep_element = loader._load_file(dep.name, rewritable, ticker, dep.provenance) else: dep_element = self._elements.get(dep.name) @@ -350,14 +349,16 @@ class Loader(): loader_queue.append((dep_element, list(reversed(dep_deps)), [])) # Pylint is not very happy about Cython and can't understand 'node' is a 'MappingNode' - if dep_element.node.get_str(Symbol.KIND) == 'junction': # pylint: disable=no-member - raise LoadError("{}: Cannot depend on junction" .format(dep.provenance), - LoadErrorReason.INVALID_DATA) + if dep_element.node.get_str(Symbol.KIND) == "junction": # pylint: disable=no-member + raise LoadError( + "{}: Cannot depend on junction".format(dep.provenance), LoadErrorReason.INVALID_DATA + ) # All is well, push the dependency onto the LoadElement # Pylint is not very happy with Cython and can't understand 'dependencies' is a list current_element[0].dependencies.append( # pylint: disable=no-member - Dependency(dep_element, dep.dep_type, dep.strict)) + Dependency(dep_element, dep.dep_type, dep.strict) + ) else: # We do not have any more dependencies to load for this # element on the queue, report any invalid dep names @@ -397,12 +398,14 @@ class Loader(): # Create `chain`, the loop of element dependencies from this # element back to itself, by trimming everything before this # element from the sequence under consideration. - chain = [element.full_name for element in sequence[sequence.index(element):]] + chain = [element.full_name for element in sequence[sequence.index(element) :]] chain.append(element.full_name) - raise LoadError(("Circular dependency detected at element: {}\n" + - "Dependency chain: {}") - .format(element.full_name, " -> ".join(chain)), - LoadErrorReason.CIRCULAR_DEPENDENCY) + raise LoadError( + ("Circular dependency detected at element: {}\n" + "Dependency chain: {}").format( + element.full_name, " -> ".join(chain) + ), + LoadErrorReason.CIRCULAR_DEPENDENCY, + ) if element not in validated: # We've not already validated this element, so let's # descend into it to check it out @@ -447,9 +450,9 @@ class Loader(): workspace = self._context.get_workspaces().get_workspace(element.name) skip_workspace = True if workspace: - workspace_node = {'kind': 'workspace'} - workspace_node['path'] = workspace.get_absolute_path() - workspace_node['ref'] = str(workspace.to_dict().get('last_successful', 'ignored')) + workspace_node = {"kind": "workspace"} + workspace_node["path"] = workspace.get_absolute_path() + workspace_node["ref"] = str(workspace.to_dict().get("last_successful", "ignored")) node[Symbol.SOURCES] = [workspace_node] skip_workspace = False @@ -457,7 +460,7 @@ class Loader(): for index, source in enumerate(sources): kind = source.get_str(Symbol.KIND) # the workspace source plugin cannot be used unless the element is workspaced - if kind == 'workspace' and skip_workspace: + if kind == "workspace" and skip_workspace: continue del source[Symbol.KIND] @@ -469,15 +472,20 @@ class Loader(): meta_source = MetaSource(element.name, index, element_kind, kind, source, directory) meta_sources.append(meta_source) - meta_element = MetaElement(self.project, element.name, element_kind, - elt_provenance, meta_sources, - node.get_mapping(Symbol.CONFIG, default={}), - node.get_mapping(Symbol.VARIABLES, default={}), - node.get_mapping(Symbol.ENVIRONMENT, default={}), - node.get_str_list(Symbol.ENV_NOCACHE, default=[]), - node.get_mapping(Symbol.PUBLIC, default={}), - node.get_mapping(Symbol.SANDBOX, default={}), - element_kind == 'junction') + meta_element = MetaElement( + self.project, + element.name, + element_kind, + elt_provenance, + meta_sources, + node.get_mapping(Symbol.CONFIG, default={}), + node.get_mapping(Symbol.VARIABLES, default={}), + node.get_mapping(Symbol.ENVIRONMENT, default={}), + node.get_str_list(Symbol.ENV_NOCACHE, default=[]), + node.get_mapping(Symbol.PUBLIC, default={}), + node.get_mapping(Symbol.SANDBOX, default={}), + element_kind == "junction", + ) # Cache it now, make sure it's already there before recursing self._meta_elements[element.name] = meta_element @@ -522,9 +530,9 @@ class Loader(): else: meta_dep = loader._meta_elements[name] - if dep.dep_type != 'runtime': + if dep.dep_type != "runtime": meta_element.build_dependencies.append(meta_dep) - if dep.dep_type != 'build': + if dep.dep_type != "build": meta_element.dependencies.append(meta_dep) if dep.strict: meta_element.strict_dependencies.append(meta_dep) @@ -543,8 +551,7 @@ class Loader(): # Raises: LoadError # # Returns: A Loader or None if specified junction does not exist - def _get_loader(self, filename, *, rewritable=False, ticker=None, level=0, - provenance=None): + def _get_loader(self, filename, *, rewritable=False, ticker=None, level=0, provenance=None): provenance_str = "" if provenance is not None: @@ -557,17 +564,21 @@ class Loader(): if loader is None: # do not allow junctions with the same name in different # subprojects - raise LoadError("{}Conflicting junction {} in subprojects, define junction in {}" - .format(provenance_str, filename, self.project.name), - LoadErrorReason.CONFLICTING_JUNCTION) + raise LoadError( + "{}Conflicting junction {} in subprojects, define junction in {}".format( + provenance_str, filename, self.project.name + ), + LoadErrorReason.CONFLICTING_JUNCTION, + ) return loader if self._parent: # junctions in the parent take precedence over junctions defined # in subprojects - loader = self._parent._get_loader(filename, rewritable=rewritable, ticker=ticker, - level=level + 1, provenance=provenance) + loader = self._parent._get_loader( + filename, rewritable=rewritable, ticker=ticker, level=level + 1, provenance=provenance + ) if loader: self._loaders[filename] = loader return loader @@ -599,10 +610,11 @@ class Loader(): # Any task counting *inside* the junction will be handled by # its loader. meta_element = self._collect_element_no_deps(self._elements[filename], _NO_PROGRESS) - if meta_element.kind != 'junction': - raise LoadError("{}{}: Expected junction but element kind is {}" - .format(provenance_str, filename, meta_element.kind), - LoadErrorReason.INVALID_DATA) + if meta_element.kind != "junction": + raise LoadError( + "{}{}: Expected junction but element kind is {}".format(provenance_str, filename, meta_element.kind), + LoadErrorReason.INVALID_DATA, + ) # We check that junctions have no dependencies a little # early. This is cheating, since we don't technically know @@ -618,9 +630,7 @@ class Loader(): # would be nice if this could be done for *all* element types, # but since we haven't loaded those yet that's impossible. if self._elements[filename].dependencies: - raise LoadError( - "Dependencies are forbidden for 'junction' elements", - LoadErrorReason.INVALID_JUNCTION) + raise LoadError("Dependencies are forbidden for 'junction' elements", LoadErrorReason.INVALID_JUNCTION) element = Element._new_from_meta(meta_element) element._update_state() @@ -628,10 +638,12 @@ class Loader(): # If this junction element points to a sub-sub-project, we need to # find loader for that project. if element.target: - subproject_loader = self._get_loader(element.target_junction, rewritable=rewritable, ticker=ticker, - level=level, provenance=provenance) - loader = subproject_loader._get_loader(element.target_element, rewritable=rewritable, ticker=ticker, - level=level, provenance=provenance) + subproject_loader = self._get_loader( + element.target_junction, rewritable=rewritable, ticker=ticker, level=level, provenance=provenance + ) + loader = subproject_loader._get_loader( + element.target_element, rewritable=rewritable, ticker=ticker, level=level, provenance=provenance + ) self._loaders[filename] = loader return loader @@ -639,15 +651,18 @@ class Loader(): # if element._get_consistency() >= Consistency.RESOLVED and not element._source_cached(): if ticker: - ticker(filename, 'Fetching subproject') + ticker(filename, "Fetching subproject") self._fetch_subprojects([element]) # Handle the case where a subproject has no ref # elif element._get_consistency() == Consistency.INCONSISTENT: detail = "Try tracking the junction element with `bst source track {}`".format(filename) - raise LoadError("{}Subproject has no ref for junction: {}".format(provenance_str, filename), - LoadErrorReason.SUBPROJECT_INCONSISTENT, detail=detail) + raise LoadError( + "{}Subproject has no ref for junction: {}".format(provenance_str, filename), + LoadErrorReason.SUBPROJECT_INCONSISTENT, + detail=detail, + ) sources = list(element.sources()) if len(sources) == 1 and sources[0]._get_local_path(): @@ -656,8 +671,9 @@ class Loader(): else: # Stage sources element._set_required() - basedir = os.path.join(self.project.directory, ".bst", "staged-junctions", - filename, element._get_cache_key()) + basedir = os.path.join( + self.project.directory, ".bst", "staged-junctions", filename, element._get_cache_key() + ) if not os.path.exists(basedir): os.makedirs(basedir, exist_ok=True) element._stage_sources_at(basedir) @@ -666,9 +682,15 @@ class Loader(): project_dir = os.path.join(basedir, element.path) try: from .._project import Project # pylint: disable=cyclic-import - project = Project(project_dir, self._context, junction=element, - parent_loader=self, search_for_project=False, - fetch_subprojects=self._fetch_subprojects) + + project = Project( + project_dir, + self._context, + junction=element, + parent_loader=self, + search_for_project=False, + fetch_subprojects=self._fetch_subprojects, + ) except LoadError as e: if e.reason == LoadErrorReason.MISSING_PROJECT_CONF: message = ( @@ -706,7 +728,7 @@ class Loader(): # We allow to split only once since deep junctions names are forbidden. # Users who want to refer to elements in sub-sub-projects are required # to create junctions on the top level project. - junction_path = name.rsplit(':', 1) + junction_path = name.rsplit(":", 1) if len(junction_path) == 1: return None, junction_path[-1], self else: @@ -760,11 +782,17 @@ class Loader(): invalid_elements[CoreWarnings.BAD_CHARACTERS_IN_NAME].append(filename) if invalid_elements[CoreWarnings.BAD_ELEMENT_SUFFIX]: - self._warn("Target elements '{}' do not have expected file extension `.bst` " - "Improperly named elements will not be discoverable by commands" - .format(invalid_elements[CoreWarnings.BAD_ELEMENT_SUFFIX]), - warning_token=CoreWarnings.BAD_ELEMENT_SUFFIX) + self._warn( + "Target elements '{}' do not have expected file extension `.bst` " + "Improperly named elements will not be discoverable by commands".format( + invalid_elements[CoreWarnings.BAD_ELEMENT_SUFFIX] + ), + warning_token=CoreWarnings.BAD_ELEMENT_SUFFIX, + ) if invalid_elements[CoreWarnings.BAD_CHARACTERS_IN_NAME]: - self._warn("Target elements '{}' have invalid characerts in their name." - .format(invalid_elements[CoreWarnings.BAD_CHARACTERS_IN_NAME]), - warning_token=CoreWarnings.BAD_CHARACTERS_IN_NAME) + self._warn( + "Target elements '{}' have invalid characerts in their name.".format( + invalid_elements[CoreWarnings.BAD_CHARACTERS_IN_NAME] + ), + warning_token=CoreWarnings.BAD_CHARACTERS_IN_NAME, + ) diff --git a/src/buildstream/_loader/metaelement.py b/src/buildstream/_loader/metaelement.py index 00d8560f8..97b0de242 100644 --- a/src/buildstream/_loader/metaelement.py +++ b/src/buildstream/_loader/metaelement.py @@ -20,7 +20,7 @@ from ..node import Node -class MetaElement(): +class MetaElement: # MetaElement() # @@ -40,9 +40,21 @@ class MetaElement(): # sandbox: Configuration specific to the sandbox environment # first_pass: The element is to be loaded with first pass configuration (junction) # - def __init__(self, project, name, kind=None, provenance=None, sources=None, config=None, - variables=None, environment=None, env_nocache=None, public=None, - sandbox=None, first_pass=False): + def __init__( + self, + project, + name, + kind=None, + provenance=None, + sources=None, + config=None, + variables=None, + environment=None, + env_nocache=None, + public=None, + sandbox=None, + first_pass=False, + ): self.project = project self.name = name self.kind = kind diff --git a/src/buildstream/_loader/metasource.py b/src/buildstream/_loader/metasource.py index da2c0e292..5466d3aa5 100644 --- a/src/buildstream/_loader/metasource.py +++ b/src/buildstream/_loader/metasource.py @@ -18,7 +18,7 @@ # Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> -class MetaSource(): +class MetaSource: # MetaSource() # diff --git a/src/buildstream/_message.py b/src/buildstream/_message.py index f4f342a6a..a2844ddce 100644 --- a/src/buildstream/_message.py +++ b/src/buildstream/_message.py @@ -23,57 +23,55 @@ import os # Types of status messages. # -class MessageType(): - DEBUG = "debug" # Debugging message - STATUS = "status" # Status message, verbose details - INFO = "info" # Informative messages - WARN = "warning" # Warning messages - ERROR = "error" # Error messages - BUG = "bug" # An unhandled exception was raised in a plugin - LOG = "log" # Messages for log files _only_, never in the frontend +class MessageType: + DEBUG = "debug" # Debugging message + STATUS = "status" # Status message, verbose details + INFO = "info" # Informative messages + WARN = "warning" # Warning messages + ERROR = "error" # Error messages + BUG = "bug" # An unhandled exception was raised in a plugin + LOG = "log" # Messages for log files _only_, never in the frontend # Timed Messages: SUCCESS and FAIL have duration timestamps - START = "start" # Status start message - SUCCESS = "success" # Successful status complete message - FAIL = "failure" # Failing status complete message + START = "start" # Status start message + SUCCESS = "success" # Successful status complete message + FAIL = "failure" # Failing status complete message SKIPPED = "skipped" # Messages which should be reported regardless of whether # they are currently silenced or not -unconditional_messages = [ - MessageType.INFO, - MessageType.WARN, - MessageType.FAIL, - MessageType.ERROR, - MessageType.BUG -] +unconditional_messages = [MessageType.INFO, MessageType.WARN, MessageType.FAIL, MessageType.ERROR, MessageType.BUG] # Message object # -class Message(): - - def __init__(self, message_type, message, *, - element_name=None, - element_key=None, - detail=None, - action_name=None, - elapsed=None, - logfile=None, - sandbox=False, - scheduler=False): +class Message: + def __init__( + self, + message_type, + message, + *, + element_name=None, + element_key=None, + detail=None, + action_name=None, + elapsed=None, + logfile=None, + sandbox=False, + scheduler=False + ): self.message_type = message_type # Message type - self.message = message # The message string - self.element_name = element_name # The instance element name of the issuing plugin - self.element_key = element_key # The display key of the issuing plugin element - self.detail = detail # An additional detail string - self.action_name = action_name # Name of the task queue (fetch, refresh, build, etc) - self.elapsed = elapsed # The elapsed time, in timed messages - self.logfile = logfile # The log file path where commands took place - self.sandbox = sandbox # Whether the error that caused this message used a sandbox - self.pid = os.getpid() # The process pid - self.scheduler = scheduler # Whether this is a scheduler level message + self.message = message # The message string + self.element_name = element_name # The instance element name of the issuing plugin + self.element_key = element_key # The display key of the issuing plugin element + self.detail = detail # An additional detail string + self.action_name = action_name # Name of the task queue (fetch, refresh, build, etc) + self.elapsed = elapsed # The elapsed time, in timed messages + self.logfile = logfile # The log file path where commands took place + self.sandbox = sandbox # Whether the error that caused this message used a sandbox + self.pid = os.getpid() # The process pid + self.scheduler = scheduler # Whether this is a scheduler level message self.creation_time = datetime.datetime.now() if message_type in (MessageType.SUCCESS, MessageType.FAIL): assert elapsed is not None diff --git a/src/buildstream/_messenger.py b/src/buildstream/_messenger.py index 20c327728..03b2833ec 100644 --- a/src/buildstream/_messenger.py +++ b/src/buildstream/_messenger.py @@ -39,15 +39,14 @@ if "BST_TEST_SUITE" in os.environ: # TimeData class to contain times in an object that can be passed around # and updated from different places -class _TimeData(): - __slots__ = ['start_time'] +class _TimeData: + __slots__ = ["start_time"] def __init__(self, start_time): self.start_time = start_time -class Messenger(): - +class Messenger: def __init__(self): self._message_handler = None self._silence_scope_depth = 0 @@ -238,8 +237,9 @@ class Messenger(): detail = "{} of {} subtasks processed".format(task.current_progress, task.maximum_progress) else: detail = "{} subtasks processed".format(task.current_progress) - message = Message(MessageType.SUCCESS, activity_name, elapsed=elapsed, detail=detail, - element_name=element_name) + message = Message( + MessageType.SUCCESS, activity_name, elapsed=elapsed, detail=detail, element_name=element_name + ) self.message(message) # recorded_messages() @@ -274,14 +274,13 @@ class Messenger(): # Create the fully qualified logfile in the log directory, # appending the pid and .log extension at the end. - self._log_filename = os.path.join(logdir, - '{}.{}.log'.format(filename, os.getpid())) + self._log_filename = os.path.join(logdir, "{}.{}.log".format(filename, os.getpid())) # Ensure the directory exists first directory = os.path.dirname(self._log_filename) os.makedirs(directory, exist_ok=True) - with open(self._log_filename, 'a') as logfile: + with open(self._log_filename, "a") as logfile: # Write one last line to the log and flush it to disk def flush_log(): @@ -291,7 +290,7 @@ class Messenger(): # # So just try to flush as well as we can at SIGTERM time try: - logfile.write('\n\nForcefully terminated\n') + logfile.write("\n\nForcefully terminated\n") logfile.flush() except RuntimeError: os.fsync(logfile.fileno()) @@ -352,26 +351,28 @@ class Messenger(): template += ": {message}" - detail = '' + detail = "" if message.detail is not None: template += "\n\n{detail}" - detail = message.detail.rstrip('\n') + detail = message.detail.rstrip("\n") detail = INDENT + INDENT.join(detail.splitlines(True)) timecode = EMPTYTIME if message.message_type in (MessageType.SUCCESS, MessageType.FAIL): - hours, remainder = divmod(int(message.elapsed.total_seconds()), 60**2) + hours, remainder = divmod(int(message.elapsed.total_seconds()), 60 ** 2) minutes, seconds = divmod(remainder, 60) timecode = "{0:02d}:{1:02d}:{2:02d}".format(hours, minutes, seconds) - text = template.format(timecode=timecode, - element_name=element_name, - type=message.message_type.upper(), - message=message.message, - detail=detail) + text = template.format( + timecode=timecode, + element_name=element_name, + type=message.message_type.upper(), + message=message.message, + detail=detail, + ) # Write to the open log file - self._log_handle.write('{}\n'.format(text)) + self._log_handle.write("{}\n".format(text)) self._log_handle.flush() # get_state_for_child_job_pickling(self) @@ -399,21 +400,21 @@ class Messenger(): # access to private details of Messenger, but it would open up a window # where messagesw wouldn't be handled as expected. # - del state['_message_handler'] + del state["_message_handler"] # The render status callback is only used in the main process # - del state['_render_status_cb'] + del state["_render_status_cb"] # The "simple_task" context manager is not needed outside the main # process. During testing we override it to something that cannot # pickle, so just drop it when pickling to a child job. Note that it # will only appear in 'state' if it has been overridden. # - state.pop('simple_task', None) + state.pop("simple_task", None) # The State object is not needed outside the main process - del state['_state'] + del state["_state"] return state diff --git a/src/buildstream/_options/option.py b/src/buildstream/_options/option.py index 51017be22..71d2f12f3 100644 --- a/src/buildstream/_options/option.py +++ b/src/buildstream/_options/option.py @@ -27,11 +27,7 @@ if TYPE_CHECKING: # Shared symbols for validation purposes # -OPTION_SYMBOLS = [ - 'type', - 'description', - 'variable' -] +OPTION_SYMBOLS = ["type", "description", "variable"] # Option() @@ -42,7 +38,7 @@ OPTION_SYMBOLS = [ # the loaded project options is a collection of typed Option # instances. # -class Option(): +class Option: # Subclasses use this to specify the type name used # for the yaml format and error messages @@ -66,12 +62,12 @@ class Option(): def load(self, node): # We don't use the description, but we do require that options have a # description. - node.get_str('description') - self.variable = node.get_str('variable', default=None) + node.get_str("description") + self.variable = node.get_str("variable", default=None) # Assert valid symbol name for variable name if self.variable is not None: - _assert_symbol_name(self.variable, 'variable name', ref_node=node.get_node('variable')) + _assert_symbol_name(self.variable, "variable name", ref_node=node.get_node("variable")) # load_value() # diff --git a/src/buildstream/_options/optionarch.py b/src/buildstream/_options/optionarch.py index cbe360f9e..2d663f0ef 100644 --- a/src/buildstream/_options/optionarch.py +++ b/src/buildstream/_options/optionarch.py @@ -36,7 +36,7 @@ from .optionenum import OptionEnum # class OptionArch(OptionEnum): - OPTION_TYPE = 'arch' + OPTION_TYPE = "arch" def load(self, node): super().load_special(node, allow_default_definition=False) @@ -54,12 +54,14 @@ class OptionArch(OptionEnum): # Do not terminate the loop early to ensure we validate # all values in the list. except PlatformError as e: - provenance = node.get_sequence('values').scalar_at(index).get_provenance() + provenance = node.get_sequence("values").scalar_at(index).get_provenance() prefix = "" if provenance: prefix = "{}: ".format(provenance) - raise LoadError("{}Invalid value for {} option '{}': {}" - .format(prefix, self.OPTION_TYPE, self.name, e), LoadErrorReason.INVALID_DATA) + raise LoadError( + "{}Invalid value for {} option '{}': {}".format(prefix, self.OPTION_TYPE, self.name, e), + LoadErrorReason.INVALID_DATA, + ) if default_value is None: # Host architecture is not supported by the project. diff --git a/src/buildstream/_options/optionbool.py b/src/buildstream/_options/optionbool.py index f91cb257d..c7289b936 100644 --- a/src/buildstream/_options/optionbool.py +++ b/src/buildstream/_options/optionbool.py @@ -27,13 +27,13 @@ from .option import Option, OPTION_SYMBOLS # class OptionBool(Option): - OPTION_TYPE = 'bool' + OPTION_TYPE = "bool" def load(self, node): super().load(node) - node.validate_keys(OPTION_SYMBOLS + ['default']) - self.value = node.get_bool('default') + node.validate_keys(OPTION_SYMBOLS + ["default"]) + self.value = node.get_bool("default") def load_value(self, node, *, transform=None): if transform: @@ -42,13 +42,14 @@ class OptionBool(Option): self.value = node.get_bool(self.name) def set_value(self, value): - if value in ('True', 'true'): + if value in ("True", "true"): self.value = True - elif value in ('False', 'false'): + elif value in ("False", "false"): self.value = False else: - raise LoadError("Invalid value for boolean option {}: {}".format(self.name, value), - LoadErrorReason.INVALID_DATA) + raise LoadError( + "Invalid value for boolean option {}: {}".format(self.name, value), LoadErrorReason.INVALID_DATA + ) def get_value(self): if self.value: diff --git a/src/buildstream/_options/optioneltmask.py b/src/buildstream/_options/optioneltmask.py index 178999fa1..5a0d15f8e 100644 --- a/src/buildstream/_options/optioneltmask.py +++ b/src/buildstream/_options/optioneltmask.py @@ -28,7 +28,7 @@ from .optionflags import OptionFlags # class OptionEltMask(OptionFlags): - OPTION_TYPE = 'element-mask' + OPTION_TYPE = "element-mask" def load(self, node): # Ask the parent constructor to disallow value definitions, @@ -41,6 +41,6 @@ class OptionEltMask(OptionFlags): def load_valid_values(self, node): values = [] for filename in utils.list_relative_paths(self.pool.element_path): - if filename.endswith('.bst'): + if filename.endswith(".bst"): values.append(filename) return values diff --git a/src/buildstream/_options/optionenum.py b/src/buildstream/_options/optionenum.py index 4a0941369..d30f45696 100644 --- a/src/buildstream/_options/optionenum.py +++ b/src/buildstream/_options/optionenum.py @@ -27,7 +27,7 @@ from .option import Option, OPTION_SYMBOLS # class OptionEnum(Option): - OPTION_TYPE = 'enum' + OPTION_TYPE = "enum" def __init__(self, name, definition, pool): self.values = None @@ -39,17 +39,20 @@ class OptionEnum(Option): def load_special(self, node, allow_default_definition=True): super().load(node) - valid_symbols = OPTION_SYMBOLS + ['values'] + valid_symbols = OPTION_SYMBOLS + ["values"] if allow_default_definition: - valid_symbols += ['default'] + valid_symbols += ["default"] node.validate_keys(valid_symbols) - self.values = node.get_str_list('values', default=[]) + self.values = node.get_str_list("values", default=[]) if not self.values: - raise LoadError("{}: No values specified for {} option '{}'" - .format(node.get_provenance(), self.OPTION_TYPE, self.name), - LoadErrorReason.INVALID_DATA,) + raise LoadError( + "{}: No values specified for {} option '{}'".format( + node.get_provenance(), self.OPTION_TYPE, self.name + ), + LoadErrorReason.INVALID_DATA, + ) # Allow subclass to define the default value self.value = self.load_default_value(node) @@ -77,13 +80,14 @@ class OptionEnum(Option): prefix = "{}: ".format(provenance) else: prefix = "" - raise LoadError("{}Invalid value for {} option '{}': {}\n" - .format(prefix, self.OPTION_TYPE, self.name, value) + - "Valid values: {}".format(", ".join(self.values)), - LoadErrorReason.INVALID_DATA) + raise LoadError( + "{}Invalid value for {} option '{}': {}\n".format(prefix, self.OPTION_TYPE, self.name, value) + + "Valid values: {}".format(", ".join(self.values)), + LoadErrorReason.INVALID_DATA, + ) def load_default_value(self, node): - value_node = node.get_scalar('default') + value_node = node.get_scalar("default") value = value_node.as_str() self.validate(value, value_node) return value diff --git a/src/buildstream/_options/optionflags.py b/src/buildstream/_options/optionflags.py index e5217a718..82ede5649 100644 --- a/src/buildstream/_options/optionflags.py +++ b/src/buildstream/_options/optionflags.py @@ -27,7 +27,7 @@ from .option import Option, OPTION_SYMBOLS # class OptionFlags(Option): - OPTION_TYPE = 'flags' + OPTION_TYPE = "flags" def __init__(self, name, definition, pool): self.values = None @@ -39,20 +39,23 @@ class OptionFlags(Option): def load_special(self, node, allow_value_definitions=True): super().load(node) - valid_symbols = OPTION_SYMBOLS + ['default'] + valid_symbols = OPTION_SYMBOLS + ["default"] if allow_value_definitions: - valid_symbols += ['values'] + valid_symbols += ["values"] node.validate_keys(valid_symbols) # Allow subclass to define the valid values self.values = self.load_valid_values(node) if not self.values: - raise LoadError("{}: No values specified for {} option '{}'" - .format(node.get_provenance(), self.OPTION_TYPE, self.name), - LoadErrorReason.INVALID_DATA) - - value_node = node.get_sequence('default', default=[]) + raise LoadError( + "{}: No values specified for {} option '{}'".format( + node.get_provenance(), self.OPTION_TYPE, self.name + ), + LoadErrorReason.INVALID_DATA, + ) + + value_node = node.get_sequence("default", default=[]) self.value = value_node.as_str_list() self.validate(self.value, value_node) @@ -70,7 +73,7 @@ class OptionFlags(Option): stripped = "".join(value.split()) # Get the comma separated values - list_value = stripped.split(',') + list_value = stripped.split(",") self.validate(list_value) self.value = sorted(list_value) @@ -86,12 +89,13 @@ class OptionFlags(Option): prefix = "{}: ".format(provenance) else: prefix = "" - raise LoadError("{}Invalid value for flags option '{}': {}\n" - .format(prefix, self.name, value) + - "Valid values: {}".format(", ".join(self.values)), - LoadErrorReason.INVALID_DATA) + raise LoadError( + "{}Invalid value for flags option '{}': {}\n".format(prefix, self.name, value) + + "Valid values: {}".format(", ".join(self.values)), + LoadErrorReason.INVALID_DATA, + ) def load_valid_values(self, node): # Allow the more descriptive error to raise when no values # exist rather than bailing out here (by specifying default_value) - return node.get_str_list('values', default=[]) + return node.get_str_list("values", default=[]) diff --git a/src/buildstream/_options/optionos.py b/src/buildstream/_options/optionos.py index fcf4552f5..3f4e902c9 100644 --- a/src/buildstream/_options/optionos.py +++ b/src/buildstream/_options/optionos.py @@ -1,4 +1,3 @@ - # # Copyright (C) 2017 Codethink Limited # @@ -26,7 +25,7 @@ from .optionenum import OptionEnum # class OptionOS(OptionEnum): - OPTION_TYPE = 'os' + OPTION_TYPE = "os" def load(self, node): super().load_special(node, allow_default_definition=False) diff --git a/src/buildstream/_options/optionpool.py b/src/buildstream/_options/optionpool.py index a0730c617..f105bb12c 100644 --- a/src/buildstream/_options/optionpool.py +++ b/src/buildstream/_options/optionpool.py @@ -50,8 +50,7 @@ class OptionTypes(FastEnum): OS = OptionOS.OPTION_TYPE -class OptionPool(): - +class OptionPool: def __init__(self, element_path): # We hold on to the element path for the sake of OptionEltMask self.element_path = element_path @@ -59,7 +58,7 @@ class OptionPool(): # # Private members # - self._options = {} # The Options + self._options = {} # The Options self._variables = None # The Options resolved into typed variables self._environment = None @@ -69,7 +68,7 @@ class OptionPool(): state = self.__dict__.copy() # Jinja2 Environments don't appear to be serializable. It is easy # enough for us to reconstruct this one anyway, so no need to pickle it. - del state['_environment'] + del state["_environment"] return state def __setstate__(self, state): @@ -90,7 +89,7 @@ class OptionPool(): # Assert that the option name is a valid symbol _assert_symbol_name(option_name, "option name", ref_node=option_definition, allow_dashes=False) - opt_type_name = option_definition.get_enum('type', OptionTypes) + opt_type_name = option_definition.get_enum("type", OptionTypes) opt_type = _OPTION_TYPES[opt_type_name.value] option = opt_type(option_name, option_definition, self) @@ -110,8 +109,9 @@ class OptionPool(): option = self._options[option_name] except KeyError as e: p = option_value.get_provenance() - raise LoadError("{}: Unknown option '{}' specified" - .format(p, option_name), LoadErrorReason.INVALID_DATA) from e + raise LoadError( + "{}: Unknown option '{}' specified".format(p, option_name), LoadErrorReason.INVALID_DATA + ) from e option.load_value(node, transform=transform) # load_cli_values() @@ -129,8 +129,10 @@ class OptionPool(): option = self._options[option_name] except KeyError as e: if not ignore_unknown: - raise LoadError("Unknown option '{}' specified on the command line" - .format(option_name), LoadErrorReason.INVALID_DATA) from e + raise LoadError( + "Unknown option '{}' specified on the command line".format(option_name), + LoadErrorReason.INVALID_DATA, + ) from e else: option.set_value(option_value) @@ -239,11 +241,13 @@ class OptionPool(): elif val == "False": return False else: # pragma: nocover - raise LoadError("Failed to evaluate expression: {}".format(expression), - LoadErrorReason.EXPRESSION_FAILED) + raise LoadError( + "Failed to evaluate expression: {}".format(expression), LoadErrorReason.EXPRESSION_FAILED + ) except jinja2.exceptions.TemplateError as e: - raise LoadError("Failed to evaluate expression ({}): {}".format(expression, e), - LoadErrorReason.EXPRESSION_FAILED) + raise LoadError( + "Failed to evaluate expression ({}): {}".format(expression, e), LoadErrorReason.EXPRESSION_FAILED + ) # Recursion assistent for lists, in case there # are lists of lists. @@ -262,25 +266,27 @@ class OptionPool(): # Return true if a conditional was processed. # def _process_one_node(self, node): - conditions = node.get_sequence('(?)', default=None) - assertion = node.get_str('(!)', default=None) + conditions = node.get_sequence("(?)", default=None) + assertion = node.get_str("(!)", default=None) # Process assersions first, we want to abort on the first encountered # assertion in a given dictionary, and not lose an assertion due to # it being overwritten by a later assertion which might also trigger. if assertion is not None: - p = node.get_scalar('(!)').get_provenance() + p = node.get_scalar("(!)").get_provenance() raise LoadError("{}: {}".format(p, assertion.strip()), LoadErrorReason.USER_ASSERTION) if conditions is not None: - del node['(?)'] + del node["(?)"] for condition in conditions: tuples = list(condition.items()) if len(tuples) > 1: provenance = condition.get_provenance() - raise LoadError("{}: Conditional statement has more than one key".format(provenance), - LoadErrorReason.INVALID_DATA) + raise LoadError( + "{}: Conditional statement has more than one key".format(provenance), + LoadErrorReason.INVALID_DATA, + ) expression, value = tuples[0] try: @@ -292,8 +298,10 @@ class OptionPool(): if type(value) is not MappingNode: # pylint: disable=unidiomatic-typecheck provenance = condition.get_provenance() - raise LoadError("{}: Only values of type 'dict' can be composed.".format(provenance), - LoadErrorReason.ILLEGAL_COMPOSITE) + raise LoadError( + "{}: Only values of type 'dict' can be composed.".format(provenance), + LoadErrorReason.ILLEGAL_COMPOSITE, + ) # Apply the yaml fragment if its condition evaluates to true if apply_fragment: diff --git a/src/buildstream/_pipeline.py b/src/buildstream/_pipeline.py index b9efc7826..0b9ab5f24 100644 --- a/src/buildstream/_pipeline.py +++ b/src/buildstream/_pipeline.py @@ -40,27 +40,27 @@ from ._project import ProjectRefStorage # # These values correspond to the CLI `--deps` arguments for convenience. # -class PipelineSelection(): +class PipelineSelection: # Select only the target elements in the associated targets - NONE = 'none' + NONE = "none" # As NONE, but redirect elements that are capable of it - REDIRECT = 'redirect' + REDIRECT = "redirect" # Select elements which must be built for the associated targets to be built - PLAN = 'plan' + PLAN = "plan" # All dependencies of all targets, including the targets - ALL = 'all' + ALL = "all" # All direct build dependencies and their recursive runtime dependencies, # excluding the targets - BUILD = 'build' + BUILD = "build" # All direct runtime dependencies and their recursive runtime dependencies, # including the targets - RUN = 'run' + RUN = "run" # Pipeline() @@ -70,12 +70,11 @@ class PipelineSelection(): # context (Context): The Context object # artifacts (Context): The ArtifactCache object # -class Pipeline(): - +class Pipeline: def __init__(self, context, project, artifacts): - self._context = context # The Context - self._project = project # The toplevel project + self._context = context # The Context + self._project = project # The toplevel project # # Private members @@ -108,10 +107,7 @@ class Pipeline(): # Now create element groups to match the input target groups elt_iter = iter(elements) - element_groups = [ - [next(elt_iter) for i in range(len(group))] - for group in target_groups - ] + element_groups = [[next(elt_iter) for i in range(len(group))] for group in target_groups] return tuple(element_groups) @@ -240,8 +236,7 @@ class Pipeline(): for t in targets: new_elm = t._get_source_element() if new_elm != t and not silent: - self._message(MessageType.INFO, "Element '{}' redirected to '{}'" - .format(t.name, new_elm.name)) + self._message(MessageType.INFO, "Element '{}' redirected to '{}'".format(t.name, new_elm.name)) if new_elm not in elements: elements.append(new_elm) elif mode == PipelineSelection.PLAN: @@ -296,9 +291,7 @@ class Pipeline(): # Build a list of 'intersection' elements, i.e. the set of # elements that lie on the border closest to excepted elements # between excepted and target elements. - intersection = list(itertools.chain.from_iterable( - find_intersection(element) for element in except_targets - )) + intersection = list(itertools.chain.from_iterable(find_intersection(element) for element in except_targets)) # Now use this set of elements to traverse the targeted # elements, except 'intersection' elements and their unique @@ -354,10 +347,7 @@ class Pipeline(): # def subtract_elements(self, elements, subtract): subtract_set = set(subtract) - return [ - e for e in elements - if e not in subtract_set - ] + return [e for e in elements if e not in subtract_set] # add_elements() # @@ -426,14 +416,13 @@ class Pipeline(): for source in element.sources(): if source._get_consistency() == Consistency.INCONSISTENT: detail += " {} is missing ref\n".format(source) - detail += '\n' + detail += "\n" detail += "Try tracking these elements first with `bst source track`\n" raise PipelineError("Inconsistent pipeline", detail=detail, reason="inconsistent-pipeline") if inconsistent_workspaced: - detail = "Some workspaces exist but are not closed\n" + \ - "Try closing them with `bst workspace close`\n\n" + detail = "Some workspaces exist but are not closed\n" + "Try closing them with `bst workspace close`\n\n" for element in inconsistent_workspaced: detail += " " + element._get_full_name() + "\n" raise PipelineError("Inconsistent pipeline", detail=detail, reason="inconsistent-pipeline-workspaced") @@ -449,8 +438,7 @@ class Pipeline(): uncached = [] with self._context.messenger.timed_activity("Checking sources"): for element in elements: - if element._get_consistency() < Consistency.CACHED and \ - not element._source_cached(): + if element._get_consistency() < Consistency.CACHED and not element._source_cached(): uncached.append(element) if uncached: @@ -460,9 +448,11 @@ class Pipeline(): for source in element.sources(): if source._get_consistency() < Consistency.CACHED: detail += " {}\n".format(source) - detail += '\n' - detail += "Try fetching these elements first with `bst source fetch`,\n" + \ - "or run this command with `--fetch` option\n" + detail += "\n" + detail += ( + "Try fetching these elements first with `bst source fetch`,\n" + + "or run this command with `--fetch` option\n" + ) raise PipelineError("Uncached sources", detail=detail, reason="uncached-sources") @@ -483,10 +473,7 @@ class Pipeline(): # not contain any cross junction elements. # def _filter_cross_junctions(self, project, elements): - return [ - element for element in elements - if element._get_project() is project - ] + return [element for element in elements if element._get_project() is project] # _assert_junction_tracking() # @@ -511,8 +498,10 @@ class Pipeline(): for element in elements: element_project = element._get_project() if element_project is not self._project: - detail = "Requested to track sources across junction boundaries\n" + \ - "in a project which does not use project.refs ref-storage." + detail = ( + "Requested to track sources across junction boundaries\n" + + "in a project which does not use project.refs ref-storage." + ) raise PipelineError("Untrackable sources", detail=detail, reason="untrackable-sources") @@ -522,8 +511,7 @@ class Pipeline(): # def _message(self, message_type, message, **kwargs): args = dict(kwargs) - self._context.messenger.message( - Message(message_type, message, **args)) + self._context.messenger.message(Message(message_type, message, **args)) # _Planner() @@ -533,7 +521,7 @@ class Pipeline(): # parts need to be built depending on build only dependencies # being cached, and depth sorting for more efficient processing. # -class _Planner(): +class _Planner: def __init__(self): self.depth_map = OrderedDict() self.visiting_elements = set() diff --git a/src/buildstream/_platform/darwin.py b/src/buildstream/_platform/darwin.py index f23535373..06491e8b4 100644 --- a/src/buildstream/_platform/darwin.py +++ b/src/buildstream/_platform/darwin.py @@ -59,9 +59,9 @@ class Darwin(Platform): @staticmethod def _create_dummy_sandbox(*args, **kwargs): - kwargs['dummy_reason'] = \ - "OSXFUSE is not supported and there are no supported sandbox " + \ - "technologies for MacOS at this time" + kwargs["dummy_reason"] = ( + "OSXFUSE is not supported and there are no supported sandbox " + "technologies for MacOS at this time" + ) return SandboxDummy(*args, **kwargs) def _setup_dummy_sandbox(self): diff --git a/src/buildstream/_platform/fallback.py b/src/buildstream/_platform/fallback.py index 4f7ff8086..b9e9f520d 100644 --- a/src/buildstream/_platform/fallback.py +++ b/src/buildstream/_platform/fallback.py @@ -20,15 +20,15 @@ from .platform import Platform class Fallback(Platform): - def _check_dummy_sandbox_config(self, config): return True def _create_dummy_sandbox(self, *args, **kwargs): - kwargs['dummy_reason'] = \ - ("FallBack platform only implements dummy sandbox, " - "Buildstream may be having issues correctly detecting your platform, " - "platform can be forced with BST_FORCE_BACKEND") + kwargs["dummy_reason"] = ( + "FallBack platform only implements dummy sandbox, " + "Buildstream may be having issues correctly detecting your platform, " + "platform can be forced with BST_FORCE_BACKEND" + ) return SandboxDummy(*args, **kwargs) def _setup_dummy_sandbox(self): diff --git a/src/buildstream/_platform/linux.py b/src/buildstream/_platform/linux.py index b400bfaac..bdc2e0df1 100644 --- a/src/buildstream/_platform/linux.py +++ b/src/buildstream/_platform/linux.py @@ -28,17 +28,16 @@ from .._exceptions import PlatformError class Linux(Platform): - def _setup_sandbox(self, force_sandbox): sandbox_setups = { - 'bwrap': self._setup_bwrap_sandbox, - 'buildbox': self._setup_buildbox_sandbox, - 'chroot': self._setup_chroot_sandbox, - 'dummy': self._setup_dummy_sandbox, + "bwrap": self._setup_bwrap_sandbox, + "buildbox": self._setup_buildbox_sandbox, + "chroot": self._setup_chroot_sandbox, + "dummy": self._setup_dummy_sandbox, } preferred_sandboxes = [ - 'bwrap', + "bwrap", ] self._try_sandboxes(force_sandbox, sandbox_setups, preferred_sandboxes) @@ -54,11 +53,12 @@ class Linux(Platform): def can_crossbuild(self, config): host_arch = self.get_host_arch() - if ((config.build_arch == "x86-32" and host_arch == "x86-64") or - (config.build_arch == "aarch32" and host_arch == "aarch64")): + if (config.build_arch == "x86-32" and host_arch == "x86-64") or ( + config.build_arch == "aarch32" and host_arch == "aarch64" + ): if self.linux32 is None: try: - utils.get_host_tool('linux32') + utils.get_host_tool("linux32") self.linux32 = True except utils.ProgramNotFoundError: self.linux32 = False @@ -76,7 +76,7 @@ class Linux(Platform): def _create_dummy_sandbox(self, *args, **kwargs): dummy_reasons = " and ".join(self.dummy_reasons) - kwargs['dummy_reason'] = dummy_reasons + kwargs["dummy_reason"] = dummy_reasons return SandboxDummy(*args, **kwargs) def _setup_dummy_sandbox(self): @@ -87,11 +87,13 @@ class Linux(Platform): # Bubble-wrap sandbox methods def _check_sandbox_config_bwrap(self, config): from ..sandbox._sandboxbwrap import SandboxBwrap + return SandboxBwrap.check_sandbox_config(self, config) def _create_bwrap_sandbox(self, *args, **kwargs): from ..sandbox._sandboxbwrap import SandboxBwrap - kwargs['linux32'] = self.linux32 + + kwargs["linux32"] = self.linux32 return SandboxBwrap(*args, **kwargs) def _setup_bwrap_sandbox(self): @@ -110,15 +112,18 @@ class Linux(Platform): # Chroot sandbox methods def _check_sandbox_config_chroot(self, config): from ..sandbox._sandboxchroot import SandboxChroot + return SandboxChroot.check_sandbox_config(self, config) @staticmethod def _create_chroot_sandbox(*args, **kwargs): from ..sandbox._sandboxchroot import SandboxChroot + return SandboxChroot(*args, **kwargs) def _setup_chroot_sandbox(self): from ..sandbox._sandboxchroot import SandboxChroot + self._check_sandbox(SandboxChroot) self.check_sandbox_config = self._check_sandbox_config_chroot self.create_sandbox = Linux._create_chroot_sandbox @@ -127,18 +132,23 @@ class Linux(Platform): # Buildbox sandbox methods def _check_sandbox_config_buildbox(self, config): from ..sandbox._sandboxbuildbox import SandboxBuildBox + return SandboxBuildBox.check_sandbox_config(self, config) @staticmethod def _create_buildbox_sandbox(*args, **kwargs): from ..sandbox._sandboxbuildbox import SandboxBuildBox - if kwargs.get('allow_real_directory'): - raise PlatformError("The BuildBox Sandbox does not support real directories.", - reason="You are using BuildBox sandbox because BST_FORCE_SANBOX=buildbox") + + if kwargs.get("allow_real_directory"): + raise PlatformError( + "The BuildBox Sandbox does not support real directories.", + reason="You are using BuildBox sandbox because BST_FORCE_SANBOX=buildbox", + ) return SandboxBuildBox(*args, **kwargs) def _setup_buildbox_sandbox(self): from ..sandbox._sandboxbuildbox import SandboxBuildBox + self._check_sandbox(SandboxBuildBox) self.check_sandbox_config = self._check_sandbox_config_buildbox self.create_sandbox = self._create_buildbox_sandbox diff --git a/src/buildstream/_platform/platform.py b/src/buildstream/_platform/platform.py index af49b9e82..1fddbe82c 100644 --- a/src/buildstream/_platform/platform.py +++ b/src/buildstream/_platform/platform.py @@ -29,7 +29,7 @@ from .._exceptions import PlatformError, ImplError, SandboxError from .. import utils -class Platform(): +class Platform: # Platform() # # A class to manage platform-specific details. Currently holds the @@ -45,7 +45,7 @@ class Platform(): self._setup_sandbox(force_sandbox) def _setup_sandbox(self, force_sandbox): - sandbox_setups = {'dummy': self._setup_dummy_sandbox} + sandbox_setups = {"dummy": self._setup_dummy_sandbox} preferred_sandboxes = [] self._try_sandboxes(force_sandbox, sandbox_setups, preferred_sandboxes) @@ -58,12 +58,16 @@ class Platform(): try: sandbox_setups[force_sandbox]() except KeyError: - raise PlatformError("Forced Sandbox is unavailable on this platform: BST_FORCE_SANDBOX" - " is set to {} but it is not available".format(force_sandbox)) + raise PlatformError( + "Forced Sandbox is unavailable on this platform: BST_FORCE_SANDBOX" + " is set to {} but it is not available".format(force_sandbox) + ) except SandboxError as Error: - raise PlatformError("Forced Sandbox Error: BST_FORCE_SANDBOX" - " is set to {} but cannot be setup".format(force_sandbox), - detail=" and ".join(self.dummy_reasons)) from Error + raise PlatformError( + "Forced Sandbox Error: BST_FORCE_SANDBOX" + " is set to {} but cannot be setup".format(force_sandbox), + detail=" and ".join(self.dummy_reasons), + ) from Error else: for good_sandbox in preferred_sandboxes: try: @@ -73,7 +77,7 @@ class Platform(): continue except utils.ProgramNotFoundError: continue - sandbox_setups['dummy']() + sandbox_setups["dummy"]() def _check_sandbox(self, Sandbox): try: @@ -87,29 +91,29 @@ class Platform(): # Meant for testing purposes and therefore hidden in the # deepest corners of the source code. Try not to abuse this, # please? - if os.getenv('BST_FORCE_SANDBOX'): - force_sandbox = os.getenv('BST_FORCE_SANDBOX') + if os.getenv("BST_FORCE_SANDBOX"): + force_sandbox = os.getenv("BST_FORCE_SANDBOX") else: force_sandbox = None - if os.getenv('BST_FORCE_BACKEND'): - backend = os.getenv('BST_FORCE_BACKEND') - elif sys.platform.startswith('darwin'): - backend = 'darwin' - elif sys.platform.startswith('linux'): - backend = 'linux' - elif sys.platform == 'win32': - backend = 'win32' + if os.getenv("BST_FORCE_BACKEND"): + backend = os.getenv("BST_FORCE_BACKEND") + elif sys.platform.startswith("darwin"): + backend = "darwin" + elif sys.platform.startswith("linux"): + backend = "linux" + elif sys.platform == "win32": + backend = "win32" else: - backend = 'fallback' + backend = "fallback" - if backend == 'linux': + if backend == "linux": from .linux import Linux as PlatformImpl # pylint: disable=cyclic-import - elif backend == 'darwin': + elif backend == "darwin": from .darwin import Darwin as PlatformImpl # pylint: disable=cyclic-import - elif backend == 'win32': + elif backend == "win32": from .win32 import Win32 as PlatformImpl # pylint: disable=cyclic-import - elif backend == 'fallback': + elif backend == "fallback": from .fallback import Fallback as PlatformImpl # pylint: disable=cyclic-import else: raise PlatformError("No such platform: '{}'".format(backend)) @@ -156,11 +160,11 @@ class Platform(): "sparc64": "sparc-v9", "sparc-v9": "sparc-v9", "x86-32": "x86-32", - "x86-64": "x86-64" + "x86-64": "x86-64", } try: - return aliases[arch.replace('_', '-').lower()] + return aliases[arch.replace("_", "-").lower()] except KeyError: raise PlatformError("Unknown architecture: {}".format(arch)) @@ -188,7 +192,7 @@ class Platform(): def does_multiprocessing_start_require_pickling(self): # Note that if the start method has not been set before now, it will be # set to the platform default by `get_start_method`. - return multiprocessing.get_start_method() != 'fork' + return multiprocessing.get_start_method() != "fork" ################################################################## # Sandbox functions # @@ -206,12 +210,12 @@ class Platform(): # (Sandbox) A sandbox # def create_sandbox(self, *args, **kwargs): - raise ImplError("Platform {platform} does not implement create_sandbox()" - .format(platform=type(self).__name__)) + raise ImplError("Platform {platform} does not implement create_sandbox()".format(platform=type(self).__name__)) def check_sandbox_config(self, config): - raise ImplError("Platform {platform} does not implement check_sandbox_config()" - .format(platform=type(self).__name__)) + raise ImplError( + "Platform {platform} does not implement check_sandbox_config()".format(platform=type(self).__name__) + ) def maximize_open_file_limit(self): # Need to set resources for _frontend/app.py as this is dependent on the platform @@ -230,5 +234,6 @@ class Platform(): resource.setrlimit(resource.RLIMIT_NOFILE, (hard_limit, hard_limit)) def _setup_dummy_sandbox(self): - raise ImplError("Platform {platform} does not implement _setup_dummy_sandbox()" - .format(platform=type(self).__name__)) + raise ImplError( + "Platform {platform} does not implement _setup_dummy_sandbox()".format(platform=type(self).__name__) + ) diff --git a/src/buildstream/_platform/win32.py b/src/buildstream/_platform/win32.py index 36680019d..a2529d8f6 100644 --- a/src/buildstream/_platform/win32.py +++ b/src/buildstream/_platform/win32.py @@ -20,7 +20,6 @@ from .platform import Platform class Win32(Platform): - def maximize_open_file_limit(self): # Note that on Windows, we don't have the 'resource' module to help us # configure open file limits. @@ -50,7 +49,7 @@ class Win32(Platform): @staticmethod def _create_dummy_sandbox(*args, **kwargs): - kwargs['dummy_reason'] = "There are no supported sandbox technologies for Win32 at this time." + kwargs["dummy_reason"] = "There are no supported sandbox technologies for Win32 at this time." return SandboxDummy(*args, **kwargs) def _setup_dummy_sandbox(self): diff --git a/src/buildstream/_plugincontext.py b/src/buildstream/_plugincontext.py index b07c2b31a..54839e16b 100644 --- a/src/buildstream/_plugincontext.py +++ b/src/buildstream/_plugincontext.py @@ -41,10 +41,8 @@ from . import utils # a given BuildStream project are isolated to their respective # Pipelines. # -class PluginContext(): - - def __init__(self, plugin_base, base_type, site_plugin_path, *, - plugin_origins=None, format_versions={}): +class PluginContext: + def __init__(self, plugin_base, base_type, site_plugin_path, *, plugin_origins=None, format_versions={}): # For pickling across processes, make sure this context has a unique # identifier, which we prepend to the identifier of each PluginSource. @@ -59,7 +57,7 @@ class PluginContext(): # Private members # self._base_type = base_type # The base class plugins derive from - self._types = {} # Plugin type lookup table by kind + self._types = {} # Plugin type lookup table by kind self._plugin_origins = plugin_origins or [] # The PluginSource object @@ -72,8 +70,7 @@ class PluginContext(): def _init_site_source(self): self._site_source = self._plugin_base.make_plugin_source( - searchpath=self._site_plugin_path, - identifier=self._identifier + 'site', + searchpath=self._site_plugin_path, identifier=self._identifier + "site", ) def __getstate__(self): @@ -93,11 +90,11 @@ class PluginContext(): # this by making sure we are not creating new members, only clearing # existing ones. # - del state['_site_source'] - assert '_types' in state - state['_types'] = {} - assert '_alternate_sources' in state - state['_alternate_sources'] = {} + del state["_site_source"] + assert "_types" in state + state["_types"] = {} + assert "_alternate_sources" in state + state["_alternate_sources"] = {} return state @@ -133,60 +130,51 @@ class PluginContext(): return self._types.values() def _get_local_plugin_source(self, path): - if ('local', path) not in self._alternate_sources: + if ("local", path) not in self._alternate_sources: # key by a tuple to avoid collision - source = self._plugin_base.make_plugin_source( - searchpath=[path], - identifier=self._identifier + path, - ) + source = self._plugin_base.make_plugin_source(searchpath=[path], identifier=self._identifier + path,) # Ensure that sources never get garbage collected, # as they'll take the plugins with them. - self._alternate_sources[('local', path)] = source + self._alternate_sources[("local", path)] = source else: - source = self._alternate_sources[('local', path)] + source = self._alternate_sources[("local", path)] return source def _get_pip_plugin_source(self, package_name, kind): defaults = None - if ('pip', package_name) not in self._alternate_sources: + if ("pip", package_name) not in self._alternate_sources: import pkg_resources + # key by a tuple to avoid collision try: - package = pkg_resources.get_entry_info(package_name, - 'buildstream.plugins', - kind) + package = pkg_resources.get_entry_info(package_name, "buildstream.plugins", kind) except pkg_resources.DistributionNotFound as e: - raise PluginError("Failed to load {} plugin '{}': {}" - .format(self._base_type.__name__, kind, e)) from e + raise PluginError("Failed to load {} plugin '{}': {}".format(self._base_type.__name__, kind, e)) from e if package is None: - raise PluginError("Pip package {} does not contain a plugin named '{}'" - .format(package_name, kind)) + raise PluginError("Pip package {} does not contain a plugin named '{}'".format(package_name, kind)) location = package.dist.get_resource_filename( - pkg_resources._manager, - package.module_name.replace('.', os.sep) + '.py' + pkg_resources._manager, package.module_name.replace(".", os.sep) + ".py" ) # Also load the defaults - required since setuptools # may need to extract the file. try: defaults = package.dist.get_resource_filename( - pkg_resources._manager, - package.module_name.replace('.', os.sep) + '.yaml' + pkg_resources._manager, package.module_name.replace(".", os.sep) + ".yaml" ) except KeyError: # The plugin didn't have an accompanying YAML file defaults = None source = self._plugin_base.make_plugin_source( - searchpath=[os.path.dirname(location)], - identifier=self._identifier + os.path.dirname(location), + searchpath=[os.path.dirname(location)], identifier=self._identifier + os.path.dirname(location), ) - self._alternate_sources[('pip', package_name)] = source + self._alternate_sources[("pip", package_name)] = source else: - source = self._alternate_sources[('pip', package_name)] + source = self._alternate_sources[("pip", package_name)] return source, defaults @@ -199,27 +187,27 @@ class PluginContext(): loaded_dependency = False for origin in self._plugin_origins: - if kind not in origin.get_str_list('plugins'): + if kind not in origin.get_str_list("plugins"): continue - if origin.get_str('origin') == 'local': - local_path = origin.get_str('path') + if origin.get_str("origin") == "local": + local_path = origin.get_str("path") source = self._get_local_plugin_source(local_path) - elif origin.get_str('origin') == 'pip': - package_name = origin.get_str('package-name') + elif origin.get_str("origin") == "pip": + package_name = origin.get_str("package-name") source, defaults = self._get_pip_plugin_source(package_name, kind) else: - raise PluginError("Failed to load plugin '{}': " - "Unexpected plugin origin '{}'" - .format(kind, origin.get_str('origin'))) + raise PluginError( + "Failed to load plugin '{}': " + "Unexpected plugin origin '{}'".format(kind, origin.get_str("origin")) + ) loaded_dependency = True break # Fall back to getting the source from site if not source: if kind not in self._site_source.list_plugins(): - raise PluginError("No {} type registered for kind '{}'" - .format(self._base_type.__name__, kind)) + raise PluginError("No {} type registered for kind '{}'".format(self._base_type.__name__, kind)) source = self._site_source @@ -241,17 +229,18 @@ class PluginContext(): defaults = os.path.join(plugin_dir, plugin_conf_name) except ImportError as e: - raise PluginError("Failed to load {} plugin '{}': {}" - .format(self._base_type.__name__, kind, e)) from e + raise PluginError("Failed to load {} plugin '{}': {}".format(self._base_type.__name__, kind, e)) from e try: plugin_type = plugin.setup() except AttributeError as e: - raise PluginError("{} plugin '{}' did not provide a setup() function" - .format(self._base_type.__name__, kind)) from e + raise PluginError( + "{} plugin '{}' did not provide a setup() function".format(self._base_type.__name__, kind) + ) from e except TypeError as e: - raise PluginError("setup symbol in {} plugin '{}' is not a function" - .format(self._base_type.__name__, kind)) from e + raise PluginError( + "setup symbol in {} plugin '{}' is not a function".format(self._base_type.__name__, kind) + ) from e self._assert_plugin(kind, plugin_type) self._assert_version(kind, plugin_type) @@ -259,19 +248,23 @@ class PluginContext(): def _assert_plugin(self, kind, plugin_type): if kind in self._types: - raise PluginError("Tried to register {} plugin for existing kind '{}' " - "(already registered {})" - .format(self._base_type.__name__, kind, self._types[kind].__name__)) + raise PluginError( + "Tried to register {} plugin for existing kind '{}' " + "(already registered {})".format(self._base_type.__name__, kind, self._types[kind].__name__) + ) try: if not issubclass(plugin_type, self._base_type): - raise PluginError("{} plugin '{}' returned type '{}', which is not a subclass of {}" - .format(self._base_type.__name__, kind, - plugin_type.__name__, - self._base_type.__name__)) + raise PluginError( + "{} plugin '{}' returned type '{}', which is not a subclass of {}".format( + self._base_type.__name__, kind, plugin_type.__name__, self._base_type.__name__ + ) + ) except TypeError as e: - raise PluginError("{} plugin '{}' returned something that is not a type (expected subclass of {})" - .format(self._base_type.__name__, kind, - self._base_type.__name__)) from e + raise PluginError( + "{} plugin '{}' returned something that is not a type (expected subclass of {})".format( + self._base_type.__name__, kind, self._base_type.__name__ + ) + ) from e def _assert_version(self, kind, plugin_type): @@ -282,12 +275,16 @@ class PluginContext(): req_minor = plugin_type.BST_REQUIRED_VERSION_MINOR if (bst_major, bst_minor) < (req_major, req_minor): - raise PluginError("BuildStream {}.{} is too old for {} plugin '{}' (requires {}.{})" - .format( - bst_major, bst_minor, - self._base_type.__name__, kind, - plugin_type.BST_REQUIRED_VERSION_MAJOR, - plugin_type.BST_REQUIRED_VERSION_MINOR)) + raise PluginError( + "BuildStream {}.{} is too old for {} plugin '{}' (requires {}.{})".format( + bst_major, + bst_minor, + self._base_type.__name__, + kind, + plugin_type.BST_REQUIRED_VERSION_MAJOR, + plugin_type.BST_REQUIRED_VERSION_MINOR, + ) + ) # _assert_plugin_format() # @@ -296,6 +293,9 @@ class PluginContext(): # def _assert_plugin_format(self, plugin, version): if plugin.BST_FORMAT_VERSION < version: - raise LoadError("{}: Format version {} is too old for requested version {}" - .format(plugin, plugin.BST_FORMAT_VERSION, version), - LoadErrorReason.UNSUPPORTED_PLUGIN) + raise LoadError( + "{}: Format version {} is too old for requested version {}".format( + plugin, plugin.BST_FORMAT_VERSION, version + ), + LoadErrorReason.UNSUPPORTED_PLUGIN, + ) diff --git a/src/buildstream/_profile.py b/src/buildstream/_profile.py index c68d058ad..fdde04ab7 100644 --- a/src/buildstream/_profile.py +++ b/src/buildstream/_profile.py @@ -40,15 +40,15 @@ from ._exceptions import ProfileError # BST_PROFILE=circ-dep-check:sort-deps bst <command> <args> # # The special 'all' value will enable all profiles. -class Topics(): - CIRCULAR_CHECK = 'circ-dep-check' - SORT_DEPENDENCIES = 'sort-deps' - LOAD_CONTEXT = 'load-context' - LOAD_PROJECT = 'load-project' - LOAD_PIPELINE = 'load-pipeline' - LOAD_SELECTION = 'load-selection' - SCHEDULER = 'scheduler' - ALL = 'all' +class Topics: + CIRCULAR_CHECK = "circ-dep-check" + SORT_DEPENDENCIES = "sort-deps" + LOAD_CONTEXT = "load-context" + LOAD_PROJECT = "load-project" + LOAD_PIPELINE = "load-pipeline" + LOAD_SELECTION = "load-selection" + SCHEDULER = "scheduler" + ALL = "all" class _Profile: @@ -64,8 +64,8 @@ class _Profile: os.getcwd(), "profile-{}-{}".format( datetime.datetime.fromtimestamp(self.start_time).strftime("%Y%m%dT%H%M%S"), - self.key.replace("/", "-").replace(".", "-") - ) + self.key.replace("/", "-").replace(".", "-"), + ), ) self.log_filename = "{}.log".format(filename_template) self.cprofile_filename = "{}.cprofile".format(filename_template) @@ -87,14 +87,16 @@ class _Profile: self.profiler.disable() def save(self): - heading = "\n".join([ - "-" * 64, - "Profile for key: {}".format(self.key), - "Started at: {}".format(self.start_time), - "\n\t{}".format(self.message) if self.message else "", - "-" * 64, - "" # for a final new line - ]) + heading = "\n".join( + [ + "-" * 64, + "Profile for key: {}".format(self.key), + "Started at: {}".format(self.start_time), + "\n\t{}".format(self.message) if self.message else "", + "-" * 64, + "", # for a final new line + ] + ) with open(self.log_filename, "a") as fp: stats = pstats.Stats(self.profiler, *self._additional_pstats_files, stream=fp) @@ -116,10 +118,7 @@ class _Profiler: self._valid_topics = False if settings: - self.enabled_topics = { - topic - for topic in settings.split(":") - } + self.enabled_topics = {topic for topic in settings.split(":")} @contextlib.contextmanager def profile(self, topic, key, message=None): @@ -170,8 +169,7 @@ class _Profiler: non_valid_topics = [topic for topic in self.enabled_topics if topic not in vars(Topics).values()] if non_valid_topics: - raise ProfileError("Provided BST_PROFILE topics do not exist: {}" - .format(", ".join(non_valid_topics))) + raise ProfileError("Provided BST_PROFILE topics do not exist: {}".format(", ".join(non_valid_topics))) self._valid_topics = True diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py index 54a011e0d..67d41a6b5 100644 --- a/src/buildstream/_project.py +++ b/src/buildstream/_project.py @@ -49,7 +49,7 @@ from ._workspaces import WORKSPACE_PROJECT_FILE # Project Configuration file -_PROJECT_CONF_FILE = 'project.conf' +_PROJECT_CONF_FILE = "project.conf" # List of all places plugins can come from @@ -64,8 +64,7 @@ class PluginOrigins(FastEnum): # A simple object describing the behavior of # a host mount. # -class HostMount(): - +class HostMount: def __init__(self, path, host_path=None, optional=False): # Support environment variable expansion in host mounts @@ -73,9 +72,9 @@ class HostMount(): if host_path is not None: host_path = os.path.expandvars(host_path) - self.path = path # Path inside the sandbox - self.host_path = host_path # Path on the host - self.optional = optional # Optional mounts do not incur warnings or errors + self.path = path # Path inside the sandbox + self.host_path = host_path # Path on the host + self.optional = optional # Optional mounts do not incur warnings or errors if self.host_path is None: self.host_path = self.path @@ -86,24 +85,32 @@ class ProjectConfig: def __init__(self): self.element_factory = None self.source_factory = None - self.options = None # OptionPool - self.base_variables = {} # The base set of variables - self.element_overrides = {} # Element specific configurations - self.source_overrides = {} # Source specific configurations - self.mirrors = OrderedDict() # contains dicts of alias-mappings to URIs. - self.default_mirror = None # The name of the preferred mirror. - self._aliases = None # Aliases dictionary + self.options = None # OptionPool + self.base_variables = {} # The base set of variables + self.element_overrides = {} # Element specific configurations + self.source_overrides = {} # Source specific configurations + self.mirrors = OrderedDict() # contains dicts of alias-mappings to URIs. + self.default_mirror = None # The name of the preferred mirror. + self._aliases = None # Aliases dictionary # Project() # # The Project Configuration # -class Project(): - - def __init__(self, directory, context, *, junction=None, cli_options=None, - default_mirror=None, parent_loader=None, - search_for_project=True, fetch_subprojects=None): +class Project: + def __init__( + self, + directory, + context, + *, + junction=None, + cli_options=None, + default_mirror=None, + parent_loader=None, + search_for_project=True, + fetch_subprojects=None + ): # The project name self.name = None @@ -125,31 +132,31 @@ class Project(): self._default_targets = None # ProjectRefs for the main refs and also for junctions - self.refs = ProjectRefs(self.directory, 'project.refs') - self.junction_refs = ProjectRefs(self.directory, 'junction.refs') + self.refs = ProjectRefs(self.directory, "project.refs") + self.junction_refs = ProjectRefs(self.directory, "junction.refs") self.config = ProjectConfig() self.first_pass_config = ProjectConfig() - self.junction = junction # The junction Element object, if this is a subproject + self.junction = junction # The junction Element object, if this is a subproject - self.ref_storage = None # ProjectRefStorage setting - self.base_environment = {} # The base set of environment variables - self.base_env_nocache = None # The base nocache mask (list) for the environment + self.ref_storage = None # ProjectRefStorage setting + self.base_environment = {} # The base set of environment variables + self.base_env_nocache = None # The base nocache mask (list) for the environment # # Private Members # - self._default_mirror = default_mirror # The name of the preferred mirror. + self._default_mirror = default_mirror # The name of the preferred mirror. self._cli_options = cli_options - self._fatal_warnings = [] # A list of warnings which should trigger an error + self._fatal_warnings = [] # A list of warnings which should trigger an error - self._shell_command = [] # The default interactive shell command + self._shell_command = [] # The default interactive shell command self._shell_environment = {} # Statically set environment vars - self._shell_host_files = [] # A list of HostMount objects + self._shell_host_files = [] # A list of HostMount objects self.artifact_cache_specs = None self.source_cache_specs = None @@ -163,7 +170,7 @@ class Project(): self._fully_loaded = False self._project_includes = None - with PROFILER.profile(Topics.LOAD_PROJECT, self.directory.replace(os.sep, '-')): + with PROFILER.profile(Topics.LOAD_PROJECT, self.directory.replace(os.sep, "-")): self._load(parent_loader=parent_loader, fetch_subprojects=fetch_subprojects) self._partially_loaded = True @@ -252,23 +259,24 @@ class Project(): # (LoadError): In case that the project path is not valid or does not # exist # - def get_path_from_node(self, node, *, - check_is_file=False, check_is_dir=False): + def get_path_from_node(self, node, *, check_is_file=False, check_is_dir=False): path_str = node.as_str() path = Path(path_str) full_path = self._absolute_directory_path / path if full_path.is_symlink(): provenance = node.get_provenance() - raise LoadError("{}: Specified path '{}' must not point to " - "symbolic links ".format(provenance, path_str), - LoadErrorReason.PROJ_PATH_INVALID_KIND) + raise LoadError( + "{}: Specified path '{}' must not point to " "symbolic links ".format(provenance, path_str), + LoadErrorReason.PROJ_PATH_INVALID_KIND, + ) - if path.parts and path.parts[0] == '..': + if path.parts and path.parts[0] == "..": provenance = node.get_provenance() - raise LoadError("{}: Specified path '{}' first component must " - "not be '..'".format(provenance, path_str), - LoadErrorReason.PROJ_PATH_INVALID) + raise LoadError( + "{}: Specified path '{}' first component must " "not be '..'".format(provenance, path_str), + LoadErrorReason.PROJ_PATH_INVALID, + ) try: if sys.version_info[0] == 3 and sys.version_info[1] < 6: @@ -277,55 +285,81 @@ class Project(): full_resolved_path = full_path.resolve(strict=True) # pylint: disable=unexpected-keyword-arg except FileNotFoundError: provenance = node.get_provenance() - raise LoadError("{}: Specified path '{}' does not exist".format(provenance, path_str), - LoadErrorReason.MISSING_FILE) + raise LoadError( + "{}: Specified path '{}' does not exist".format(provenance, path_str), LoadErrorReason.MISSING_FILE + ) is_inside = self._absolute_directory_path in full_resolved_path.parents or ( - full_resolved_path == self._absolute_directory_path) + full_resolved_path == self._absolute_directory_path + ) if not is_inside: provenance = node.get_provenance() - raise LoadError("{}: Specified path '{}' must not lead outside of the " - "project directory".format(provenance, path_str), - LoadErrorReason.PROJ_PATH_INVALID) + raise LoadError( + "{}: Specified path '{}' must not lead outside of the " + "project directory".format(provenance, path_str), + LoadErrorReason.PROJ_PATH_INVALID, + ) if path.is_absolute(): provenance = node.get_provenance() - raise LoadError("{}: Absolute path: '{}' invalid.\n" - "Please specify a path relative to the project's root." - .format(provenance, path), LoadErrorReason.PROJ_PATH_INVALID) + raise LoadError( + "{}: Absolute path: '{}' invalid.\n" + "Please specify a path relative to the project's root.".format(provenance, path), + LoadErrorReason.PROJ_PATH_INVALID, + ) - if full_resolved_path.is_socket() or ( - full_resolved_path.is_fifo() or - full_resolved_path.is_block_device()): + if full_resolved_path.is_socket() or (full_resolved_path.is_fifo() or full_resolved_path.is_block_device()): provenance = node.get_provenance() - raise LoadError("{}: Specified path '{}' points to an unsupported " - "file kind".format(provenance, path_str), LoadErrorReason.PROJ_PATH_INVALID_KIND) + raise LoadError( + "{}: Specified path '{}' points to an unsupported " "file kind".format(provenance, path_str), + LoadErrorReason.PROJ_PATH_INVALID_KIND, + ) if check_is_file and not full_resolved_path.is_file(): provenance = node.get_provenance() - raise LoadError("{}: Specified path '{}' is not a regular file" - .format(provenance, path_str), LoadErrorReason.PROJ_PATH_INVALID_KIND) + raise LoadError( + "{}: Specified path '{}' is not a regular file".format(provenance, path_str), + LoadErrorReason.PROJ_PATH_INVALID_KIND, + ) if check_is_dir and not full_resolved_path.is_dir(): provenance = node.get_provenance() - raise LoadError("{}: Specified path '{}' is not a directory" - .format(provenance, path_str), LoadErrorReason.PROJ_PATH_INVALID_KIND) + raise LoadError( + "{}: Specified path '{}' is not a directory".format(provenance, path_str), + LoadErrorReason.PROJ_PATH_INVALID_KIND, + ) return path_str def _validate_node(self, node): - node.validate_keys([ - 'format-version', - 'element-path', 'variables', - 'environment', 'environment-nocache', - 'split-rules', 'elements', 'plugins', - 'aliases', 'name', 'defaults', - 'artifacts', 'options', - 'fail-on-overlap', 'shell', 'fatal-warnings', - 'ref-storage', 'sandbox', 'mirrors', 'remote-execution', - 'sources', 'source-caches', '(@)' - ]) + node.validate_keys( + [ + "format-version", + "element-path", + "variables", + "environment", + "environment-nocache", + "split-rules", + "elements", + "plugins", + "aliases", + "name", + "defaults", + "artifacts", + "options", + "fail-on-overlap", + "shell", + "fatal-warnings", + "ref-storage", + "sandbox", + "mirrors", + "remote-execution", + "sources", + "source-caches", + "(@)", + ] + ) # create_element() # @@ -438,10 +472,7 @@ class Project(): with self._context.messenger.simple_task("Resolving elements") as task: if task: task.set_maximum_progress(self.loader.loaded) - elements = [ - Element._new_from_meta(meta, task) - for meta in meta_elements - ] + elements = [Element._new_from_meta(meta, task) for meta in meta_elements] Element._clear_meta_elements_cache() @@ -450,13 +481,11 @@ class Project(): redundant_refs = Element._get_redundant_source_refs() if redundant_refs: detail = "The following inline specified source references will be ignored:\n\n" - lines = [ - "{}:{}".format(source._get_provenance(), ref) - for source, ref in redundant_refs - ] + lines = ["{}:{}".format(source._get_provenance(), ref) for source, ref in redundant_refs] detail += "\n".join(lines) self._context.messenger.message( - Message(MessageType.WARN, "Ignoring redundant source references", detail=detail)) + Message(MessageType.WARN, "Ignoring redundant source references", detail=detail) + ) return elements @@ -590,49 +619,49 @@ class Project(): self._project_conf._composite(pre_config_node) # Assert project's format version early, before validating toplevel keys - format_version = pre_config_node.get_int('format-version') + format_version = pre_config_node.get_int("format-version") if format_version < BST_FORMAT_VERSION_MIN: major, minor = utils.get_bst_version() raise LoadError( "Project requested format version {}, but BuildStream {}.{} only supports format version {} or above." - "Use latest 1.x release" - .format(format_version, major, minor, BST_FORMAT_VERSION_MIN), LoadErrorReason.UNSUPPORTED_PROJECT) + "Use latest 1.x release".format(format_version, major, minor, BST_FORMAT_VERSION_MIN), + LoadErrorReason.UNSUPPORTED_PROJECT, + ) if BST_FORMAT_VERSION < format_version: major, minor = utils.get_bst_version() raise LoadError( - "Project requested format version {}, but BuildStream {}.{} only supports up until format version {}" - .format(format_version, major, minor, BST_FORMAT_VERSION), LoadErrorReason.UNSUPPORTED_PROJECT) + "Project requested format version {}, but BuildStream {}.{} only supports up until format version {}".format( + format_version, major, minor, BST_FORMAT_VERSION + ), + LoadErrorReason.UNSUPPORTED_PROJECT, + ) self._validate_node(pre_config_node) # The project name, element path and option declarations # are constant and cannot be overridden by option conditional statements # FIXME: we should be keeping node information for further composition here - self.name = self._project_conf.get_str('name') + self.name = self._project_conf.get_str("name") # Validate that project name is a valid symbol name - _assert_symbol_name(self.name, "project name", - ref_node=pre_config_node.get_node('name')) + _assert_symbol_name(self.name, "project name", ref_node=pre_config_node.get_node("name")) self.element_path = os.path.join( - self.directory, - self.get_path_from_node(pre_config_node.get_scalar('element-path'), - check_is_dir=True) + self.directory, self.get_path_from_node(pre_config_node.get_scalar("element-path"), check_is_dir=True) ) self.config.options = OptionPool(self.element_path) self.first_pass_config.options = OptionPool(self.element_path) - defaults = pre_config_node.get_mapping('defaults') - defaults.validate_keys(['targets']) + defaults = pre_config_node.get_mapping("defaults") + defaults.validate_keys(["targets"]) self._default_targets = defaults.get_str_list("targets") # Fatal warnings - self._fatal_warnings = pre_config_node.get_str_list('fatal-warnings', default=[]) + self._fatal_warnings = pre_config_node.get_str_list("fatal-warnings", default=[]) - self.loader = Loader(self._context, self, - parent=parent_loader, fetch_subprojects=fetch_subprojects) + self.loader = Loader(self._context, self, parent=parent_loader, fetch_subprojects=fetch_subprojects) self._project_includes = Includes(self.loader, copy_tree=False) @@ -641,16 +670,17 @@ class Project(): config_no_include = self._default_config_node.clone() project_conf_first_pass._composite(config_no_include) - self._load_pass(config_no_include, self.first_pass_config, - ignore_unknown=True) + self._load_pass(config_no_include, self.first_pass_config, ignore_unknown=True) # Use separate file for storing source references - ref_storage_node = pre_config_node.get_scalar('ref-storage') + ref_storage_node = pre_config_node.get_scalar("ref-storage") self.ref_storage = ref_storage_node.as_str() if self.ref_storage not in [ProjectRefStorage.INLINE, ProjectRefStorage.PROJECT_REFS]: p = ref_storage_node.get_provenance() - raise LoadError("{}: Invalid value '{}' specified for ref-storage" - .format(p, self.ref_storage), LoadErrorReason.INVALID_DATA) + raise LoadError( + "{}: Invalid value '{}' specified for ref-storage".format(p, self.ref_storage), + LoadErrorReason.INVALID_DATA, + ) if self.ref_storage == ProjectRefStorage.PROJECT_REFS: self.junction_refs.load(self.first_pass_config.options) @@ -692,8 +722,7 @@ class Project(): # Load remote-execution configuration for this project project_specs = SandboxRemote.specs_from_config_node(config, self.directory) - override_specs = SandboxRemote.specs_from_config_node( - self._context.get_overrides(self.name), self.directory) + override_specs = SandboxRemote.specs_from_config_node(self._context.get_overrides(self.name), self.directory) if override_specs is not None: self.remote_execution_specs = override_specs @@ -703,25 +732,25 @@ class Project(): self.remote_execution_specs = self._context.remote_execution_specs # Load sandbox environment variables - self.base_environment = config.get_mapping('environment') - self.base_env_nocache = config.get_str_list('environment-nocache') + self.base_environment = config.get_mapping("environment") + self.base_env_nocache = config.get_str_list("environment-nocache") # Load sandbox configuration - self._sandbox = config.get_mapping('sandbox') + self._sandbox = config.get_mapping("sandbox") # Load project split rules - self._splits = config.get_mapping('split-rules') + self._splits = config.get_mapping("split-rules") # Support backwards compatibility for fail-on-overlap - fail_on_overlap = config.get_scalar('fail-on-overlap', None) + fail_on_overlap = config.get_scalar("fail-on-overlap", None) # Deprecation check if not fail_on_overlap.is_none(): self._context.messenger.message( Message( MessageType.WARN, - "Use of fail-on-overlap within project.conf " + - "is deprecated. Consider using fatal-warnings instead." + "Use of fail-on-overlap within project.conf " + + "is deprecated. Consider using fatal-warnings instead.", ) ) @@ -733,29 +762,29 @@ class Project(): self.refs.load(self.options) # Parse shell options - shell_options = config.get_mapping('shell') - shell_options.validate_keys(['command', 'environment', 'host-files']) - self._shell_command = shell_options.get_str_list('command') + shell_options = config.get_mapping("shell") + shell_options.validate_keys(["command", "environment", "host-files"]) + self._shell_command = shell_options.get_str_list("command") # Perform environment expansion right away - shell_environment = shell_options.get_mapping('environment', default={}) + shell_environment = shell_options.get_mapping("environment", default={}) for key in shell_environment.keys(): value = shell_environment.get_str(key) self._shell_environment[key] = os.path.expandvars(value) # Host files is parsed as a list for convenience - host_files = shell_options.get_sequence('host-files', default=[]) + host_files = shell_options.get_sequence("host-files", default=[]) for host_file in host_files: if isinstance(host_file, ScalarNode): mount = HostMount(host_file) else: # Some validation - host_file.validate_keys(['path', 'host_path', 'optional']) + host_file.validate_keys(["path", "host_path", "optional"]) # Parse the host mount - path = host_file.get_str('path') - host_path = host_file.get_str('host_path', default=None) - optional = host_file.get_bool('optional', default=False) + path = host_file.get_str("path") + host_path = host_file.get_str("host_path", default=None) + optional = host_file.get_bool("optional", default=False) mount = HostMount(path, host_path, optional) self._shell_host_files.append(mount) @@ -770,22 +799,21 @@ class Project(): # output (ProjectConfig) - ProjectConfig to load configuration onto. # ignore_unknown (bool) - Whether option loader shoud ignore unknown options. # - def _load_pass(self, config, output, *, - ignore_unknown=False): + def _load_pass(self, config, output, *, ignore_unknown=False): # Element and Source type configurations will be composited later onto # element/source types, so we delete it from here and run our final # assertion after. - output.element_overrides = config.get_mapping('elements', default={}) - output.source_overrides = config.get_mapping('sources', default={}) - config.safe_del('elements') - config.safe_del('sources') + output.element_overrides = config.get_mapping("elements", default={}) + output.source_overrides = config.get_mapping("sources", default={}) + config.safe_del("elements") + config.safe_del("sources") config._assert_fully_composited() self._load_plugin_factories(config, output) # Load project options - options_node = config.get_mapping('options', default={}) + options_node = config.get_mapping("options", default={}) output.options.load(options_node) if self.junction: # load before user configuration @@ -793,7 +821,7 @@ class Project(): # Collect option values specified in the user configuration overrides = self._context.get_overrides(self.name) - override_options = overrides.get_mapping('options', default={}) + override_options = overrides.get_mapping("options", default={}) output.options.load_yaml_values(override_options) if self._cli_options: output.options.load_cli_values(self._cli_options, ignore_unknown=ignore_unknown) @@ -812,10 +840,10 @@ class Project(): output.options.process_node(output.source_overrides) # Load base variables - output.base_variables = config.get_mapping('variables') + output.base_variables = config.get_mapping("variables") # Add the project name as a default variable - output.base_variables['project-name'] = self.name + output.base_variables["project-name"] = self.name # Extend variables with automatic variables and option exports # Initialize it as a string as all variables are processed as strings. @@ -825,27 +853,24 @@ class Project(): if self._context.build_max_jobs == 0: # User requested automatic max-jobs platform = self._context.platform - output.base_variables['max-jobs'] = str(platform.get_cpu_count(8)) + output.base_variables["max-jobs"] = str(platform.get_cpu_count(8)) else: # User requested explicit max-jobs setting - output.base_variables['max-jobs'] = str(self._context.build_max_jobs) + output.base_variables["max-jobs"] = str(self._context.build_max_jobs) # Export options into variables, if that was requested output.options.export_variables(output.base_variables) # Override default_mirror if not set by command-line - output.default_mirror = self._default_mirror or overrides.get_str( - 'default-mirror', default=None) + output.default_mirror = self._default_mirror or overrides.get_str("default-mirror", default=None) - mirrors = config.get_sequence('mirrors', default=[]) + mirrors = config.get_sequence("mirrors", default=[]) for mirror in mirrors: - allowed_mirror_fields = [ - 'name', 'aliases' - ] + allowed_mirror_fields = ["name", "aliases"] mirror.validate_keys(allowed_mirror_fields) - mirror_name = mirror.get_str('name') + mirror_name = mirror.get_str("name") alias_mappings = {} - for alias_mapping, uris in mirror.get_mapping('aliases').items(): + for alias_mapping, uris in mirror.get_mapping("aliases").items(): assert type(uris) is SequenceNode # pylint: disable=unidiomatic-typecheck alias_mappings[alias_mapping] = uris.as_str_list() output.mirrors[mirror_name] = alias_mappings @@ -853,7 +878,7 @@ class Project(): output.default_mirror = mirror_name # Source url aliases - output._aliases = config.get_mapping('aliases', default={}) + output._aliases = config.get_mapping("aliases", default={}) # _find_project_dir() # @@ -873,9 +898,7 @@ class Project(): def _find_project_dir(self, directory): workspace_element = None config_filenames = [_PROJECT_CONF_FILE, WORKSPACE_PROJECT_FILE] - found_directory, filename = utils._search_upward_for_files( - directory, config_filenames - ) + found_directory, filename = utils._search_upward_for_files(directory, config_filenames) if filename == _PROJECT_CONF_FILE: project_directory = found_directory elif filename == WORKSPACE_PROJECT_FILE: @@ -885,57 +908,62 @@ class Project(): project_directory = workspace_project.get_default_project_path() workspace_element = workspace_project.get_default_element() else: - raise LoadError("None of {names} found in '{path}' or any of its parent directories" - .format(names=config_filenames, path=directory), LoadErrorReason.MISSING_PROJECT_CONF) + raise LoadError( + "None of {names} found in '{path}' or any of its parent directories".format( + names=config_filenames, path=directory + ), + LoadErrorReason.MISSING_PROJECT_CONF, + ) return project_directory, workspace_element def _load_plugin_factories(self, config, output): - plugin_source_origins = [] # Origins of custom sources + plugin_source_origins = [] # Origins of custom sources plugin_element_origins = [] # Origins of custom elements # Plugin origins and versions - origins = config.get_sequence('plugins', default=[]) + origins = config.get_sequence("plugins", default=[]) source_format_versions = {} element_format_versions = {} for origin in origins: allowed_origin_fields = [ - 'origin', 'sources', 'elements', - 'package-name', 'path', + "origin", + "sources", + "elements", + "package-name", + "path", ] origin.validate_keys(allowed_origin_fields) # Store source versions for checking later - source_versions = origin.get_mapping('sources', default={}) + source_versions = origin.get_mapping("sources", default={}) for key in source_versions.keys(): if key in source_format_versions: - raise LoadError("Duplicate listing of source '{}'".format(key), - LoadErrorReason.INVALID_YAML) + raise LoadError("Duplicate listing of source '{}'".format(key), LoadErrorReason.INVALID_YAML) source_format_versions[key] = source_versions.get_int(key) # Store element versions for checking later - element_versions = origin.get_mapping('elements', default={}) + element_versions = origin.get_mapping("elements", default={}) for key in element_versions.keys(): if key in element_format_versions: - raise LoadError("Duplicate listing of element '{}'".format(key), - LoadErrorReason.INVALID_YAML) + raise LoadError("Duplicate listing of element '{}'".format(key), LoadErrorReason.INVALID_YAML) element_format_versions[key] = element_versions.get_int(key) # Store the origins if they're not 'core'. # core elements are loaded by default, so storing is unnecessary. - origin_value = origin.get_enum('origin', PluginOrigins) + origin_value = origin.get_enum("origin", PluginOrigins) if origin_value != PluginOrigins.CORE: - self._store_origin(origin, 'sources', plugin_source_origins) - self._store_origin(origin, 'elements', plugin_element_origins) + self._store_origin(origin, "sources", plugin_source_origins) + self._store_origin(origin, "elements", plugin_element_origins) - pluginbase = PluginBase(package='buildstream.plugins') - output.element_factory = ElementFactory(pluginbase, - plugin_origins=plugin_element_origins, - format_versions=element_format_versions) - output.source_factory = SourceFactory(pluginbase, - plugin_origins=plugin_source_origins, - format_versions=source_format_versions) + pluginbase = PluginBase(package="buildstream.plugins") + output.element_factory = ElementFactory( + pluginbase, plugin_origins=plugin_element_origins, format_versions=element_format_versions + ) + output.source_factory = SourceFactory( + pluginbase, plugin_origins=plugin_source_origins, format_versions=source_format_versions + ) # _store_origin() # @@ -951,25 +979,25 @@ class Project(): # Raises: # LoadError if 'origin' is an unexpected value def _store_origin(self, origin, plugin_group, destination): - expected_groups = ['sources', 'elements'] + expected_groups = ["sources", "elements"] if plugin_group not in expected_groups: - raise LoadError("Unexpected plugin group: {}, expecting {}" - .format(plugin_group, expected_groups), - LoadErrorReason.INVALID_DATA) + raise LoadError( + "Unexpected plugin group: {}, expecting {}".format(plugin_group, expected_groups), + LoadErrorReason.INVALID_DATA, + ) if plugin_group in origin.keys(): origin_node = origin.clone() plugins = origin.get_mapping(plugin_group, default={}) - origin_node['plugins'] = plugins.keys() + origin_node["plugins"] = plugins.keys() for group in expected_groups: if group in origin_node: del origin_node[group] - if origin_node.get_enum('origin', PluginOrigins) == PluginOrigins.LOCAL: - path = self.get_path_from_node(origin.get_scalar('path'), - check_is_dir=True) + if origin_node.get_enum("origin", PluginOrigins) == PluginOrigins.LOCAL: + path = self.get_path_from_node(origin.get_scalar("path"), check_is_dir=True) # paths are passed in relative to the project, but must be absolute - origin_node['path'] = os.path.join(self.directory, path) + origin_node["path"] = os.path.join(self.directory, path) destination.append(origin_node) # _warning_is_fatal(): diff --git a/src/buildstream/_projectrefs.py b/src/buildstream/_projectrefs.py index 0555488c8..aca7c6712 100644 --- a/src/buildstream/_projectrefs.py +++ b/src/buildstream/_projectrefs.py @@ -26,15 +26,15 @@ from ._exceptions import LoadError, LoadErrorReason # ProjectRefStorage() # # Indicates the type of ref storage -class ProjectRefStorage(): +class ProjectRefStorage: # Source references are stored inline # - INLINE = 'inline' + INLINE = "inline" # Source references are stored in a central project.refs file # - PROJECT_REFS = 'project.refs' + PROJECT_REFS = "project.refs" # ProjectRefs() @@ -45,8 +45,7 @@ class ProjectRefStorage(): # directory (str): The project directory # base_name (str): The project.refs basename # -class ProjectRefs(): - +class ProjectRefs: def __init__(self, directory, base_name): directory = os.path.abspath(directory) self._fullpath = os.path.join(directory, base_name) @@ -83,12 +82,12 @@ class ProjectRefs(): self._toplevel_node = _new_synthetic_file(self._fullpath) self._toplevel_save = self._toplevel_node - self._toplevel_node.validate_keys(['projects']) + self._toplevel_node.validate_keys(["projects"]) # Ensure we create our toplevel entry point on the fly here for node in [self._toplevel_node, self._toplevel_save]: - if 'projects' not in node: - node['projects'] = {} + if "projects" not in node: + node["projects"] = {} # lookup_ref() # @@ -122,7 +121,7 @@ class ProjectRefs(): # Looks up a ref node in the project.refs file, creates one if ensure is True. # def _lookup(self, toplevel, project, element, source_index, *, ensure=False): - projects = toplevel.get_mapping('projects') + projects = toplevel.get_mapping("projects") # Fetch the project try: diff --git a/src/buildstream/_remote.py b/src/buildstream/_remote.py index ab1dc1924..78f67726a 100644 --- a/src/buildstream/_remote.py +++ b/src/buildstream/_remote.py @@ -35,14 +35,14 @@ class RemoteType(FastEnum): ALL = "all" def __str__(self): - return self.name.lower().replace('_', '-') + return self.name.lower().replace("_", "-") # RemoteSpec(): # # Defines the basic structure of a remote specification. # -class RemoteSpec(namedtuple('RemoteSpec', 'url push server_cert client_key client_cert instance_name type')): +class RemoteSpec(namedtuple("RemoteSpec", "url push server_cert client_key client_cert instance_name type")): # new_from_config_node # @@ -60,15 +60,15 @@ class RemoteSpec(namedtuple('RemoteSpec', 'url push server_cert client_key clien # @classmethod def new_from_config_node(cls, spec_node, basedir=None): - spec_node.validate_keys(['url', 'push', 'server-cert', 'client-key', 'client-cert', 'instance-name', 'type']) + spec_node.validate_keys(["url", "push", "server-cert", "client-key", "client-cert", "instance-name", "type"]) - url = spec_node.get_str('url') + url = spec_node.get_str("url") if not url: - provenance = spec_node.get_node('url').get_provenance() + provenance = spec_node.get_node("url").get_provenance() raise LoadError("{}: empty artifact cache URL".format(provenance), LoadErrorReason.INVALID_DATA) - push = spec_node.get_bool('push', default=False) - instance_name = spec_node.get_str('instance-name', default=None) + push = spec_node.get_bool("push", default=False) + instance_name = spec_node.get_str("instance-name", default=None) def parse_cert(key): cert = spec_node.get_str(key, default=None) @@ -80,20 +80,22 @@ class RemoteSpec(namedtuple('RemoteSpec', 'url push server_cert client_key clien return cert - cert_keys = ('server-cert', 'client-key', 'client-cert') + cert_keys = ("server-cert", "client-key", "client-cert") server_cert, client_key, client_cert = tuple(parse_cert(key) for key in cert_keys) if client_key and not client_cert: - provenance = spec_node.get_node('client-key').get_provenance() - raise LoadError("{}: 'client-key' was specified without 'client-cert'".format(provenance), - LoadErrorReason.INVALID_DATA) + provenance = spec_node.get_node("client-key").get_provenance() + raise LoadError( + "{}: 'client-key' was specified without 'client-cert'".format(provenance), LoadErrorReason.INVALID_DATA + ) if client_cert and not client_key: - provenance = spec_node.get_node('client-cert').get_provenance() - raise LoadError("{}: 'client-cert' was specified without 'client-key'".format(provenance), - LoadErrorReason.INVALID_DATA) + provenance = spec_node.get_node("client-cert").get_provenance() + raise LoadError( + "{}: 'client-cert' was specified without 'client-key'".format(provenance), LoadErrorReason.INVALID_DATA + ) - type_ = spec_node.get_enum('type', RemoteType, default=RemoteType.ALL) + type_ = spec_node.get_enum("type", RemoteType, default=RemoteType.ALL) return cls(url, push, server_cert, client_key, client_cert, instance_name, type_) @@ -108,11 +110,11 @@ class RemoteSpec(namedtuple('RemoteSpec', 'url push server_cert client_key clien RemoteSpec.__new__.__defaults__ = ( # type: ignore # mandatory # url - The url of the remote # mandatory # push - Whether the remote should be used for pushing - None, # server_cert - The server certificate - None, # client_key - The (private) client key - None, # client_cert - The (public) client certificate - None, # instance_name - The (grpc) instance name of the remote - RemoteType.ALL # type - The type of the remote (index, storage, both) + None, # server_cert - The server certificate + None, # client_key - The (private) client key + None, # client_cert - The (public) client certificate + None, # instance_name - The (grpc) instance name of the remote + RemoteType.ALL, # type - The type of the remote (index, storage, both) ) @@ -126,7 +128,7 @@ RemoteSpec.__new__.__defaults__ = ( # type: ignore # Customization for the particular protocol is expected to be # performed in children. # -class BaseRemote(): +class BaseRemote: key_name = None def __init__(self, spec): @@ -154,25 +156,24 @@ class BaseRemote(): # Set up the communcation channel url = urlparse(self.spec.url) - if url.scheme == 'http': + if url.scheme == "http": port = url.port or 80 - self.channel = grpc.insecure_channel('{}:{}'.format(url.hostname, port)) - elif url.scheme == 'https': + self.channel = grpc.insecure_channel("{}:{}".format(url.hostname, port)) + elif url.scheme == "https": port = url.port or 443 try: server_cert, client_key, client_cert = _read_files( - self.spec.server_cert, - self.spec.client_key, - self.spec.client_cert) + self.spec.server_cert, self.spec.client_key, self.spec.client_cert + ) except FileNotFoundError as e: raise RemoteError("Could not read certificates: {}".format(e)) from e self.server_cert = server_cert self.client_key = client_key self.client_cert = client_cert - credentials = grpc.ssl_channel_credentials(root_certificates=self.server_cert, - private_key=self.client_key, - certificate_chain=self.client_cert) - self.channel = grpc.secure_channel('{}:{}'.format(url.hostname, port), credentials) + credentials = grpc.ssl_channel_credentials( + root_certificates=self.server_cert, private_key=self.client_key, certificate_chain=self.client_cert + ) + self.channel = grpc.secure_channel("{}:{}".format(url.hostname, port), credentials) else: raise RemoteError("Unsupported URL: {}".format(self.spec.url)) @@ -258,7 +259,8 @@ class BaseRemote(): def _read_files(*files): def read_file(f): if f: - with open(f, 'rb') as data: + with open(f, "rb") as data: return data.read() return None + return (read_file(f) for f in files) diff --git a/src/buildstream/_scheduler/jobs/elementjob.py b/src/buildstream/_scheduler/jobs/elementjob.py index 246eb75c6..6e035be9c 100644 --- a/src/buildstream/_scheduler/jobs/elementjob.py +++ b/src/buildstream/_scheduler/jobs/elementjob.py @@ -69,9 +69,9 @@ class ElementJob(Job): super().__init__(*args, **kwargs) self.set_name(element._get_full_name()) self.queue = queue - self._element = element # Set the Element pertaining to the job - self._action_cb = action_cb # The action callable function - self._complete_cb = complete_cb # The complete callable function + self._element = element # Set the Element pertaining to the job + self._action_cb = action_cb # The action callable function + self._complete_cb = complete_cb # The complete callable function # Set the plugin element name & key for logging purposes self.set_message_element_name(self.name) @@ -97,9 +97,7 @@ class ChildElementJob(ChildJob): # This should probably be omitted for non-build tasks but it's harmless here elt_env = self._element.get_environment() env_dump = yaml.round_trip_dump(elt_env, default_flow_style=False, allow_unicode=True) - self.message(MessageType.LOG, - "Build environment for element {}".format(self._element.name), - detail=env_dump) + self.message(MessageType.LOG, "Build environment for element {}".format(self._element.name), detail=env_dump) # Run the action return self._action_cb(self._element) @@ -109,6 +107,6 @@ class ChildElementJob(ChildJob): workspace = self._element._get_workspace() if workspace is not None: - data['workspace'] = workspace.to_dict() + data["workspace"] = workspace.to_dict() return data diff --git a/src/buildstream/_scheduler/jobs/job.py b/src/buildstream/_scheduler/jobs/job.py index 3363d7b60..e7866bcd4 100644 --- a/src/buildstream/_scheduler/jobs/job.py +++ b/src/buildstream/_scheduler/jobs/job.py @@ -66,7 +66,7 @@ class JobStatus(FastEnum): # Used to distinguish between status messages and return values -class _Envelope(): +class _Envelope: def __init__(self, message_type, message): self.message_type = message_type self.message = message @@ -115,35 +115,34 @@ class _MessageType(FastEnum): # that should be used - should contain {pid}. # max_retries (int): The maximum number of retries # -class Job(): - +class Job: def __init__(self, scheduler, action_name, logfile, *, max_retries=0): # # Public members # - self.name = None # The name of the job, set by the job's subclass - self.action_name = action_name # The action name for the Queue - self.child_data = None # Data to be sent to the main process + self.name = None # The name of the job, set by the job's subclass + self.action_name = action_name # The action name for the Queue + self.child_data = None # Data to be sent to the main process # # Private members # - self._scheduler = scheduler # The scheduler - self._queue = None # A message passing queue - self._process = None # The Process object - self._watcher = None # Child process watcher - self._listening = False # Whether the parent is currently listening - self._suspended = False # Whether this job is currently suspended - self._max_retries = max_retries # Maximum number of automatic retries - self._result = None # Return value of child action in the parent - self._tries = 0 # Try count, for retryable jobs - self._terminated = False # Whether this job has been explicitly terminated + self._scheduler = scheduler # The scheduler + self._queue = None # A message passing queue + self._process = None # The Process object + self._watcher = None # Child process watcher + self._listening = False # Whether the parent is currently listening + self._suspended = False # Whether this job is currently suspended + self._max_retries = max_retries # Maximum number of automatic retries + self._result = None # Return value of child action in the parent + self._tries = 0 # Try count, for retryable jobs + self._terminated = False # Whether this job has been explicitly terminated self._logfile = logfile - self._message_element_name = None # The plugin instance element name for messaging - self._message_element_key = None # The element key for messaging - self._element = None # The Element() passed to the Job() constructor, if applicable + self._message_element_name = None # The plugin instance element name for messaging + self._message_element_key = None # The element key for messaging + self._element = None # The Element() passed to the Job() constructor, if applicable # set_name() # @@ -170,23 +169,16 @@ class Job(): self._max_retries, self._tries, self._message_element_name, - self._message_element_key + self._message_element_key, ) if self._scheduler.context.platform.does_multiprocessing_start_require_pickling(): - pickled = pickle_child_job( - child_job, - self._scheduler.context.get_projects(), - ) + pickled = pickle_child_job(child_job, self._scheduler.context.get_projects(),) self._process = _multiprocessing.AsyncioSafeProcess( - target=do_pickled_child_job, - args=[pickled, self._queue], + target=do_pickled_child_job, args=[pickled, self._queue], ) else: - self._process = _multiprocessing.AsyncioSafeProcess( - target=child_job.child_action, - args=[self._queue], - ) + self._process = _multiprocessing.AsyncioSafeProcess(target=child_job.child_action, args=[self._queue],) # Block signals which are handled in the main process such that # the child process does not inherit the parent's state, but the main @@ -257,8 +249,7 @@ class Job(): # def kill(self): # Force kill - self.message(MessageType.WARN, - "{} did not terminate gracefully, killing".format(self.action_name)) + self.message(MessageType.WARN, "{} did not terminate gracefully, killing".format(self.action_name)) utils._kill_process_tree(self._process.pid) # suspend() @@ -267,8 +258,7 @@ class Job(): # def suspend(self): if not self._suspended: - self.message(MessageType.STATUS, - "{} suspending".format(self.action_name)) + self.message(MessageType.STATUS, "{} suspending".format(self.action_name)) try: # Use SIGTSTP so that child processes may handle and propagate @@ -292,8 +282,7 @@ class Job(): def resume(self, silent=False): if self._suspended: if not silent and not self._scheduler.terminated: - self.message(MessageType.STATUS, - "{} resuming".format(self.action_name)) + self.message(MessageType.STATUS, "{} resuming".format(self.action_name)) os.kill(self._process.pid, signal.SIGCONT) self._suspended = False @@ -335,7 +324,7 @@ class Job(): # override 'element_name' and 'element_key' this way. # def message(self, message_type, message, element_name=None, element_key=None, **kwargs): - kwargs['scheduler'] = True + kwargs["scheduler"] = True # If default name & key values not provided, set as given job attributes if element_name is None: element_name = self._message_element_name @@ -373,8 +362,7 @@ class Job(): # lists, dicts, numbers, but not Element instances). # def handle_message(self, message): - raise ImplError("Job '{kind}' does not implement handle_message()" - .format(kind=type(self).__name__)) + raise ImplError("Job '{kind}' does not implement handle_message()".format(kind=type(self).__name__)) # parent_complete() # @@ -386,8 +374,7 @@ class Job(): # result (any): The result returned by child_process(). # def parent_complete(self, status, result): - raise ImplError("Job '{kind}' does not implement parent_complete()" - .format(kind=type(self).__name__)) + raise ImplError("Job '{kind}' does not implement parent_complete()".format(kind=type(self).__name__)) # create_child_job() # @@ -405,8 +392,7 @@ class Job(): # (ChildJob): An instance of a subclass of ChildJob. # def create_child_job(self, *args, **kwargs): - raise ImplError("Job '{kind}' does not implement create_child_job()" - .format(kind=type(self).__name__)) + raise ImplError("Job '{kind}' does not implement create_child_job()".format(kind=type(self).__name__)) ####################################################### # Local Private Methods # @@ -437,9 +423,11 @@ class Job(): returncode = _ReturnCode(returncode) except ValueError: # An unexpected return code was returned; fail permanently and report - self.message(MessageType.ERROR, - "Internal job process unexpectedly died with exit code {}".format(returncode), - logfile=self._logfile) + self.message( + MessageType.ERROR, + "Internal job process unexpectedly died with exit code {}".format(returncode), + logfile=self._logfile, + ) returncode = _ReturnCode.PERM_FAIL # We don't want to retry if we got OK or a permanent fail. @@ -503,8 +491,7 @@ class Job(): # For regression tests only, save the last error domain / reason # reported from a child task in the main process, this global state # is currently managed in _exceptions.py - set_last_task_error(envelope.message['domain'], - envelope.message['reason']) + set_last_task_error(envelope.message["domain"], envelope.message["reason"]) elif envelope.message_type is _MessageType.RESULT: assert self._result is None self._result = envelope.message @@ -514,8 +501,7 @@ class Job(): elif envelope.message_type is _MessageType.SUBCLASS_CUSTOM_MESSAGE: self.handle_message(envelope.message) else: - assert False, "Unhandled message type '{}': {}".format( - envelope.message_type, envelope.message) + assert False, "Unhandled message type '{}': {}".format(envelope.message_type, envelope.message) # _parent_process_queue() # @@ -552,8 +538,7 @@ class Job(): # http://bugs.python.org/issue3831 # if not self._listening: - self._scheduler.loop.add_reader( - self._queue._reader.fileno(), self._parent_recv) + self._scheduler.loop.add_reader(self._queue._reader.fileno(), self._parent_recv) self._listening = True # _parent_stop_listening() @@ -589,11 +574,10 @@ class Job(): # message_element_key (tuple): None, or the element display key tuple # to be supplied to the Message() constructor. # -class ChildJob(): - +class ChildJob: def __init__( - self, action_name, messenger, logdir, logfile, max_retries, tries, - message_element_name, message_element_key): + self, action_name, messenger, logdir, logfile, max_retries, tries, message_element_name, message_element_key + ): self.action_name = action_name @@ -624,14 +608,15 @@ class ChildJob(): # overriden here. # def message(self, message_type, message, element_name=None, element_key=None, **kwargs): - kwargs['scheduler'] = True + kwargs["scheduler"] = True # If default name & key values not provided, set as given job attributes if element_name is None: element_name = self._message_element_name if element_key is None: element_key = self._message_element_key - self._messenger.message(Message(message_type, message, element_name=element_name, - element_key=element_key, **kwargs)) + self._messenger.message( + Message(message_type, message, element_name=element_name, element_key=element_key, **kwargs) + ) # send_message() # @@ -668,8 +653,7 @@ class ChildJob(): # the result of the Job. # def child_process(self): - raise ImplError("ChildJob '{kind}' does not implement child_process()" - .format(kind=type(self).__name__)) + raise ImplError("ChildJob '{kind}' does not implement child_process()".format(kind=type(self).__name__)) # child_process_data() # @@ -723,12 +707,13 @@ class ChildJob(): def resume_time(): nonlocal stopped_time nonlocal starttime - starttime += (datetime.datetime.now() - stopped_time) + starttime += datetime.datetime.now() - stopped_time # Time, log and and run the action function # - with _signals.suspendable(stop_time, resume_time), \ - self._messenger.recorded_messages(self._logfile, self._logdir) as filename: + with _signals.suspendable(stop_time, resume_time), self._messenger.recorded_messages( + self._logfile, self._logdir + ) as filename: # Graciously handle sigterms. def handle_sigterm(_signum, _sigframe): @@ -743,8 +728,7 @@ class ChildJob(): result = self.child_process() # pylint: disable=assignment-from-no-return except SkipJob as e: elapsed = datetime.datetime.now() - starttime - self.message(MessageType.SKIPPED, str(e), - elapsed=elapsed, logfile=filename) + self.message(MessageType.SKIPPED, str(e), elapsed=elapsed, logfile=filename) # Alert parent of skip by return code self._child_shutdown(_ReturnCode.SKIPPED) @@ -753,13 +737,16 @@ class ChildJob(): retry_flag = e.temporary if retry_flag and (self._tries <= self._max_retries): - self.message(MessageType.FAIL, - "Try #{} failed, retrying".format(self._tries), - elapsed=elapsed, logfile=filename) + self.message( + MessageType.FAIL, + "Try #{} failed, retrying".format(self._tries), + elapsed=elapsed, + logfile=filename, + ) else: - self.message(MessageType.FAIL, str(e), - elapsed=elapsed, detail=e.detail, - logfile=filename, sandbox=e.sandbox) + self.message( + MessageType.FAIL, str(e), elapsed=elapsed, detail=e.detail, logfile=filename, sandbox=e.sandbox + ) self._send_message(_MessageType.CHILD_DATA, self.child_process_data()) @@ -770,7 +757,7 @@ class ChildJob(): # self._child_shutdown(_ReturnCode.FAIL if retry_flag else _ReturnCode.PERM_FAIL) - except Exception: # pylint: disable=broad-except + except Exception: # pylint: disable=broad-except # If an unhandled (not normalized to BstError) occurs, that's a bug, # send the traceback and formatted exception back to the frontend @@ -779,9 +766,7 @@ class ChildJob(): elapsed = datetime.datetime.now() - starttime detail = "An unhandled exception occured:\n\n{}".format(traceback.format_exc()) - self.message(MessageType.BUG, self.action_name, - elapsed=elapsed, detail=detail, - logfile=filename) + self.message(MessageType.BUG, self.action_name, elapsed=elapsed, detail=detail, logfile=filename) # Unhandled exceptions should permenantly fail self._child_shutdown(_ReturnCode.PERM_FAIL) @@ -791,8 +776,7 @@ class ChildJob(): self._child_send_result(result) elapsed = datetime.datetime.now() - starttime - self.message(MessageType.SUCCESS, self.action_name, elapsed=elapsed, - logfile=filename) + self.message(MessageType.SUCCESS, self.action_name, elapsed=elapsed, logfile=filename) # Shutdown needs to stay outside of the above context manager, # make sure we dont try to handle SIGTERM while the process @@ -831,10 +815,7 @@ class ChildJob(): domain = e.domain reason = e.reason - self._send_message(_MessageType.ERROR, { - 'domain': domain, - 'reason': reason - }) + self._send_message(_MessageType.ERROR, {"domain": domain, "reason": reason}) # _child_send_result() # diff --git a/src/buildstream/_scheduler/jobs/jobpickler.py b/src/buildstream/_scheduler/jobs/jobpickler.py index b0465ec9e..1d47f67db 100644 --- a/src/buildstream/_scheduler/jobs/jobpickler.py +++ b/src/buildstream/_scheduler/jobs/jobpickler.py @@ -37,9 +37,7 @@ _NAME_TO_PROTO_CLASS = { "digest": DigestProto, } -_PROTO_CLASS_TO_NAME = { - cls: name for name, cls in _NAME_TO_PROTO_CLASS.items() -} +_PROTO_CLASS_TO_NAME = {cls: name for name, cls in _NAME_TO_PROTO_CLASS.items()} # pickle_child_job() @@ -57,10 +55,7 @@ def pickle_child_job(child_job, projects): # necessary for the job, this includes e.g. the global state of the node # module. node_module_state = node._get_state_for_pickling() - return _pickle_child_job_data( - (child_job, node_module_state), - projects, - ) + return _pickle_child_job_data((child_job, node_module_state), projects,) # do_pickled_child_job() @@ -146,10 +141,7 @@ def _pickle_child_job_data(child_job_data, projects): ] plugin_class_to_factory = { - cls: factory - for factory in factory_list - if factory is not None - for cls, _ in factory.all_loaded_plugins() + cls: factory for factory in factory_list if factory is not None for cls, _ in factory.all_loaded_plugins() } pickled_data = io.BytesIO() diff --git a/src/buildstream/_scheduler/queues/buildqueue.py b/src/buildstream/_scheduler/queues/buildqueue.py index dc33e6510..d98b49476 100644 --- a/src/buildstream/_scheduler/queues/buildqueue.py +++ b/src/buildstream/_scheduler/queues/buildqueue.py @@ -50,10 +50,15 @@ class BuildQueue(Queue): self._tried.add(element) _, description, detail = element._get_build_result() logfile = element._get_build_log() - self._message(element, MessageType.FAIL, description, - detail=detail, action_name=self.action_name, - elapsed=timedelta(seconds=0), - logfile=logfile) + self._message( + element, + MessageType.FAIL, + description, + detail=detail, + action_name=self.action_name, + elapsed=timedelta(seconds=0), + logfile=logfile, + ) self._done_queue.append(element) element_name = element._get_full_name() self._task_group.add_failed_task(element_name) diff --git a/src/buildstream/_scheduler/queues/queue.py b/src/buildstream/_scheduler/queues/queue.py index 49fae5677..986ac6c0a 100644 --- a/src/buildstream/_scheduler/queues/queue.py +++ b/src/buildstream/_scheduler/queues/queue.py @@ -57,11 +57,11 @@ class QueueStatus(FastEnum): # Args: # scheduler (Scheduler): The Scheduler # -class Queue(): +class Queue: # These should be overridden on class data of of concrete Queue implementations - action_name = None # type: Optional[str] - complete_name = None # type: Optional[str] + action_name = None # type: Optional[str] + complete_name = None # type: Optional[str] # Resources this queues' jobs want resources = [] # type: List[int] @@ -72,11 +72,11 @@ class Queue(): # self._scheduler = scheduler self._resources = scheduler.resources # Shared resource pool - self._ready_queue = [] # Ready elements - self._done_queue = deque() # Processed / Skipped elements + self._ready_queue = [] # Ready elements + self._done_queue = deque() # Processed / Skipped elements self._max_retries = 0 - self._required_element_check = False # Whether we should check that elements are required before enqueuing + self._required_element_check = False # Whether we should check that elements are required before enqueuing # Assert the subclass has setup class data assert self.action_name is not None @@ -162,8 +162,7 @@ class Queue(): # element (Element): The element waiting to be pushed into the queue # def register_pending_element(self, element): - raise ImplError("Queue type: {} does not implement register_pending_element()" - .format(self.action_name)) + raise ImplError("Queue type: {} does not implement register_pending_element()".format(self.action_name)) ##################################################### # Scheduler / Pipeline facing APIs # @@ -229,12 +228,16 @@ class Queue(): ready.append(element) return [ - ElementJob(self._scheduler, self.action_name, - self._element_log_path(element), - element=element, queue=self, - action_cb=self.get_process_func(), - complete_cb=self._job_done, - max_retries=self._max_retries) + ElementJob( + self._scheduler, + self.action_name, + self._element_log_path(element), + element=element, + queue=self, + action_cb=self.get_process_func(), + complete_cb=self._job_done, + max_retries=self._max_retries, + ) for element in ready ] @@ -267,7 +270,7 @@ class Queue(): def _update_workspaces(self, element, job): workspace_dict = None if job.child_data: - workspace_dict = job.child_data.get('workspace', None) + workspace_dict = job.child_data.get("workspace", None) # Handle any workspace modifications now # @@ -279,10 +282,13 @@ class Queue(): workspaces.save_config() except BstError as e: self._message(element, MessageType.ERROR, "Error saving workspaces", detail=str(e)) - except Exception: # pylint: disable=broad-except - self._message(element, MessageType.BUG, - "Unhandled exception while saving workspaces", - detail=traceback.format_exc()) + except Exception: # pylint: disable=broad-except + self._message( + element, + MessageType.BUG, + "Unhandled exception while saving workspaces", + detail=traceback.format_exc(), + ) # _job_done() # @@ -322,13 +328,13 @@ class Queue(): # set_last_task_error(e.domain, e.reason) - except Exception: # pylint: disable=broad-except + except Exception: # pylint: disable=broad-except # Report unhandled exceptions and mark as failed # - self._message(element, MessageType.BUG, - "Unhandled exception in post processing", - detail=traceback.format_exc()) + self._message( + element, MessageType.BUG, "Unhandled exception in post processing", detail=traceback.format_exc() + ) self._task_group.add_failed_task(element._get_full_name()) else: # All elements get placed on the done queue for later processing. @@ -372,7 +378,7 @@ class Queue(): if status == QueueStatus.SKIP: # Place skipped elements into the done queue immediately self._task_group.add_skipped_task() - self._done_queue.append(element) # Elements to proceed to the next queue + self._done_queue.append(element) # Elements to proceed to the next queue elif status == QueueStatus.READY: # Push elements which are ready to be processed immediately into the queue heapq.heappush(self._ready_queue, (element._depth, element)) diff --git a/src/buildstream/_scheduler/resources.py b/src/buildstream/_scheduler/resources.py index 73bf66b4a..e76158779 100644 --- a/src/buildstream/_scheduler/resources.py +++ b/src/buildstream/_scheduler/resources.py @@ -1,17 +1,17 @@ -class ResourceType(): +class ResourceType: CACHE = 0 DOWNLOAD = 1 PROCESS = 2 UPLOAD = 3 -class Resources(): +class Resources: def __init__(self, num_builders, num_fetchers, num_pushers): self._max_resources = { ResourceType.CACHE: 0, ResourceType.DOWNLOAD: num_fetchers, ResourceType.PROCESS: num_builders, - ResourceType.UPLOAD: num_pushers + ResourceType.UPLOAD: num_pushers, } # Resources jobs are currently using. @@ -19,7 +19,7 @@ class Resources(): ResourceType.CACHE: 0, ResourceType.DOWNLOAD: 0, ResourceType.PROCESS: 0, - ResourceType.UPLOAD: 0 + ResourceType.UPLOAD: 0, } # Resources jobs currently want exclusive access to. The set @@ -31,7 +31,7 @@ class Resources(): ResourceType.CACHE: set(), ResourceType.DOWNLOAD: set(), ResourceType.PROCESS: set(), - ResourceType.UPLOAD: set() + ResourceType.UPLOAD: set(), } # reserve() @@ -90,8 +90,7 @@ class Resources(): # available. If we don't have enough, the job cannot be # scheduled. for resource in resources: - if (self._max_resources[resource] > 0 and - self._used_resources[resource] >= self._max_resources[resource]): + if self._max_resources[resource] > 0 and self._used_resources[resource] >= self._max_resources[resource]: return False # Now we register the fact that our job is using the resources diff --git a/src/buildstream/_scheduler/scheduler.py b/src/buildstream/_scheduler/scheduler.py index 86e3af021..171281bd9 100644 --- a/src/buildstream/_scheduler/scheduler.py +++ b/src/buildstream/_scheduler/scheduler.py @@ -76,17 +76,18 @@ class NotificationType(FastEnum): # required. NOTE: The notification object should be lightweight # and all attributes must be picklable. # -class Notification(): - - def __init__(self, - notification_type, - *, - full_name=None, - job_action=None, - job_status=None, - time=None, - element=None, - message=None): +class Notification: + def __init__( + self, + notification_type, + *, + full_name=None, + job_action=None, + job_status=None, + time=None, + element=None, + message=None + ): self.notification_type = notification_type self.full_name = full_name self.job_action = job_action @@ -116,40 +117,36 @@ class Notification(): # interrupt_callback: A callback to handle ^C # ticker_callback: A callback call once per second # -class Scheduler(): - - def __init__(self, context, - start_time, state, notification_queue, notifier): +class Scheduler: + def __init__(self, context, start_time, state, notification_queue, notifier): # # Public members # - self.queues = None # Exposed for the frontend to print summaries - self.context = context # The Context object shared with Queues - self.terminated = False # Whether the scheduler was asked to terminate or has terminated - self.suspended = False # Whether the scheduler is currently suspended + self.queues = None # Exposed for the frontend to print summaries + self.context = context # The Context object shared with Queues + self.terminated = False # Whether the scheduler was asked to terminate or has terminated + self.suspended = False # Whether the scheduler is currently suspended # These are shared with the Job, but should probably be removed or made private in some way. - self.loop = None # Shared for Job access to observe the message queue - self.internal_stops = 0 # Amount of SIGSTP signals we've introduced, this is shared with job.py + self.loop = None # Shared for Job access to observe the message queue + self.internal_stops = 0 # Amount of SIGSTP signals we've introduced, this is shared with job.py # # Private members # - self._active_jobs = [] # Jobs currently being run in the scheduler - self._starttime = start_time # Initial application start time - self._suspendtime = None # Session time compensation for suspended state - self._queue_jobs = True # Whether we should continue to queue jobs + self._active_jobs = [] # Jobs currently being run in the scheduler + self._starttime = start_time # Initial application start time + self._suspendtime = None # Session time compensation for suspended state + self._queue_jobs = True # Whether we should continue to queue jobs self._state = state - self._casd_process = None # handle to the casd process for monitoring purpose + self._casd_process = None # handle to the casd process for monitoring purpose # Bidirectional queue to send notifications back to the Scheduler's owner self._notification_queue = notification_queue self._notifier = notifier - self.resources = Resources(context.sched_builders, - context.sched_fetchers, - context.sched_pushers) + self.resources = Resources(context.sched_builders, context.sched_fetchers, context.sched_pushers) # run() # @@ -310,11 +307,13 @@ class Scheduler(): element_info = None # Now check for more jobs - notification = Notification(NotificationType.JOB_COMPLETE, - full_name=job.name, - job_action=job.action_name, - job_status=status, - element=element_info) + notification = Notification( + NotificationType.JOB_COMPLETE, + full_name=job.name, + job_action=job.action_name, + job_status=status, + element=element_info, + ) self._notify(notification) self._sched() @@ -360,10 +359,12 @@ class Scheduler(): # def _start_job(self, job): self._active_jobs.append(job) - notification = Notification(NotificationType.JOB_START, - full_name=job.name, - job_action=job.action_name, - time=self._state.elapsed_time(start_time=self._starttime)) + notification = Notification( + NotificationType.JOB_START, + full_name=job.name, + job_action=job.action_name, + time=self._state.elapsed_time(start_time=self._starttime), + ) self._notify(notification) job.start() @@ -399,9 +400,7 @@ class Scheduler(): # to fetch tasks for elements which failed to pull, and # thus need all the pulls to complete before ever starting # a build - ready.extend(chain.from_iterable( - q.harvest_jobs() for q in reversed(self.queues) - )) + ready.extend(chain.from_iterable(q.harvest_jobs() for q in reversed(self.queues))) # harvest_jobs() may have decided to skip some jobs, making # them eligible for promotion to the next queue as a side effect. @@ -471,7 +470,7 @@ class Scheduler(): self.suspended = False # Notify that we're unsuspended self._notify(Notification(NotificationType.SUSPENDED)) - self._starttime += (datetime.datetime.now() - self._suspendtime) + self._starttime += datetime.datetime.now() - self._suspendtime self._notify(Notification(NotificationType.SCHED_START_TIME, time=self._starttime)) self._suspendtime = None diff --git a/src/buildstream/_signals.py b/src/buildstream/_signals.py index 31982c199..425a57239 100644 --- a/src/buildstream/_signals.py +++ b/src/buildstream/_signals.py @@ -37,8 +37,8 @@ if TYPE_CHECKING: # typing.MutableSequence. However, that is only available in Python versions # 3.5.4 onward and 3.6.1 onward. # Debian 9 ships with 3.5.3. -terminator_stack = deque() # type: MutableSequence[Callable] -suspendable_stack = deque() # type: MutableSequence[Callable] +terminator_stack = deque() # type: MutableSequence[Callable] +suspendable_stack = deque() # type: MutableSequence[Callable] # Per process SIGTERM handler @@ -47,16 +47,18 @@ def terminator_handler(signal_, frame): terminator_ = terminator_stack.pop() try: terminator_() - except: # noqa pylint: disable=bare-except + except: # noqa pylint: disable=bare-except # Ensure we print something if there's an exception raised when # processing the handlers. Note that the default exception # handler won't be called because we os._exit next, so we must # catch all possible exceptions with the unqualified 'except' # clause. traceback.print_exc(file=sys.stderr) - print('Error encountered in BuildStream while processing custom SIGTERM handler:', - terminator_, - file=sys.stderr) + print( + "Error encountered in BuildStream while processing custom SIGTERM handler:", + terminator_, + file=sys.stderr, + ) # Use special exit here, terminate immediately, recommended # for precisely this situation where child processes are teminated. @@ -79,7 +81,7 @@ def terminator_handler(signal_, frame): # @contextmanager def terminator(terminate_func): - global terminator_stack # pylint: disable=global-statement + global terminator_stack # pylint: disable=global-statement # Signal handling only works in the main thread if threading.current_thread() != threading.main_thread(): @@ -101,7 +103,7 @@ def terminator(terminate_func): # Just a simple object for holding on to two callbacks -class Suspender(): +class Suspender: def __init__(self, suspend_callback, resume_callback): self.suspend = suspend_callback self.resume = resume_callback @@ -144,7 +146,7 @@ def suspend_handler(sig, frame): # @contextmanager def suspendable(suspend_callback, resume_callback): - global suspendable_stack # pylint: disable=global-statement + global suspendable_stack # pylint: disable=global-statement outermost = bool(not suspendable_stack) suspender = Suspender(suspend_callback, resume_callback) diff --git a/src/buildstream/_site.py b/src/buildstream/_site.py index 8940fa34a..db0587120 100644 --- a/src/buildstream/_site.py +++ b/src/buildstream/_site.py @@ -30,22 +30,22 @@ import subprocess root = os.path.dirname(os.path.abspath(__file__)) # The Element plugin directory -element_plugins = os.path.join(root, 'plugins', 'elements') +element_plugins = os.path.join(root, "plugins", "elements") # The Source plugin directory -source_plugins = os.path.join(root, 'plugins', 'sources') +source_plugins = os.path.join(root, "plugins", "sources") # Default user configuration -default_user_config = os.path.join(root, 'data', 'userconfig.yaml') +default_user_config = os.path.join(root, "data", "userconfig.yaml") # Default project configuration -default_project_config = os.path.join(root, 'data', 'projectconfig.yaml') +default_project_config = os.path.join(root, "data", "projectconfig.yaml") # Script template to call module building scripts -build_all_template = os.path.join(root, 'data', 'build-all.sh.in') +build_all_template = os.path.join(root, "data", "build-all.sh.in") # Module building script template -build_module_template = os.path.join(root, 'data', 'build-module.sh.in') +build_module_template = os.path.join(root, "data", "build-module.sh.in") def get_bwrap_version(): @@ -53,7 +53,7 @@ def get_bwrap_version(): # # returns None if no bwrap was found # otherwise returns a tuple of 3 int: major, minor, patch - bwrap_path = shutil.which('bwrap') + bwrap_path = shutil.which("bwrap") if not bwrap_path: return None diff --git a/src/buildstream/_sourcecache.py b/src/buildstream/_sourcecache.py index 28ad82831..03e2d1830 100644 --- a/src/buildstream/_sourcecache.py +++ b/src/buildstream/_sourcecache.py @@ -26,12 +26,10 @@ from .storage._casbaseddirectory import CasBasedDirectory from ._basecache import BaseCache from ._exceptions import CASError, CASRemoteError, SourceCacheError, RemoteError from . import utils -from ._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc, \ - source_pb2, source_pb2_grpc +from ._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc, source_pb2, source_pb2_grpc class SourceRemote(BaseRemote): - def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.source_service = None @@ -65,8 +63,10 @@ class SourceRemote(BaseRemote): except grpc.RpcError as e: # Check if this remote has the artifact service if e.code() == grpc.StatusCode.UNIMPLEMENTED: - raise RemoteError("Configured remote does not have the BuildStream " - "capabilities service. Please check remote configuration.") + raise RemoteError( + "Configured remote does not have the BuildStream " + "capabilities service. Please check remote configuration." + ) raise RemoteError("Remote initialisation failed: {}".format(e.details())) if not response.source_capabilities: @@ -129,7 +129,7 @@ class SourceCache(BaseCache): def __init__(self, context): super().__init__(context) - self.sourcerefdir = os.path.join(context.cachedir, 'source_protos') + self.sourcerefdir = os.path.join(context.cachedir, "source_protos") os.makedirs(self.sourcerefdir, exist_ok=True) # list_sources() @@ -182,7 +182,7 @@ class SourceCache(BaseCache): vdir.import_files(self.export(previous_source)) if not source.BST_STAGE_VIRTUAL_DIRECTORY: - with utils._tempdir(dir=self.context.tmpdir, prefix='staging-temp') as tmpdir: + with utils._tempdir(dir=self.context.tmpdir, prefix="staging-temp") as tmpdir: if not vdir.is_empty(): vdir.export_files(tmpdir) source._stage(tmpdir) @@ -233,12 +233,12 @@ class SourceCache(BaseCache): source_proto = self._pull_source(ref, remote) if source_proto is None: - source.info("Remote source service ({}) does not have source {} cached".format( - remote, display_key)) + source.info( + "Remote source service ({}) does not have source {} cached".format(remote, display_key) + ) continue except CASError as e: - raise SourceCacheError("Failed to pull source {}: {}".format( - display_key, e)) from e + raise SourceCacheError("Failed to pull source {}: {}".format(display_key, e)) from e if not source_proto: return False @@ -255,8 +255,7 @@ class SourceCache(BaseCache): missing_blobs = self.cas.fetch_blobs(remote, missing_blobs) if missing_blobs: - source.info("Remote cas ({}) does not have source {} cached".format( - remote, display_key)) + source.info("Remote cas ({}) does not have source {} cached".format(remote, display_key)) continue source.info("Pulled source {} <- {}".format(display_key, remote)) @@ -266,8 +265,7 @@ class SourceCache(BaseCache): source.info("Remote cas ({}) does not have blob {} cached".format(remote, e.blob)) continue except CASError as e: - raise SourceCacheError("Failed to pull source {}: {}".format( - display_key, e)) from e + raise SourceCacheError("Failed to pull source {}: {}".format(display_key, e)) from e return False @@ -315,8 +313,7 @@ class SourceCache(BaseCache): # check whether cache has files already if self._pull_source(ref, remote) is not None: - source.info("Remote ({}) already has source {} cached" - .format(remote, display_key)) + source.info("Remote ({}) already has source {} cached".format(remote, display_key)) continue if not self._push_source(ref, remote): @@ -340,19 +337,18 @@ class SourceCache(BaseCache): def _store_proto(self, proto, ref): path = self._source_path(ref) os.makedirs(os.path.dirname(path), exist_ok=True) - with utils.save_file_atomic(path, 'w+b') as f: + with utils.save_file_atomic(path, "w+b") as f: f.write(proto.SerializeToString()) def _get_source(self, ref): path = self._source_path(ref) source_proto = source_pb2.Source() try: - with open(path, 'r+b') as f: + with open(path, "r+b") as f: source_proto.ParseFromString(f.read()) return source_proto except FileNotFoundError as e: - raise SourceCacheError("Attempted to access unavailable source: {}" - .format(e)) from e + raise SourceCacheError("Attempted to access unavailable source: {}".format(e)) from e def _source_path(self, ref): return os.path.join(self.sourcerefdir, ref) @@ -361,7 +357,7 @@ class SourceCache(BaseCache): for root, _, files in os.walk(self.sourcerefdir): for source_file in files: source = source_pb2.Source() - with open(os.path.join(root, source_file), 'r+b') as f: + with open(os.path.join(root, source_file), "r+b") as f: source.ParseFromString(f.read()) yield source.files diff --git a/src/buildstream/_sourcefactory.py b/src/buildstream/_sourcefactory.py index 1d959a140..7c90042a8 100644 --- a/src/buildstream/_sourcefactory.py +++ b/src/buildstream/_sourcefactory.py @@ -30,14 +30,11 @@ from .source import Source # plugin_origins (list): Data used to search for external Source plugins # class SourceFactory(PluginContext): + def __init__(self, plugin_base, *, format_versions={}, plugin_origins=None): - def __init__(self, plugin_base, *, - format_versions={}, - plugin_origins=None): - - super().__init__(plugin_base, Source, [_site.source_plugins], - format_versions=format_versions, - plugin_origins=plugin_origins) + super().__init__( + plugin_base, Source, [_site.source_plugins], format_versions=format_versions, plugin_origins=plugin_origins + ) # create(): # diff --git a/src/buildstream/_state.py b/src/buildstream/_state.py index 310e12a63..d85e348f2 100644 --- a/src/buildstream/_state.py +++ b/src/buildstream/_state.py @@ -28,7 +28,7 @@ from collections import OrderedDict # state (State): The state object # complete_name (str): Optional name for frontend status rendering, e.g. 'built' # -class TaskGroup(): +class TaskGroup: def __init__(self, name, state, complete_name=None): self.name = name self.complete_name = complete_name @@ -98,14 +98,14 @@ class TaskGroup(): # Args: # session_start (datetime): The time the session started # -class State(): +class State: def __init__(self, session_start): self._session_start = session_start self.task_groups = OrderedDict() # key is TaskGroup name # Note: A Task's full_name is technically unique, but only accidentally. - self.tasks = OrderedDict() # key is a tuple of action_name and full_name + self.tasks = OrderedDict() # key is a tuple of action_name and full_name self._task_added_cbs = [] self._task_removed_cbs = [] @@ -281,8 +281,9 @@ class State(): # def add_task(self, action_name, full_name, elapsed_offset=None): task_key = (action_name, full_name) - assert task_key not in self.tasks, \ - "Trying to add task '{}:{}' to '{}'".format(action_name, full_name, self.tasks) + assert task_key not in self.tasks, "Trying to add task '{}:{}' to '{}'".format( + action_name, full_name, self.tasks + ) if not elapsed_offset: elapsed_offset = self.elapsed_time() @@ -366,7 +367,7 @@ class State(): # e.g. an element's name. # elapsed_offset (timedelta): The time the task started, relative to # buildstream's start time. -class _Task(): +class _Task: def __init__(self, state, action_name, full_name, elapsed_offset): self._state = state self.action_name = action_name diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index f09a46185..2515fadce 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -33,8 +33,19 @@ from collections import deque from ._artifactelement import verify_artifact_ref, ArtifactElement from ._exceptions import StreamError, ImplError, BstError, ArtifactElementError, ArtifactError from ._message import Message, MessageType -from ._scheduler import Scheduler, SchedStatus, TrackQueue, FetchQueue, \ - SourcePushQueue, BuildQueue, PullQueue, ArtifactPushQueue, NotificationType, Notification, JobStatus +from ._scheduler import ( + Scheduler, + SchedStatus, + TrackQueue, + FetchQueue, + SourcePushQueue, + BuildQueue, + PullQueue, + ArtifactPushQueue, + NotificationType, + Notification, + JobStatus, +) from ._pipeline import Pipeline, PipelineSelection from ._profile import Topics, PROFILER from ._state import State @@ -55,20 +66,18 @@ from . import Scope, Consistency # interrupt_callback (callable): A callback to invoke when we get interrupted # ticker_callback (callable): Invoked every second while running the scheduler # -class Stream(): - - def __init__(self, context, session_start, *, - session_start_callback=None, - interrupt_callback=None, - ticker_callback=None): +class Stream: + def __init__( + self, context, session_start, *, session_start_callback=None, interrupt_callback=None, ticker_callback=None + ): # # Public members # - self.targets = [] # Resolved target elements - self.session_elements = [] # List of elements being processed this session - self.total_elements = [] # Total list of elements based on targets - self.queues = [] # Queue objects + self.targets = [] # Resolved target elements + self.session_elements = [] # List of elements being processed this session + self.total_elements = [] # Total list of elements based on targets + self.queues = [] # Queue objects # # Private members @@ -84,8 +93,9 @@ class Stream(): context.messenger.set_state(self._state) - self._scheduler = Scheduler(context, session_start, self._state, self._notification_queue, - self._scheduler_notification_handler) + self._scheduler = Scheduler( + context, session_start, self._state, self._notification_queue, self._scheduler_notification_handler + ) self._first_non_track_queue = None self._session_start_callback = session_start_callback self._ticker_callback = ticker_callback @@ -138,17 +148,24 @@ class Stream(): # # Returns: # (list of Element): The selected elements - def load_selection(self, targets, *, - selection=PipelineSelection.NONE, - except_targets=(), - use_artifact_config=False, - load_refs=False): + def load_selection( + self, + targets, + *, + selection=PipelineSelection.NONE, + except_targets=(), + use_artifact_config=False, + load_refs=False + ): with PROFILER.profile(Topics.LOAD_SELECTION, "_".join(t.replace(os.sep, "-") for t in targets)): - target_objects, _ = self._load(targets, (), - selection=selection, - except_targets=except_targets, - use_artifact_config=use_artifact_config, - load_refs=load_refs) + target_objects, _ = self._load( + targets, + (), + selection=selection, + except_targets=except_targets, + use_artifact_config=use_artifact_config, + load_refs=load_refs, + ) return target_objects @@ -171,14 +188,20 @@ class Stream(): # Returns: # (int): The exit code of the launched shell # - def shell(self, element, scope, prompt, *, - directory=None, - mounts=None, - isolate=False, - command=None, - usebuildtree=None, - pull_dependencies=None, - unique_id=None): + def shell( + self, + element, + scope, + prompt, + *, + directory=None, + mounts=None, + isolate=False, + command=None, + usebuildtree=None, + pull_dependencies=None, + unique_id=None + ): # Load the Element via the unique_id if given if unique_id and element is None: @@ -192,18 +215,16 @@ class Stream(): if not element._source_cached(): raise StreamError( "Sources for element {} are not cached." - "Element must be fetched.".format(element._get_full_name())) + "Element must be fetched.".format(element._get_full_name()) + ) - missing_deps = [ - dep for dep in self._pipeline.dependencies([element], scope) - if not dep._cached() - ] + missing_deps = [dep for dep in self._pipeline.dependencies([element], scope) if not dep._cached()] if missing_deps: if not pull_dependencies: raise StreamError( "Elements need to be built or downloaded before staging a shell environment", - detail="\n" - .join(list(map(lambda x: x._get_full_name(), missing_deps)))) + detail="\n".join(list(map(lambda x: x._get_full_name(), missing_deps))), + ) self._message(MessageType.INFO, "Attempting to fetch missing or incomplete artifacts") self._scheduler.clear_queues() self._add_queue(PullQueue(self._scheduler)) @@ -236,8 +257,9 @@ class Stream(): else: buildtree = True - return element._shell(scope, directory, mounts=mounts, isolate=isolate, prompt=prompt, command=command, - usebuildtree=buildtree) + return element._shell( + scope, directory, mounts=mounts, isolate=isolate, prompt=prompt, command=command, usebuildtree=buildtree + ) # build() # @@ -252,23 +274,22 @@ class Stream(): # If `remote` specified as None, then regular configuration will be used # to determine where to push artifacts to. # - def build(self, targets, *, - selection=PipelineSelection.PLAN, - ignore_junction_targets=False, - remote=None): + def build(self, targets, *, selection=PipelineSelection.PLAN, ignore_junction_targets=False, remote=None): use_config = True if remote: use_config = False - elements, _ = \ - self._load(targets, [], - selection=selection, - ignore_junction_targets=ignore_junction_targets, - use_artifact_config=use_config, - artifact_remote_url=remote, - use_source_config=True, - dynamic_plan=True) + elements, _ = self._load( + targets, + [], + selection=selection, + ignore_junction_targets=ignore_junction_targets, + use_artifact_config=use_config, + artifact_remote_url=remote, + use_source_config=True, + dynamic_plan=True, + ) # Assert that the elements are consistent self._pipeline.assert_consistent(elements) @@ -317,12 +338,16 @@ class Stream(): # track_cross_junctions (bool): Whether tracking should cross junction boundaries # remote (str|None): The URL of a specific remote server to pull from. # - def fetch(self, targets, *, - selection=PipelineSelection.PLAN, - except_targets=None, - track_targets=False, - track_cross_junctions=False, - remote=None): + def fetch( + self, + targets, + *, + selection=PipelineSelection.PLAN, + except_targets=None, + track_targets=False, + track_cross_junctions=False, + remote=None + ): if track_targets: track_targets = targets @@ -337,14 +362,17 @@ class Stream(): if remote: use_source_config = False - elements, track_elements = \ - self._load(targets, track_targets, - selection=selection, track_selection=track_selection, - except_targets=except_targets, - track_except_targets=track_except_targets, - track_cross_junctions=track_cross_junctions, - use_source_config=use_source_config, - source_remote_url=remote) + elements, track_elements = self._load( + targets, + track_targets, + selection=selection, + track_selection=track_selection, + except_targets=except_targets, + track_except_targets=track_except_targets, + track_cross_junctions=track_cross_junctions, + use_source_config=use_source_config, + source_remote_url=remote, + ) # Delegated to a shared fetch method self._fetch(elements, track_elements=track_elements) @@ -362,20 +390,20 @@ class Stream(): # If no error is encountered while tracking, then the project files # are rewritten inline. # - def track(self, targets, *, - selection=PipelineSelection.REDIRECT, - except_targets=None, - cross_junctions=False): + def track(self, targets, *, selection=PipelineSelection.REDIRECT, except_targets=None, cross_junctions=False): # We pass no target to build. Only to track. Passing build targets # would fully load project configuration which might not be # possible before tracking is done. - _, elements = \ - self._load([], targets, - selection=selection, track_selection=selection, - except_targets=except_targets, - track_except_targets=except_targets, - track_cross_junctions=cross_junctions) + _, elements = self._load( + [], + targets, + selection=selection, + track_selection=selection, + except_targets=except_targets, + track_except_targets=except_targets, + track_cross_junctions=cross_junctions, + ) # FIXME: this can be refactored after element._update_state is simplified/removed elements = [element for element in elements if element._schedule_tracking()] @@ -400,21 +428,21 @@ class Stream(): # If `remote` specified as None, then regular configuration will be used # to determine where to pull artifacts from. # - def pull(self, targets, *, - selection=PipelineSelection.NONE, - ignore_junction_targets=False, - remote=None): + def pull(self, targets, *, selection=PipelineSelection.NONE, ignore_junction_targets=False, remote=None): use_config = True if remote: use_config = False - elements, _ = self._load(targets, (), - selection=selection, - ignore_junction_targets=ignore_junction_targets, - use_artifact_config=use_config, - artifact_remote_url=remote, - load_refs=True) + elements, _ = self._load( + targets, + (), + selection=selection, + ignore_junction_targets=ignore_junction_targets, + use_artifact_config=use_config, + artifact_remote_url=remote, + load_refs=True, + ) if not self._artifacts.has_fetch_remotes(): raise StreamError("No artifact caches available for pulling artifacts") @@ -442,21 +470,21 @@ class Stream(): # a pull queue will be created if user context and available remotes allow for # attempting to fetch them. # - def push(self, targets, *, - selection=PipelineSelection.NONE, - ignore_junction_targets=False, - remote=None): + def push(self, targets, *, selection=PipelineSelection.NONE, ignore_junction_targets=False, remote=None): use_config = True if remote: use_config = False - elements, _ = self._load(targets, (), - selection=selection, - ignore_junction_targets=ignore_junction_targets, - use_artifact_config=use_config, - artifact_remote_url=remote, - load_refs=True) + elements, _ = self._load( + targets, + (), + selection=selection, + ignore_junction_targets=ignore_junction_targets, + use_artifact_config=use_config, + artifact_remote_url=remote, + load_refs=True, + ) if not self._artifacts.has_push_remotes(): raise StreamError("No artifact caches available for pushing artifacts") @@ -500,8 +528,10 @@ class Stream(): # ready an uncached element in the PushQueue. if self._context.sched_error_action == _SchedulerErrorAction.CONTINUE and uncached_elements: names = [element.name for element in uncached_elements] - fail_str = "Error while pushing. The following elements were not pushed as they are " \ + fail_str = ( + "Error while pushing. The following elements were not pushed as they are " "not yet cached:\n\n\t{}\n".format("\n\t".join(names)) + ) raise StreamError(fail_str) @@ -525,15 +555,19 @@ class Stream(): # pull (bool): If true will attempt to pull any missing or incomplete # artifacts. # - def checkout(self, target, *, - location=None, - force=False, - selection=PipelineSelection.RUN, - integrate=True, - hardlinks=False, - compression='', - pull=False, - tar=False): + def checkout( + self, + target, + *, + location=None, + force=False, + selection=PipelineSelection.RUN, + integrate=True, + hardlinks=False, + compression="", + pull=False, + tar=False + ): elements, _ = self._load((target,), (), selection=selection, use_artifact_config=True, load_refs=True) @@ -554,15 +588,15 @@ class Stream(): self._run() try: - scope = {'run': Scope.RUN, 'build': Scope.BUILD, 'none': Scope.NONE, 'all': Scope.ALL} - with target._prepare_sandbox(scope=scope[selection], directory=None, - integrate=integrate) as sandbox: + scope = {"run": Scope.RUN, "build": Scope.BUILD, "none": Scope.NONE, "all": Scope.ALL} + with target._prepare_sandbox(scope=scope[selection], directory=None, integrate=integrate) as sandbox: # Copy or move the sandbox to the target directory virdir = sandbox.get_virtual_directory() self._export_artifact(tar, location, compression, target, hardlinks, virdir) except BstError as e: - raise StreamError("Error while staging dependencies into a sandbox" - ": '{}'".format(e), detail=e.detail, reason=e.reason) from e + raise StreamError( + "Error while staging dependencies into a sandbox" ": '{}'".format(e), detail=e.detail, reason=e.reason + ) from e # _export_artifact() # @@ -578,34 +612,32 @@ class Stream(): # def _export_artifact(self, tar, location, compression, target, hardlinks, virdir): if not tar: - with target.timed_activity("Checking out files in '{}'" - .format(location)): + with target.timed_activity("Checking out files in '{}'".format(location)): try: if hardlinks: self._checkout_hardlinks(virdir, location) else: virdir.export_files(location) except OSError as e: - raise StreamError("Failed to checkout files: '{}'" - .format(e)) from e + raise StreamError("Failed to checkout files: '{}'".format(e)) from e else: - to_stdout = location == '-' + to_stdout = location == "-" mode = _handle_compression(compression, to_stream=to_stdout) with target.timed_activity("Creating tarball"): if to_stdout: # Save the stdout FD to restore later saved_fd = os.dup(sys.stdout.fileno()) try: - with os.fdopen(sys.stdout.fileno(), 'wb') as fo: + with os.fdopen(sys.stdout.fileno(), "wb") as fo: with tarfile.open(fileobj=fo, mode=mode) as tf: - virdir.export_to_tar(tf, '.') + virdir.export_to_tar(tf, ".") finally: # No matter what, restore stdout for further use os.dup2(saved_fd, sys.stdout.fileno()) os.close(saved_fd) else: with tarfile.open(location, mode=mode) as tf: - virdir.export_to_tar(tf, '.') + virdir.export_to_tar(tf, ".") # artifact_show() # @@ -614,13 +646,9 @@ class Stream(): # Args: # targets (str): Targets to show the cached state of # - def artifact_show(self, targets, *, - selection=PipelineSelection.NONE): + def artifact_show(self, targets, *, selection=PipelineSelection.NONE): # Obtain list of Element and/or ArtifactElement objects - target_objects = self.load_selection(targets, - selection=selection, - use_artifact_config=True, - load_refs=True) + target_objects = self.load_selection(targets, selection=selection, use_artifact_config=True, load_refs=True) if self._artifacts.has_fetch_remotes(): self._pipeline.check_remotes(target_objects) @@ -695,8 +723,7 @@ class Stream(): # Args: # targets (str): Targets to remove # - def artifact_delete(self, targets, *, - selection=PipelineSelection.NONE): + def artifact_delete(self, targets, *, selection=PipelineSelection.NONE): # Return list of Element and/or ArtifactElement objects target_objects = self.load_selection(targets, selection=selection, load_refs=True) @@ -736,20 +763,22 @@ class Stream(): # compression (str): The type of compression for tarball # include_build_scripts (bool): Whether to include build scripts in the checkout # - def source_checkout(self, target, *, - location=None, - force=False, - deps='none', - except_targets=(), - tar=False, - compression=None, - include_build_scripts=False): + def source_checkout( + self, + target, + *, + location=None, + force=False, + deps="none", + except_targets=(), + tar=False, + compression=None, + include_build_scripts=False + ): self._check_location_writable(location, force=force, tar=tar) - elements, _ = self._load((target,), (), - selection=deps, - except_targets=except_targets) + elements, _ = self._load((target,), (), selection=deps, except_targets=except_targets) # Assert all sources are cached in the source dir self._fetch(elements) @@ -757,11 +786,11 @@ class Stream(): # Stage all sources determined by scope try: - self._source_checkout(elements, location, force, deps, - tar, compression, include_build_scripts) + self._source_checkout(elements, location, force, deps, tar, compression, include_build_scripts) except BstError as e: - raise StreamError("Error while writing sources" - ": '{}'".format(e), detail=e.detail, reason=e.reason) from e + raise StreamError( + "Error while writing sources" ": '{}'".format(e), detail=e.detail, reason=e.reason + ) from e self._message(MessageType.INFO, "Checked out sources to '{}'".format(location)) @@ -776,11 +805,7 @@ class Stream(): # force (bool): Whether to ignore contents in an existing directory # custom_dir (str): Custom location to create a workspace or false to use default location. # - def workspace_open(self, targets, *, - no_checkout, - track_first, - force, - custom_dir): + def workspace_open(self, targets, *, no_checkout, track_first, force, custom_dir): # This function is a little funny but it is trying to be as atomic as possible. if track_first: @@ -788,9 +813,9 @@ class Stream(): else: track_targets = () - elements, track_elements = self._load(targets, track_targets, - selection=PipelineSelection.REDIRECT, - track_selection=PipelineSelection.REDIRECT) + elements, track_elements = self._load( + targets, track_targets, selection=PipelineSelection.REDIRECT, track_selection=PipelineSelection.REDIRECT + ) workspaces = self._context.get_workspaces() @@ -819,33 +844,44 @@ class Stream(): workspace = workspaces.get_workspace(target._get_full_name()) if workspace: if not force: - raise StreamError("Element '{}' already has an open workspace defined at: {}" - .format(target.name, workspace.get_absolute_path())) + raise StreamError( + "Element '{}' already has an open workspace defined at: {}".format( + target.name, workspace.get_absolute_path() + ) + ) if not no_checkout: - target.warn("Replacing existing workspace for element '{}' defined at: {}" - .format(target.name, workspace.get_absolute_path())) + target.warn( + "Replacing existing workspace for element '{}' defined at: {}".format( + target.name, workspace.get_absolute_path() + ) + ) self.workspace_close(target._get_full_name(), remove_dir=not no_checkout) target_consistency = target._get_consistency() - if not no_checkout and target_consistency < Consistency.CACHED and \ - target_consistency._source_cached(): - raise StreamError("Could not stage uncached source. For {} ".format(target.name) + - "Use `--track` to track and " + - "fetch the latest version of the " + - "source.") + if not no_checkout and target_consistency < Consistency.CACHED and target_consistency._source_cached(): + raise StreamError( + "Could not stage uncached source. For {} ".format(target.name) + + "Use `--track` to track and " + + "fetch the latest version of the " + + "source." + ) if not custom_dir: directory = os.path.abspath(os.path.join(self._context.workspacedir, target.name)) - if directory[-4:] == '.bst': + if directory[-4:] == ".bst": directory = directory[:-4] expanded_directories.append(directory) if custom_dir: if len(elements) != 1: - raise StreamError("Exactly one element can be given if --directory is used", - reason='directory-with-multiple-elements') + raise StreamError( + "Exactly one element can be given if --directory is used", + reason="directory-with-multiple-elements", + ) directory = os.path.abspath(custom_dir) - expanded_directories = [directory, ] + expanded_directories = [ + directory, + ] else: # If this fails it is a bug in what ever calls this, usually cli.py and so can not be tested for via the # run bst test mechanism. @@ -854,12 +890,16 @@ class Stream(): for target, directory in zip(elements, expanded_directories): if os.path.exists(directory): if not os.path.isdir(directory): - raise StreamError("For element '{}', Directory path is not a directory: {}" - .format(target.name, directory), reason='bad-directory') + raise StreamError( + "For element '{}', Directory path is not a directory: {}".format(target.name, directory), + reason="bad-directory", + ) if not (no_checkout or force) and os.listdir(directory): - raise StreamError("For element '{}', Directory path is not empty: {}" - .format(target.name, directory), reason='bad-directory') + raise StreamError( + "For element '{}', Directory path is not empty: {}".format(target.name, directory), + reason="bad-directory", + ) if os.listdir(directory): if force and not no_checkout: utils._force_rmtree(directory) @@ -868,8 +908,7 @@ class Stream(): # Now it does the bits that can not be made atomic. targetGenerator = zip(elements, expanded_directories) for target, directory in targetGenerator: - self._message(MessageType.INFO, "Creating workspace for element {}" - .format(target.name)) + self._message(MessageType.INFO, "Creating workspace for element {}".format(target.name)) workspace = workspaces.get_workspace(target._get_full_name()) if workspace and not no_checkout: @@ -886,8 +925,7 @@ class Stream(): raise StreamError("Failed to create workspace directory: {}".format(e) + todo_elements) from e workspaces.create_workspace(target, directory, checkout=not no_checkout) - self._message(MessageType.INFO, "Created a workspace for element: {}" - .format(target._get_full_name())) + self._message(MessageType.INFO, "Created a workspace for element: {}".format(target._get_full_name())) # workspace_close # @@ -903,13 +941,13 @@ class Stream(): # Remove workspace directory if prompted if remove_dir: - with self._context.messenger.timed_activity("Removing workspace directory {}" - .format(workspace.get_absolute_path())): + with self._context.messenger.timed_activity( + "Removing workspace directory {}".format(workspace.get_absolute_path()) + ): try: shutil.rmtree(workspace.get_absolute_path()) except OSError as e: - raise StreamError("Could not remove '{}': {}" - .format(workspace.get_absolute_path(), e)) from e + raise StreamError("Could not remove '{}': {}".format(workspace.get_absolute_path(), e)) from e # Delete the workspace and save the configuration workspaces.delete_workspace(element_name) @@ -928,9 +966,9 @@ class Stream(): # def workspace_reset(self, targets, *, soft, track_first): - elements, _ = self._load(targets, [], - selection=PipelineSelection.REDIRECT, - track_selection=PipelineSelection.REDIRECT) + elements, _ = self._load( + targets, [], selection=PipelineSelection.REDIRECT, track_selection=PipelineSelection.REDIRECT + ) nonexisting = [] for element in elements: @@ -946,14 +984,20 @@ class Stream(): if soft: workspace.prepared = False - self._message(MessageType.INFO, "Reset workspace state for {} at: {}" - .format(element.name, workspace_path)) + self._message( + MessageType.INFO, "Reset workspace state for {} at: {}".format(element.name, workspace_path) + ) continue self.workspace_close(element._get_full_name(), remove_dir=True) workspaces.save_config() - self.workspace_open([element._get_full_name()], - no_checkout=False, track_first=track_first, force=True, custom_dir=workspace_path) + self.workspace_open( + [element._get_full_name()], + no_checkout=False, + track_first=track_first, + force=True, + custom_dir=workspace_path, + ) # workspace_exists # @@ -1001,14 +1045,12 @@ class Stream(): workspaces = [] for element_name, workspace_ in self._context.get_workspaces().list(): workspace_detail = { - 'element': element_name, - 'directory': workspace_.get_absolute_path(), + "element": element_name, + "directory": workspace_.get_absolute_path(), } workspaces.append(workspace_detail) - _yaml.roundtrip_dump({ - 'workspaces': workspaces - }) + _yaml.roundtrip_dump({"workspaces": workspaces}) # redirect_element_names() # @@ -1034,9 +1076,9 @@ class Stream(): else: output_elements.add(e) if load_elements: - loaded_elements, _ = self._load(load_elements, (), - selection=PipelineSelection.REDIRECT, - track_selection=PipelineSelection.REDIRECT) + loaded_elements, _ = self._load( + load_elements, (), selection=PipelineSelection.REDIRECT, track_selection=PipelineSelection.REDIRECT + ) for e in loaded_elements: output_elements.add(e.name) @@ -1166,26 +1208,31 @@ class Stream(): # (list of Element): The primary element selection # (list of Element): The tracking element selection # - def _load(self, targets, track_targets, *, - selection=PipelineSelection.NONE, - track_selection=PipelineSelection.NONE, - except_targets=(), - track_except_targets=(), - track_cross_junctions=False, - ignore_junction_targets=False, - use_artifact_config=False, - use_source_config=False, - artifact_remote_url=None, - source_remote_url=None, - dynamic_plan=False, - load_refs=False): + def _load( + self, + targets, + track_targets, + *, + selection=PipelineSelection.NONE, + track_selection=PipelineSelection.NONE, + except_targets=(), + track_except_targets=(), + track_cross_junctions=False, + ignore_junction_targets=False, + use_artifact_config=False, + use_source_config=False, + artifact_remote_url=None, + source_remote_url=None, + dynamic_plan=False, + load_refs=False + ): # Classify element and artifact strings target_elements, target_artifacts = self._classify_artifacts(targets) if target_artifacts: if not load_refs: - detail = '\n'.join(target_artifacts) + detail = "\n".join(target_artifacts) raise ArtifactElementError("Cannot perform this operation with artifact refs:", detail=detail) if selection in (PipelineSelection.ALL, PipelineSelection.RUN): raise StreamError("Error: '--deps {}' is not supported for artifact refs".format(selection)) @@ -1198,8 +1245,9 @@ class Stream(): # Load all target elements loadable = [target_elements, except_targets, track_targets, track_except_targets] if any(loadable): - elements, except_elements, track_elements, track_except_elements = \ - self._pipeline.load(loadable, rewritable=rewritable) + elements, except_elements, track_elements, track_except_elements = self._pipeline.load( + loadable, rewritable=rewritable + ) else: elements, except_elements, track_elements, track_except_elements = [], [], [], [] @@ -1208,7 +1256,7 @@ class Stream(): # Optionally filter out junction elements if ignore_junction_targets: - elements = [e for e in elements if e.get_kind() != 'junction'] + elements = [e for e in elements if e.get_kind() != "junction"] # Hold on to the targets self.targets = elements + artifacts @@ -1233,14 +1281,10 @@ class Stream(): for project, project_elements in track_projects.items(): selected = self._pipeline.get_selection(project_elements, track_selection) - selected = self._pipeline.track_cross_junction_filter(project, - selected, - track_cross_junctions) + selected = self._pipeline.track_cross_junction_filter(project, selected, track_cross_junctions) track_selected.extend(selected) - track_selected = self._pipeline.except_elements(track_elements, - track_selected, - track_except_elements) + track_selected = self._pipeline.except_elements(track_elements, track_selected, track_except_elements) if not targets: return [], track_selected @@ -1257,9 +1301,7 @@ class Stream(): # self._pipeline.resolve_elements(self.targets) selected = self._pipeline.get_selection(self.targets, selection, silent=False) - selected = self._pipeline.except_elements(self.targets, - selected, - except_elements) + selected = self._pipeline.except_elements(self.targets, selected, except_elements) if selection == PipelineSelection.PLAN and dynamic_plan: # We use a dynamic build plan, only request artifacts of top-level targets, @@ -1279,8 +1321,7 @@ class Stream(): # def _message(self, message_type, message, **kwargs): args = dict(kwargs) - self._context.messenger.message( - Message(message_type, message, **args)) + self._context.messenger.message(Message(message_type, message, **args)) # _add_queue() # @@ -1321,9 +1362,7 @@ class Stream(): # unique_id (str): A unique_id to load an Element instance # def _failure_retry(self, action_name, unique_id): - notification = Notification(NotificationType.RETRY, - job_action=action_name, - element=unique_id) + notification = Notification(NotificationType.RETRY, job_action=action_name, element=unique_id) self._notify(notification) # _run() @@ -1370,8 +1409,7 @@ class Stream(): # Filter out elements with cached sources, only from the fetch plan # let the track plan resolve new refs. - cached = [elt for elt in fetch_plan - if not elt._should_fetch(fetch_original)] + cached = [elt for elt in fetch_plan if not elt._should_fetch(fetch_original)] fetch_plan = self._pipeline.subtract_elements(fetch_plan, cached) # Construct queues, enqueue and run @@ -1406,21 +1444,16 @@ class Stream(): try: os.makedirs(location, exist_ok=True) except OSError as e: - raise StreamError("Failed to create destination directory: '{}'" - .format(e)) from e + raise StreamError("Failed to create destination directory: '{}'".format(e)) from e if not os.access(location, os.W_OK): - raise StreamError("Destination directory '{}' not writable" - .format(location)) + raise StreamError("Destination directory '{}' not writable".format(location)) if not force and os.listdir(location): - raise StreamError("Destination directory '{}' not empty" - .format(location)) - elif os.path.exists(location) and location != '-': + raise StreamError("Destination directory '{}' not empty".format(location)) + elif os.path.exists(location) and location != "-": if not os.access(location, os.W_OK): - raise StreamError("Output file '{}' not writable" - .format(location)) + raise StreamError("Output file '{}' not writable".format(location)) if not force and os.path.exists(location): - raise StreamError("Output file '{}' already exists" - .format(location)) + raise StreamError("Output file '{}' already exists".format(location)) # Helper function for checkout() # @@ -1433,13 +1466,16 @@ class Stream(): sandbox_vroot.export_files(directory, can_link=True, can_destroy=True) # Helper function for source_checkout() - def _source_checkout(self, elements, - location=None, - force=False, - deps='none', - tar=False, - compression=None, - include_build_scripts=False): + def _source_checkout( + self, + elements, + location=None, + force=False, + deps="none", + tar=False, + compression=None, + include_build_scripts=False, + ): location = os.path.abspath(location) # Stage all our sources in a temporary directory. The this @@ -1455,8 +1491,7 @@ class Stream(): else: self._move_directory(temp_source_dir.name, location, force) except OSError as e: - raise StreamError("Failed to checkout sources to {}: {}" - .format(location, e)) from e + raise StreamError("Failed to checkout sources to {}: {}".format(location, e)) from e finally: with suppress(FileNotFoundError): temp_source_dir.cleanup() @@ -1498,10 +1533,10 @@ class Stream(): # Create a tarball from the content of directory def _create_tarball(self, directory, tar_name, compression): if compression is None: - compression = '' + compression = "" mode = _handle_compression(compression) try: - with utils.save_file_atomic(tar_name, mode='wb') as f: + with utils.save_file_atomic(tar_name, mode="wb") as f: tarball = tarfile.open(fileobj=f, mode=mode) for item in os.listdir(str(directory)): file_to_add = os.path.join(directory, item) @@ -1598,7 +1633,7 @@ class Stream(): artifact_globs = [] for target in targets: - if target.endswith('.bst'): + if target.endswith(".bst"): if any(c in "*?[" for c in target): element_globs.append(target) else: @@ -1628,7 +1663,7 @@ class Stream(): for glob in artifact_globs: artifact_refs.extend(self._artifacts.list_artifacts(glob=glob)) if not artifact_refs: - self._message(MessageType.WARN, "No artifacts found for globs: {}".format(', '.join(artifact_globs))) + self._message(MessageType.WARN, "No artifacts found for globs: {}".format(", ".join(artifact_globs))) return element_targets, artifact_refs @@ -1648,8 +1683,7 @@ class Stream(): elif notification.notification_type == NotificationType.JOB_COMPLETE: self._state.remove_task(notification.job_action, notification.full_name) if notification.job_status == JobStatus.FAIL: - self._state.fail_task(notification.job_action, notification.full_name, - notification.element) + self._state.fail_task(notification.job_action, notification.full_name, notification.element) elif notification.notification_type == NotificationType.SCHED_START_TIME: self._starttime = notification.time elif notification.notification_type == NotificationType.RUNNING: @@ -1694,5 +1728,5 @@ class Stream(): # (str): The tarfile mode string # def _handle_compression(compression, *, to_stream=False): - mode_prefix = 'w|' if to_stream else 'w:' + mode_prefix = "w|" if to_stream else "w:" return mode_prefix + compression diff --git a/src/buildstream/_version.py b/src/buildstream/_version.py index 03f946cb8..10905c4ea 100644 --- a/src/buildstream/_version.py +++ b/src/buildstream/_version.py @@ -60,17 +60,18 @@ HANDLERS = {} def register_vcs_handler(vcs, method): # decorator """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): """Store f in HANDLERS[vcs][method].""" if vcs not in HANDLERS: HANDLERS[vcs] = {} HANDLERS[vcs][method] = f return f + return decorate -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None): """Call the given command(s).""" assert isinstance(commands, list) p = None @@ -78,10 +79,9 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, try: dispcmd = str([c] + args) # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) + p = subprocess.Popen( + [c] + args, cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=(subprocess.PIPE if hide_stderr else None) + ) break except EnvironmentError: e = sys.exc_info()[1] @@ -118,16 +118,19 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): for i in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} + return { + "version": dirname[len(parentdir_prefix) :], + "full-revisionid": None, + "dirty": False, + "error": None, + "date": None, + } else: rootdirs.append(root) root = os.path.dirname(root) # up a level if verbose: - print("Tried directories %s but none started with prefix %s" % - (str(rootdirs), parentdir_prefix)) + print("Tried directories %s but none started with prefix %s" % (str(rootdirs), parentdir_prefix)) raise NotThisMethod("rootdir doesn't start with parentdir_prefix") @@ -183,7 +186,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d @@ -192,7 +195,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) + tags = set([r for r in refs if re.search(r"\d", r)]) if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: @@ -200,19 +203,26 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): for ref in sorted(tags): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] + r = ref[len(tag_prefix) :] if verbose: print("picking %s" % r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} + return { + "version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": None, + "date": date, + } # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} + return { + "version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": "no suitable tags", + "date": None, + } @register_vcs_handler("git", "pieces_from_vcs") @@ -227,8 +237,7 @@ def git_pieces_from_vcs(tag_prefix, tag_regex, root, verbose, run_command=run_co if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) + out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True) if rc != 0: if verbose: print("Directory %s not under git control" % root) @@ -236,10 +245,11 @@ def git_pieces_from_vcs(tag_prefix, tag_regex, root, verbose, run_command=run_co # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%s%s" % (tag_prefix, tag_regex)], - cwd=root) + describe_out, rc = run_command( + GITS, + ["describe", "--tags", "--dirty", "--always", "--long", "--match", "%s%s" % (tag_prefix, tag_regex)], + cwd=root, + ) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") @@ -262,17 +272,16 @@ def git_pieces_from_vcs(tag_prefix, tag_regex, root, verbose, run_command=run_co dirty = git_describe.endswith("-dirty") pieces["dirty"] = dirty if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] + git_describe = git_describe[: git_describe.rindex("-dirty")] # now we have TAG-NUM-gHEX or HEX if "-" in git_describe: # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) if not mo: # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%s'" - % describe_out) + pieces["error"] = "unable to parse git-describe output: '%s'" % describe_out return pieces # tag @@ -281,10 +290,9 @@ def git_pieces_from_vcs(tag_prefix, tag_regex, root, verbose, run_command=run_co if verbose: fmt = "tag '%s' doesn't start with prefix '%s'" print(fmt % (full_tag, tag_prefix)) - pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" - % (full_tag, tag_prefix)) + pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % (full_tag, tag_prefix) return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] + pieces["closest-tag"] = full_tag[len(tag_prefix) :] # distance: number of commits since tag pieces["distance"] = int(mo.group(2)) @@ -295,13 +303,11 @@ def git_pieces_from_vcs(tag_prefix, tag_regex, root, verbose, run_command=run_co else: # HEX: no tags pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) + count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], cwd=root) pieces["distance"] = int(count_out) # total number of commits # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], - cwd=root)[0].strip() + date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[0].strip() pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) return pieces @@ -332,8 +338,7 @@ def render_pep440(pieces): rendered += ".dirty" else: # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) + rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered @@ -447,11 +452,13 @@ def render_git_describe_long(pieces): def render(pieces, style): """Render the given version pieces into the requested style.""" if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} + return { + "version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None, + } if not style or style == "default": style = "pep440" # the default @@ -471,9 +478,13 @@ def render(pieces, style): else: raise ValueError("unknown style '%s'" % style) - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} + return { + "version": rendered, + "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], + "error": None, + "date": pieces.get("date"), + } def get_versions(): @@ -487,8 +498,7 @@ def get_versions(): verbose = cfg.verbose try: - return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, - verbose) + return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, verbose) except NotThisMethod: pass @@ -497,13 +507,16 @@ def get_versions(): # versionfile_source is the relative path from the top of the source # tree (where the .git directory might live) to this file. Invert # this to find the root from __file__. - for i in cfg.versionfile_source.split('/'): + for i in cfg.versionfile_source.split("/"): root = os.path.dirname(root) except NameError: - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None} + return { + "version": "0+unknown", + "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree", + "date": None, + } try: pieces = git_pieces_from_vcs(cfg.tag_prefix, cfg.tag_regex, root, verbose) @@ -517,6 +530,10 @@ def get_versions(): except NotThisMethod: pass - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", "date": None} + return { + "version": "0+unknown", + "full-revisionid": None, + "dirty": None, + "error": "unable to compute version", + "date": None, + } diff --git a/src/buildstream/_workspaces.py b/src/buildstream/_workspaces.py index f9023dc54..3d50fd9c0 100644 --- a/src/buildstream/_workspaces.py +++ b/src/buildstream/_workspaces.py @@ -38,7 +38,7 @@ WORKSPACE_PROJECT_FILE = ".bstproject.yaml" # Args: # directory (str): The directory that the workspace exists in. # -class WorkspaceProject(): +class WorkspaceProject: def __init__(self, directory): self._projects = [] self._directory = directory @@ -51,7 +51,7 @@ class WorkspaceProject(): # (str): The path to a project # def get_default_project_path(self): - return self._projects[0]['project-path'] + return self._projects[0]["project-path"] # get_default_element() # @@ -61,7 +61,7 @@ class WorkspaceProject(): # (str): The name of an element # def get_default_element(self): - return self._projects[0]['element-name'] + return self._projects[0]["element-name"] # to_dict() # @@ -72,8 +72,8 @@ class WorkspaceProject(): # def to_dict(self): ret = { - 'projects': self._projects, - 'format-version': BST_WORKSPACE_PROJECT_FORMAT_VERSION, + "projects": self._projects, + "format-version": BST_WORKSPACE_PROJECT_FORMAT_VERSION, } return ret @@ -91,13 +91,14 @@ class WorkspaceProject(): @classmethod def from_dict(cls, directory, dictionary): # Only know how to handle one format-version at the moment. - format_version = int(dictionary['format-version']) - assert format_version == BST_WORKSPACE_PROJECT_FORMAT_VERSION, \ - "Format version {} not found in {}".format(BST_WORKSPACE_PROJECT_FORMAT_VERSION, dictionary) + format_version = int(dictionary["format-version"]) + assert format_version == BST_WORKSPACE_PROJECT_FORMAT_VERSION, "Format version {} not found in {}".format( + BST_WORKSPACE_PROJECT_FORMAT_VERSION, dictionary + ) workspace_project = cls(directory) - for item in dictionary['projects']: - workspace_project.add_project(item['project-path'], item['element-name']) + for item in dictionary["projects"]: + workspace_project.add_project(item["project-path"], item["element-name"]) return workspace_project @@ -145,15 +146,15 @@ class WorkspaceProject(): # element_name (str): The name of the element that the workspace belongs to. # def add_project(self, project_path, element_name): - assert (project_path and element_name) - self._projects.append({'project-path': project_path, 'element-name': element_name}) + assert project_path and element_name + self._projects.append({"project-path": project_path, "element-name": element_name}) # WorkspaceProjectCache() # # A class to manage workspace project data for multiple workspaces. # -class WorkspaceProjectCache(): +class WorkspaceProjectCache: def __init__(self): self._projects = {} # Mapping of a workspace directory to its WorkspaceProject @@ -216,8 +217,9 @@ class WorkspaceProjectCache(): def remove(self, directory): workspace_project = self.get(directory) if not workspace_project: - raise LoadError("Failed to find a {} file to remove".format(WORKSPACE_PROJECT_FILE), - LoadErrorReason.MISSING_FILE) + raise LoadError( + "Failed to find a {} file to remove".format(WORKSPACE_PROJECT_FILE), LoadErrorReason.MISSING_FILE + ) path = workspace_project.get_filename() try: os.unlink(path) @@ -242,7 +244,7 @@ class WorkspaceProjectCache(): # changed between failed builds. Should be # made obsolete with failed build artifacts. # -class Workspace(): +class Workspace: def __init__(self, toplevel_project, *, last_successful=None, path=None, prepared=False, running_files=None): self.prepared = prepared self.last_successful = last_successful @@ -260,11 +262,7 @@ class Workspace(): # (dict) A dict representation of the workspace # def to_dict(self): - ret = { - 'prepared': self.prepared, - 'path': self._path, - 'running_files': self.running_files - } + ret = {"prepared": self.prepared, "path": self._path, "running_files": self.running_files} if self.last_successful is not None: ret["last_successful"] = self.last_successful return ret @@ -363,8 +361,7 @@ class Workspace(): try: stat = os.lstat(filename) except OSError as e: - raise LoadError("Failed to stat file in workspace: {}".format(e), - LoadErrorReason.MISSING_FILE) + raise LoadError("Failed to stat file in workspace: {}".format(e), LoadErrorReason.MISSING_FILE) # Use the mtime of any file with sub second precision return stat.st_mtime_ns @@ -378,8 +375,7 @@ class Workspace(): if os.path.isdir(fullpath): filelist = utils.list_relative_paths(fullpath) filelist = [ - (relpath, os.path.join(fullpath, relpath)) for relpath in filelist - if relpath not in excluded_files + (relpath, os.path.join(fullpath, relpath)) for relpath in filelist if relpath not in excluded_files ] else: filelist = [(self.get_absolute_path(), fullpath)] @@ -404,7 +400,7 @@ class Workspace(): # toplevel_project (Project): Top project used to resolve paths. # workspace_project_cache (WorkspaceProjectCache): The cache of WorkspaceProjects # -class Workspaces(): +class Workspaces: def __init__(self, toplevel_project, workspace_project_cache): self._toplevel_project = toplevel_project self._bst_directory = os.path.join(toplevel_project.directory, ".bst") @@ -525,11 +521,8 @@ class Workspaces(): assert utils._is_main_process() config = { - 'format-version': BST_WORKSPACE_FORMAT_VERSION, - 'workspaces': { - element: workspace.to_dict() - for element, workspace in self._workspaces.items() - } + "format-version": BST_WORKSPACE_FORMAT_VERSION, + "workspaces": {element: workspace.to_dict() for element, workspace in self._workspaces.items()}, } os.makedirs(self._bst_directory, exist_ok=True) _yaml.roundtrip_dump(config, self._get_filename()) @@ -572,10 +565,11 @@ class Workspaces(): # def _parse_workspace_config(self, workspaces): try: - version = workspaces.get_int('format-version', default=0) + version = workspaces.get_int("format-version", default=0) except ValueError: - raise LoadError("Format version is not an integer in workspace configuration", - LoadErrorReason.INVALID_DATA) + raise LoadError( + "Format version is not an integer in workspace configuration", LoadErrorReason.INVALID_DATA + ) if version == 0: # Pre-versioning format can be of two forms @@ -588,17 +582,17 @@ class Workspaces(): elif config_type is MappingNode: sources = list(config.values()) if len(sources) > 1: - detail = "There are multiple workspaces open for '{}'.\n" + \ - "This is not supported anymore.\n" + \ - "Please remove this element from '{}'." - raise LoadError(detail.format(element, self._get_filename()), - LoadErrorReason.INVALID_DATA) + detail = ( + "There are multiple workspaces open for '{}'.\n" + + "This is not supported anymore.\n" + + "Please remove this element from '{}'." + ) + raise LoadError(detail.format(element, self._get_filename()), LoadErrorReason.INVALID_DATA) workspaces[element] = sources[0] else: - raise LoadError("Workspace config is in unexpected format.", - LoadErrorReason.INVALID_DATA) + raise LoadError("Workspace config is in unexpected format.", LoadErrorReason.INVALID_DATA) res = { element: Workspace(self._toplevel_project, path=config.as_str()) @@ -607,13 +601,16 @@ class Workspaces(): elif 1 <= version <= BST_WORKSPACE_FORMAT_VERSION: workspaces = workspaces.get_mapping("workspaces", default={}) - res = {element: self._load_workspace(node) - for element, node in workspaces.items()} + res = {element: self._load_workspace(node) for element, node in workspaces.items()} else: - raise LoadError("Workspace configuration format version {} not supported." - "Your version of buildstream may be too old. Max supported version: {}" - .format(version, BST_WORKSPACE_FORMAT_VERSION), LoadErrorReason.INVALID_DATA) + raise LoadError( + "Workspace configuration format version {} not supported." + "Your version of buildstream may be too old. Max supported version: {}".format( + version, BST_WORKSPACE_FORMAT_VERSION + ), + LoadErrorReason.INVALID_DATA, + ) return res @@ -628,15 +625,15 @@ class Workspaces(): # (Workspace): A newly instantiated Workspace # def _load_workspace(self, node): - running_files = node.get_mapping('running_files', default=None) + running_files = node.get_mapping("running_files", default=None) if running_files: running_files = running_files.strip_node_info() dictionary = { - 'prepared': node.get_bool('prepared', default=False), - 'path': node.get_str('path'), - 'last_successful': node.get_str('last_successful', default=None), - 'running_files': running_files, + "prepared": node.get_bool("prepared", default=False), + "path": node.get_str("path"), + "last_successful": node.get_str("last_successful", default=None), + "running_files": running_files, } return Workspace.from_dict(self._toplevel_project, dictionary) diff --git a/src/buildstream/buildelement.py b/src/buildstream/buildelement.py index 7fe97c168..f04d3b0dc 100644 --- a/src/buildstream/buildelement.py +++ b/src/buildstream/buildelement.py @@ -144,17 +144,16 @@ from .types import Scope # This list is preserved because of an unfortunate situation, we # need to remove these older commands which were secret and never # documented, but without breaking the cache keys. -_legacy_command_steps = ['bootstrap-commands', - 'configure-commands', - 'build-commands', - 'test-commands', - 'install-commands', - 'strip-commands'] +_legacy_command_steps = [ + "bootstrap-commands", + "configure-commands", + "build-commands", + "test-commands", + "install-commands", + "strip-commands", +] -_command_steps = ['configure-commands', - 'build-commands', - 'install-commands', - 'strip-commands'] +_command_steps = ["configure-commands", "build-commands", "install-commands", "strip-commands"] class BuildElement(Element): @@ -190,21 +189,21 @@ class BuildElement(Element): # cache key, while having the side effect of setting max-jobs to 1, # which is normally automatically resolved and does not affect # the cache key. - if self.get_variable('notparallel'): - dictionary['notparallel'] = True + if self.get_variable("notparallel"): + dictionary["notparallel"] = True return dictionary def configure_sandbox(self, sandbox): - build_root = self.get_variable('build-root') - install_root = self.get_variable('install-root') + build_root = self.get_variable("build-root") + install_root = self.get_variable("install-root") # Tell the sandbox to mount the build root and install root sandbox.mark_directory(build_root) sandbox.mark_directory(install_root) # Allow running all commands in a specified subdirectory - command_subdir = self.get_variable('command-subdir') + command_subdir = self.get_variable("command-subdir") if command_subdir: command_dir = os.path.join(build_root, command_subdir) else: @@ -230,13 +229,13 @@ class BuildElement(Element): dep.integrate(sandbox) # Stage sources in the build root - self.stage_sources(sandbox, self.get_variable('build-root')) + self.stage_sources(sandbox, self.get_variable("build-root")) def assemble(self, sandbox): # Run commands for command_name in _command_steps: commands = self.__commands[command_name] - if not commands or command_name == 'configure-commands': + if not commands or command_name == "configure-commands": continue with sandbox.batch(SandboxFlags.ROOT_READ_ONLY, label="Running {}".format(command_name)): @@ -247,21 +246,22 @@ class BuildElement(Element): # to - if an element later attempts to stage to a location # that is not empty, we abort the build - in this case this # will almost certainly happen. - staged_build = os.path.join(self.get_variable('install-root'), - self.get_variable('build-root')) + staged_build = os.path.join(self.get_variable("install-root"), self.get_variable("build-root")) if os.path.isdir(staged_build) and os.listdir(staged_build): - self.warn("Writing to %{install-root}/%{build-root}.", - detail="Writing to this directory will almost " + - "certainly cause an error, since later elements " + - "will not be allowed to stage to %{build-root}.") + self.warn( + "Writing to %{install-root}/%{build-root}.", + detail="Writing to this directory will almost " + + "certainly cause an error, since later elements " + + "will not be allowed to stage to %{build-root}.", + ) # Return the payload, this is configurable but is generally # always the /buildstream-install directory - return self.get_variable('install-root') + return self.get_variable("install-root") def prepare(self, sandbox): - commands = self.__commands['configure-commands'] + commands = self.__commands["configure-commands"] if commands: with sandbox.batch(SandboxFlags.ROOT_READ_ONLY, label="Running configure-commands"): for cmd in commands: @@ -282,15 +282,10 @@ class BuildElement(Element): ############################################################# def __get_commands(self, node, name): raw_commands = node.get_sequence(name, []) - return [ - self.node_subst_vars(command) - for command in raw_commands - ] + return [self.node_subst_vars(command) for command in raw_commands] def __run_command(self, sandbox, cmd): # Note the -e switch to 'sh' means to exit with an error # if any untested command fails. # - sandbox.run(['sh', '-c', '-e', cmd + '\n'], - SandboxFlags.ROOT_READ_ONLY, - label=cmd) + sandbox.run(["sh", "-c", "-e", cmd + "\n"], SandboxFlags.ROOT_READ_ONLY, label=cmd) diff --git a/src/buildstream/element.py b/src/buildstream/element.py index 5fa8f14df..5028cc5fa 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -90,8 +90,7 @@ from pyroaring import BitMap # pylint: disable=no-name-in-module from . import _yaml from ._variables import Variables from ._versions import BST_CORE_ARTIFACT_VERSION -from ._exceptions import BstError, LoadError, LoadErrorReason, ImplError, \ - ErrorDomain, SourceCacheError +from ._exceptions import BstError, LoadError, LoadErrorReason, ImplError, ErrorDomain, SourceCacheError from .utils import FileListResult from . import utils from . import _cachekey @@ -122,6 +121,7 @@ if TYPE_CHECKING: from ._context import Context from ._loader.metaelement import MetaElement from ._project import Project + # pylint: enable=cyclic-import @@ -136,13 +136,10 @@ class ElementError(BstError): collect: An optional directory containing partial install contents temporary: An indicator to whether the error may occur if the operation was run again. (*Since: 1.2*) """ - def __init__(self, - message: str, - *, - detail: str = None, - reason: str = None, - collect: str = None, - temporary: bool = False): + + def __init__( + self, message: str, *, detail: str = None, reason: str = None, collect: str = None, temporary: bool = False + ): super().__init__(message, detail=detail, domain=ErrorDomain.ELEMENT, reason=reason, temporary=temporary) self.collect = collect @@ -156,12 +153,13 @@ class Element(Plugin): All elements derive from this class, this interface defines how the core will be interacting with Elements. """ + # The defaults from the yaml file and project __defaults = None # A hash of Element by MetaElement - __instantiated_elements = {} # type: Dict[MetaElement, Element] + __instantiated_elements = {} # type: Dict[MetaElement, Element] # A list of (source, ref) tuples which were redundantly specified - __redundant_source_refs = [] # type: List[Tuple[Source, SourceRef]] + __redundant_source_refs = [] # type: List[Tuple[Source, SourceRef]] BST_ARTIFACT_VERSION = 0 """The element plugin's artifact version @@ -215,10 +213,10 @@ class Element(Plugin): *Since: 1.90* """ - def __init__(self, context: 'Context', project: 'Project', meta: 'MetaElement', plugin_conf: Dict[str, Any]): + def __init__(self, context: "Context", project: "Project", meta: "MetaElement", plugin_conf: Dict[str, Any]): - self.__cache_key_dict = None # Dict for cache key calculation - self.__cache_key = None # Our cached cache key + self.__cache_key_dict = None # Dict for cache key calculation + self.__cache_key = None # Our cached cache key super().__init__(meta.name, context, project, meta.provenance, "element") @@ -236,75 +234,75 @@ class Element(Plugin): """ # Direct runtime dependency Elements - self.__runtime_dependencies = [] # type: List[Element] + self.__runtime_dependencies = [] # type: List[Element] # Direct build dependency Elements - self.__build_dependencies = [] # type: List[Element] + self.__build_dependencies = [] # type: List[Element] # Direct build dependency subset which require strict rebuilds - self.__strict_dependencies = [] # type: List[Element] + self.__strict_dependencies = [] # type: List[Element] # Direct reverse build dependency Elements - self.__reverse_build_deps = set() # type: Set[Element] + self.__reverse_build_deps = set() # type: Set[Element] # Direct reverse runtime dependency Elements - self.__reverse_runtime_deps = set() # type: Set[Element] - self.__build_deps_without_strict_cache_key = None # Number of build dependencies without a strict key + self.__reverse_runtime_deps = set() # type: Set[Element] + self.__build_deps_without_strict_cache_key = None # Number of build dependencies without a strict key self.__runtime_deps_without_strict_cache_key = None # Number of runtime dependencies without a strict key - self.__build_deps_without_cache_key = None # Number of build dependencies without a cache key + self.__build_deps_without_cache_key = None # Number of build dependencies without a cache key self.__runtime_deps_without_cache_key = None # Number of runtime dependencies without a cache key - self.__build_deps_uncached = None # Build dependencies which are not yet cached + self.__build_deps_uncached = None # Build dependencies which are not yet cached self.__runtime_deps_uncached = None # Runtime dependencies which are not yet cached self.__updated_strict_cache_keys_of_rdeps = False # Whether we've updated strict cache keys of rdeps - self.__ready_for_runtime = False # Whether the element and its runtime dependencies have cache keys + self.__ready_for_runtime = False # Whether the element and its runtime dependencies have cache keys self.__ready_for_runtime_and_cached = False # Whether all runtime deps are cached, as well as the element - self.__cached_remotely = None # Whether the element is cached remotely + self.__cached_remotely = None # Whether the element is cached remotely # List of Sources - self.__sources = [] # type: List[Source] - self.__weak_cache_key = None # Our cached weak cache key - self.__strict_cache_key = None # Our cached cache key for strict builds + self.__sources = [] # type: List[Source] + self.__weak_cache_key = None # Our cached weak cache key + self.__strict_cache_key = None # Our cached cache key for strict builds self.__artifacts = context.artifactcache # Artifact cache self.__sourcecache = context.sourcecache # Source cache self.__consistency = Consistency.INCONSISTENT # Cached overall consistency state - self.__assemble_scheduled = False # Element is scheduled to be assembled - self.__assemble_done = False # Element is assembled - self.__tracking_scheduled = False # Sources are scheduled to be tracked - self.__pull_done = False # Whether pull was attempted - self.__cached_successfully = None # If the Element is known to be successfully cached - self.__source_cached = None # If the sources are known to be successfully cached - self.__splits = None # Resolved regex objects for computing split domains - self.__whitelist_regex = None # Resolved regex object to check if file is allowed to overlap + self.__assemble_scheduled = False # Element is scheduled to be assembled + self.__assemble_done = False # Element is assembled + self.__tracking_scheduled = False # Sources are scheduled to be tracked + self.__pull_done = False # Whether pull was attempted + self.__cached_successfully = None # If the Element is known to be successfully cached + self.__source_cached = None # If the sources are known to be successfully cached + self.__splits = None # Resolved regex objects for computing split domains + self.__whitelist_regex = None # Resolved regex object to check if file is allowed to overlap # Location where Element.stage_sources() was called self.__staged_sources_directory = None # type: Optional[str] - self.__tainted = None # Whether the artifact is tainted and should not be shared - self.__required = False # Whether the artifact is required in the current session + self.__tainted = None # Whether the artifact is tainted and should not be shared + self.__required = False # Whether the artifact is required in the current session self.__artifact_files_required = False # Whether artifact files are required in the local cache - self.__build_result = None # The result of assembling this Element (success, description, detail) - self._build_log_path = None # The path of the build log for this Element + self.__build_result = None # The result of assembling this Element (success, description, detail) + self._build_log_path = None # The path of the build log for this Element # Artifact class for direct artifact composite interaction - self.__artifact = None # type: Optional[Artifact] - self.__strict_artifact = None # Artifact for strict cache key - self.__meta_kind = meta.kind # The kind of this source, required for unpickling + self.__artifact = None # type: Optional[Artifact] + self.__strict_artifact = None # Artifact for strict cache key + self.__meta_kind = meta.kind # The kind of this source, required for unpickling # the index of the last source in this element that requires previous # sources for staging self.__last_source_requires_previous_ix = None - self.__batch_prepare_assemble = False # Whether batching across prepare()/assemble() is configured - self.__batch_prepare_assemble_flags = 0 # Sandbox flags for batching across prepare()/assemble() + self.__batch_prepare_assemble = False # Whether batching across prepare()/assemble() is configured + self.__batch_prepare_assemble_flags = 0 # Sandbox flags for batching across prepare()/assemble() # Collect dir for batching across prepare()/assemble() self.__batch_prepare_assemble_collect = None # type: Optional[str] # Callbacks - self.__required_callback = None # Callback to Queues - self.__can_query_cache_callback = None # Callback to PullQueue/FetchQueue - self.__buildable_callback = None # Callback to BuildQueue + self.__required_callback = None # Callback to Queues + self.__can_query_cache_callback = None # Callback to PullQueue/FetchQueue + self.__buildable_callback = None # Callback to BuildQueue - self._depth = None # Depth of Element in its current dependency graph - self._resolved_initial_state = False # Whether the initial state of the Element has been resolved + self._depth = None # Depth of Element in its current dependency graph + self._resolved_initial_state = False # Whether the initial state of the Element has been resolved # Ensure we have loaded this class's defaults self.__init_defaults(project, plugin_conf, meta.kind, meta.is_junction) # Collect the composited variables and resolve them variables = self.__extract_variables(project, meta) - variables['element-name'] = self.name + variables["element-name"] = self.name self.__variables = Variables(variables) # Collect the composited environment now that we have variables @@ -348,7 +346,7 @@ class Element(Plugin): ############################################################# # Abstract Methods # ############################################################# - def configure_sandbox(self, sandbox: 'Sandbox') -> None: + def configure_sandbox(self, sandbox: "Sandbox") -> None: """Configures the the sandbox for execution Args: @@ -360,10 +358,9 @@ class Element(Plugin): Elements must implement this method to configure the sandbox object for execution. """ - raise ImplError("element plugin '{kind}' does not implement configure_sandbox()".format( - kind=self.get_kind())) + raise ImplError("element plugin '{kind}' does not implement configure_sandbox()".format(kind=self.get_kind())) - def stage(self, sandbox: 'Sandbox') -> None: + def stage(self, sandbox: "Sandbox") -> None: """Stage inputs into the sandbox directories Args: @@ -377,10 +374,9 @@ class Element(Plugin): objects, by staging the artifacts of the elements this element depends on, or both. """ - raise ImplError("element plugin '{kind}' does not implement stage()".format( - kind=self.get_kind())) + raise ImplError("element plugin '{kind}' does not implement stage()".format(kind=self.get_kind())) - def prepare(self, sandbox: 'Sandbox') -> None: + def prepare(self, sandbox: "Sandbox") -> None: """Run one-off preparation commands. This is run before assemble(), but is guaranteed to run only @@ -400,7 +396,7 @@ class Element(Plugin): *Since: 1.2* """ - def assemble(self, sandbox: 'Sandbox') -> str: + def assemble(self, sandbox: "Sandbox") -> str: """Assemble the output artifact Args: @@ -415,8 +411,7 @@ class Element(Plugin): Elements must implement this method to create an output artifact from its sources and dependencies. """ - raise ImplError("element plugin '{kind}' does not implement assemble()".format( - kind=self.get_kind())) + raise ImplError("element plugin '{kind}' does not implement assemble()".format(kind=self.get_kind())) def generate_script(self) -> str: """Generate a build (sh) script to build this element @@ -437,13 +432,12 @@ class Element(Plugin): If the script fails, it is expected to return with an exit code != 0. """ - raise ImplError("element plugin '{kind}' does not implement write_script()".format( - kind=self.get_kind())) + raise ImplError("element plugin '{kind}' does not implement write_script()".format(kind=self.get_kind())) ############################################################# # Public Methods # ############################################################# - def sources(self) -> Iterator['Source']: + def sources(self) -> Iterator["Source"]: """A generator function to enumerate the element sources Yields: @@ -452,7 +446,7 @@ class Element(Plugin): for source in self.__sources: yield source - def dependencies(self, scope: Scope, *, recurse: bool = True, visited=None) -> Iterator['Element']: + def dependencies(self, scope: Scope, *, recurse: bool = True, visited=None) -> Iterator["Element"]: """dependencies(scope, *, recurse=True) A generator function which yields the dependencies of the given element. @@ -479,6 +473,7 @@ class Element(Plugin): if scope in (Scope.RUN, Scope.ALL): yield from self.__runtime_dependencies else: + def visit(element, scope, visited): if scope == Scope.ALL: visited[0].add(element._unique_id) @@ -519,7 +514,7 @@ class Element(Plugin): yield from visit(self, scope, visited) - def search(self, scope: Scope, name: str) -> Optional['Element']: + def search(self, scope: Scope, name: str) -> Optional["Element"]: """Search for a dependency by name Args: @@ -535,7 +530,7 @@ class Element(Plugin): return None - def node_subst_vars(self, node: 'ScalarNode') -> str: + def node_subst_vars(self, node: "ScalarNode") -> str: """Replace any variables in the string contained in the node and returns it. Args: @@ -559,9 +554,9 @@ class Element(Plugin): return self.__variables.subst(node.as_str()) except LoadError as e: provenance = node.get_provenance() - raise LoadError('{}: {}'.format(provenance, e), e.reason, detail=e.detail) from e + raise LoadError("{}: {}".format(provenance, e), e.reason, detail=e.detail) from e - def node_subst_sequence_vars(self, node: 'SequenceNode[ScalarNode]') -> List[str]: + def node_subst_sequence_vars(self, node: "SequenceNode[ScalarNode]") -> List[str]: """Substitute any variables in the given sequence Args: @@ -580,14 +575,12 @@ class Element(Plugin): ret.append(self.__variables.subst(value.as_str())) except LoadError as e: provenance = value.get_provenance() - raise LoadError('{}: {}'.format(provenance, e), e.reason, detail=e.detail) from e + raise LoadError("{}: {}".format(provenance, e), e.reason, detail=e.detail) from e return ret - def compute_manifest(self, - *, - include: Optional[List[str]] = None, - exclude: Optional[List[str]] = None, - orphans: bool = True) -> str: + def compute_manifest( + self, *, include: Optional[List[str]] = None, exclude: Optional[List[str]] = None, orphans: bool = True + ) -> str: """Compute and return this element's selective manifest The manifest consists on the list of file paths in the @@ -630,14 +623,16 @@ class Element(Plugin): return _compose_artifact_name(self.project_name, self.normal_name, key) - def stage_artifact(self, - sandbox: 'Sandbox', - *, - path: str = None, - include: Optional[List[str]] = None, - exclude: Optional[List[str]] = None, - orphans: bool = True, - update_mtimes: Optional[List[str]] = None) -> FileListResult: + def stage_artifact( + self, + sandbox: "Sandbox", + *, + path: str = None, + include: Optional[List[str]] = None, + exclude: Optional[List[str]] = None, + orphans: bool = True, + update_mtimes: Optional[List[str]] = None + ) -> FileListResult: """Stage this element's output artifact in the sandbox This will stage the files from the artifact to the sandbox at specified location. @@ -675,10 +670,11 @@ class Element(Plugin): """ if not self._cached(): - detail = "No artifacts have been cached yet for that element\n" + \ - "Try building the element first with `bst build`\n" - raise ElementError("No artifacts to stage", - detail=detail, reason="uncached-checkout-attempt") + detail = ( + "No artifacts have been cached yet for that element\n" + + "Try building the element first with `bst build`\n" + ) + raise ElementError("No artifacts to stage", detail=detail, reason="uncached-checkout-attempt") if update_mtimes is None: update_mtimes = [] @@ -689,47 +685,49 @@ class Element(Plugin): with self.timed_activity("Staging {}/{}".format(self.name, self._get_brief_display_key())): # Disable type checking since we can't easily tell mypy that # `self.__artifact` can't be None at this stage. - files_vdir = self.__artifact.get_files() # type: ignore + files_vdir = self.__artifact.get_files() # type: ignore # Hard link it into the staging area # vbasedir = sandbox.get_virtual_directory() - vstagedir = vbasedir \ - if path is None \ - else vbasedir.descend(*path.lstrip(os.sep).split(os.sep)) + vstagedir = vbasedir if path is None else vbasedir.descend(*path.lstrip(os.sep).split(os.sep)) split_filter = self.__split_filter_func(include, exclude, orphans) # We must not hardlink files whose mtimes we want to update if update_mtimes: + def link_filter(path): - return ((split_filter is None or split_filter(path)) and - path not in update_mtimes) + return (split_filter is None or split_filter(path)) and path not in update_mtimes def copy_filter(path): - return ((split_filter is None or split_filter(path)) and - path in update_mtimes) + return (split_filter is None or split_filter(path)) and path in update_mtimes + else: link_filter = split_filter - result = vstagedir.import_files(files_vdir, filter_callback=link_filter, - report_written=True, can_link=True) + result = vstagedir.import_files( + files_vdir, filter_callback=link_filter, report_written=True, can_link=True + ) if update_mtimes: - copy_result = vstagedir.import_files(files_vdir, filter_callback=copy_filter, - report_written=True, update_mtime=True) + copy_result = vstagedir.import_files( + files_vdir, filter_callback=copy_filter, report_written=True, update_mtime=True + ) result = result.combine(copy_result) return result - def stage_dependency_artifacts(self, - sandbox: 'Sandbox', - scope: Scope, - *, - path: str = None, - include: Optional[List[str]] = None, - exclude: Optional[List[str]] = None, - orphans: bool = True) -> None: + def stage_dependency_artifacts( + self, + sandbox: "Sandbox", + scope: Scope, + *, + path: str = None, + include: Optional[List[str]] = None, + exclude: Optional[List[str]] = None, + orphans: bool = True + ) -> None: """Stage element dependencies in scope This is primarily a convenience wrapper around @@ -751,8 +749,8 @@ class Element(Plugin): occur. """ ignored = {} - overlaps = OrderedDict() # type: OrderedDict[str, List[str]] - files_written = {} # type: Dict[str, List[str]] + overlaps = OrderedDict() # type: OrderedDict[str, List[str]] + files_written = {} # type: Dict[str, List[str]] old_dep_keys = None workspace = self._get_workspace() context = self._get_context() @@ -803,12 +801,9 @@ class Element(Plugin): if utils._is_main_process(): context.get_workspaces().save_config() - result = dep.stage_artifact(sandbox, - path=path, - include=include, - exclude=exclude, - orphans=orphans, - update_mtimes=to_update) + result = dep.stage_artifact( + sandbox, path=path, include=include, exclude=exclude, orphans=orphans, update_mtimes=to_update + ) if result.overwritten: for overwrite in result.overwritten: # Completely new overwrite @@ -841,8 +836,9 @@ class Element(Plugin): warning_detail += _overlap_error_detail(f, overlap_warning_elements, elements) if overlap_warning: - self.warn("Non-whitelisted overlaps detected", detail=warning_detail, - warning_token=CoreWarnings.OVERLAPS) + self.warn( + "Non-whitelisted overlaps detected", detail=warning_detail, warning_token=CoreWarnings.OVERLAPS + ) if ignored: detail = "Not staging files which would replace non-empty directories:\n" @@ -851,7 +847,7 @@ class Element(Plugin): detail += " " + " ".join(["/" + f + "\n" for f in value]) self.warn("Ignored files", detail=detail) - def integrate(self, sandbox: 'Sandbox') -> None: + def integrate(self, sandbox: "Sandbox") -> None: """Integrate currently staged filesystem against this artifact. Args: @@ -863,19 +859,18 @@ class Element(Plugin): commands will create and update important system cache files required for running the installed software (such as the ld.so.cache). """ - bstdata = self.get_public_data('bst') + bstdata = self.get_public_data("bst") environment = self.get_environment() if bstdata is not None: with sandbox.batch(SandboxFlags.NONE): - commands = bstdata.get_sequence('integration-commands', []) + commands = bstdata.get_sequence("integration-commands", []) for command in commands: cmd = self.node_subst_vars(command) - sandbox.run(['sh', '-e', '-c', cmd], 0, env=environment, cwd='/', - label=cmd) + sandbox.run(["sh", "-e", "-c", cmd], 0, env=environment, cwd="/", label=cmd) - def stage_sources(self, sandbox: 'Sandbox', directory: str) -> None: + def stage_sources(self, sandbox: "Sandbox", directory: str) -> None: """Stage this element's sources to a directory in the sandbox Args: @@ -892,7 +887,7 @@ class Element(Plugin): self._stage_sources_in_sandbox(sandbox, directory) - def get_public_data(self, domain: str) -> 'MappingNode[Any, Any]': + def get_public_data(self, domain: str) -> "MappingNode[Any, Any]": """Fetch public data on this element Args: @@ -911,13 +906,13 @@ class Element(Plugin): # Disable type-checking since we can't easily tell mypy that # `self.__dynamic_public` can't be None here. - data = self.__dynamic_public.get_mapping(domain, default=None) # type: ignore + data = self.__dynamic_public.get_mapping(domain, default=None) # type: ignore if data is not None: data = data.clone() return data - def set_public_data(self, domain: str, data: 'MappingNode[Any, Any]') -> None: + def set_public_data(self, domain: str, data: "MappingNode[Any, Any]") -> None: """Set public data on this element Args: @@ -935,7 +930,7 @@ class Element(Plugin): if data is not None: data = data.clone() - self.__dynamic_public[domain] = data # type: ignore + self.__dynamic_public[domain] = data # type: ignore def get_environment(self) -> Dict[str, str]: """Fetch the environment suitable for running in the sandbox @@ -1016,8 +1011,7 @@ class Element(Plugin): # Instantiate sources and generate their keys for meta_source in meta.sources: meta_source.first_pass = meta.is_junction - source = meta.project.create_source(meta_source, - first_pass=meta.first_pass) + source = meta.project.create_source(meta_source, first_pass=meta.first_pass) redundant_ref = source._load_ref() @@ -1190,8 +1184,7 @@ class Element(Plugin): # (bool): Whether this element can currently be built # def _buildable(self): - if self._get_consistency() < Consistency.CACHED and \ - not self._source_cached(): + if self._get_consistency() < Consistency.CACHED and not self._source_cached(): return False if not self.__assemble_scheduled: @@ -1261,11 +1254,14 @@ class Element(Plugin): # If the element wasn't assembled and isn't scheduled to be assemble, # or cached, or waiting to be pulled but has an artifact then schedule # the assembly. - if (not self.__assemble_scheduled and not self.__assemble_done and - self.__artifact and - self._is_required() and - not self._cached() and - not self._pull_pending()): + if ( + not self.__assemble_scheduled + and not self.__assemble_done + and self.__artifact + and self._is_required() + and not self._cached() + and not self._pull_pending() + ): self._schedule_assemble() # If a build has been scheduled, we know that the element @@ -1298,7 +1294,7 @@ class Element(Plugin): cache_key = self._get_cache_key() if not cache_key: - cache_key = "{:?<64}".format('') + cache_key = "{:?<64}".format("") elif cache_key == self.__strict_cache_key: # Strong cache key used in this session matches cache key # that would be used in strict build mode @@ -1378,8 +1374,10 @@ class Element(Plugin): # Complimentary warning that the new ref will be unused. if old_ref != new_ref and self._get_workspace(): - detail = "This source has an open workspace.\n" \ + detail = ( + "This source has an open workspace.\n" + "To start using the new reference, please close the existing workspace." + ) source.warn("Updated reference will be ignored as source has open workspace", detail=detail) return refs @@ -1393,8 +1391,9 @@ class Element(Plugin): def _prepare_sandbox(self, scope, directory, shell=False, integrate=True, usebuildtree=False): # bst shell and bst artifact checkout require a local sandbox. bare_directory = bool(directory) - with self.__sandbox(directory, config=self.__sandbox_config, allow_remote=False, - bare_directory=bare_directory) as sandbox: + with self.__sandbox( + directory, config=self.__sandbox_config, allow_remote=False, bare_directory=bare_directory + ) as sandbox: sandbox._usebuildtree = usebuildtree # Configure always comes first, and we need it. @@ -1452,8 +1451,9 @@ class Element(Plugin): # It's advantageous to have this temporary directory on # the same file system as the rest of our cache. - with self.timed_activity("Staging sources", silent_nested=True), \ - utils._tempdir(dir=context.tmpdir, prefix='staging-temp') as temp_staging_directory: + with self.timed_activity("Staging sources", silent_nested=True), utils._tempdir( + dir=context.tmpdir, prefix="staging-temp" + ) as temp_staging_directory: import_dir = temp_staging_directory @@ -1488,12 +1488,12 @@ class Element(Plugin): import_dir.import_files(source_dir) except SourceCacheError as e: - raise ElementError("Error trying to export source for {}: {}" - .format(self.name, e)) + raise ElementError("Error trying to export source for {}: {}".format(self.name, e)) except VirtualDirectoryError as e: - raise ElementError("Error trying to import sources together for {}: {}" - .format(self.name, e), - reason="import-source-files-fail") + raise ElementError( + "Error trying to import sources together for {}: {}".format(self.name, e), + reason="import-source-files-fail", + ) with utils._deterministic_umask(): vdirectory.import_files(import_dir) @@ -1601,8 +1601,7 @@ class Element(Plugin): self._update_ready_for_runtime_and_cached() if self._get_workspace() and self._cached_success(): - assert utils._is_main_process(), \ - "Attempted to save workspace configuration from child process" + assert utils._is_main_process(), "Attempted to save workspace configuration from child process" # # Note that this block can only happen in the # main process, since `self._cached_success()` cannot @@ -1638,9 +1637,12 @@ class Element(Plugin): with self._output_file() as output_file: if not self.__sandbox_config_supported: - self.warn("Sandbox configuration is not supported by the platform.", - detail="Falling back to UID {} GID {}. Artifact will not be pushed." - .format(self.__sandbox_config.build_uid, self.__sandbox_config.build_gid)) + self.warn( + "Sandbox configuration is not supported by the platform.", + detail="Falling back to UID {} GID {}. Artifact will not be pushed.".format( + self.__sandbox_config.build_uid, self.__sandbox_config.build_gid + ), + ) # Explicitly clean it up, keep the build dir around if exceptions are raised os.makedirs(context.builddir, exist_ok=True) @@ -1650,13 +1652,14 @@ class Element(Plugin): def cleanup_rootdir(): utils._force_rmtree(rootdir) - with _signals.terminator(cleanup_rootdir), \ - self.__sandbox(rootdir, output_file, output_file, self.__sandbox_config) as sandbox: # noqa + with _signals.terminator(cleanup_rootdir), self.__sandbox( + rootdir, output_file, output_file, self.__sandbox_config + ) as sandbox: # noqa # Let the sandbox know whether the buildtree will be required. # This allows the remote execution sandbox to skip buildtree # download when it's not needed. - buildroot = self.get_variable('build-root') + buildroot = self.get_variable("build-root") cache_buildtrees = context.cache_buildtrees if cache_buildtrees != _CacheBuildTrees.NEVER: always_cache_buildtrees = cache_buildtrees == _CacheBuildTrees.ALWAYS @@ -1681,8 +1684,9 @@ class Element(Plugin): self.stage(sandbox) try: if self.__batch_prepare_assemble: - cm = sandbox.batch(self.__batch_prepare_assemble_flags, - collect=self.__batch_prepare_assemble_collect) + cm = sandbox.batch( + self.__batch_prepare_assemble_flags, collect=self.__batch_prepare_assemble_collect + ) else: cm = contextlib.suppress() @@ -1724,11 +1728,13 @@ class Element(Plugin): # result. Element types without a build-root dir will be cached # with an empty buildtreedir regardless of this configuration. - if cache_buildtrees == _CacheBuildTrees.ALWAYS or \ - (cache_buildtrees == _CacheBuildTrees.AUTO and not build_success): + if cache_buildtrees == _CacheBuildTrees.ALWAYS or ( + cache_buildtrees == _CacheBuildTrees.AUTO and not build_success + ): try: sandbox_build_dir = sandbox_vroot.descend( - *self.get_variable('build-root').lstrip(os.sep).split(os.sep)) + *self.get_variable("build-root").lstrip(os.sep).split(os.sep) + ) sandbox._fetch_missing_blobs(sandbox_build_dir) except VirtualDirectoryError: # Directory could not be found. Pre-virtual @@ -1747,14 +1753,13 @@ class Element(Plugin): self._assemble_done() with self.timed_activity("Caching artifact"): - artifact_size = self.__artifact.cache(rootdir, sandbox_build_dir, collectvdir, - buildresult, publicdata) + artifact_size = self.__artifact.cache(rootdir, sandbox_build_dir, collectvdir, buildresult, publicdata) if collect is not None and collectvdir is None: raise ElementError( "Directory '{}' was not found inside the sandbox, " - "unable to collect artifact contents" - .format(collect)) + "unable to collect artifact contents".format(collect) + ) return artifact_size @@ -1855,8 +1860,7 @@ class Element(Plugin): def _skip_source_push(self): if not self.__sources or self._get_workspace(): return True - return not (self.__sourcecache.has_push_remotes(plugin=self) and - self._source_cached()) + return not (self.__sourcecache.has_push_remotes(plugin=self) and self._source_cached()) def _source_push(self): # try and push sources if we've got them @@ -1931,8 +1935,9 @@ class Element(Plugin): # Returns: Exit code # # If directory is not specified, one will be staged using scope - def _shell(self, scope=None, directory=None, *, mounts=None, isolate=False, prompt=None, command=None, - usebuildtree=False): + def _shell( + self, scope=None, directory=None, *, mounts=None, isolate=False, prompt=None, command=None, usebuildtree=False + ): with self._prepare_sandbox(scope, directory, shell=True, usebuildtree=usebuildtree) as sandbox: environment = self.get_environment() @@ -1946,7 +1951,7 @@ class Element(Plugin): shell_command, shell_environment, shell_host_files = project.get_shell_config() if prompt is not None: - environment['PS1'] = prompt + environment["PS1"] = prompt # Special configurations for non-isolated sandboxes if not isolate: @@ -2002,8 +2007,7 @@ class Element(Plugin): # additional support from Source implementations. # os.makedirs(context.builddir, exist_ok=True) - with utils._tempdir(dir=context.builddir, prefix='workspace-{}' - .format(self.normal_name)) as temp: + with utils._tempdir(dir=context.builddir, prefix="workspace-{}".format(self.normal_name)) as temp: for source in self.sources(): source._init_workspace(temp) @@ -2032,10 +2036,10 @@ class Element(Plugin): script = script_template.format( name=self.normal_name, - build_root=self.get_variable('build-root'), - install_root=self.get_variable('install-root'), + build_root=self.get_variable("build-root"), + install_root=self.get_variable("install-root"), variables=variable_string, - commands=self.generate_script() + commands=self.generate_script(), ) os.makedirs(directory, exist_ok=True) @@ -2120,8 +2124,7 @@ class Element(Plugin): continue # try and fetch from source cache - if source._get_consistency() < Consistency.CACHED and \ - self.__sourcecache.has_fetch_remotes(): + if source._get_consistency() < Consistency.CACHED and self.__sourcecache.has_fetch_remotes(): if self.__sourcecache.pull(source): continue @@ -2154,35 +2157,31 @@ class Element(Plugin): # Generate dict that is used as base for all cache keys if self.__cache_key_dict is None: # Filter out nocache variables from the element's environment - cache_env = { - key: value - for key, value in self.__environment.items() - if key not in self.__env_nocache - } + cache_env = {key: value for key, value in self.__environment.items() if key not in self.__env_nocache} project = self._get_project() self.__cache_key_dict = { - 'core-artifact-version': BST_CORE_ARTIFACT_VERSION, - 'element-plugin-key': self.get_unique_key(), - 'element-plugin-name': self.get_kind(), - 'element-plugin-version': self.BST_ARTIFACT_VERSION, - 'sandbox': self.__sandbox_config.get_unique_key(), - 'environment': cache_env, - 'public': self.__public.strip_node_info() + "core-artifact-version": BST_CORE_ARTIFACT_VERSION, + "element-plugin-key": self.get_unique_key(), + "element-plugin-name": self.get_kind(), + "element-plugin-version": self.BST_ARTIFACT_VERSION, + "sandbox": self.__sandbox_config.get_unique_key(), + "environment": cache_env, + "public": self.__public.strip_node_info(), } - self.__cache_key_dict['sources'] = [] + self.__cache_key_dict["sources"] = [] for source in self.__sources: - self.__cache_key_dict['sources'].append( - {'key': source._get_unique_key(), - 'name': source._get_source_name()}) + self.__cache_key_dict["sources"].append( + {"key": source._get_unique_key(), "name": source._get_source_name()} + ) - self.__cache_key_dict['fatal-warnings'] = sorted(project._fatal_warnings) + self.__cache_key_dict["fatal-warnings"] = sorted(project._fatal_warnings) cache_key_dict = self.__cache_key_dict.copy() - cache_key_dict['dependencies'] = dependencies + cache_key_dict["dependencies"] = dependencies return _cachekey.generate_key(cache_key_dict) @@ -2216,8 +2215,9 @@ class Element(Plugin): Args: fetch_original (bool): whether we need to original unstaged source """ - if (self._get_consistency() == Consistency.CACHED and fetch_original) or \ - (self._source_cached() and not fetch_original): + if (self._get_consistency() == Consistency.CACHED and fetch_original) or ( + self._source_cached() and not fetch_original + ): return False else: return True @@ -2299,8 +2299,7 @@ class Element(Plugin): # def _update_ready_for_runtime_and_cached(self): if not self.__ready_for_runtime_and_cached: - if self.__runtime_deps_uncached == 0 and self._cached_success() and \ - self.__cache_key: + if self.__runtime_deps_uncached == 0 and self._cached_success() and self.__cache_key: self.__ready_for_runtime_and_cached = True # Notify reverse dependencies @@ -2450,6 +2449,7 @@ class Element(Plugin): self.prepare(sandbox) if workspace: + def mark_workspace_prepared(): workspace.prepared = True @@ -2466,23 +2466,31 @@ class Element(Plugin): if self.BST_FORBID_RDEPENDS and self.BST_FORBID_BDEPENDS: if any(self.dependencies(Scope.RUN, recurse=False)) or any(self.dependencies(Scope.BUILD, recurse=False)): - raise ElementError("{}: Dependencies are forbidden for '{}' elements" - .format(self, self.get_kind()), reason="element-forbidden-depends") + raise ElementError( + "{}: Dependencies are forbidden for '{}' elements".format(self, self.get_kind()), + reason="element-forbidden-depends", + ) if self.BST_FORBID_RDEPENDS: if any(self.dependencies(Scope.RUN, recurse=False)): - raise ElementError("{}: Runtime dependencies are forbidden for '{}' elements" - .format(self, self.get_kind()), reason="element-forbidden-rdepends") + raise ElementError( + "{}: Runtime dependencies are forbidden for '{}' elements".format(self, self.get_kind()), + reason="element-forbidden-rdepends", + ) if self.BST_FORBID_BDEPENDS: if any(self.dependencies(Scope.BUILD, recurse=False)): - raise ElementError("{}: Build dependencies are forbidden for '{}' elements" - .format(self, self.get_kind()), reason="element-forbidden-bdepends") + raise ElementError( + "{}: Build dependencies are forbidden for '{}' elements".format(self, self.get_kind()), + reason="element-forbidden-bdepends", + ) if self.BST_FORBID_SOURCES: if any(self.sources()): - raise ElementError("{}: Sources are forbidden for '{}' elements" - .format(self, self.get_kind()), reason="element-forbidden-sources") + raise ElementError( + "{}: Sources are forbidden for '{}' elements".format(self, self.get_kind()), + reason="element-forbidden-sources", + ) try: self.preflight() @@ -2492,9 +2500,10 @@ class Element(Plugin): # Ensure that the first source does not need access to previous soruces if self.__sources and self.__sources[0]._requires_previous_sources(): - raise ElementError("{}: {} cannot be the first source of an element " - "as it requires access to previous sources" - .format(self, self.__sources[0])) + raise ElementError( + "{}: {} cannot be the first source of an element " + "as it requires access to previous sources".format(self, self.__sources[0]) + ) # Preflight the sources for source in self.sources(): @@ -2505,8 +2514,7 @@ class Element(Plugin): # Raises an error if the artifact is not cached. # def __assert_cached(self): - assert self._cached(), "{}: Missing artifact {}".format( - self, self._get_brief_display_key()) + assert self._cached(), "{}: Missing artifact {}".format(self, self._get_brief_display_key()) # __get_tainted(): # @@ -2532,8 +2540,7 @@ class Element(Plugin): workspaced_dependencies = self.__artifact.get_metadata_workspaced_dependencies() # Other conditions should be or-ed - self.__tainted = (workspaced or workspaced_dependencies or - not self.__sandbox_config_supported) + self.__tainted = workspaced or workspaced_dependencies or not self.__sandbox_config_supported return self.__tainted @@ -2572,36 +2579,45 @@ class Element(Plugin): if directory is not None and allow_remote and self.__use_remote_execution(): if not self.BST_VIRTUAL_DIRECTORY: - raise ElementError("Element {} is configured to use remote execution but plugin does not support it." - .format(self.name), detail="Plugin '{kind}' does not support virtual directories." - .format(kind=self.get_kind())) + raise ElementError( + "Element {} is configured to use remote execution but plugin does not support it.".format( + self.name + ), + detail="Plugin '{kind}' does not support virtual directories.".format(kind=self.get_kind()), + ) self.info("Using a remote sandbox for artifact {} with directory '{}'".format(self.name, directory)) output_files_required = context.require_artifact_files or self._artifact_files_required() - sandbox = SandboxRemote(context, project, - directory, - plugin=self, - stdout=stdout, - stderr=stderr, - config=config, - specs=self.__remote_execution_specs, - bare_directory=bare_directory, - allow_real_directory=False, - output_files_required=output_files_required) + sandbox = SandboxRemote( + context, + project, + directory, + plugin=self, + stdout=stdout, + stderr=stderr, + config=config, + specs=self.__remote_execution_specs, + bare_directory=bare_directory, + allow_real_directory=False, + output_files_required=output_files_required, + ) yield sandbox elif directory is not None and os.path.exists(directory): - sandbox = platform.create_sandbox(context, project, - directory, - plugin=self, - stdout=stdout, - stderr=stderr, - config=config, - bare_directory=bare_directory, - allow_real_directory=not self.BST_VIRTUAL_DIRECTORY) + sandbox = platform.create_sandbox( + context, + project, + directory, + plugin=self, + stdout=stdout, + stderr=stderr, + config=config, + bare_directory=bare_directory, + allow_real_directory=not self.BST_VIRTUAL_DIRECTORY, + ) yield sandbox else: @@ -2609,8 +2625,9 @@ class Element(Plugin): rootdir = tempfile.mkdtemp(prefix="{}-".format(self.normal_name), dir=context.builddir) # Recursive contextmanager... - with self.__sandbox(rootdir, stdout=stdout, stderr=stderr, config=config, - allow_remote=allow_remote, bare_directory=False) as sandbox: + with self.__sandbox( + rootdir, stdout=stdout, stderr=stderr, config=config, allow_remote=allow_remote, bare_directory=False + ) as sandbox: yield sandbox # Cleanup the build dir @@ -2632,9 +2649,9 @@ class Element(Plugin): # Extend project wide split rules with any split rules defined by the element element_splits._composite(splits) - element_bst['split-rules'] = splits - element_public['bst'] = element_bst - defaults['public'] = element_public + element_bst["split-rules"] = splits + element_public["bst"] = element_bst + defaults["public"] = element_public @classmethod def __init_defaults(cls, project, plugin_conf, kind, is_junction): @@ -2704,7 +2721,7 @@ class Element(Plugin): else: project_nocache = project.base_env_nocache - default_nocache = cls.__defaults.get_str_list('environment-nocache', default=[]) + default_nocache = cls.__defaults.get_str_list("environment-nocache", default=[]) element_nocache = meta.env_nocache # Accumulate values from the element default, the project and the element @@ -2719,7 +2736,7 @@ class Element(Plugin): # @classmethod def __extract_variables(cls, project, meta): - default_vars = cls.__defaults.get_mapping('variables', default={}) + default_vars = cls.__defaults.get_mapping("variables", default={}) if meta.is_junction: variables = project.first_pass_config.base_variables.clone() @@ -2730,7 +2747,7 @@ class Element(Plugin): meta.variables._composite(variables) variables._assert_fully_composited() - for var in ('project-name', 'element-name', 'max-jobs'): + for var in ("project-name", "element-name", "max-jobs"): node = variables.get_node(var, allow_none=True) if node is None: @@ -2738,8 +2755,10 @@ class Element(Plugin): provenance = node.get_provenance() if not provenance._is_synthetic: - raise LoadError("{}: invalid redefinition of protected variable '{}'" - .format(provenance, var), LoadErrorReason.PROTECTED_VARIABLE_REDEFINED) + raise LoadError( + "{}: invalid redefinition of protected variable '{}'".format(provenance, var), + LoadErrorReason.PROTECTED_VARIABLE_REDEFINED, + ) return variables @@ -2750,7 +2769,7 @@ class Element(Plugin): def __extract_config(cls, meta): # The default config is already composited with the project overrides - config = cls.__defaults.get_mapping('config', default={}) + config = cls.__defaults.get_mapping("config", default={}) config = config.clone() meta.config._composite(config) @@ -2763,10 +2782,7 @@ class Element(Plugin): @classmethod def __extract_sandbox_config(cls, context, project, meta): if meta.is_junction: - sandbox_config = Node.from_dict({ - 'build-uid': 0, - 'build-gid': 0 - }) + sandbox_config = Node.from_dict({"build-uid": 0, "build-gid": 0}) else: sandbox_config = project._sandbox.clone() @@ -2776,7 +2792,7 @@ class Element(Plugin): host_os = platform.get_host_os() # The default config is already composited with the project overrides - sandbox_defaults = cls.__defaults.get_mapping('sandbox', default={}) + sandbox_defaults = cls.__defaults.get_mapping("sandbox", default={}) sandbox_defaults = sandbox_defaults.clone() sandbox_defaults._composite(sandbox_config) @@ -2784,41 +2800,42 @@ class Element(Plugin): sandbox_config._assert_fully_composited() # Sandbox config, unlike others, has fixed members so we should validate them - sandbox_config.validate_keys(['build-uid', 'build-gid', 'build-os', 'build-arch']) + sandbox_config.validate_keys(["build-uid", "build-gid", "build-os", "build-arch"]) - build_arch = sandbox_config.get_str('build-arch', default=None) + build_arch = sandbox_config.get_str("build-arch", default=None) if build_arch: build_arch = Platform.canonicalize_arch(build_arch) else: build_arch = host_arch return SandboxConfig( - sandbox_config.get_int('build-uid'), - sandbox_config.get_int('build-gid'), - sandbox_config.get_str('build-os', default=host_os), - build_arch) + sandbox_config.get_int("build-uid"), + sandbox_config.get_int("build-gid"), + sandbox_config.get_str("build-os", default=host_os), + build_arch, + ) # This makes a special exception for the split rules, which # elements may extend but whos defaults are defined in the project. # @classmethod def __extract_public(cls, meta): - base_public = cls.__defaults.get_mapping('public', default={}) + base_public = cls.__defaults.get_mapping("public", default={}) base_public = base_public.clone() - base_bst = base_public.get_mapping('bst', default={}) - base_splits = base_bst.get_mapping('split-rules', default={}) + base_bst = base_public.get_mapping("bst", default={}) + base_splits = base_bst.get_mapping("split-rules", default={}) element_public = meta.public.clone() - element_bst = element_public.get_mapping('bst', default={}) - element_splits = element_bst.get_mapping('split-rules', default={}) + element_bst = element_public.get_mapping("bst", default={}) + element_splits = element_bst.get_mapping("split-rules", default={}) # Allow elements to extend the default splits defined in their project or # element specific defaults element_splits._composite(base_splits) - element_bst['split-rules'] = base_splits - element_public['bst'] = element_bst + element_bst["split-rules"] = base_splits + element_public["bst"] = element_bst element_public._assert_fully_composited() @@ -2826,24 +2843,21 @@ class Element(Plugin): # Expand the splits in the public data using the Variables in the element def __expand_splits(self, element_public): - element_bst = element_public.get_mapping('bst', default={}) - element_splits = element_bst.get_mapping('split-rules', default={}) + element_bst = element_public.get_mapping("bst", default={}) + element_splits = element_bst.get_mapping("split-rules", default={}) # Resolve any variables in the public split rules directly for domain, splits in element_splits.items(): - splits = [ - self.__variables.subst(split.strip()) - for split in splits.as_str_list() - ] + splits = [self.__variables.subst(split.strip()) for split in splits.as_str_list()] element_splits[domain] = splits return element_public def __init_splits(self): - bstdata = self.get_public_data('bst') - splits = bstdata.get_mapping('split-rules') + bstdata = self.get_public_data("bst") + splits = bstdata.get_mapping("split-rules") self.__splits = { - domain: re.compile('^(?:' + '|'.join([utils._glob2re(r) for r in rules.as_str_list()]) + ')$') + domain: re.compile("^(?:" + "|".join([utils._glob2re(r) for r in rules.as_str_list()]) + ")$") for domain, rules in splits.items() } @@ -2944,10 +2958,10 @@ class Element(Plugin): # the build, but I can think of no reason to change it mid-build. # If this ever changes, things will go wrong unexpectedly. if not self.__whitelist_regex: - bstdata = self.get_public_data('bst') - whitelist = bstdata.get_str_list('overlap-whitelist', default=[]) + bstdata = self.get_public_data("bst") + whitelist = bstdata.get_str_list("overlap-whitelist", default=[]) whitelist_expressions = [utils._glob2re(self.__variables.subst(exp.strip())) for exp in whitelist] - expression = ('^(?:' + '|'.join(whitelist_expressions) + ')$') + expression = "^(?:" + "|".join(whitelist_expressions) + ")$" self.__whitelist_regex = re.compile(expression) return self.__whitelist_regex.match(os.path.join(os.sep, path)) @@ -3005,8 +3019,7 @@ class Element(Plugin): # def __pull_weak(self, *, pull_buildtrees): weak_key = self._get_cache_key(strength=_KeyStrength.WEAK) - if not self.__artifacts.pull(self, weak_key, - pull_buildtrees=pull_buildtrees): + if not self.__artifacts.pull(self, weak_key, pull_buildtrees=pull_buildtrees): return False # extract strong cache key from this newly fetched artifact @@ -3159,8 +3172,9 @@ class Element(Plugin): return if not self.__strict_artifact: - self.__strict_artifact = Artifact(self, context, strong_key=self.__strict_cache_key, - weak_key=self.__weak_cache_key) + self.__strict_artifact = Artifact( + self, context, strong_key=self.__strict_cache_key, weak_key=self.__weak_cache_key + ) if context.get_strict(): self.__artifact = self.__strict_artifact @@ -3192,9 +3206,7 @@ class Element(Plugin): self.__cache_key = strong_key elif self.__assemble_scheduled or self.__assemble_done: # Artifact will or has been built, not downloaded - dependencies = [ - e._get_cache_key() for e in self.dependencies(Scope.BUILD) - ] + dependencies = [e._get_cache_key() for e in self.dependencies(Scope.BUILD)] self.__cache_key = self._calculate_cache_key(dependencies) if self.__cache_key is None: @@ -3216,8 +3228,7 @@ class Element(Plugin): # def __update_strict_cache_key_of_rdeps(self): if not self.__updated_strict_cache_keys_of_rdeps: - if self.__runtime_deps_without_strict_cache_key == 0 and \ - self.__strict_cache_key is not None: + if self.__runtime_deps_without_strict_cache_key == 0 and self.__strict_cache_key is not None: self.__updated_strict_cache_keys_of_rdeps = True # Notify reverse dependencies @@ -3251,8 +3262,7 @@ class Element(Plugin): # def __update_ready_for_runtime(self): if not self.__ready_for_runtime: - if self.__runtime_deps_without_cache_key == 0 and \ - self.__cache_key is not None: + if self.__runtime_deps_without_cache_key == 0 and self.__cache_key is not None: self.__ready_for_runtime = True # Notify reverse dependencies @@ -3279,10 +3289,12 @@ class Element(Plugin): def _overlap_error_detail(f, forbidden_overlap_elements, elements): if forbidden_overlap_elements: - return ("/{}: {} {} not permitted to overlap other elements, order {} \n" - .format(f, " and ".join(forbidden_overlap_elements), - "is" if len(forbidden_overlap_elements) == 1 else "are", - " above ".join(reversed(elements)))) + return "/{}: {} {} not permitted to overlap other elements, order {} \n".format( + f, + " and ".join(forbidden_overlap_elements), + "is" if len(forbidden_overlap_elements) == 1 else "are", + " above ".join(reversed(elements)), + ) else: return "" @@ -3299,7 +3311,7 @@ def _overlap_error_detail(f, forbidden_overlap_elements, elements): # (str): The normalised element name # def _get_normal_name(element_name): - return os.path.splitext(element_name.replace(os.sep, '-'))[0] + return os.path.splitext(element_name.replace(os.sep, "-"))[0] # _compose_artifact_name(): @@ -3315,12 +3327,9 @@ def _get_normal_name(element_name): # (str): The constructed artifact name path # def _compose_artifact_name(project_name, normal_name, cache_key): - valid_chars = string.digits + string.ascii_letters + '-._' - normal_name = ''.join([ - x if x in valid_chars else '_' - for x in normal_name - ]) + valid_chars = string.digits + string.ascii_letters + "-._" + normal_name = "".join([x if x in valid_chars else "_" for x in normal_name]) # Note that project names are not allowed to contain slashes. Element names containing # a '/' will have this replaced with a '-' upon Element object instantiation. - return '{0}/{1}/{2}'.format(project_name, normal_name, cache_key) + return "{0}/{1}/{2}".format(project_name, normal_name, cache_key) diff --git a/src/buildstream/plugin.py b/src/buildstream/plugin.py index c1ee333f7..0cbd72e27 100644 --- a/src/buildstream/plugin.py +++ b/src/buildstream/plugin.py @@ -127,10 +127,11 @@ if TYPE_CHECKING: # pylint: disable=cyclic-import from ._context import Context from ._project import Project + # pylint: enable=cyclic-import -class Plugin(): +class Plugin: """Plugin() Base Plugin class. @@ -210,15 +211,17 @@ class Plugin(): # # Note that Plugins can only be instantiated in the main process before # scheduling tasks. - __TABLE = WeakValueDictionary() # type: WeakValueDictionary[int, Plugin] - - def __init__(self, - name: str, - context: 'Context', - project: 'Project', - provenance: ProvenanceInformation, - type_tag: str, - unique_id: Optional[int] = None): + __TABLE = WeakValueDictionary() # type: WeakValueDictionary[int, Plugin] + + def __init__( + self, + name: str, + context: "Context", + project: "Project", + provenance: ProvenanceInformation, + type_tag: str, + unique_id: Optional[int] = None, + ): self.name = name """The plugin name @@ -248,30 +251,29 @@ class Plugin(): # plugin in a subprocess and should use the same ID. self._unique_id = unique_id - self.__context = context # The Context object + self.__context = context # The Context object # Note that when pickling jobs over to a child process, we rely on this # reference to the Project, it keeps the plugin factory alive. If the # factory were to be GC'd then we would see undefined behaviour. Make # sure to test plugin pickling if this reference is to be removed. - self.__project = project # The Project object + self.__project = project # The Project object self.__provenance = provenance # The Provenance information - self.__type_tag = type_tag # The type of plugin (element or source) - self.__configuring = False # Whether we are currently configuring + self.__type_tag = type_tag # The type of plugin (element or source) + self.__configuring = False # Whether we are currently configuring # Get the full_name as project & type_tag are resolved self.__full_name = self.__get_full_name() # Infer the kind identifier modulename = type(self).__module__ - self.__kind = modulename.split('.')[-1] + self.__kind = modulename.split(".")[-1] self.debug("Created: {}".format(self)) # If this plugin has been deprecated, emit a warning. if self.BST_PLUGIN_DEPRECATED and not self.__deprecation_warning_silenced(): - detail = "Using deprecated plugin {}: {}".format(self.__kind, - self.BST_PLUGIN_DEPRECATION_MESSAGE) + detail = "Using deprecated plugin {}: {}".format(self.__kind, self.BST_PLUGIN_DEPRECATION_MESSAGE) self.__message(MessageType.WARN, detail) def __del__(self): @@ -282,9 +284,8 @@ class Plugin(): def __str__(self): return "{kind} {typetag} at {provenance}".format( - kind=self.__kind, - typetag=self.__type_tag, - provenance=self.__provenance) + kind=self.__kind, typetag=self.__type_tag, provenance=self.__provenance + ) ############################################################# # Abstract Methods # @@ -312,8 +313,9 @@ class Plugin(): :func:`Element.node_subst_member() <buildstream.element.Element.node_subst_member>` method can be used. """ - raise ImplError("{tag} plugin '{kind}' does not implement configure()".format( - tag=self.__type_tag, kind=self.get_kind())) + raise ImplError( + "{tag} plugin '{kind}' does not implement configure()".format(tag=self.__type_tag, kind=self.get_kind()) + ) def preflight(self) -> None: """Preflight Check @@ -333,8 +335,9 @@ class Plugin(): them with :func:`utils.get_host_tool() <buildstream.utils.get_host_tool>` which will raise an error automatically informing the user that a host tool is needed. """ - raise ImplError("{tag} plugin '{kind}' does not implement preflight()".format( - tag=self.__type_tag, kind=self.get_kind())) + raise ImplError( + "{tag} plugin '{kind}' does not implement preflight()".format(tag=self.__type_tag, kind=self.get_kind()) + ) def get_unique_key(self) -> SourceRef: """Return something which uniquely identifies the plugin input @@ -355,8 +358,11 @@ class Plugin(): which is to say that the Source is expected to have an exact *ref* indicating exactly what source is going to be staged. """ - raise ImplError("{tag} plugin '{kind}' does not implement get_unique_key()".format( - tag=self.__type_tag, kind=self.get_kind())) + raise ImplError( + "{tag} plugin '{kind}' does not implement get_unique_key()".format( + tag=self.__type_tag, kind=self.get_kind() + ) + ) ############################################################# # Public Methods # @@ -369,8 +375,7 @@ class Plugin(): """ return self.__kind - def node_get_project_path(self, node, *, - check_is_file=False, check_is_dir=False): + def node_get_project_path(self, node, *, check_is_file=False, check_is_dir=False): """Fetches a project path from a dictionary node and validates it Paths are asserted to never lead to a directory outside of the @@ -408,9 +413,7 @@ class Plugin(): """ - return self.__project.get_path_from_node(node, - check_is_file=check_is_file, - check_is_dir=check_is_dir) + return self.__project.get_path_from_node(node, check_is_file=check_is_file, check_is_dir=check_is_dir) def debug(self, brief: str, *, detail: Optional[str] = None) -> None: """Print a debugging message @@ -485,11 +488,9 @@ class Plugin(): self.__message(MessageType.LOG, brief, detail=detail) @contextmanager - def timed_activity(self, - activity_name: str, - *, - detail: Optional[str] = None, - silent_nested: bool = False) -> Generator[None, None, None]: + def timed_activity( + self, activity_name: str, *, detail: Optional[str] = None, silent_nested: bool = False + ) -> Generator[None, None, None]: """Context manager for performing timed activities in plugins Args: @@ -511,10 +512,9 @@ class Plugin(): # This will raise SourceError on its own self.call(... command which takes time ...) """ - with self.__context.messenger.timed_activity(activity_name, - element_name=self._get_full_name(), - detail=detail, - silent_nested=silent_nested): + with self.__context.messenger.timed_activity( + activity_name, element_name=self._get_full_name(), detail=detail, silent_nested=silent_nested + ): yield def call(self, *popenargs, fail: Optional[str] = None, fail_temporarily: bool = False, **kwargs) -> int: @@ -722,8 +722,11 @@ class Plugin(): # so it's not an ImplError - those apply to custom plugins. Direct # descendants of Plugin must implement this, e.g. Element and Source. # Raise NotImplementedError as this would be an internal bug. - raise NotImplementedError("{tag} plugin '{kind}' does not implement _get_args_for_child_job_pickling()".format( - tag=self.__type_tag, kind=self.get_kind())) + raise NotImplementedError( + "{tag} plugin '{kind}' does not implement _get_args_for_child_job_pickling()".format( + tag=self.__type_tag, kind=self.get_kind() + ) + ) ############################################################# # Local Private Methods # @@ -734,20 +737,19 @@ class Plugin(): def __call(self, *popenargs, collect_stdout=False, fail=None, fail_temporarily=False, **kwargs): with self._output_file() as output_file: - if 'stdout' not in kwargs: - kwargs['stdout'] = output_file - if 'stderr' not in kwargs: - kwargs['stderr'] = output_file + if "stdout" not in kwargs: + kwargs["stdout"] = output_file + if "stderr" not in kwargs: + kwargs["stderr"] = output_file if collect_stdout: - kwargs['stdout'] = subprocess.PIPE + kwargs["stdout"] = subprocess.PIPE self.__note_command(output_file, *popenargs, **kwargs) exit_code, output = utils._call(*popenargs, **kwargs) if fail and exit_code: - raise PluginError("{plugin}: {message}".format(plugin=self, message=fail), - temporary=fail_temporarily) + raise PluginError("{plugin}: {message}".format(plugin=self, message=fail), temporary=fail_temporarily) return (exit_code, output) @@ -756,11 +758,11 @@ class Plugin(): self.__context.messenger.message(message) def __note_command(self, output, *popenargs, **kwargs): - workdir = kwargs.get('cwd', os.getcwd()) + workdir = kwargs.get("cwd", os.getcwd()) command = " ".join(popenargs[0]) - output.write('Running host command {}: {}\n'.format(workdir, command)) + output.write("Running host command {}: {}\n".format(workdir, command)) output.flush() - self.status('Running host command', detail=command) + self.status("Running host command", detail=command) def __deprecation_warning_silenced(self): if not self.BST_PLUGIN_DEPRECATED: @@ -770,10 +772,10 @@ class Plugin(): project = self.__project for key, value in project.element_overrides.items(): - if value.get_bool('suppress-deprecation-warnings', default=False): + if value.get_bool("suppress-deprecation-warnings", default=False): silenced_warnings.add(key) for key, value in project.source_overrides.items(): - if value.get_bool('suppress-deprecation-warnings', default=False): + if value.get_bool("suppress-deprecation-warnings", default=False): silenced_warnings.add(key) return self.get_kind() in silenced_warnings @@ -783,18 +785,14 @@ class Plugin(): # Set the name, depending on element or source plugin type name = self._element_name if self.__type_tag == "source" else self.name # pylint: disable=no-member if project.junction: - return '{}:{}'.format(project.junction.name, name) + return "{}:{}".format(project.junction.name, name) else: return name # A local table for _prefix_warning() # -__CORE_WARNINGS = [ - value - for name, value in CoreWarnings.__dict__.items() - if not name.startswith("__") -] +__CORE_WARNINGS = [value for name, value in CoreWarnings.__dict__.items() if not name.startswith("__")] # _prefix_warning(): diff --git a/src/buildstream/plugins/elements/autotools.py b/src/buildstream/plugins/elements/autotools.py index 7a05336b7..089c9bca0 100644 --- a/src/buildstream/plugins/elements/autotools.py +++ b/src/buildstream/plugins/elements/autotools.py @@ -66,8 +66,7 @@ class AutotoolsElement(BuildElement): # Enable command batching across prepare() and assemble() def configure_sandbox(self, sandbox): super().configure_sandbox(sandbox) - self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY, - collect=self.get_variable('install-root')) + self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY, collect=self.get_variable("install-root")) # Plugin entry point diff --git a/src/buildstream/plugins/elements/compose.py b/src/buildstream/plugins/elements/compose.py index 511925731..063c5d44f 100644 --- a/src/buildstream/plugins/elements/compose.py +++ b/src/buildstream/plugins/elements/compose.py @@ -62,27 +62,23 @@ class ComposeElement(Element): BST_RUN_COMMANDS = False def configure(self, node): - node.validate_keys([ - 'integrate', 'include', 'exclude', 'include-orphans' - ]) + node.validate_keys(["integrate", "include", "exclude", "include-orphans"]) # We name this variable 'integration' only to avoid # collision with the Element.integrate() method. - self.integration = node.get_bool('integrate') - self.include = node.get_str_list('include') - self.exclude = node.get_str_list('exclude') - self.include_orphans = node.get_bool('include-orphans') + self.integration = node.get_bool("integrate") + self.include = node.get_str_list("include") + self.exclude = node.get_str_list("exclude") + self.include_orphans = node.get_bool("include-orphans") def preflight(self): pass def get_unique_key(self): - key = {'integrate': self.integration, - 'include': sorted(self.include), - 'orphans': self.include_orphans} + key = {"integrate": self.integration, "include": sorted(self.include), "orphans": self.include_orphans} if self.exclude: - key['exclude'] = sorted(self.exclude) + key["exclude"] = sorted(self.exclude) return key @@ -104,9 +100,9 @@ class ComposeElement(Element): if require_split: with self.timed_activity("Computing split", silent_nested=True): for dep in self.dependencies(Scope.BUILD): - files = dep.compute_manifest(include=self.include, - exclude=self.exclude, - orphans=self.include_orphans) + files = dep.compute_manifest( + include=self.include, exclude=self.exclude, orphans=self.include_orphans + ) manifest.update(files) # Make a snapshot of all the files. @@ -141,13 +137,16 @@ class ComposeElement(Element): for path in basedir_contents: if path not in snapshot: added_files.add(path) - self.info("Integration modified {}, added {} and removed {} files" - .format(len(modified_files), len(added_files), len(removed_files))) + self.info( + "Integration modified {}, added {} and removed {} files".format( + len(modified_files), len(added_files), len(removed_files) + ) + ) # The remainder of this is expensive, make an early exit if # we're not being selective about what is to be included. if not require_split: - return '/' + return "/" # Do we want to force include files which were modified by # the integration commands, even if they were not added ? @@ -159,7 +158,7 @@ class ComposeElement(Element): # instead of into a subdir. The element assemble() method should # support this in some way. # - installdir = vbasedir.descend('buildstream', 'install', create=True) + installdir = vbasedir.descend("buildstream", "install", create=True) # We already saved the manifest for created files in the integration phase, # now collect the rest of the manifest. @@ -189,7 +188,7 @@ class ComposeElement(Element): installdir.import_files(vbasedir, filter_callback=import_filter, can_link=True) # And we're done - return os.path.join(os.sep, 'buildstream', 'install') + return os.path.join(os.sep, "buildstream", "install") # Plugin entry point diff --git a/src/buildstream/plugins/elements/filter.py b/src/buildstream/plugins/elements/filter.py index d808c9e5a..71ed1f6cb 100644 --- a/src/buildstream/plugins/elements/filter.py +++ b/src/buildstream/plugins/elements/filter.py @@ -167,17 +167,15 @@ class FilterElement(Element): BST_RUN_COMMANDS = False def configure(self, node): - node.validate_keys([ - 'include', 'exclude', 'include-orphans', 'pass-integration' - ]) + node.validate_keys(["include", "exclude", "include-orphans", "pass-integration"]) - self.include_node = node.get_sequence('include') - self.exclude_node = node.get_sequence('exclude') + self.include_node = node.get_sequence("include") + self.exclude_node = node.get_sequence("exclude") self.include = self.include_node.as_str_list() self.exclude = self.exclude_node.as_str_list() - self.include_orphans = node.get_bool('include-orphans') - self.pass_integration = node.get_bool('pass-integration', False) + self.include_orphans = node.get_bool("include-orphans") + self.pass_integration = node.get_bool("pass-integration", False) def preflight(self): # Exactly one build-depend is permitted @@ -186,9 +184,13 @@ class FilterElement(Element): detail = "Full list of build-depends:\n" deps_list = " \n".join([x.name for x in build_deps]) detail += deps_list - raise ElementError("{}: {} element must have exactly 1 build-dependency, actually have {}" - .format(self, type(self).__name__, len(build_deps)), - detail=detail, reason="filter-bdepend-wrong-count") + raise ElementError( + "{}: {} element must have exactly 1 build-dependency, actually have {}".format( + self, type(self).__name__, len(build_deps) + ), + detail=detail, + reason="filter-bdepend-wrong-count", + ) # That build-depend must not also be a runtime-depend runtime_deps = list(self.dependencies(Scope.RUN, recurse=False)) @@ -196,23 +198,29 @@ class FilterElement(Element): detail = "Full list of runtime depends:\n" deps_list = " \n".join([x.name for x in runtime_deps]) detail += deps_list - raise ElementError("{}: {} element's build dependency must not also be a runtime dependency" - .format(self, type(self).__name__), - detail=detail, reason="filter-bdepend-also-rdepend") + raise ElementError( + "{}: {} element's build dependency must not also be a runtime dependency".format( + self, type(self).__name__ + ), + detail=detail, + reason="filter-bdepend-also-rdepend", + ) # If a parent does not produce an artifact, fail and inform user that the dependency # must produce artifacts if not build_deps[0].BST_ELEMENT_HAS_ARTIFACT: detail = "{} does not produce an artifact, so there is nothing to filter".format(build_deps[0].name) - raise ElementError("{}: {} element's build dependency must produce an artifact" - .format(self, type(self).__name__), - detail=detail, reason="filter-bdepend-no-artifact") + raise ElementError( + "{}: {} element's build dependency must produce an artifact".format(self, type(self).__name__), + detail=detail, + reason="filter-bdepend-no-artifact", + ) def get_unique_key(self): key = { - 'include': sorted(self.include), - 'exclude': sorted(self.exclude), - 'orphans': self.include_orphans, + "include": sorted(self.include), + "exclude": sorted(self.exclude), + "orphans": self.include_orphans, } return key @@ -226,8 +234,8 @@ class FilterElement(Element): with self.timed_activity("Staging artifact", silent_nested=True): for dep in self.dependencies(Scope.BUILD, recurse=False): # Check that all the included/excluded domains exist - pub_data = dep.get_public_data('bst') - split_rules = pub_data.get_mapping('split-rules', {}) + pub_data = dep.get_public_data("bst") + split_rules = pub_data.get_mapping("split-rules", {}) unfound_includes = [] for domain in self.include: if domain not in split_rules: @@ -240,18 +248,17 @@ class FilterElement(Element): detail = [] if unfound_includes: detail.append("Unknown domains were used in {}".format(self.include_node.get_provenance())) - detail.extend([' - {}'.format(domain) for domain in unfound_includes]) + detail.extend([" - {}".format(domain) for domain in unfound_includes]) if unfound_excludes: detail.append("Unknown domains were used in {}".format(self.exclude_node.get_provenance())) - detail.extend([' - {}'.format(domain) for domain in unfound_excludes]) + detail.extend([" - {}".format(domain) for domain in unfound_excludes]) if detail: - detail = '\n'.join(detail) + detail = "\n".join(detail) raise ElementError("Unknown domains declared.", detail=detail) - dep.stage_artifact(sandbox, include=self.include, - exclude=self.exclude, orphans=self.include_orphans) + dep.stage_artifact(sandbox, include=self.include, exclude=self.exclude, orphans=self.include_orphans) return "" def _get_source_element(self): diff --git a/src/buildstream/plugins/elements/import.py b/src/buildstream/plugins/elements/import.py index 404a0f4ee..2b68197a7 100644 --- a/src/buildstream/plugins/elements/import.py +++ b/src/buildstream/plugins/elements/import.py @@ -45,12 +45,10 @@ class ImportElement(Element): BST_RUN_COMMANDS = False def configure(self, node): - node.validate_keys([ - 'source', 'target' - ]) + node.validate_keys(["source", "target"]) - self.source = self.node_subst_vars(node.get_scalar('source')) - self.target = self.node_subst_vars(node.get_scalar('target')) + self.source = self.node_subst_vars(node.get_scalar("source")) + self.target = self.node_subst_vars(node.get_scalar("target")) def preflight(self): # Assert that we have at least one source to fetch. @@ -60,10 +58,7 @@ class ImportElement(Element): raise ElementError("{}: An import element must have at least one source.".format(self)) def get_unique_key(self): - return { - 'source': self.source, - 'target': self.target - } + return {"source": self.source, "target": self.target} def configure_sandbox(self, sandbox): pass @@ -74,11 +69,11 @@ class ImportElement(Element): def assemble(self, sandbox): # Stage sources into the input directory - self.stage_sources(sandbox, 'input') + self.stage_sources(sandbox, "input") rootdir = sandbox.get_virtual_directory() - inputdir = rootdir.descend('input') - outputdir = rootdir.descend('output', create=True) + inputdir = rootdir.descend("input") + outputdir = rootdir.descend("output", create=True) # The directory to grab inputdir = inputdir.descend(*self.source.strip(os.sep).split(os.sep)) @@ -87,18 +82,17 @@ class ImportElement(Element): outputdir = outputdir.descend(*self.target.strip(os.sep).split(os.sep), create=True) if inputdir.is_empty(): - raise ElementError("{}: No files were found inside directory '{}'" - .format(self, self.source)) + raise ElementError("{}: No files were found inside directory '{}'".format(self, self.source)) # Move it over outputdir.import_files(inputdir) # And we're done - return '/output' + return "/output" def generate_script(self): - build_root = self.get_variable('build-root') - install_root = self.get_variable('install-root') + build_root = self.get_variable("build-root") + install_root = self.get_variable("install-root") commands = [] # The directory to grab diff --git a/src/buildstream/plugins/elements/junction.py b/src/buildstream/plugins/elements/junction.py index aec32516b..42b9ef08e 100644 --- a/src/buildstream/plugins/elements/junction.py +++ b/src/buildstream/plugins/elements/junction.py @@ -187,13 +187,13 @@ class JunctionElement(Element): BST_FORBID_RDEPENDS = True def configure(self, node): - self.path = node.get_str('path', default='') - self.options = node.get_mapping('options', default={}) - self.target = node.get_str('target', default=None) + self.path = node.get_str("path", default="") + self.options = node.get_mapping("options", default={}) + self.target = node.get_str("target", default=None) self.target_element = None self.target_junction = None - self.cache_junction_elements = node.get_bool('cache-junction-elements', default=False) - self.ignore_junction_remotes = node.get_bool('ignore-junction-remotes', default=False) + self.cache_junction_elements = node.get_bool("cache-junction-elements", default=False) + self.ignore_junction_remotes = node.get_bool("ignore-junction-remotes", default=False) def preflight(self): # "target" cannot be used in conjunction with: diff --git a/src/buildstream/plugins/elements/manual.py b/src/buildstream/plugins/elements/manual.py index bbda65312..97da41615 100644 --- a/src/buildstream/plugins/elements/manual.py +++ b/src/buildstream/plugins/elements/manual.py @@ -42,8 +42,7 @@ class ManualElement(BuildElement): # Enable command batching across prepare() and assemble() def configure_sandbox(self, sandbox): super().configure_sandbox(sandbox) - self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY, - collect=self.get_variable('install-root')) + self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY, collect=self.get_variable("install-root")) # Plugin entry point diff --git a/src/buildstream/plugins/elements/pip.py b/src/buildstream/plugins/elements/pip.py index 4a9eefde1..93303748d 100644 --- a/src/buildstream/plugins/elements/pip.py +++ b/src/buildstream/plugins/elements/pip.py @@ -42,8 +42,7 @@ class PipElement(BuildElement): # Enable command batching across prepare() and assemble() def configure_sandbox(self, sandbox): super().configure_sandbox(sandbox) - self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY, - collect=self.get_variable('install-root')) + self.batch_prepare_assemble(SandboxFlags.ROOT_READ_ONLY, collect=self.get_variable("install-root")) # Plugin entry point diff --git a/src/buildstream/plugins/elements/script.py b/src/buildstream/plugins/elements/script.py index f3f0a2f7a..abfb7b3b0 100644 --- a/src/buildstream/plugins/elements/script.py +++ b/src/buildstream/plugins/elements/script.py @@ -46,21 +46,19 @@ class ScriptElement(buildstream.ScriptElement): BST_VIRTUAL_DIRECTORY = True def configure(self, node): - for n in node.get_sequence('layout', []): - dst = self.node_subst_vars(n.get_scalar('destination')) - elm = self.node_subst_vars(n.get_scalar('element', None)) + for n in node.get_sequence("layout", []): + dst = self.node_subst_vars(n.get_scalar("destination")) + elm = self.node_subst_vars(n.get_scalar("element", None)) self.layout_add(elm, dst) - node.validate_keys([ - 'commands', 'root-read-only', 'layout' - ]) + node.validate_keys(["commands", "root-read-only", "layout"]) cmds = self.node_subst_sequence_vars(node.get_sequence("commands")) self.add_commands("commands", cmds) self.set_work_dir() self.set_install_root() - self.set_root_read_only(node.get_bool('root-read-only', default=False)) + self.set_root_read_only(node.get_bool("root-read-only", default=False)) # Plugin entry point diff --git a/src/buildstream/plugins/elements/stack.py b/src/buildstream/plugins/elements/stack.py index ae15af63a..f569199b6 100644 --- a/src/buildstream/plugins/elements/stack.py +++ b/src/buildstream/plugins/elements/stack.py @@ -64,10 +64,10 @@ class StackElement(Element): # Just create a dummy empty artifact, its existence is a statement # that all this stack's dependencies are built. vrootdir = sandbox.get_virtual_directory() - vrootdir.descend('output', create=True) + vrootdir.descend("output", create=True) # And we're done - return '/output' + return "/output" # Plugin entry point diff --git a/src/buildstream/plugins/sources/_downloadablefilesource.py b/src/buildstream/plugins/sources/_downloadablefilesource.py index 1e759b94f..4e43ee3e3 100644 --- a/src/buildstream/plugins/sources/_downloadablefilesource.py +++ b/src/buildstream/plugins/sources/_downloadablefilesource.py @@ -12,7 +12,6 @@ from buildstream import utils class _NetrcFTPOpener(urllib.request.FTPHandler): - def __init__(self, netrc_config): self.netrc = netrc_config @@ -28,11 +27,11 @@ class _NetrcFTPOpener(urllib.request.FTPHandler): def _unsplit(self, host, port, user, passwd): if port: - host = '{}:{}'.format(host, port) + host = "{}:{}".format(host, port) if user: if passwd: - user = '{}:{}'.format(user, passwd) - host = '{}@{}'.format(user, host) + user = "{}:{}".format(user, passwd) + host = "{}@{}".format(user, host) return host @@ -50,7 +49,6 @@ class _NetrcFTPOpener(urllib.request.FTPHandler): class _NetrcPasswordManager: - def __init__(self, netrc_config): self.netrc = netrc_config @@ -72,17 +70,16 @@ class _NetrcPasswordManager: class DownloadableFileSource(Source): # pylint: disable=attribute-defined-outside-init - COMMON_CONFIG_KEYS = Source.COMMON_CONFIG_KEYS + ['url', 'ref', 'etag'] + COMMON_CONFIG_KEYS = Source.COMMON_CONFIG_KEYS + ["url", "ref", "etag"] __urlopener = None __default_mirror_file = None def configure(self, node): - self.original_url = node.get_str('url') - self.ref = node.get_str('ref', None) + self.original_url = node.get_str("url") + self.ref = node.get_str("ref", None) self.url = self.translate_url(self.original_url) - self._mirror_dir = os.path.join(self.get_mirror_directory(), - utils.url_directory_name(self.original_url)) + self._mirror_dir = os.path.join(self.get_mirror_directory(), utils.url_directory_name(self.original_url)) self._warn_deprecated_etag(node) def preflight(self): @@ -102,28 +99,29 @@ class DownloadableFileSource(Source): return Consistency.RESOLVED def load_ref(self, node): - self.ref = node.get_str('ref', None) + self.ref = node.get_str("ref", None) self._warn_deprecated_etag(node) def get_ref(self): return self.ref def set_ref(self, ref, node): - node['ref'] = self.ref = ref + node["ref"] = self.ref = ref def track(self): # pylint: disable=arguments-differ # there is no 'track' field in the source to determine what/whether # or not to update refs, because tracking a ref is always a conscious # decision by the user. - with self.timed_activity("Tracking {}".format(self.url), - silent_nested=True): + with self.timed_activity("Tracking {}".format(self.url), silent_nested=True): new_ref = self._ensure_mirror() if self.ref and self.ref != new_ref: - detail = "When tracking, new ref differs from current ref:\n" \ - + " Tracked URL: {}\n".format(self.url) \ - + " Current ref: {}\n".format(self.ref) \ + detail = ( + "When tracking, new ref differs from current ref:\n" + + " Tracked URL: {}\n".format(self.url) + + " Current ref: {}\n".format(self.ref) + " New ref: {}\n".format(new_ref) + ) self.warn("Potential man-in-the-middle attack!", detail=detail) return new_ref @@ -142,25 +140,26 @@ class DownloadableFileSource(Source): with self.timed_activity("Fetching {}".format(self.url), silent_nested=True): sha256 = self._ensure_mirror() if sha256 != self.ref: - raise SourceError("File downloaded from {} has sha256sum '{}', not '{}'!" - .format(self.url, sha256, self.ref)) + raise SourceError( + "File downloaded from {} has sha256sum '{}', not '{}'!".format(self.url, sha256, self.ref) + ) def _warn_deprecated_etag(self, node): - etag = node.get_str('etag', None) + etag = node.get_str("etag", None) if etag: provenance = node.get_scalar(etag).get_provenance() self.warn('{} "etag" is deprecated and ignored.'.format(provenance)) def _get_etag(self, ref): - etagfilename = os.path.join(self._mirror_dir, '{}.etag'.format(ref)) + etagfilename = os.path.join(self._mirror_dir, "{}.etag".format(ref)) if os.path.exists(etagfilename): - with open(etagfilename, 'r') as etagfile: + with open(etagfilename, "r") as etagfile: return etagfile.read() return None def _store_etag(self, ref, etag): - etagfilename = os.path.join(self._mirror_dir, '{}.etag'.format(ref)) + etagfilename = os.path.join(self._mirror_dir, "{}.etag".format(ref)) with utils.save_file_atomic(etagfilename) as etagfile: etagfile.write(etag) @@ -170,7 +169,7 @@ class DownloadableFileSource(Source): with self.tempdir() as td: default_name = os.path.basename(self.url) request = urllib.request.Request(self.url) - request.add_header('Accept', '*/*') + request.add_header("Accept", "*/*") # We do not use etag in case what we have in cache is # not matching ref in order to be able to recover from @@ -180,18 +179,18 @@ class DownloadableFileSource(Source): # Do not re-download the file if the ETag matches. if etag and self.get_consistency() == Consistency.CACHED: - request.add_header('If-None-Match', etag) + request.add_header("If-None-Match", etag) opener = self.__get_urlopener() with contextlib.closing(opener.open(request)) as response: info = response.info() - etag = info['ETag'] if 'ETag' in info else None + etag = info["ETag"] if "ETag" in info else None filename = info.get_filename(default_name) filename = os.path.basename(filename) local_file = os.path.join(td, filename) - with open(local_file, 'wb') as dest: + with open(local_file, "wb") as dest: shutil.copyfileobj(response, dest) # Make sure url-specific mirror dir exists. @@ -214,14 +213,12 @@ class DownloadableFileSource(Source): # Because we use etag only for matching ref, currently specified ref is what # we would have downloaded. return self.ref - raise SourceError("{}: Error mirroring {}: {}" - .format(self, self.url, e), temporary=True) from e + raise SourceError("{}: Error mirroring {}: {}".format(self, self.url, e), temporary=True) from e except (urllib.error.URLError, urllib.error.ContentTooShortError, OSError, ValueError) as e: # Note that urllib.request.Request in the try block may throw a # ValueError for unknown url types, so we handle it here. - raise SourceError("{}: Error mirroring {}: {}" - .format(self, self.url, e), temporary=True) from e + raise SourceError("{}: Error mirroring {}: {}".format(self, self.url, e), temporary=True) from e def _get_mirror_file(self, sha=None): if sha is not None: @@ -245,7 +242,7 @@ class DownloadableFileSource(Source): # DownloadableFileSource.__urlopener = urllib.request.build_opener() except netrc.NetrcParseError as e: - self.warn('{}: While reading .netrc: {}'.format(self, e)) + self.warn("{}: While reading .netrc: {}".format(self, e)) return urllib.request.build_opener() else: netrc_pw_mgr = _NetrcPasswordManager(netrc_config) diff --git a/src/buildstream/plugins/sources/bzr.py b/src/buildstream/plugins/sources/bzr.py index 88dba7dc2..30ce55585 100644 --- a/src/buildstream/plugins/sources/bzr.py +++ b/src/buildstream/plugins/sources/bzr.py @@ -67,16 +67,16 @@ class BzrSource(Source): # pylint: disable=attribute-defined-outside-init def configure(self, node): - node.validate_keys(['url', 'track', 'ref', *Source.COMMON_CONFIG_KEYS]) + node.validate_keys(["url", "track", "ref", *Source.COMMON_CONFIG_KEYS]) - self.original_url = node.get_str('url') - self.tracking = node.get_str('track') - self.ref = node.get_str('ref', None) + self.original_url = node.get_str("url") + self.tracking = node.get_str("track") + self.ref = node.get_str("ref", None) self.url = self.translate_url(self.original_url) def preflight(self): # Check if bzr is installed, get the binary at the same time. - self.host_bzr = utils.get_host_tool('bzr') + self.host_bzr = utils.get_host_tool("bzr") def get_unique_key(self): return [self.original_url, self.tracking, self.ref] @@ -93,39 +93,44 @@ class BzrSource(Source): return Consistency.RESOLVED def load_ref(self, node): - self.ref = node.get_str('ref', None) + self.ref = node.get_str("ref", None) def get_ref(self): return self.ref def set_ref(self, ref, node): - node['ref'] = self.ref = ref + node["ref"] = self.ref = ref def track(self): # pylint: disable=arguments-differ - with self.timed_activity("Tracking {}".format(self.url), - silent_nested=True), self._locked(): + with self.timed_activity("Tracking {}".format(self.url), silent_nested=True), self._locked(): self._ensure_mirror(skip_ref_check=True) - ret, out = self.check_output([self.host_bzr, "version-info", - "--custom", "--template={revno}", - self._get_branch_dir()], - fail="Failed to read the revision number at '{}'" - .format(self._get_branch_dir())) + ret, out = self.check_output( + [self.host_bzr, "version-info", "--custom", "--template={revno}", self._get_branch_dir()], + fail="Failed to read the revision number at '{}'".format(self._get_branch_dir()), + ) if ret != 0: raise SourceError("{}: Failed to get ref for tracking {}".format(self, self.tracking)) return out def fetch(self): # pylint: disable=arguments-differ - with self.timed_activity("Fetching {}".format(self.url), - silent_nested=True), self._locked(): + with self.timed_activity("Fetching {}".format(self.url), silent_nested=True), self._locked(): self._ensure_mirror() def stage(self, directory): - self.call([self.host_bzr, "checkout", "--lightweight", - "--revision=revno:{}".format(self.ref), - self._get_branch_dir(), directory], - fail="Failed to checkout revision {} from branch {} to {}" - .format(self.ref, self._get_branch_dir(), directory)) + self.call( + [ + self.host_bzr, + "checkout", + "--lightweight", + "--revision=revno:{}".format(self.ref), + self._get_branch_dir(), + directory, + ], + fail="Failed to checkout revision {} from branch {} to {}".format( + self.ref, self._get_branch_dir(), directory + ), + ) # Remove .bzr dir shutil.rmtree(os.path.join(directory, ".bzr")) @@ -133,16 +138,24 @@ class BzrSource(Source): url = os.path.join(self.url, self.tracking) with self.timed_activity('Setting up workspace "{}"'.format(directory), silent_nested=True): # Checkout from the cache - self.call([self.host_bzr, "branch", - "--use-existing-dir", - "--revision=revno:{}".format(self.ref), - self._get_branch_dir(), directory], - fail="Failed to branch revision {} from branch {} to {}" - .format(self.ref, self._get_branch_dir(), directory)) + self.call( + [ + self.host_bzr, + "branch", + "--use-existing-dir", + "--revision=revno:{}".format(self.ref), + self._get_branch_dir(), + directory, + ], + fail="Failed to branch revision {} from branch {} to {}".format( + self.ref, self._get_branch_dir(), directory + ), + ) # Switch the parent branch to the source's origin - self.call([self.host_bzr, "switch", - "--directory={}".format(directory), url], - fail="Failed to switch workspace's parent branch to {}".format(url)) + self.call( + [self.host_bzr, "switch", "--directory={}".format(directory), url], + fail="Failed to switch workspace's parent branch to {}".format(url), + ) # _locked() # @@ -151,13 +164,10 @@ class BzrSource(Source): # @contextmanager def _locked(self): - lockdir = os.path.join(self.get_mirror_directory(), 'locks') - lockfile = os.path.join( - lockdir, - utils.url_directory_name(self.original_url) + '.lock' - ) + lockdir = os.path.join(self.get_mirror_directory(), "locks") + lockfile = os.path.join(lockdir, utils.url_directory_name(self.original_url) + ".lock") os.makedirs(lockdir, exist_ok=True) - with open(lockfile, 'w') as lock: + with open(lockfile, "w") as lock: fcntl.flock(lock, fcntl.LOCK_EX) try: yield @@ -169,41 +179,42 @@ class BzrSource(Source): if not os.path.exists(self._get_branch_dir()): return False - return self.call([self.host_bzr, "revno", - "--revision=revno:{}".format(self.ref), - self._get_branch_dir()]) == 0 + return self.call([self.host_bzr, "revno", "--revision=revno:{}".format(self.ref), self._get_branch_dir()]) == 0 def _get_branch_dir(self): return os.path.join(self._get_mirror_dir(), self.tracking) def _get_mirror_dir(self): - return os.path.join(self.get_mirror_directory(), - utils.url_directory_name(self.original_url)) + return os.path.join(self.get_mirror_directory(), utils.url_directory_name(self.original_url)) def _ensure_mirror(self, skip_ref_check=False): mirror_dir = self._get_mirror_dir() bzr_metadata_dir = os.path.join(mirror_dir, ".bzr") if not os.path.exists(bzr_metadata_dir): - self.call([self.host_bzr, "init-repo", "--no-trees", mirror_dir], - fail="Failed to initialize bzr repository") + self.call( + [self.host_bzr, "init-repo", "--no-trees", mirror_dir], fail="Failed to initialize bzr repository" + ) branch_dir = os.path.join(mirror_dir, self.tracking) branch_url = self.url + "/" + self.tracking if not os.path.exists(branch_dir): # `bzr branch` the branch if it doesn't exist # to get the upstream code - self.call([self.host_bzr, "branch", branch_url, branch_dir], - fail="Failed to branch from {} to {}".format(branch_url, branch_dir)) + self.call( + [self.host_bzr, "branch", branch_url, branch_dir], + fail="Failed to branch from {} to {}".format(branch_url, branch_dir), + ) else: # `bzr pull` the branch if it does exist # to get any changes to the upstream code - self.call([self.host_bzr, "pull", "--directory={}".format(branch_dir), branch_url], - fail="Failed to pull new changes for {}".format(branch_dir)) + self.call( + [self.host_bzr, "pull", "--directory={}".format(branch_dir), branch_url], + fail="Failed to pull new changes for {}".format(branch_dir), + ) if not skip_ref_check and not self._check_ref(): - raise SourceError("Failed to ensure ref '{}' was mirrored".format(self.ref), - reason="ref-not-mirrored") + raise SourceError("Failed to ensure ref '{}' was mirrored".format(self.ref), reason="ref-not-mirrored") def setup(): diff --git a/src/buildstream/plugins/sources/deb.py b/src/buildstream/plugins/sources/deb.py index cc88cf53c..a7437b150 100644 --- a/src/buildstream/plugins/sources/deb.py +++ b/src/buildstream/plugins/sources/deb.py @@ -50,7 +50,7 @@ details on common configuration options for sources. import tarfile from contextlib import contextmanager -import arpy # pylint: disable=import-error +import arpy # pylint: disable=import-error from .tar import TarSource @@ -61,14 +61,14 @@ class DebSource(TarSource): def configure(self, node): super().configure(node) - self.base_dir = node.get_str('base-dir', None) + self.base_dir = node.get_str("base-dir", None) def preflight(self): return @contextmanager def _get_tar(self): - with open(self._get_mirror_file(), 'rb') as deb_file: + with open(self._get_mirror_file(), "rb") as deb_file: arpy_archive = arpy.Archive(fileobj=deb_file) arpy_archive.read_all_headers() data_tar_arpy = [v for k, v in arpy_archive.archived_files.items() if b"data.tar" in k][0] diff --git a/src/buildstream/plugins/sources/local.py b/src/buildstream/plugins/sources/local.py index f40fd79c0..90d8a8f6f 100644 --- a/src/buildstream/plugins/sources/local.py +++ b/src/buildstream/plugins/sources/local.py @@ -54,8 +54,8 @@ class LocalSource(Source): self.__unique_key = None def configure(self, node): - node.validate_keys(['path', *Source.COMMON_CONFIG_KEYS]) - self.path = self.node_get_project_path(node.get_scalar('path')) + node.validate_keys(["path", *Source.COMMON_CONFIG_KEYS]) + self.path = self.node_get_project_path(node.get_scalar("path")) self.fullpath = os.path.join(self.get_project_directory(), self.path) def preflight(self): @@ -89,8 +89,8 @@ class LocalSource(Source): if result.overwritten or result.ignored: raise SourceError( - "Failed to stage source: files clash with existing directory", - reason='ensure-stage-dir-fail') + "Failed to stage source: files clash with existing directory", reason="ensure-stage-dir-fail" + ) def _get_local_path(self): return self.fullpath diff --git a/src/buildstream/plugins/sources/patch.py b/src/buildstream/plugins/sources/patch.py index 86811cb4d..082983023 100644 --- a/src/buildstream/plugins/sources/patch.py +++ b/src/buildstream/plugins/sources/patch.py @@ -56,8 +56,7 @@ class PatchSource(Source): def configure(self, node): node.validate_keys(["path", "strip-level", *Source.COMMON_CONFIG_KEYS]) - self.path = self.node_get_project_path(node.get_scalar('path'), - check_is_file=True) + self.path = self.node_get_project_path(node.get_scalar("path"), check_is_file=True) self.strip_level = node.get_int("strip-level", default=1) self.fullpath = os.path.join(self.get_project_directory(), self.path) @@ -89,12 +88,13 @@ class PatchSource(Source): # Bail out with a comprehensive message if the target directory is empty if not os.listdir(directory): - raise SourceError("Nothing to patch in directory '{}'".format(directory), - reason="patch-no-files") + raise SourceError("Nothing to patch in directory '{}'".format(directory), reason="patch-no-files") strip_level_option = "-p{}".format(self.strip_level) - self.call([self.host_patch, strip_level_option, "-i", self.fullpath, "-d", directory], - fail="Failed to apply patch {}".format(self.path)) + self.call( + [self.host_patch, strip_level_option, "-i", self.fullpath, "-d", directory], + fail="Failed to apply patch {}".format(self.path), + ) # Plugin entry point diff --git a/src/buildstream/plugins/sources/pip.py b/src/buildstream/plugins/sources/pip.py index 758ef665f..2c9773787 100644 --- a/src/buildstream/plugins/sources/pip.py +++ b/src/buildstream/plugins/sources/pip.py @@ -74,30 +74,28 @@ import re from buildstream import Consistency, Source, SourceError, utils -_OUTPUT_DIRNAME = '.bst_pip_downloads' -_PYPI_INDEX_URL = 'https://pypi.org/simple/' +_OUTPUT_DIRNAME = ".bst_pip_downloads" +_PYPI_INDEX_URL = "https://pypi.org/simple/" # Used only for finding pip command _PYTHON_VERSIONS = [ - 'python', # when running in a venv, we might not have the exact version - 'python2.7', - 'python3.0', - 'python3.1', - 'python3.2', - 'python3.3', - 'python3.4', - 'python3.5', - 'python3.6', - 'python3.7', + "python", # when running in a venv, we might not have the exact version + "python2.7", + "python3.0", + "python3.1", + "python3.2", + "python3.3", + "python3.4", + "python3.5", + "python3.6", + "python3.7", ] # List of allowed extensions taken from # https://docs.python.org/3/distutils/sourcedist.html. # Names of source distribution archives must be of the form # '%{package-name}-%{version}.%{extension}'. -_SDIST_RE = re.compile( - r'^([\w.-]+?)-((?:[\d.]+){2,})\.(?:tar|tar.bz2|tar.gz|tar.xz|tar.Z|zip)$', - re.IGNORECASE) +_SDIST_RE = re.compile(r"^([\w.-]+?)-((?:[\d.]+){2,})\.(?:tar|tar.bz2|tar.gz|tar.xz|tar.Z|zip)$", re.IGNORECASE) class PipSource(Source): @@ -109,16 +107,15 @@ class PipSource(Source): BST_REQUIRES_PREVIOUS_SOURCES_TRACK = True def configure(self, node): - node.validate_keys(['url', 'packages', 'ref', 'requirements-files'] + - Source.COMMON_CONFIG_KEYS) - self.ref = node.get_str('ref', None) - self.original_url = node.get_str('url', _PYPI_INDEX_URL) + node.validate_keys(["url", "packages", "ref", "requirements-files"] + Source.COMMON_CONFIG_KEYS) + self.ref = node.get_str("ref", None) + self.original_url = node.get_str("url", _PYPI_INDEX_URL) self.index_url = self.translate_url(self.original_url) - self.packages = node.get_str_list('packages', []) - self.requirements_files = node.get_str_list('requirements-files', []) + self.packages = node.get_str_list("packages", []) + self.requirements_files = node.get_str_list("requirements-files", []) if not (self.packages or self.requirements_files): - raise SourceError("{}: Either 'packages' or 'requirements-files' must be specified". format(self)) + raise SourceError("{}: Either 'packages' or 'requirements-files' must be specified".format(self)) def preflight(self): # Try to find a pip version that spports download command @@ -126,9 +123,9 @@ class PipSource(Source): for python in reversed(_PYTHON_VERSIONS): try: host_python = utils.get_host_tool(python) - rc = self.call([host_python, '-m', 'pip', 'download', '--help']) + rc = self.call([host_python, "-m", "pip", "download", "--help"]) if rc == 0: - self.host_pip = [host_python, '-m', 'pip'] + self.host_pip = [host_python, "-m", "pip"] break except utils.ProgramNotFoundError: pass @@ -150,10 +147,10 @@ class PipSource(Source): return self.ref def load_ref(self, node): - self.ref = node.get_str('ref', None) + self.ref = node.get_str("ref", None) def set_ref(self, ref, node): - node['ref'] = self.ref = ref + node["ref"] = self.ref = ref def track(self, previous_sources_dir): # pylint: disable=arguments-differ # XXX pip does not offer any public API other than the CLI tool so it @@ -163,32 +160,44 @@ class PipSource(Source): # for details. # As a result, we have to wastefully install the packages during track. with self.tempdir() as tmpdir: - install_args = self.host_pip + ['download', - '--no-binary', ':all:', - '--index-url', self.index_url, - '--dest', tmpdir] + install_args = self.host_pip + [ + "download", + "--no-binary", + ":all:", + "--index-url", + self.index_url, + "--dest", + tmpdir, + ] for requirement_file in self.requirements_files: fpath = os.path.join(previous_sources_dir, requirement_file) - install_args += ['-r', fpath] + install_args += ["-r", fpath] install_args += self.packages self.call(install_args, fail="Failed to install python packages") reqs = self._parse_sdist_names(tmpdir) - return '\n'.join(["{}=={}".format(pkg, ver) for pkg, ver in reqs]) + return "\n".join(["{}=={}".format(pkg, ver) for pkg, ver in reqs]) def fetch(self): # pylint: disable=arguments-differ with self.tempdir() as tmpdir: - packages = self.ref.strip().split('\n') - package_dir = os.path.join(tmpdir, 'packages') + packages = self.ref.strip().split("\n") + package_dir = os.path.join(tmpdir, "packages") os.makedirs(package_dir) - self.call([*self.host_pip, - 'download', - '--no-binary', ':all:', - '--index-url', self.index_url, - '--dest', package_dir, - *packages], - fail="Failed to install python packages: {}".format(packages)) + self.call( + [ + *self.host_pip, + "download", + "--no-binary", + ":all:", + "--index-url", + self.index_url, + "--dest", + package_dir, + *packages, + ], + fail="Failed to install python packages: {}".format(packages), + ) # If the mirror directory already exists, assume that some other # process has fetched the sources before us and ensure that we do @@ -200,8 +209,11 @@ class PipSource(Source): # before us. pass except OSError as e: - raise SourceError("{}: Failed to move downloaded pip packages from '{}' to '{}': {}" - .format(self, package_dir, self._mirror, e)) from e + raise SourceError( + "{}: Failed to move downloaded pip packages from '{}' to '{}': {}".format( + self, package_dir, self._mirror, e + ) + ) from e def stage(self, directory): with self.timed_activity("Staging Python packages", silent_nested=True): @@ -213,9 +225,11 @@ class PipSource(Source): def _mirror(self): if not self.ref: return None - return os.path.join(self.get_mirror_directory(), - utils.url_directory_name(self.original_url), - hashlib.sha256(self.ref.encode()).hexdigest()) + return os.path.join( + self.get_mirror_directory(), + utils.url_directory_name(self.original_url), + hashlib.sha256(self.ref.encode()).hexdigest(), + ) # Parse names of downloaded source distributions # diff --git a/src/buildstream/plugins/sources/remote.py b/src/buildstream/plugins/sources/remote.py index 68aa577fc..da1a1f964 100644 --- a/src/buildstream/plugins/sources/remote.py +++ b/src/buildstream/plugins/sources/remote.py @@ -62,13 +62,14 @@ class RemoteSource(DownloadableFileSource): def configure(self, node): super().configure(node) - self.filename = node.get_str('filename', os.path.basename(self.url)) - self.executable = node.get_bool('executable', default=False) + self.filename = node.get_str("filename", os.path.basename(self.url)) + self.executable = node.get_bool("executable", default=False) if os.sep in self.filename: - raise SourceError('{}: filename parameter cannot contain directories'.format(self), - reason="filename-contains-directory") - node.validate_keys(DownloadableFileSource.COMMON_CONFIG_KEYS + ['filename', 'executable']) + raise SourceError( + "{}: filename parameter cannot contain directories".format(self), reason="filename-contains-directory" + ) + node.validate_keys(DownloadableFileSource.COMMON_CONFIG_KEYS + ["filename", "executable"]) def get_unique_key(self): return super().get_unique_key() + [self.filename, self.executable] diff --git a/src/buildstream/plugins/sources/tar.py b/src/buildstream/plugins/sources/tar.py index 60d464457..658cc2735 100644 --- a/src/buildstream/plugins/sources/tar.py +++ b/src/buildstream/plugins/sources/tar.py @@ -73,6 +73,7 @@ class ReadableTarInfo(tarfile.TarInfo): `mode` attribute in `TarInfo`, the class that encapsulates the internal meta-data of the tarball, so that the owner-read bit is always set. """ + @property def mode(self): # ensure file is readable by owner @@ -89,13 +90,13 @@ class TarSource(DownloadableFileSource): def configure(self, node): super().configure(node) - self.base_dir = node.get_str('base-dir', '*') - node.validate_keys(DownloadableFileSource.COMMON_CONFIG_KEYS + ['base-dir']) + self.base_dir = node.get_str("base-dir", "*") + node.validate_keys(DownloadableFileSource.COMMON_CONFIG_KEYS + ["base-dir"]) def preflight(self): self.host_lzip = None - if self.url.endswith('.lz'): - self.host_lzip = utils.get_host_tool('lzip') + if self.url.endswith(".lz"): + self.host_lzip = utils.get_host_tool("lzip") def get_unique_key(self): return super().get_unique_key() + [self.base_dir] @@ -104,19 +105,17 @@ class TarSource(DownloadableFileSource): def _run_lzip(self): assert self.host_lzip with TemporaryFile() as lzip_stdout: - with open(self._get_mirror_file(), 'r') as lzip_file: - self.call([self.host_lzip, '-d'], - stdin=lzip_file, - stdout=lzip_stdout) + with open(self._get_mirror_file(), "r") as lzip_file: + self.call([self.host_lzip, "-d"], stdin=lzip_file, stdout=lzip_stdout) lzip_stdout.seek(0, 0) yield lzip_stdout @contextmanager def _get_tar(self): - if self.url.endswith('.lz'): + if self.url.endswith(".lz"): with self._run_lzip() as lzip_dec: - with tarfile.open(fileobj=lzip_dec, mode='r:', tarinfo=ReadableTarInfo) as tar: + with tarfile.open(fileobj=lzip_dec, mode="r:", tarinfo=ReadableTarInfo) as tar: yield tar else: with tarfile.open(self._get_mirror_file(), tarinfo=ReadableTarInfo) as tar: @@ -147,14 +146,18 @@ class TarSource(DownloadableFileSource): def assert_safe(member): final_path = os.path.abspath(os.path.join(target_dir, member.path)) if not final_path.startswith(target_dir): - raise SourceError("{}: Tarfile attempts to extract outside the staging area: " - "{} -> {}".format(self, member.path, final_path)) + raise SourceError( + "{}: Tarfile attempts to extract outside the staging area: " + "{} -> {}".format(self, member.path, final_path) + ) if member.islnk(): linked_path = os.path.abspath(os.path.join(target_dir, member.linkname)) if not linked_path.startswith(target_dir): - raise SourceError("{}: Tarfile attempts to hardlink outside the staging area: " - "{} -> {}".format(self, member.path, final_path)) + raise SourceError( + "{}: Tarfile attempts to hardlink outside the staging area: " + "{} -> {}".format(self, member.path, final_path) + ) # Don't need to worry about symlinks because they're just # files here and won't be able to do much harm once we are @@ -167,9 +170,9 @@ class TarSource(DownloadableFileSource): for member in tar.getmembers(): # First, ensure that a member never starts with `./` - if member.path.startswith('./'): + if member.path.startswith("./"): member.path = member.path[2:] - if member.islnk() and member.linkname.startswith('./'): + if member.islnk() and member.linkname.startswith("./"): member.linkname = member.linkname[2:] # Now extract only the paths which match the normalized path @@ -202,16 +205,16 @@ class TarSource(DownloadableFileSource): # Remove any possible leading './', offer more consistent behavior # across tarballs encoded with or without a leading '.' - member_name = member.name.lstrip('./') + member_name = member.name.lstrip("./") if not member.isdir(): # Loop over the components of a path, for a path of a/b/c/d # we will first visit 'a', then 'a/b' and then 'a/b/c', excluding # the final component - components = member_name.split('/') + components = member_name.split("/") for i in range(len(components) - 1): - dir_component = '/'.join([components[j] for j in range(i + 1)]) + dir_component = "/".join([components[j] for j in range(i + 1)]) if dir_component not in visited: visited.add(dir_component) try: @@ -219,7 +222,7 @@ class TarSource(DownloadableFileSource): # exist in the archive _ = tar.getmember(dir_component) except KeyError: - if dir_component != '.': + if dir_component != ".": yield dir_component continue @@ -227,7 +230,7 @@ class TarSource(DownloadableFileSource): # Avoid considering the '.' directory, if any is included in the archive # this is to avoid the default 'base-dir: *' value behaving differently # depending on whether the tarball was encoded with a leading '.' or not - elif member_name == '.': + elif member_name == ".": continue yield member_name diff --git a/src/buildstream/plugins/sources/workspace.py b/src/buildstream/plugins/sources/workspace.py index f40f5fae8..a845fd440 100644 --- a/src/buildstream/plugins/sources/workspace.py +++ b/src/buildstream/plugins/sources/workspace.py @@ -59,9 +59,9 @@ class WorkspaceSource(Source): return None def configure(self, node: MappingNode) -> None: - node.validate_keys(['path', 'ref', 'kind']) - self.path = node.get_str('path') - self.__digest = node.get_str('ref') + node.validate_keys(["path", "ref", "kind"]) + self.path = node.get_str("path") + self.__digest = node.get_str("ref") def preflight(self) -> None: pass # pragma: nocover @@ -79,7 +79,7 @@ class WorkspaceSource(Source): # # Raises AssertionError: existing workspaces should not be reinitialized def init_workspace(self, directory: Directory) -> None: - raise AssertionError('Attempting to re-open an existing workspace') + raise AssertionError("Attempting to re-open an existing workspace") def get_consistency(self): # always return cached state @@ -95,8 +95,8 @@ class WorkspaceSource(Source): if result.overwritten or result.ignored: raise SourceError( - "Failed to stage source: files clash with existing directory", - reason='ensure-stage-dir-fail') + "Failed to stage source: files clash with existing directory", reason="ensure-stage-dir-fail" + ) def _get_local_path(self) -> str: return self.path diff --git a/src/buildstream/plugins/sources/zip.py b/src/buildstream/plugins/sources/zip.py index 322be58d7..47933c8eb 100644 --- a/src/buildstream/plugins/sources/zip.py +++ b/src/buildstream/plugins/sources/zip.py @@ -72,8 +72,8 @@ class ZipSource(DownloadableFileSource): def configure(self, node): super().configure(node) - self.base_dir = node.get_str('base-dir', '*') - node.validate_keys(DownloadableFileSource.COMMON_CONFIG_KEYS + ['base-dir']) + self.base_dir = node.get_str("base-dir", "*") + node.validate_keys(DownloadableFileSource.COMMON_CONFIG_KEYS + ["base-dir"]) def get_unique_key(self): return super().get_unique_key() + [self.base_dir] @@ -139,14 +139,14 @@ class ZipSource(DownloadableFileSource): # ZipInfo.is_dir() is only available in python >= 3.6, but all # it does is check for a trailing '/' in the name # - if not member.filename.endswith('/'): + if not member.filename.endswith("/"): # Loop over the components of a path, for a path of a/b/c/d # we will first visit 'a', then 'a/b' and then 'a/b/c', excluding # the final component - components = member.filename.split('/') + components = member.filename.split("/") for i in range(len(components) - 1): - dir_component = '/'.join([components[j] for j in range(i + 1)]) + dir_component = "/".join([components[j] for j in range(i + 1)]) if dir_component not in visited: visited[dir_component] = True try: @@ -154,7 +154,7 @@ class ZipSource(DownloadableFileSource): # exist in the archive _ = archive.getinfo(dir_component) except KeyError: - if dir_component != '.': + if dir_component != ".": yield dir_component continue @@ -162,7 +162,7 @@ class ZipSource(DownloadableFileSource): # Avoid considering the '.' directory, if any is included in the archive # this is to avoid the default 'base-dir: *' value behaving differently # depending on whether the archive was encoded with a leading '.' or not - elif member.filename == '.' or member.filename == './': + elif member.filename == "." or member.filename == "./": continue yield member.filename diff --git a/src/buildstream/sandbox/_config.py b/src/buildstream/sandbox/_config.py index 457f92b3c..614f22063 100644 --- a/src/buildstream/sandbox/_config.py +++ b/src/buildstream/sandbox/_config.py @@ -22,7 +22,7 @@ # # A container for sandbox configuration data. We want the internals # of this to be opaque, hence putting it in its own private file. -class SandboxConfig(): +class SandboxConfig: def __init__(self, build_uid, build_gid, build_os=None, build_arch=None): self.build_uid = build_uid self.build_gid = build_gid @@ -46,17 +46,14 @@ class SandboxConfig(): # However this should be the right place to support # such configurations in the future. # - unique_key = { - 'os': self.build_os, - 'arch': self.build_arch - } + unique_key = {"os": self.build_os, "arch": self.build_arch} # Avoid breaking cache key calculation with # the addition of configurabuild build uid/gid if self.build_uid != 0: - unique_key['build-uid'] = self.build_uid + unique_key["build-uid"] = self.build_uid if self.build_gid != 0: - unique_key['build-gid'] = self.build_gid + unique_key["build-gid"] = self.build_gid return unique_key diff --git a/src/buildstream/sandbox/_mount.py b/src/buildstream/sandbox/_mount.py index c0f26c8d7..18751dde5 100644 --- a/src/buildstream/sandbox/_mount.py +++ b/src/buildstream/sandbox/_mount.py @@ -29,7 +29,7 @@ from .._fuse import SafeHardlinks # # Helper data object representing a single mount point in the mount map # -class Mount(): +class Mount: def __init__(self, sandbox, mount_point, safe_hardlinks, fuse_mount_options=None): # Getting _get_underlying_directory() here is acceptable as # we're part of the sandbox code. This will fail if our @@ -54,8 +54,8 @@ class Mount(): # Redirected mount self.mount_origin = os.path.join(root_directory, mount_point.lstrip(os.sep)) self.mount_base = os.path.join(scratch_directory, utils.url_directory_name(mount_point)) - self.mount_source = os.path.join(self.mount_base, 'mount') - self.mount_tempdir = os.path.join(self.mount_base, 'temp') + self.mount_source = os.path.join(self.mount_base, "mount") + self.mount_tempdir = os.path.join(self.mount_base, "temp") os.makedirs(self.mount_origin, exist_ok=True) os.makedirs(self.mount_tempdir, exist_ok=True) else: @@ -74,10 +74,10 @@ class Mount(): # When mounting a regular file, ensure the parent # directory exists in the sandbox; and that an empty # file is created at the mount location. - parent_dir = os.path.dirname(self.mount_source.rstrip('/')) + parent_dir = os.path.dirname(self.mount_source.rstrip("/")) os.makedirs(parent_dir, exist_ok=True) if not os.path.exists(self.mount_source): - with open(self.mount_source, 'w'): + with open(self.mount_source, "w"): pass @contextmanager @@ -99,8 +99,7 @@ class Mount(): # sandbox (Sandbox): The sandbox object # root_readonly (bool): Whether the sandbox root is readonly # -class MountMap(): - +class MountMap: def __init__(self, sandbox, root_readonly, fuse_mount_options=None): # We will be doing the mounts in the order in which they were declared. self.mounts = OrderedDict() @@ -109,11 +108,11 @@ class MountMap(): fuse_mount_options = {} # We want safe hardlinks on rootfs whenever root is not readonly - self.mounts['/'] = Mount(sandbox, '/', not root_readonly, fuse_mount_options) + self.mounts["/"] = Mount(sandbox, "/", not root_readonly, fuse_mount_options) for mark in sandbox._get_marked_directories(): - directory = mark['directory'] - artifact = mark['artifact'] + directory = mark["directory"] + artifact = mark["artifact"] # We want safe hardlinks for any non-root directory where # artifacts will be staged to diff --git a/src/buildstream/sandbox/_mounter.py b/src/buildstream/sandbox/_mounter.py index 4e31ef67a..3adf8ff5b 100644 --- a/src/buildstream/sandbox/_mounter.py +++ b/src/buildstream/sandbox/_mounter.py @@ -25,22 +25,20 @@ from .. import utils, _signals # A class to wrap the `mount` and `umount` system commands -class Mounter(): +class Mounter: @classmethod - def _mount(cls, dest, src=None, mount_type=None, - stdout=None, stderr=None, options=None, - flags=None): + def _mount(cls, dest, src=None, mount_type=None, stdout=None, stderr=None, options=None, flags=None): if stdout is None: stdout = sys.stdout if stderr is None: stderr = sys.stderr - argv = [utils.get_host_tool('mount')] + argv = [utils.get_host_tool("mount")] if mount_type: - argv.extend(['-t', mount_type]) + argv.extend(["-t", mount_type]) if options: - argv.extend(['-o', options]) + argv.extend(["-o", options]) if flags: argv.extend(flags) @@ -48,16 +46,10 @@ class Mounter(): argv += [src] argv += [dest] - status, _ = utils._call( - argv, - terminate=True, - stdout=stdout, - stderr=stderr - ) + status, _ = utils._call(argv, terminate=True, stdout=stdout, stderr=stderr) if status != 0: - raise SandboxError('`{}` failed with exit code {}' - .format(' '.join(argv), status)) + raise SandboxError("`{}` failed with exit code {}".format(" ".join(argv), status)) return dest @@ -68,17 +60,11 @@ class Mounter(): if stderr is None: stderr = sys.stderr - cmd = [utils.get_host_tool('umount'), '-R', path] - status, _ = utils._call( - cmd, - terminate=True, - stdout=stdout, - stderr=stderr - ) + cmd = [utils.get_host_tool("umount"), "-R", path] + status, _ = utils._call(cmd, terminate=True, stdout=stdout, stderr=stderr) if status != 0: - raise SandboxError('`{}` failed with exit code {}' - .format(' '.join(cmd), status)) + raise SandboxError("`{}` failed with exit code {}".format(" ".join(cmd), status)) # mount() # @@ -98,8 +84,7 @@ class Mounter(): # @classmethod @contextmanager - def mount(cls, dest, src=None, stdout=None, - stderr=None, mount_type=None, **kwargs): + def mount(cls, dest, src=None, stdout=None, stderr=None, mount_type=None, **kwargs): if stdout is None: stdout = sys.stdout if stderr is None: @@ -108,7 +93,7 @@ class Mounter(): def kill_proc(): cls._umount(dest, stdout, stderr) - options = ','.join([key for key, val in kwargs.items() if val]) + options = ",".join([key for key, val in kwargs.items() if val]) path = cls._mount(dest, src, mount_type, stdout=stdout, stderr=stderr, options=options) try: @@ -139,8 +124,7 @@ class Mounter(): # @classmethod @contextmanager - def bind_mount(cls, dest, src=None, stdout=None, - stderr=None, **kwargs): + def bind_mount(cls, dest, src=None, stdout=None, stderr=None, **kwargs): if stdout is None: stdout = sys.stdout if stderr is None: @@ -149,8 +133,8 @@ class Mounter(): def kill_proc(): cls._umount(dest, stdout, stderr) - kwargs['rbind'] = True - options = ','.join([key for key, val in kwargs.items() if val]) + kwargs["rbind"] = True + options = ",".join([key for key, val in kwargs.items() if val]) path = cls._mount(dest, src, None, stdout, stderr, options) @@ -158,7 +142,7 @@ class Mounter(): with _signals.terminator(kill_proc): # Make the rbind a slave to avoid unmounting vital devices in # /proc - cls._mount(dest, flags=['--make-rslave']) + cls._mount(dest, flags=["--make-rslave"]) yield path finally: cls._umount(dest, stdout, stderr) diff --git a/src/buildstream/sandbox/_sandboxbuildbox.py b/src/buildstream/sandbox/_sandboxbuildbox.py index 4258ee26d..c34d95223 100644 --- a/src/buildstream/sandbox/_sandboxbuildbox.py +++ b/src/buildstream/sandbox/_sandboxbuildbox.py @@ -34,22 +34,20 @@ from .._exceptions import SandboxError # BuildBox-based sandbox implementation. # class SandboxBuildBox(Sandbox): - def __init__(self, context, project, directory, **kwargs): - if kwargs.get('allow_real_directory'): + if kwargs.get("allow_real_directory"): raise SandboxError("BuildBox does not support real directories") - kwargs['allow_real_directory'] = False + kwargs["allow_real_directory"] = False super().__init__(context, project, directory, **kwargs) @classmethod def check_available(cls): try: - utils.get_host_tool('buildbox') + utils.get_host_tool("buildbox") except utils.ProgramNotFoundError as Error: cls._dummy_reasons += ["buildbox not found"] - raise SandboxError(" and ".join(cls._dummy_reasons), - reason="unavailable-local-sandbox") from Error + raise SandboxError(" and ".join(cls._dummy_reasons), reason="unavailable-local-sandbox") from Error @classmethod def check_sandbox_config(cls, platform, config): @@ -73,42 +71,42 @@ class SandboxBuildBox(Sandbox): scratch_directory = self._get_scratch_directory() if not self._has_command(command[0], env): - raise SandboxCommandError("Staged artifacts do not provide command " - "'{}'".format(command[0]), - reason='missing-command') + raise SandboxCommandError( + "Staged artifacts do not provide command " "'{}'".format(command[0]), reason="missing-command" + ) # Grab the full path of the buildbox binary try: - buildbox_command = [utils.get_host_tool('buildbox')] + buildbox_command = [utils.get_host_tool("buildbox")] except ProgramNotFoundError as Err: - raise SandboxError(("BuildBox not on path, you are using the BuildBox sandbox because " - "BST_FORCE_SANDBOX=buildbox")) from Err + raise SandboxError( + ("BuildBox not on path, you are using the BuildBox sandbox because " "BST_FORCE_SANDBOX=buildbox") + ) from Err for mark in self._get_marked_directories(): - path = mark['directory'] - assert path.startswith('/') and len(path) > 1 + path = mark["directory"] + assert path.startswith("/") and len(path) > 1 root_directory.descend(*path[1:].split(os.path.sep), create=True) digest = root_directory._get_digest() - with open(os.path.join(scratch_directory, 'in'), 'wb') as input_digest_file: + with open(os.path.join(scratch_directory, "in"), "wb") as input_digest_file: input_digest_file.write(digest.SerializeToString()) buildbox_command += ["--local=" + root_directory.cas_cache.casdir] buildbox_command += ["--input-digest=in"] buildbox_command += ["--output-digest=out"] - common_details = ("BuildBox is a experimental sandbox and does not support the requested feature.\n" - "You are using this feature because BST_FORCE_SANDBOX=buildbox.") + common_details = ( + "BuildBox is a experimental sandbox and does not support the requested feature.\n" + "You are using this feature because BST_FORCE_SANDBOX=buildbox." + ) if not flags & SandboxFlags.NETWORK_ENABLED: # TODO - self._issue_warning( - "BuildBox sandbox does not have Networking yet", - detail=common_details - ) + self._issue_warning("BuildBox sandbox does not have Networking yet", detail=common_details) if cwd is not None: - buildbox_command += ['--chdir=' + cwd] + buildbox_command += ["--chdir=" + cwd] # In interactive mode, we want a complete devpts inside # the container, so there is a /dev/console and such. In @@ -118,27 +116,24 @@ class SandboxBuildBox(Sandbox): if flags & SandboxFlags.INTERACTIVE: # TODO self._issue_warning( - "BuildBox sandbox does not fully support BuildStream shells yet", - detail=common_details + "BuildBox sandbox does not fully support BuildStream shells yet", detail=common_details ) if flags & SandboxFlags.ROOT_READ_ONLY: # TODO self._issue_warning( - "BuildBox sandbox does not fully support BuildStream `Read only Root`", - detail=common_details + "BuildBox sandbox does not fully support BuildStream `Read only Root`", detail=common_details ) # Set UID and GID if not flags & SandboxFlags.INHERIT_UID: # TODO self._issue_warning( - "BuildBox sandbox does not fully support BuildStream Inherit UID", - detail=common_details + "BuildBox sandbox does not fully support BuildStream Inherit UID", detail=common_details ) - os.makedirs(os.path.join(scratch_directory, 'mnt'), exist_ok=True) - buildbox_command += ['mnt'] + os.makedirs(os.path.join(scratch_directory, "mnt"), exist_ok=True) + buildbox_command += ["mnt"] # Add the command buildbox_command += command @@ -150,7 +145,7 @@ class SandboxBuildBox(Sandbox): with ExitStack() as stack: # Ensure the cwd exists if cwd is not None and len(cwd) > 1: - assert cwd.startswith('/') + assert cwd.startswith("/") root_directory.descend(*cwd[1:].split(os.path.sep), create=True) # If we're interactive, we want to inherit our stdin, @@ -162,12 +157,18 @@ class SandboxBuildBox(Sandbox): stdin = stack.enter_context(open(os.devnull, "r")) # Run buildbox ! - exit_code = self.run_buildbox(buildbox_command, stdin, stdout, stderr, env, - interactive=(flags & SandboxFlags.INTERACTIVE), - cwd=scratch_directory) + exit_code = self.run_buildbox( + buildbox_command, + stdin, + stdout, + stderr, + env, + interactive=(flags & SandboxFlags.INTERACTIVE), + cwd=scratch_directory, + ) if exit_code == 0: - with open(os.path.join(scratch_directory, 'out'), 'rb') as output_digest_file: + with open(os.path.join(scratch_directory, "out"), "rb") as output_digest_file: output_digest = remote_execution_pb2.Digest() output_digest.ParseFromString(output_digest_file.read()) self._vdir = CasBasedDirectory(root_directory.cas_cache, digest=output_digest) @@ -203,7 +204,7 @@ class SandboxBuildBox(Sandbox): stdout=stdout, stderr=stderr, cwd=cwd, - start_new_session=interactive + start_new_session=interactive, ) # Wait for the child process to finish, ensuring that diff --git a/src/buildstream/sandbox/_sandboxbwrap.py b/src/buildstream/sandbox/_sandboxbwrap.py index 5c4b9a295..1405611bc 100644 --- a/src/buildstream/sandbox/_sandboxbwrap.py +++ b/src/buildstream/sandbox/_sandboxbwrap.py @@ -48,34 +48,27 @@ class SandboxBwrap(Sandbox): _have_good_bwrap = None # Minimal set of devices for the sandbox - DEVICES = [ - '/dev/full', - '/dev/null', - '/dev/urandom', - '/dev/random', - '/dev/zero' - ] + DEVICES = ["/dev/full", "/dev/null", "/dev/urandom", "/dev/random", "/dev/zero"] def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.linux32 = kwargs['linux32'] + self.linux32 = kwargs["linux32"] @classmethod def check_available(cls): cls._have_fuse = os.path.exists("/dev/fuse") if not cls._have_fuse: - cls._dummy_reasons += ['Fuse is unavailable'] + cls._dummy_reasons += ["Fuse is unavailable"] try: - utils.get_host_tool('bwrap') + utils.get_host_tool("bwrap") except utils.ProgramNotFoundError as Error: cls._bwrap_exists = False cls._have_good_bwrap = False cls._die_with_parent_available = False cls._json_status_available = False - cls._dummy_reasons += ['Bubblewrap not found'] - raise SandboxError(" and ".join(cls._dummy_reasons), - reason="unavailable-local-sandbox") from Error + cls._dummy_reasons += ["Bubblewrap not found"] + raise SandboxError(" and ".join(cls._dummy_reasons), reason="unavailable-local-sandbox") from Error bwrap_version = _site.get_bwrap_version() @@ -84,7 +77,7 @@ class SandboxBwrap(Sandbox): cls._die_with_parent_available = (0, 1, 8) <= bwrap_version cls._json_status_available = (0, 3, 2) <= bwrap_version if not cls._have_good_bwrap: - cls._dummy_reasons += ['Bubblewrap is too old'] + cls._dummy_reasons += ["Bubblewrap is too old"] raise SandboxError(" and ".join(cls._dummy_reasons)) cls._uid = os.geteuid() @@ -98,29 +91,26 @@ class SandboxBwrap(Sandbox): # issue a warning if it's not available, and save the state # locally so that we can inform the sandbox to not try it # later on. - bwrap = utils.get_host_tool('bwrap') + bwrap = utils.get_host_tool("bwrap") try: - whoami = utils.get_host_tool('whoami') - output = subprocess.check_output([ - bwrap, - '--ro-bind', '/', '/', - '--unshare-user', - '--uid', '0', '--gid', '0', - whoami, - ], universal_newlines=True).strip() + whoami = utils.get_host_tool("whoami") + output = subprocess.check_output( + [bwrap, "--ro-bind", "/", "/", "--unshare-user", "--uid", "0", "--gid", "0", whoami,], + universal_newlines=True, + ).strip() except subprocess.CalledProcessError: - output = '' + output = "" except utils.ProgramNotFoundError: - output = '' + output = "" - return output == 'root' + return output == "root" @classmethod def check_sandbox_config(cls, local_platform, config): if cls.user_ns_available: # User namespace support allows arbitrary build UID/GID settings. pass - elif (config.build_uid != local_platform._uid or config.build_gid != local_platform._gid): + elif config.build_uid != local_platform._uid or config.build_gid != local_platform._gid: # Without user namespace support, the UID/GID in the sandbox # will match the host UID/GID. return False @@ -141,9 +131,9 @@ class SandboxBwrap(Sandbox): root_directory = self.get_virtual_directory()._get_underlying_directory() if not self._has_command(command[0], env): - raise SandboxCommandError("Staged artifacts do not provide command " - "'{}'".format(command[0]), - reason='missing-command') + raise SandboxCommandError( + "Staged artifacts do not provide command " "'{}'".format(command[0]), reason="missing-command" + ) # NOTE: MountMap transitively imports `_fuse/fuse.py` which raises an # EnvironmentError when fuse is not found. Since this module is @@ -154,29 +144,29 @@ class SandboxBwrap(Sandbox): # Create the mount map, this will tell us where # each mount point needs to be mounted from and to mount_map = MountMap(self, flags & SandboxFlags.ROOT_READ_ONLY) - root_mount_source = mount_map.get_mount_source('/') + root_mount_source = mount_map.get_mount_source("/") # start command with linux32 if needed if self.linux32: - bwrap_command = [utils.get_host_tool('linux32')] + bwrap_command = [utils.get_host_tool("linux32")] else: bwrap_command = [] # Grab the full path of the bwrap binary - bwrap_command += [utils.get_host_tool('bwrap')] + bwrap_command += [utils.get_host_tool("bwrap")] for k, v in env.items(): - bwrap_command += ['--setenv', k, v] + bwrap_command += ["--setenv", k, v] for k in os.environ.keys() - env.keys(): - bwrap_command += ['--unsetenv', k] + bwrap_command += ["--unsetenv", k] # Create a new pid namespace, this also ensures that any subprocesses # are cleaned up when the bwrap process exits. - bwrap_command += ['--unshare-pid'] + bwrap_command += ["--unshare-pid"] # Ensure subprocesses are cleaned up when the bwrap parent dies. if self._die_with_parent_available: - bwrap_command += ['--die-with-parent'] + bwrap_command += ["--die-with-parent"] # Add in the root filesystem stuff first. # @@ -186,15 +176,12 @@ class SandboxBwrap(Sandbox): bwrap_command += ["--bind", root_mount_source, "/"] if not flags & SandboxFlags.NETWORK_ENABLED: - bwrap_command += ['--unshare-net'] - bwrap_command += ['--unshare-uts', '--hostname', 'buildstream'] - bwrap_command += ['--unshare-ipc'] + bwrap_command += ["--unshare-net"] + bwrap_command += ["--unshare-uts", "--hostname", "buildstream"] + bwrap_command += ["--unshare-ipc"] # Give it a proc and tmpfs - bwrap_command += [ - '--proc', '/proc', - '--tmpfs', '/tmp' - ] + bwrap_command += ["--proc", "/proc", "--tmpfs", "/tmp"] # In interactive mode, we want a complete devpts inside # the container, so there is a /dev/console and such. In @@ -202,21 +189,21 @@ class SandboxBwrap(Sandbox): # a minimal set of devices to expose to the sandbox. # if flags & SandboxFlags.INTERACTIVE: - bwrap_command += ['--dev', '/dev'] + bwrap_command += ["--dev", "/dev"] else: for device in self.DEVICES: - bwrap_command += ['--dev-bind', device, device] + bwrap_command += ["--dev-bind", device, device] # Create a tmpfs for /dev/shm, if we're in interactive this # is handled by `--dev /dev` # - bwrap_command += ['--tmpfs', '/dev/shm'] + bwrap_command += ["--tmpfs", "/dev/shm"] # Add bind mounts to any marked directories marked_directories = self._get_marked_directories() mount_source_overrides = self._get_mount_sources() for mark in marked_directories: - mount_point = mark['directory'] + mount_point = mark["directory"] if mount_point in mount_source_overrides: # pylint: disable=consider-using-get mount_source = mount_source_overrides[mount_point] else: @@ -230,22 +217,22 @@ class SandboxBwrap(Sandbox): # harmless to do in a build environment where the directories # we mount just never contain device files. # - bwrap_command += ['--dev-bind', mount_source, mount_point] + bwrap_command += ["--dev-bind", mount_source, mount_point] if flags & SandboxFlags.ROOT_READ_ONLY: bwrap_command += ["--remount-ro", "/"] if cwd is not None: - bwrap_command += ['--dir', cwd] - bwrap_command += ['--chdir', cwd] + bwrap_command += ["--dir", cwd] + bwrap_command += ["--chdir", cwd] # Set UID and GUI if self.user_ns_available: - bwrap_command += ['--unshare-user'] + bwrap_command += ["--unshare-user"] if not flags & SandboxFlags.INHERIT_UID: uid = self._get_config().build_uid gid = self._get_config().build_gid - bwrap_command += ['--uid', str(uid), '--gid', str(gid)] + bwrap_command += ["--uid", str(uid), "--gid", str(gid)] with ExitStack() as stack: pass_fds = () @@ -253,7 +240,7 @@ class SandboxBwrap(Sandbox): if self._json_status_available: json_status_file = stack.enter_context(TemporaryFile()) pass_fds = (json_status_file.fileno(),) - bwrap_command += ['--json-status-fd', str(json_status_file.fileno())] + bwrap_command += ["--json-status-fd", str(json_status_file.fileno())] # Add the command bwrap_command += command @@ -265,7 +252,7 @@ class SandboxBwrap(Sandbox): # existing_basedirs = { directory: os.path.exists(os.path.join(root_directory, directory)) - for directory in ['dev/shm', 'tmp', 'dev', 'proc'] + for directory in ["dev/shm", "tmp", "dev", "proc"] } # Use the MountMap context manager to ensure that any redirected @@ -283,15 +270,16 @@ class SandboxBwrap(Sandbox): stdin = stack.enter_context(open(os.devnull, "r")) # Run bubblewrap ! - exit_code = self.run_bwrap(bwrap_command, stdin, stdout, stderr, - (flags & SandboxFlags.INTERACTIVE), pass_fds) + exit_code = self.run_bwrap( + bwrap_command, stdin, stdout, stderr, (flags & SandboxFlags.INTERACTIVE), pass_fds + ) # Cleanup things which bwrap might have left behind, while # everything is still mounted because bwrap can be creating # the devices on the fuse mount, so we should remove it there. if not flags & SandboxFlags.INTERACTIVE: for device in self.DEVICES: - device_path = os.path.join(root_mount_source, device.lstrip('/')) + device_path = os.path.join(root_mount_source, device.lstrip("/")) # This will remove the device in a loop, allowing some # retries in case the device file leaked by bubblewrap is still busy @@ -299,7 +287,7 @@ class SandboxBwrap(Sandbox): # Remove /tmp, this is a bwrap owned thing we want to be sure # never ends up in an artifact - for basedir in ['dev/shm', 'tmp', 'dev', 'proc']: + for basedir in ["dev/shm", "tmp", "dev", "proc"]: # Skip removal of directories which already existed before # launching bwrap @@ -341,12 +329,14 @@ class SandboxBwrap(Sandbox): for line in json_status_file: with suppress(json.decoder.JSONDecodeError): o = json.loads(line.decode()) - if isinstance(o, collections.abc.Mapping) and 'exit-code' in o: - child_exit_code = o['exit-code'] + if isinstance(o, collections.abc.Mapping) and "exit-code" in o: + child_exit_code = o["exit-code"] break if child_exit_code is None: - raise SandboxError("`bwrap' terminated during sandbox setup with exitcode {}".format(exit_code), - reason="bwrap-sandbox-fail") + raise SandboxError( + "`bwrap' terminated during sandbox setup with exitcode {}".format(exit_code), + reason="bwrap-sandbox-fail", + ) exit_code = child_exit_code self._vdir._mark_changed() @@ -432,7 +422,7 @@ class SandboxBwrap(Sandbox): stdin=stdin, stdout=stdout, stderr=stderr, - start_new_session=new_session + start_new_session=new_session, ) # Wait for the child process to finish, ensuring that diff --git a/src/buildstream/sandbox/_sandboxchroot.py b/src/buildstream/sandbox/_sandboxchroot.py index 8d4c54058..1805131b1 100644 --- a/src/buildstream/sandbox/_sandboxchroot.py +++ b/src/buildstream/sandbox/_sandboxchroot.py @@ -35,7 +35,7 @@ from . import Sandbox, SandboxFlags, SandboxCommandError class SandboxChroot(Sandbox): - _FUSE_MOUNT_OPTIONS = {'dev': True} + _FUSE_MOUNT_OPTIONS = {"dev": True} def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -43,8 +43,10 @@ class SandboxChroot(Sandbox): uid = self._get_config().build_uid gid = self._get_config().build_gid if uid != 0 or gid != 0: - raise SandboxError("Chroot sandboxes cannot specify a non-root uid/gid " - "({},{} were supplied via config)".format(uid, gid)) + raise SandboxError( + "Chroot sandboxes cannot specify a non-root uid/gid " + "({},{} were supplied via config)".format(uid, gid) + ) self.mount_map = None @@ -78,20 +80,19 @@ class SandboxChroot(Sandbox): def _run(self, command, flags, *, cwd, env): if not self._has_command(command[0], env): - raise SandboxCommandError("Staged artifacts do not provide command " - "'{}'".format(command[0]), - reason='missing-command') + raise SandboxCommandError( + "Staged artifacts do not provide command " "'{}'".format(command[0]), reason="missing-command" + ) stdout, stderr = self._get_output() # Create the mount map, this will tell us where # each mount point needs to be mounted from and to - self.mount_map = MountMap(self, flags & SandboxFlags.ROOT_READ_ONLY, - self._FUSE_MOUNT_OPTIONS) + self.mount_map = MountMap(self, flags & SandboxFlags.ROOT_READ_ONLY, self._FUSE_MOUNT_OPTIONS) # Create a sysroot and run the command inside it with ExitStack() as stack: - os.makedirs('/var/run/buildstream', exist_ok=True) + os.makedirs("/var/run/buildstream", exist_ok=True) # FIXME: While we do not currently do anything to prevent # network access, we also don't copy /etc/resolv.conf to @@ -104,21 +105,20 @@ class SandboxChroot(Sandbox): # # Nonetheless a better solution could perhaps be found. - rootfs = stack.enter_context(utils._tempdir(dir='/var/run/buildstream')) + rootfs = stack.enter_context(utils._tempdir(dir="/var/run/buildstream")) stack.enter_context(self.create_devices(self._root, flags)) stack.enter_context(self.mount_dirs(rootfs, flags, stdout, stderr)) if flags & SandboxFlags.INTERACTIVE: stdin = sys.stdin else: - stdin = stack.enter_context(open(os.devnull, 'r')) + stdin = stack.enter_context(open(os.devnull, "r")) # Ensure the cwd exists if cwd is not None: workdir = os.path.join(rootfs, cwd.lstrip(os.sep)) os.makedirs(workdir, exist_ok=True) - status = self.chroot(rootfs, command, stdin, stdout, - stderr, cwd, env, flags) + status = self.chroot(rootfs, command, stdin, stdout, stderr, cwd, env, flags) self._vdir._mark_changed() return status @@ -173,7 +173,7 @@ class SandboxChroot(Sandbox): # If you try to put gtk dialogs here Tristan (either) # will personally scald you preexec_fn=lambda: (os.chroot(rootfs), os.chdir(cwd)), - start_new_session=flags & SandboxFlags.INTERACTIVE + start_new_session=flags & SandboxFlags.INTERACTIVE, ) # Wait for the child process to finish, ensuring that @@ -214,13 +214,14 @@ class SandboxChroot(Sandbox): # Exceptions in preexec_fn are simply reported as # 'Exception occurred in preexec_fn', turn these into # a more readable message. - if str(e) == 'Exception occurred in preexec_fn.': - raise SandboxError('Could not chroot into {} or chdir into {}. ' - 'Ensure you are root and that the relevant directory exists.' - .format(rootfs, cwd)) from e + if str(e) == "Exception occurred in preexec_fn.": + raise SandboxError( + "Could not chroot into {} or chdir into {}. " + "Ensure you are root and that the relevant directory exists.".format(rootfs, cwd) + ) from e # Otherwise, raise a more general error - raise SandboxError('Could not run command {}: {}'.format(command, e)) from e + raise SandboxError("Could not run command {}: {}".format(command, e)) from e return code @@ -251,8 +252,10 @@ class SandboxChroot(Sandbox): devices.append(self.mknod(device, location)) except OSError as err: if err.errno == 1: - raise SandboxError("Permission denied while creating device node: {}.".format(err) + - "BuildStream reqiures root permissions for these setttings.") + raise SandboxError( + "Permission denied while creating device node: {}.".format(err) + + "BuildStream reqiures root permissions for these setttings." + ) raise @@ -300,16 +303,16 @@ class SandboxChroot(Sandbox): with ExitStack() as stack: stack.enter_context(self.mount_map.mounted(self)) - stack.enter_context(mount_point('/')) + stack.enter_context(mount_point("/")) if flags & SandboxFlags.INTERACTIVE: - stack.enter_context(mount_src('/dev')) + stack.enter_context(mount_src("/dev")) - stack.enter_context(mount_src('/tmp')) - stack.enter_context(mount_src('/proc')) + stack.enter_context(mount_src("/tmp")) + stack.enter_context(mount_src("/proc")) for mark in self._get_marked_directories(): - stack.enter_context(mount_point(mark['directory'])) + stack.enter_context(mount_point(mark["directory"])) # Remount root RO if necessary if flags & flags & SandboxFlags.ROOT_READ_ONLY: @@ -343,10 +346,9 @@ class SandboxChroot(Sandbox): os.mknod(target, mode=stat.S_IFCHR | dev.st_mode, device=target_dev) except PermissionError as e: - raise SandboxError('Could not create device {}, ensure that you have root permissions: {}') + raise SandboxError("Could not create device {}, ensure that you have root permissions: {}") except OSError as e: - raise SandboxError('Could not create device {}: {}' - .format(target, e)) from e + raise SandboxError("Could not create device {}: {}".format(target, e)) from e return target diff --git a/src/buildstream/sandbox/_sandboxdummy.py b/src/buildstream/sandbox/_sandboxdummy.py index ae3d5e512..0cae2b6a9 100644 --- a/src/buildstream/sandbox/_sandboxdummy.py +++ b/src/buildstream/sandbox/_sandboxdummy.py @@ -28,9 +28,10 @@ class SandboxDummy(Sandbox): def _run(self, command, flags, *, cwd, env): if not self._has_command(command[0], env): - raise SandboxCommandError("Staged artifacts do not provide command " - "'{}'".format(command[0]), - reason='missing-command') + raise SandboxCommandError( + "Staged artifacts do not provide command " "'{}'".format(command[0]), reason="missing-command" + ) - raise SandboxError("This platform does not support local builds: {}".format(self._reason), - reason="unavailable-local-sandbox") + raise SandboxError( + "This platform does not support local builds: {}".format(self._reason), reason="unavailable-local-sandbox" + ) diff --git a/src/buildstream/sandbox/_sandboxreapi.py b/src/buildstream/sandbox/_sandboxreapi.py index 31c1c9674..2d661c893 100644 --- a/src/buildstream/sandbox/_sandboxreapi.py +++ b/src/buildstream/sandbox/_sandboxreapi.py @@ -30,7 +30,6 @@ from ..storage._casbaseddirectory import CasBasedDirectory # the Remote Execution API. # class SandboxREAPI(Sandbox): - def _use_cas_based_directory(self): # Always use CasBasedDirectory for REAPI return True @@ -46,14 +45,14 @@ class SandboxREAPI(Sandbox): # Ensure working directory exists if len(cwd) > 1: - assert cwd.startswith('/') + assert cwd.startswith("/") vdir.descend(*cwd[1:].split(os.path.sep), create=True) # Create directories for all marked directories. This emulates # some of the behaviour of other sandboxes, which create these # to use as mount points. for mark in self._get_marked_directories(): - directory = mark['directory'] + directory = mark["directory"] # Create each marked directory vdir.descend(*directory.split(os.path.sep), create=True) @@ -61,21 +60,21 @@ class SandboxREAPI(Sandbox): input_root_digest = vdir._get_digest() command_proto = self._create_command(command, cwd, env) command_digest = cascache.add_object(buffer=command_proto.SerializeToString()) - action = remote_execution_pb2.Action(command_digest=command_digest, - input_root_digest=input_root_digest) + action = remote_execution_pb2.Action(command_digest=command_digest, input_root_digest=input_root_digest) action_result = self._execute_action(action) # pylint: disable=assignment-from-no-return # Get output of build - self._process_job_output(action_result.output_directories, action_result.output_files, - failure=action_result.exit_code != 0) + self._process_job_output( + action_result.output_directories, action_result.output_files, failure=action_result.exit_code != 0 + ) if stdout: if action_result.stdout_raw: - stdout.write(str(action_result.stdout_raw, 'utf-8', errors='ignore')) + stdout.write(str(action_result.stdout_raw, "utf-8", errors="ignore")) if stderr: if action_result.stderr_raw: - stderr.write(str(action_result.stderr_raw, 'utf-8', errors='ignore')) + stderr.write(str(action_result.stderr_raw, "utf-8", errors="ignore")) # Non-zero exit code means a normal error during the build: # the remote execution system has worked correctly but the command failed. @@ -83,19 +82,21 @@ class SandboxREAPI(Sandbox): def _create_command(self, command, working_directory, environment): # Creates a command proto - environment_variables = [remote_execution_pb2.Command. - EnvironmentVariable(name=k, value=v) - for (k, v) in environment.items()] + environment_variables = [ + remote_execution_pb2.Command.EnvironmentVariable(name=k, value=v) for (k, v) in environment.items() + ] # Request the whole directory tree as output output_directory = os.path.relpath(os.path.sep, start=working_directory) - return remote_execution_pb2.Command(arguments=command, - working_directory=working_directory, - environment_variables=environment_variables, - output_files=[], - output_directories=[output_directory], - platform=None) + return remote_execution_pb2.Command( + arguments=command, + working_directory=working_directory, + environment_variables=environment_variables, + output_files=[], + output_directories=[output_directory], + platform=None, + ) def _process_job_output(self, output_directories, output_files, *, failure): # Reads the remote execution server response to an execution request. @@ -124,7 +125,7 @@ class SandboxREAPI(Sandbox): # Get digest of root directory from tree digest tree = remote_execution_pb2.Tree() - with open(cascache.objpath(tree_digest), 'rb') as f: + with open(cascache.objpath(tree_digest), "rb") as f: tree.ParseFromString(f.read()) root_directory = tree.root.SerializeToString() dir_digest = utils._message_digest(root_directory) @@ -140,8 +141,7 @@ class SandboxREAPI(Sandbox): return _SandboxREAPIBatch(self, main_group, flags, collect=collect) def _execute_action(self, action): - raise ImplError("Sandbox of type '{}' does not implement _execute_action()" - .format(type(self).__name__)) + raise ImplError("Sandbox of type '{}' does not implement _execute_action()".format(type(self).__name__)) # _SandboxREAPIBatch() @@ -149,7 +149,6 @@ class SandboxREAPI(Sandbox): # Command batching by shell script generation. # class _SandboxREAPIBatch(_SandboxBatch): - def __init__(self, sandbox, main_group, flags, *, collect=None): super().__init__(sandbox, main_group, flags, collect=collect) @@ -164,7 +163,7 @@ class _SandboxREAPIBatch(_SandboxBatch): self.main_group.execute(self) first = self.first_command - if first and self.sandbox.run(['sh', '-c', '-e', self.script], self.flags, cwd=first.cwd, env=first.env) != 0: + if first and self.sandbox.run(["sh", "-c", "-e", self.script], self.flags, cwd=first.cwd, env=first.env) != 0: raise SandboxCommandError("Command execution failed", collect=self.collect) def execute_group(self, group): @@ -195,7 +194,7 @@ class _SandboxREAPIBatch(_SandboxBatch): self.env = command.env # Actual command execution - cmdline = ' '.join(shlex.quote(cmd) for cmd in command.command) + cmdline = " ".join(shlex.quote(cmd) for cmd in command.command) self.script += "(set -ex; {})".format(cmdline) # Error handling diff --git a/src/buildstream/sandbox/_sandboxremote.py b/src/buildstream/sandbox/_sandboxremote.py index fa7cc9f90..d4ffd64a1 100644 --- a/src/buildstream/sandbox/_sandboxremote.py +++ b/src/buildstream/sandbox/_sandboxremote.py @@ -39,7 +39,7 @@ from .._cas import CASRemote from .._remote import RemoteSpec -class RemoteExecutionSpec(namedtuple('RemoteExecutionSpec', 'exec_service storage_service action_service')): +class RemoteExecutionSpec(namedtuple("RemoteExecutionSpec", "exec_service storage_service action_service")): pass @@ -49,59 +49,63 @@ class RemoteExecutionSpec(namedtuple('RemoteExecutionSpec', 'exec_service storag # commands to a remote server and retrieves the results from it. # class SandboxRemote(SandboxREAPI): - def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self._output_files_required = kwargs.get('output_files_required', True) + self._output_files_required = kwargs.get("output_files_required", True) - config = kwargs['specs'] # This should be a RemoteExecutionSpec + config = kwargs["specs"] # This should be a RemoteExecutionSpec if config is None: return # gRPC doesn't support fork without exec, which is used in the main process. assert not utils._is_main_process() - self.storage_url = config.storage_service['url'] - self.exec_url = config.exec_service['url'] + self.storage_url = config.storage_service["url"] + self.exec_url = config.exec_service["url"] exec_certs = {} - for key in ['client-cert', 'client-key', 'server-cert']: + for key in ["client-cert", "client-key", "server-cert"]: if key in config.exec_service: - with open(config.exec_service[key], 'rb') as f: + with open(config.exec_service[key], "rb") as f: exec_certs[key] = f.read() self.exec_credentials = grpc.ssl_channel_credentials( - root_certificates=exec_certs.get('server-cert'), - private_key=exec_certs.get('client-key'), - certificate_chain=exec_certs.get('client-cert')) + root_certificates=exec_certs.get("server-cert"), + private_key=exec_certs.get("client-key"), + certificate_chain=exec_certs.get("client-cert"), + ) action_certs = {} - for key in ['client-cert', 'client-key', 'server-cert']: + for key in ["client-cert", "client-key", "server-cert"]: if key in config.action_service: - with open(config.action_service[key], 'rb') as f: + with open(config.action_service[key], "rb") as f: action_certs[key] = f.read() if config.action_service: - self.action_url = config.action_service['url'] - self.action_instance = config.action_service.get('instance-name', None) + self.action_url = config.action_service["url"] + self.action_instance = config.action_service.get("instance-name", None) self.action_credentials = grpc.ssl_channel_credentials( - root_certificates=action_certs.get('server-cert'), - private_key=action_certs.get('client-key'), - certificate_chain=action_certs.get('client-cert')) + root_certificates=action_certs.get("server-cert"), + private_key=action_certs.get("client-key"), + certificate_chain=action_certs.get("client-cert"), + ) else: self.action_url = None self.action_instance = None self.action_credentials = None - self.exec_instance = config.exec_service.get('instance-name', None) - self.storage_instance = config.storage_service.get('instance-name', None) - - self.storage_remote_spec = RemoteSpec(self.storage_url, push=True, - server_cert=config.storage_service.get('server-cert'), - client_key=config.storage_service.get('client-key'), - client_cert=config.storage_service.get('client-cert'), - instance_name=self.storage_instance) + self.exec_instance = config.exec_service.get("instance-name", None) + self.storage_instance = config.storage_service.get("instance-name", None) + + self.storage_remote_spec = RemoteSpec( + self.storage_url, + push=True, + server_cert=config.storage_service.get("server-cert"), + client_key=config.storage_service.get("client-key"), + client_cert=config.storage_service.get("client-cert"), + instance_name=self.storage_instance, + ) self.operation_name = None def info(self, msg): @@ -109,47 +113,49 @@ class SandboxRemote(SandboxREAPI): @staticmethod def specs_from_config_node(config_node, basedir=None): - def require_node(config, keyname): val = config.get_mapping(keyname, default=None) if val is None: provenance = remote_config.get_provenance() - raise _yaml.LoadError("{}: '{}' was not present in the remote " - "execution configuration (remote-execution). " - .format(str(provenance), keyname), - _yaml.LoadErrorReason.INVALID_DATA) + raise _yaml.LoadError( + "{}: '{}' was not present in the remote " + "execution configuration (remote-execution). ".format(str(provenance), keyname), + _yaml.LoadErrorReason.INVALID_DATA, + ) return val - remote_config = config_node.get_mapping('remote-execution', default=None) + remote_config = config_node.get_mapping("remote-execution", default=None) if remote_config is None: return None - service_keys = ['execution-service', 'storage-service', 'action-cache-service'] + service_keys = ["execution-service", "storage-service", "action-cache-service"] - remote_config.validate_keys(['url', *service_keys]) + remote_config.validate_keys(["url", *service_keys]) - exec_config = require_node(remote_config, 'execution-service') - storage_config = require_node(remote_config, 'storage-service') - action_config = remote_config.get_mapping('action-cache-service', default={}) + exec_config = require_node(remote_config, "execution-service") + storage_config = require_node(remote_config, "storage-service") + action_config = remote_config.get_mapping("action-cache-service", default={}) - tls_keys = ['client-key', 'client-cert', 'server-cert'] + tls_keys = ["client-key", "client-cert", "server-cert"] - exec_config.validate_keys(['url', 'instance-name', *tls_keys]) - storage_config.validate_keys(['url', 'instance-name', *tls_keys]) + exec_config.validate_keys(["url", "instance-name", *tls_keys]) + storage_config.validate_keys(["url", "instance-name", *tls_keys]) if action_config: - action_config.validate_keys(['url', 'instance-name', *tls_keys]) + action_config.validate_keys(["url", "instance-name", *tls_keys]) # Maintain some backwards compatibility with older configs, in which # 'url' was the only valid key for remote-execution: - if 'url' in remote_config: - if 'execution-service' not in remote_config: - exec_config = Node.from_dict({'url': remote_config['url']}) + if "url" in remote_config: + if "execution-service" not in remote_config: + exec_config = Node.from_dict({"url": remote_config["url"]}) else: - provenance = remote_config.get_node('url').get_provenance() - raise _yaml.LoadError("{}: 'url' and 'execution-service' keys were found in the remote " - "execution configuration (remote-execution). " - "You can only specify one of these." - .format(str(provenance)), _yaml.LoadErrorReason.INVALID_DATA) + provenance = remote_config.get_node("url").get_provenance() + raise _yaml.LoadError( + "{}: 'url' and 'execution-service' keys were found in the remote " + "execution configuration (remote-execution). " + "You can only specify one of these.".format(str(provenance)), + _yaml.LoadErrorReason.INVALID_DATA, + ) service_configs = [exec_config, storage_config, action_config] @@ -161,12 +167,14 @@ class SandboxRemote(SandboxREAPI): for config_key, config in zip(service_keys, service_configs): # Either both or none of the TLS client key/cert pair must be specified: - if ('client-key' in config) != ('client-cert' in config): + if ("client-key" in config) != ("client-cert" in config): provenance = remote_config.get_node(config_key).get_provenance() - raise _yaml.LoadError("{}: TLS client key/cert pair is incomplete. " - "You must specify both 'client-key' and 'client-cert' " - "for authenticated HTTPS connections." - .format(str(provenance)), _yaml.LoadErrorReason.INVALID_DATA) + raise _yaml.LoadError( + "{}: TLS client key/cert pair is incomplete. " + "You must specify both 'client-key' and 'client-cert' " + "for authenticated HTTPS connections.".format(str(provenance)), + _yaml.LoadErrorReason.INVALID_DATA, + ) for tls_key in tls_keys: if tls_key in config: @@ -182,9 +190,9 @@ class SandboxRemote(SandboxREAPI): # Try to create a communication channel to the BuildGrid server. stub = remote_execution_pb2_grpc.ExecutionStub(channel) - request = remote_execution_pb2.ExecuteRequest(instance_name=self.exec_instance, - action_digest=action_digest, - skip_cache_lookup=False) + request = remote_execution_pb2.ExecuteRequest( + instance_name=self.exec_instance, action_digest=action_digest, skip_cache_lookup=False + ) def __run_remote_command(stub, execute_request=None, running_operation=None): try: @@ -206,26 +214,30 @@ class SandboxRemote(SandboxREAPI): except grpc.RpcError as e: status_code = e.code() if status_code == grpc.StatusCode.UNAVAILABLE: - raise SandboxError("Failed contacting remote execution server at {}." - .format(self.exec_url)) - - if status_code in (grpc.StatusCode.INVALID_ARGUMENT, - grpc.StatusCode.FAILED_PRECONDITION, - grpc.StatusCode.RESOURCE_EXHAUSTED, - grpc.StatusCode.INTERNAL, - grpc.StatusCode.DEADLINE_EXCEEDED): + raise SandboxError("Failed contacting remote execution server at {}.".format(self.exec_url)) + + if status_code in ( + grpc.StatusCode.INVALID_ARGUMENT, + grpc.StatusCode.FAILED_PRECONDITION, + grpc.StatusCode.RESOURCE_EXHAUSTED, + grpc.StatusCode.INTERNAL, + grpc.StatusCode.DEADLINE_EXCEEDED, + ): raise SandboxError("{} ({}).".format(e.details(), status_code.name)) if running_operation and status_code == grpc.StatusCode.UNIMPLEMENTED: - raise SandboxError("Failed trying to recover from connection loss: " - "server does not support operation status polling recovery.") + raise SandboxError( + "Failed trying to recover from connection loss: " + "server does not support operation status polling recovery." + ) return last_operation # Set up signal handler to trigger cancel_operation on SIGTERM operation = None - with self._get_context().messenger.timed_activity("Waiting for the remote build to complete"), \ - _signals.terminator(partial(self.cancel_operation, channel)): + with self._get_context().messenger.timed_activity( + "Waiting for the remote build to complete" + ), _signals.terminator(partial(self.cancel_operation, channel)): operation = __run_remote_command(stub, execute_request=request) if operation is None: return None @@ -242,18 +254,17 @@ class SandboxRemote(SandboxREAPI): return stub = operations_pb2_grpc.OperationsStub(channel) - request = operations_pb2.CancelOperationRequest( - name=str(self.operation_name)) + request = operations_pb2.CancelOperationRequest(name=str(self.operation_name)) try: stub.CancelOperation(request) except grpc.RpcError as e: - if (e.code() == grpc.StatusCode.UNIMPLEMENTED or - e.code() == grpc.StatusCode.INVALID_ARGUMENT): + if e.code() == grpc.StatusCode.UNIMPLEMENTED or e.code() == grpc.StatusCode.INVALID_ARGUMENT: pass else: - raise SandboxError("Failed trying to send CancelOperation request: " - "{} ({})".format(e.details(), e.code().name)) + raise SandboxError( + "Failed trying to send CancelOperation request: " "{} ({})".format(e.details(), e.code().name) + ) def _fetch_missing_blobs(self, vdir): context = self._get_context() @@ -282,8 +293,9 @@ class SandboxRemote(SandboxREAPI): remote_missing_blobs = cascache.fetch_blobs(casremote, blobs_to_fetch) if remote_missing_blobs: - raise SandboxError("{} output files are missing on the CAS server" - .format(len(remote_missing_blobs))) + raise SandboxError( + "{} output files are missing on the CAS server".format(len(remote_missing_blobs)) + ) def _execute_action(self, action): context = self._get_context() @@ -301,8 +313,9 @@ class SandboxRemote(SandboxREAPI): try: casremote.init() except grpc.RpcError as e: - raise SandboxError("Failed to contact remote execution CAS endpoint at {}: {}" - .format(self.storage_url, e)) from e + raise SandboxError( + "Failed to contact remote execution CAS endpoint at {}: {}".format(self.storage_url, e) + ) from e # Determine blobs missing on remote try: @@ -333,15 +346,19 @@ class SandboxRemote(SandboxREAPI): # Next, try to create a communication channel to the BuildGrid server. url = urlparse(self.exec_url) if not url.port: - raise SandboxError("You must supply a protocol and port number in the execution-service url, " - "for example: http://buildservice:50051.") - if url.scheme == 'http': - channel = grpc.insecure_channel('{}:{}'.format(url.hostname, url.port)) - elif url.scheme == 'https': - channel = grpc.secure_channel('{}:{}'.format(url.hostname, url.port), self.exec_credentials) + raise SandboxError( + "You must supply a protocol and port number in the execution-service url, " + "for example: http://buildservice:50051." + ) + if url.scheme == "http": + channel = grpc.insecure_channel("{}:{}".format(url.hostname, url.port)) + elif url.scheme == "https": + channel = grpc.secure_channel("{}:{}".format(url.hostname, url.port), self.exec_credentials) else: - raise SandboxError("Remote execution currently only supports the 'http' protocol " - "and '{}' was supplied.".format(url.scheme)) + raise SandboxError( + "Remote execution currently only supports the 'http' protocol " + "and '{}' was supplied.".format(url.scheme) + ) # Now request to execute the action with channel: @@ -369,23 +386,25 @@ class SandboxRemote(SandboxREAPI): return None url = urlparse(self.action_url) if not url.port: - raise SandboxError("You must supply a protocol and port number in the action-cache-service url, " - "for example: http://buildservice:50051.") - if url.scheme == 'http': - channel = grpc.insecure_channel('{}:{}'.format(url.hostname, url.port)) - elif url.scheme == 'https': - channel = grpc.secure_channel('{}:{}'.format(url.hostname, url.port), self.action_credentials) + raise SandboxError( + "You must supply a protocol and port number in the action-cache-service url, " + "for example: http://buildservice:50051." + ) + if url.scheme == "http": + channel = grpc.insecure_channel("{}:{}".format(url.hostname, url.port)) + elif url.scheme == "https": + channel = grpc.secure_channel("{}:{}".format(url.hostname, url.port), self.action_credentials) with channel: - request = remote_execution_pb2.GetActionResultRequest(instance_name=self.action_instance, - action_digest=action_digest) + request = remote_execution_pb2.GetActionResultRequest( + instance_name=self.action_instance, action_digest=action_digest + ) stub = remote_execution_pb2_grpc.ActionCacheStub(channel) try: result = stub.GetActionResult(request) except grpc.RpcError as e: if e.code() != grpc.StatusCode.NOT_FOUND: - raise SandboxError("Failed to query action cache: {} ({})" - .format(e.code(), e.details())) + raise SandboxError("Failed to query action cache: {} ({})".format(e.code(), e.details())) return None else: self.info("Action result found in action cache") @@ -397,7 +416,7 @@ class SandboxRemote(SandboxREAPI): # Failure of remote execution, usually due to an error in BuildStream raise SandboxError("No response returned from server") - assert not operation.HasField('error') and operation.HasField('response') + assert not operation.HasField("error") and operation.HasField("response") execution_response = remote_execution_pb2.ExecuteResponse() # The response is expected to be an ExecutionResponse message diff --git a/src/buildstream/sandbox/sandbox.py b/src/buildstream/sandbox/sandbox.py index b4691fe3f..c88cbf977 100644 --- a/src/buildstream/sandbox/sandbox.py +++ b/src/buildstream/sandbox/sandbox.py @@ -47,10 +47,11 @@ if TYPE_CHECKING: # pylint: disable=cyclic-import from .._context import Context from .._project import Project + # pylint: enable=cyclic-import -class SandboxFlags(): +class SandboxFlags: """Flags indicating how the sandbox should be run. """ @@ -100,49 +101,45 @@ class SandboxCommandError(SandboxError): collect (str): An optional directory containing partial install contents reason (str): An optional reason string (defaults to 'command-failed') """ - def __init__(self, message, *, detail=None, collect=None, reason='command-failed'): + + def __init__(self, message, *, detail=None, collect=None, reason="command-failed"): super().__init__(message, detail=detail, reason=reason) self.collect = collect -class Sandbox(): +class Sandbox: """Sandbox() Sandbox programming interface for :class:`.Element` plugins. """ # Minimal set of devices for the sandbox - DEVICES = [ - '/dev/urandom', - '/dev/random', - '/dev/zero', - '/dev/null' - ] - _dummy_reasons = [] # type: List[str] - - def __init__(self, context: 'Context', project: 'Project', directory: str, **kwargs): + DEVICES = ["/dev/urandom", "/dev/random", "/dev/zero", "/dev/null"] + _dummy_reasons = [] # type: List[str] + + def __init__(self, context: "Context", project: "Project", directory: str, **kwargs): self.__context = context self.__project = project - self.__directories = [] # type: List[Dict[str, Union[int, str]]] - self.__cwd = None # type: Optional[str] - self.__env = None # type: Optional[Dict[str, str]] - self.__mount_sources = {} # type: Dict[str, str] - self.__allow_real_directory = kwargs['allow_real_directory'] + self.__directories = [] # type: List[Dict[str, Union[int, str]]] + self.__cwd = None # type: Optional[str] + self.__env = None # type: Optional[Dict[str, str]] + self.__mount_sources = {} # type: Dict[str, str] + self.__allow_real_directory = kwargs["allow_real_directory"] self.__allow_run = True # Plugin element full name for logging - plugin = kwargs.get('plugin', None) + plugin = kwargs.get("plugin", None) if plugin: self.__element_name = plugin._get_full_name() else: self.__element_name = None # Configuration from kwargs common to all subclasses - self.__config = kwargs['config'] - self.__stdout = kwargs['stdout'] - self.__stderr = kwargs['stderr'] - self.__bare_directory = kwargs['bare_directory'] + self.__config = kwargs["config"] + self.__stdout = kwargs["stdout"] + self.__stderr = kwargs["stderr"] + self.__bare_directory = kwargs["bare_directory"] # Setup the directories. Root and output_directory should be # available to subclasses, hence being single-underscore. The @@ -153,15 +150,15 @@ class Sandbox(): self.__scratch = None os.makedirs(self._root, exist_ok=True) else: - self._root = os.path.join(directory, 'root') - self.__scratch = os.path.join(directory, 'scratch') + self._root = os.path.join(directory, "root") + self.__scratch = os.path.join(directory, "scratch") for directory_ in [self._root, self.__scratch]: os.makedirs(directory_, exist_ok=True) - self._output_directory = None # type: Optional[str] + self._output_directory = None # type: Optional[str] self._build_directory = None self._build_directory_always = None - self._vdir = None # type: Optional[Directory] + self._vdir = None # type: Optional[Directory] self._usebuildtree = False # This is set if anyone requests access to the underlying @@ -255,18 +252,17 @@ class Sandbox(): Any marked directories will be read-write in the sandboxed environment, only the root directory is allowed to be readonly. """ - self.__directories.append({ - 'directory': directory, - 'artifact': artifact - }) - - def run(self, - command: List[str], - flags: int, - *, - cwd: Optional[str] = None, - env: Optional[Dict[str, str]] = None, - label: str = None) -> Optional[int]: + self.__directories.append({"directory": directory, "artifact": artifact}) + + def run( + self, + command: List[str], + flags: int, + *, + cwd: Optional[str] = None, + env: Optional[Dict[str, str]] = None, + label: str = None + ) -> Optional[int]: """Run a command in the sandbox. If this is called outside a batch context, the command is immediately @@ -314,8 +310,7 @@ class Sandbox(): command = [command] if self.__batch: - assert flags == self.__batch.flags, \ - "Inconsistent sandbox flags in single command batch" + assert flags == self.__batch.flags, "Inconsistent sandbox flags in single command batch" batch_command = _SandboxBatchCommand(command, cwd=cwd, env=env, label=label) @@ -352,8 +347,7 @@ class Sandbox(): if self.__batch: # Nested batch - assert flags == self.__batch.flags, \ - "Inconsistent sandbox flags in single command batch" + assert flags == self.__batch.flags, "Inconsistent sandbox flags in single command batch" parent_group = self.__batch.current_group parent_group.append(group) @@ -394,8 +388,7 @@ class Sandbox(): # (int): The program exit code. # def _run(self, command, flags, *, cwd, env): - raise ImplError("Sandbox of type '{}' does not implement _run()" - .format(type(self).__name__)) + raise ImplError("Sandbox of type '{}' does not implement _run()".format(type(self).__name__)) # _create_batch() # @@ -425,7 +418,7 @@ class Sandbox(): if not self.__allow_real_directory and not self.__allow_run: return True - return 'BST_CAS_DIRECTORIES' in os.environ + return "BST_CAS_DIRECTORIES" in os.environ # _fetch_missing_blobs() # @@ -513,7 +506,7 @@ class Sandbox(): # what directory it is in makes it unnecessary to call the faulty # getcwd. env = dict(env) - env['PWD'] = cwd + env["PWD"] = cwd return env @@ -528,7 +521,7 @@ class Sandbox(): # Returns: # (str): The sandbox work directory def _get_work_directory(self, *, cwd=None): - return cwd or self.__cwd or '/' + return cwd or self.__cwd or "/" # _get_scratch_directory() # @@ -584,7 +577,7 @@ class Sandbox(): if len(command_as_parts) > 1: return False - for path in env.get('PATH').split(':'): + for path in env.get("PATH").split(":"): path_as_parts = path.lstrip(os.sep).split(os.sep) if vroot._exists(*path_as_parts, command, follow_symlinks=True): return True @@ -649,20 +642,14 @@ class Sandbox(): # message (str): A message to issue # details (str): optional, more detatils def _issue_warning(self, message, detail=None): - self.__context.messenger.message( - Message(MessageType.WARN, - message, - detail=detail - ) - ) + self.__context.messenger.message(Message(MessageType.WARN, message, detail=detail)) # _SandboxBatch() # # A batch of sandbox commands. # -class _SandboxBatch(): - +class _SandboxBatch: def __init__(self, sandbox, main_group, flags, *, collect=None): self.sandbox = sandbox self.main_group = main_group @@ -686,16 +673,21 @@ class _SandboxBatch(): def execute_command(self, command): if command.label: context = self.sandbox._get_context() - message = Message(MessageType.STATUS, 'Running command', detail=command.label, - element_name=self.sandbox._get_element_name()) + message = Message( + MessageType.STATUS, + "Running command", + detail=command.label, + element_name=self.sandbox._get_element_name(), + ) context.messenger.message(message) exitcode = self.sandbox._run(command.command, self.flags, cwd=command.cwd, env=command.env) if exitcode != 0: - cmdline = ' '.join(shlex.quote(cmd) for cmd in command.command) + cmdline = " ".join(shlex.quote(cmd) for cmd in command.command) label = command.label or cmdline - raise SandboxCommandError("Command failed with exitcode {}".format(exitcode), - detail=label, collect=self.collect) + raise SandboxCommandError( + "Command failed with exitcode {}".format(exitcode), detail=label, collect=self.collect + ) def execute_call(self, call): call.callback() @@ -705,8 +697,7 @@ class _SandboxBatch(): # # An item in a command batch. # -class _SandboxBatchItem(): - +class _SandboxBatchItem: def __init__(self, *, label=None): self.label = label @@ -716,7 +707,6 @@ class _SandboxBatchItem(): # A command item in a command batch. # class _SandboxBatchCommand(_SandboxBatchItem): - def __init__(self, command, *, cwd, env, label=None): super().__init__(label=label) @@ -733,7 +723,6 @@ class _SandboxBatchCommand(_SandboxBatchItem): # A group in a command batch. # class _SandboxBatchGroup(_SandboxBatchItem): - def __init__(self, *, label=None): super().__init__(label=label) @@ -755,7 +744,6 @@ class _SandboxBatchGroup(_SandboxBatchItem): # A call item in a command batch. # class _SandboxBatchCall(_SandboxBatchItem): - def __init__(self, callback): super().__init__() diff --git a/src/buildstream/scriptelement.py b/src/buildstream/scriptelement.py index e78049b4a..f8deff28e 100644 --- a/src/buildstream/scriptelement.py +++ b/src/buildstream/scriptelement.py @@ -48,8 +48,8 @@ class ScriptElement(Element): __install_root = "/" __cwd = "/" __root_read_only = False - __commands = None # type: OrderedDict[str, List[str]] - __layout = [] # type: List[Dict[str, Optional[str]]] + __commands = None # type: OrderedDict[str, List[str]] + __layout = [] # type: List[Dict[str, Optional[str]]] # The compose element's output is its dependencies, so # we must rebuild if the dependencies change even when @@ -149,8 +149,7 @@ class ScriptElement(Element): # if not self.__layout: self.__layout = [] - self.__layout.append({"element": element, - "destination": destination}) + self.__layout.append({"element": element, "destination": destination}) def add_commands(self, group_name: str, command_list: List[str]) -> None: """Adds a list of commands under the group-name. @@ -183,11 +182,11 @@ class ScriptElement(Element): def get_unique_key(self): return { - 'commands': self.__commands, - 'cwd': self.__cwd, - 'install-root': self.__install_root, - 'layout': self.__layout, - 'root-read-only': self.__root_read_only + "commands": self.__commands, + "cwd": self.__cwd, + "install-root": self.__install_root, + "layout": self.__layout, + "root-read-only": self.__root_read_only, } def configure_sandbox(self, sandbox): @@ -206,14 +205,14 @@ class ScriptElement(Element): # Mark the artifact directories in the layout for item in self.__layout: - destination = item['destination'] + destination = item["destination"] was_artifact = directories.get(destination, False) - directories[destination] = item['element'] or was_artifact + directories[destination] = item["element"] or was_artifact for directory, artifact in directories.items(): # Root does not need to be marked as it is always mounted # with artifact (unless explicitly marked non-artifact) - if directory != '/': + if directory != "/": sandbox.mark_directory(directory, artifact=artifact) def stage(self, sandbox): @@ -222,8 +221,7 @@ class ScriptElement(Element): if not self.__layout: # if no layout set, stage all dependencies into / for build_dep in self.dependencies(Scope.BUILD, recurse=False): - with self.timed_activity("Staging {} at /" - .format(build_dep.name), silent_nested=True): + with self.timed_activity("Staging {} at /".format(build_dep.name), silent_nested=True): build_dep.stage_dependency_artifacts(sandbox, Scope.RUN, path="/") with sandbox.batch(SandboxFlags.NONE): @@ -236,35 +234,33 @@ class ScriptElement(Element): for item in self.__layout: # Skip layout members which dont stage an element - if not item['element']: + if not item["element"]: continue - element = self.search(Scope.BUILD, item['element']) - if item['destination'] == '/': - with self.timed_activity("Staging {} at /".format(element.name), - silent_nested=True): + element = self.search(Scope.BUILD, item["element"]) + if item["destination"] == "/": + with self.timed_activity("Staging {} at /".format(element.name), silent_nested=True): element.stage_dependency_artifacts(sandbox, Scope.RUN) else: - with self.timed_activity("Staging {} at {}" - .format(element.name, item['destination']), - silent_nested=True): + with self.timed_activity( + "Staging {} at {}".format(element.name, item["destination"]), silent_nested=True + ): virtual_dstdir = sandbox.get_virtual_directory() - virtual_dstdir.descend(*item['destination'].lstrip(os.sep).split(os.sep), create=True) - element.stage_dependency_artifacts(sandbox, Scope.RUN, path=item['destination']) + virtual_dstdir.descend(*item["destination"].lstrip(os.sep).split(os.sep), create=True) + element.stage_dependency_artifacts(sandbox, Scope.RUN, path=item["destination"]) with sandbox.batch(SandboxFlags.NONE): for item in self.__layout: # Skip layout members which dont stage an element - if not item['element']: + if not item["element"]: continue - element = self.search(Scope.BUILD, item['element']) + element = self.search(Scope.BUILD, item["element"]) # Integration commands can only be run for elements staged to / - if item['destination'] == '/': - with self.timed_activity("Integrating {}".format(element.name), - silent_nested=True): + if item["destination"] == "/": + with self.timed_activity("Integrating {}".format(element.name), silent_nested=True): for dep in element.dependencies(Scope.RUN): dep.integrate(sandbox) @@ -283,9 +279,7 @@ class ScriptElement(Element): for cmd in commands: # Note the -e switch to 'sh' means to exit with an error # if any untested command fails. - sandbox.run(['sh', '-c', '-e', cmd + '\n'], - flags, - label=cmd) + sandbox.run(["sh", "-c", "-e", cmd + "\n"], flags, label=cmd) # Return where the result can be collected from return self.__install_root @@ -297,18 +291,18 @@ class ScriptElement(Element): def __validate_layout(self): if self.__layout: # Cannot proceeed if layout is used, but none are for "/" - root_defined = any([(entry['destination'] == '/') for entry in self.__layout]) + root_defined = any([(entry["destination"] == "/") for entry in self.__layout]) if not root_defined: - raise ElementError("{}: Using layout, but none are staged as '/'" - .format(self)) + raise ElementError("{}: Using layout, but none are staged as '/'".format(self)) # Cannot proceed if layout specifies an element that isn't part # of the dependencies. for item in self.__layout: - if item['element']: - if not self.search(Scope.BUILD, item['element']): - raise ElementError("{}: '{}' in layout not found in dependencies" - .format(self, item['element'])) + if item["element"]: + if not self.search(Scope.BUILD, item["element"]): + raise ElementError( + "{}: '{}' in layout not found in dependencies".format(self, item["element"]) + ) def setup(): diff --git a/src/buildstream/source.py b/src/buildstream/source.py index 05a1ae464..2e7460439 100644 --- a/src/buildstream/source.py +++ b/src/buildstream/source.py @@ -184,6 +184,7 @@ if TYPE_CHECKING: # pylint: disable=cyclic-import from ._context import Context from ._project import Project + # pylint: enable=cyclic-import @@ -197,16 +198,14 @@ class SourceError(BstError): reason: An optional machine readable reason string, used for test cases temporary: An indicator to whether the error may occur if the operation was run again. (*Since: 1.2*) """ - def __init__(self, - message: str, - *, - detail: Optional[str] = None, - reason: Optional[str] = None, - temporary: bool = False): + + def __init__( + self, message: str, *, detail: Optional[str] = None, reason: Optional[str] = None, temporary: bool = False + ): super().__init__(message, detail=detail, domain=ErrorDomain.SOURCE, reason=reason, temporary=temporary) -class SourceFetcher(): +class SourceFetcher: """SourceFetcher() This interface exists so that a source that downloads from multiple @@ -222,6 +221,7 @@ class SourceFetcher(): for every URL found in the configuration data at :func:`Plugin.configure() <buildstream.plugin.Plugin.configure>` time. """ + def __init__(self): self.__alias = None @@ -275,8 +275,9 @@ class Source(Plugin): All Sources derive from this class, this interface defines how the core will be interacting with Sources. """ + # The defaults from the project - __defaults = None # type: Optional[Dict[str, Any]] + __defaults = None # type: Optional[Dict[str, Any]] BST_REQUIRES_PREVIOUS_SOURCES_TRACK = False """Whether access to previous sources is required during track @@ -331,32 +332,40 @@ class Source(Plugin): *Since: 1.91.2* """ - def __init__(self, - context: 'Context', - project: 'Project', - meta: MetaSource, - *, - alias_override: Optional[Tuple[str, str]] = None, - unique_id: Optional[int] = None): + def __init__( + self, + context: "Context", + project: "Project", + meta: MetaSource, + *, + alias_override: Optional[Tuple[str, str]] = None, + unique_id: Optional[int] = None + ): provenance = meta.config.get_provenance() # Set element_name member before parent init, as needed for debug messaging - self.__element_name = meta.element_name # The name of the element owning this source - super().__init__("{}-{}".format(meta.element_name, meta.element_index), - context, project, provenance, "source", unique_id=unique_id) + self.__element_name = meta.element_name # The name of the element owning this source + super().__init__( + "{}-{}".format(meta.element_name, meta.element_index), + context, + project, + provenance, + "source", + unique_id=unique_id, + ) - self.__element_index = meta.element_index # The index of the source in the owning element's source list - self.__element_kind = meta.element_kind # The kind of the element owning this source - self.__directory = meta.directory # Staging relative directory - self.__consistency = Consistency.INCONSISTENT # Cached consistency state - self.__meta_kind = meta.kind # The kind of this source, required for unpickling + self.__element_index = meta.element_index # The index of the source in the owning element's source list + self.__element_kind = meta.element_kind # The kind of the element owning this source + self.__directory = meta.directory # Staging relative directory + self.__consistency = Consistency.INCONSISTENT # Cached consistency state + self.__meta_kind = meta.kind # The kind of this source, required for unpickling - self.__key = None # Cache key for source + self.__key = None # Cache key for source # The alias_override is only set on a re-instantiated Source - self.__alias_override = alias_override # Tuple of alias and its override to use instead - self.__expected_alias = None # The primary alias + self.__alias_override = alias_override # Tuple of alias and its override to use instead + self.__expected_alias = None # The primary alias # Set of marked download URLs - self.__marked_urls = set() # type: Set[str] + self.__marked_urls = set() # type: Set[str] # Collect the composited element configuration and # ask the element to configure itself. @@ -365,12 +374,12 @@ class Source(Plugin): self.__first_pass = meta.first_pass # cached values for commonly access values on the source - self.__mirror_directory = None # type: Optional[str] + self.__mirror_directory = None # type: Optional[str] self._configure(self.__config) self.__digest = None - COMMON_CONFIG_KEYS = ['kind', 'directory'] + COMMON_CONFIG_KEYS = ["kind", "directory"] """Common source config keys Source config keys that must not be accessed in configure(), and @@ -611,8 +620,8 @@ class Source(Plugin): # specific alias, so that sources that fetch from multiple # URLs and use different aliases default to only overriding # one alias, rather than getting confused. - override_alias = self.__alias_override[0] # type: ignore - override_url = self.__alias_override[1] # type: ignore + override_alias = self.__alias_override[0] # type: ignore + override_url = self.__alias_override[1] # type: ignore if url_alias == override_alias: url = override_url + url_body return url @@ -642,9 +651,9 @@ class Source(Plugin): if primary: expected_alias = _extract_alias(url) - assert (self.__expected_alias is None or - self.__expected_alias == expected_alias), \ - "Primary URL marked twice with different URLs" + assert ( + self.__expected_alias is None or self.__expected_alias == expected_alias + ), "Primary URL marked twice with different URLs" self.__expected_alias = expected_alias @@ -664,8 +673,9 @@ class Source(Plugin): # the case for git submodules which might be automatically # discovered. # - assert (url in self.__marked_urls or not _extract_alias(url)), \ - "URL was not seen at configure time: {}".format(url) + assert url in self.__marked_urls or not _extract_alias( + url + ), "URL was not seen at configure time: {}".format(url) def get_project_directory(self) -> str: """Fetch the project base directory @@ -790,8 +800,7 @@ class Source(Plugin): if self.BST_KEY_REQUIRES_STAGE: # _get_unique_key should be called before _stage assert self.__digest is not None - cas_dir = CasBasedDirectory(self._get_context().get_cascache(), - digest=self.__digest) + cas_dir = CasBasedDirectory(self._get_context().get_cascache(), digest=self.__digest) directory.import_files(cas_dir) else: self.stage(directory) @@ -811,11 +820,11 @@ class Source(Plugin): # def _get_unique_key(self): key = {} - key['directory'] = self.__directory + key["directory"] = self.__directory if self.BST_KEY_REQUIRES_STAGE: - key['unique'] = self._stage_into_cas() + key["unique"] = self._stage_into_cas() else: - key['unique'] = self.get_unique_key() # pylint: disable=assignment-from-no-return + key["unique"] = self.get_unique_key() # pylint: disable=assignment-from-no-return return key # _project_refs(): @@ -828,7 +837,7 @@ class Source(Plugin): # def _project_refs(self, project): element_kind = self.__element_kind - if element_kind == 'junction': + if element_kind == "junction": return project.junction_refs return project.refs @@ -863,9 +872,10 @@ class Source(Plugin): try: self.load_ref(ref_node) except ImplError as e: - raise SourceError("{}: Storing refs in project.refs is not supported by '{}' sources" - .format(self, self.get_kind()), - reason="unsupported-load-ref") from e + raise SourceError( + "{}: Storing refs in project.refs is not supported by '{}' sources".format(self, self.get_kind()), + reason="unsupported-load-ref", + ) from e # If the main project overrides the ref, use the override if project is not toplevel and toplevel.ref_storage == ProjectRefStorage.PROJECT_REFS: @@ -938,12 +948,12 @@ class Source(Plugin): elif provenance._project is None: assert provenance._filename == "" assert provenance._shortname == "" - raise SourceError("{}: Error saving source reference to synthetic node." - .format(self)) + raise SourceError("{}: Error saving source reference to synthetic node.".format(self)) else: - raise SourceError("{}: Cannot track source in a fragment from a junction" - .format(provenance._shortname), - reason="tracking-junction-fragment") + raise SourceError( + "{}: Cannot track source in a fragment from a junction".format(provenance._shortname), + reason="tracking-junction-fragment", + ) # # Step 2 - Set the ref in memory, and determine changed state @@ -968,13 +978,13 @@ class Source(Plugin): actions = {} for k, v in clean.items(): if k not in to_modify: - actions[k] = 'del' + actions[k] = "del" else: if v != to_modify[k]: - actions[k] = 'mod' + actions[k] = "mod" for k in to_modify.keys(): if k not in clean: - actions[k] = 'add' + actions[k] = "add" def walk_container(container, path): # For each step along path, synthesise if we need to. @@ -1002,20 +1012,19 @@ class Source(Plugin): def process_value(action, container, path, key, new_value): container = walk_container(container, path) - if action == 'del': + if action == "del": del container[key] - elif action == 'mod': + elif action == "mod": container[key] = new_value - elif action == 'add': + elif action == "add": container[key] = new_value else: - assert False, \ - "BUG: Unknown action: {}".format(action) + assert False, "BUG: Unknown action: {}".format(action) roundtrip_cache = {} for key, action in actions.items(): # Obtain the top level node and its file - if action == 'add': + if action == "add": provenance = node.get_provenance() else: provenance = node.get_node(key).get_provenance() @@ -1023,7 +1032,7 @@ class Source(Plugin): toplevel_node = provenance._toplevel # Get the path to whatever changed - if action == 'add': + if action == "add": path = toplevel_node._find(node) else: full_path = toplevel_node._find(node.get_node(key)) @@ -1033,8 +1042,7 @@ class Source(Plugin): roundtrip_file = roundtrip_cache.get(provenance._filename) if not roundtrip_file: roundtrip_file = roundtrip_cache[provenance._filename] = _yaml.roundtrip_load( - provenance._filename, - allow_missing=True + provenance._filename, allow_missing=True ) # Get the value of the round trip file that we need to change @@ -1048,9 +1056,9 @@ class Source(Plugin): try: _yaml.roundtrip_dump(data, filename) except OSError as e: - raise SourceError("{}: Error saving source reference to '{}': {}" - .format(self, filename, e), - reason="save-ref-error") from e + raise SourceError( + "{}: Error saving source reference to '{}': {}".format(self, filename, e), reason="save-ref-error" + ) from e return True @@ -1059,7 +1067,7 @@ class Source(Plugin): # Args: # previous_sources (list): List of Sources listed prior to this source # - def _track(self, previous_sources: List['Source']) -> SourceRef: + def _track(self, previous_sources: List["Source"]) -> SourceRef: if self.BST_KEY_REQUIRES_STAGE: # ensure that these sources have a key after tracking self._get_unique_key() @@ -1067,8 +1075,7 @@ class Source(Plugin): if self.BST_REQUIRES_PREVIOUS_SOURCES_TRACK: self.__ensure_previous_sources(previous_sources) - with self.__stage_previous_sources(previous_sources) \ - as staging_directory: + with self.__stage_previous_sources(previous_sources) as staging_directory: new_ref = self.__do_track(previous_sources_dir=self.__ensure_directory(staging_directory)) else: new_ref = self.__do_track() @@ -1135,9 +1142,7 @@ class Source(Plugin): # Gives a ref path that points to where sources are kept in the CAS def _get_source_name(self): # @ is used to prevent conflicts with project names - return "{}/{}".format( - self.get_kind(), - self._key) + return "{}/{}".format(self.get_kind(), self._key) def _get_brief_display_key(self): context = self._get_context() @@ -1210,9 +1215,7 @@ class Source(Plugin): meta.first_pass = self.__first_pass - clone = source_kind(context, project, meta, - alias_override=(alias, uri), - unique_id=self._unique_id) + clone = source_kind(context, project, meta, alias_override=(alias, uri), unique_id=self._unique_id) # Do the necessary post instantiation routines here # @@ -1352,20 +1355,18 @@ class Source(Plugin): try: os.makedirs(directory, exist_ok=True) except OSError as e: - raise SourceError("Failed to create staging directory: {}" - .format(e), - reason="ensure-stage-dir-fail") from e + raise SourceError( + "Failed to create staging directory: {}".format(e), reason="ensure-stage-dir-fail" + ) from e else: if self.__directory is not None: try: - directory = directory.descend( - *self.__directory.lstrip(os.sep).split(os.sep), - create=True) + directory = directory.descend(*self.__directory.lstrip(os.sep).split(os.sep), create=True) except VirtualDirectoryError as e: - raise SourceError("Failed to descend into staging directory: {}" - .format(e), - reason="ensure-stage-dir-fail") from e + raise SourceError( + "Failed to descend into staging directory: {}".format(e), reason="ensure-stage-dir-fail" + ) from e return directory @@ -1383,7 +1384,7 @@ class Source(Plugin): # @classmethod def __extract_config(cls, meta): - config = cls.__defaults.get_mapping('config', default={}) + config = cls.__defaults.get_mapping("config", default={}) config = config.clone() meta.config._composite(config) diff --git a/src/buildstream/storage/_casbaseddirectory.py b/src/buildstream/storage/_casbaseddirectory.py index 3786f25b6..df28dc591 100644 --- a/src/buildstream/storage/_casbaseddirectory.py +++ b/src/buildstream/storage/_casbaseddirectory.py @@ -38,10 +38,20 @@ from ._filebaseddirectory import FileBasedDirectory from ..utils import FileListResult, BST_ARBITRARY_TIMESTAMP -class IndexEntry(): +class IndexEntry: """ Directory entry used in CasBasedDirectory.index """ - def __init__(self, name, entrytype, *, digest=None, target=None, is_executable=False, - buildstream_object=None, modified=False): + + def __init__( + self, + name, + entrytype, + *, + digest=None, + target=None, + is_executable=False, + buildstream_object=None, + modified=False + ): self.name = name self.type = entrytype self.digest = digest @@ -52,8 +62,9 @@ class IndexEntry(): def get_directory(self, parent): if not self.buildstream_object: - self.buildstream_object = CasBasedDirectory(parent.cas_cache, digest=self.digest, - parent=parent, filename=self.name) + self.buildstream_object = CasBasedDirectory( + parent.cas_cache, digest=self.digest, parent=parent, filename=self.name + ) self.digest = None return self.buildstream_object @@ -69,6 +80,7 @@ class IndexEntry(): # which is meant to be unimplemented. # pylint: disable=super-init-not-called + class CasBasedDirectory(Directory): """ CAS-based directories can have two names; one is a 'common name' which has no effect @@ -100,21 +112,19 @@ class CasBasedDirectory(Directory): def _populate_index(self, digest): try: pb2_directory = remote_execution_pb2.Directory() - with open(self.cas_cache.objpath(digest), 'rb') as f: + with open(self.cas_cache.objpath(digest), "rb") as f: pb2_directory.ParseFromString(f.read()) except FileNotFoundError as e: raise VirtualDirectoryError("Directory not found in local cache: {}".format(e)) from e for entry in pb2_directory.directories: - self.index[entry.name] = IndexEntry(entry.name, _FileType.DIRECTORY, - digest=entry.digest) + self.index[entry.name] = IndexEntry(entry.name, _FileType.DIRECTORY, digest=entry.digest) for entry in pb2_directory.files: - self.index[entry.name] = IndexEntry(entry.name, _FileType.REGULAR_FILE, - digest=entry.digest, - is_executable=entry.is_executable) + self.index[entry.name] = IndexEntry( + entry.name, _FileType.REGULAR_FILE, digest=entry.digest, is_executable=entry.is_executable + ) for entry in pb2_directory.symlinks: - self.index[entry.name] = IndexEntry(entry.name, _FileType.SYMLINK, - target=entry.target) + self.index[entry.name] = IndexEntry(entry.name, _FileType.SYMLINK, target=entry.target) def _find_self_in_parent(self): assert self.parent is not None @@ -136,8 +146,7 @@ class CasBasedDirectory(Directory): return newdir def _add_file(self, basename, filename, modified=False, can_link=False): - entry = IndexEntry(filename, _FileType.REGULAR_FILE, - modified=modified or filename in self.index) + entry = IndexEntry(filename, _FileType.REGULAR_FILE, modified=modified or filename in self.index) path = os.path.join(basename, filename) entry.digest = self.cas_cache.add_object(path=path, link_directly=can_link) entry.is_executable = os.access(path, os.X_OK) @@ -206,14 +215,13 @@ class CasBasedDirectory(Directory): current_dir = current_dir.descend(*newpaths, follow_symlinks=True) else: error = "Cannot descend into {}, which is a '{}' in the directory {}" - raise VirtualDirectoryError(error.format(path, - current_dir.index[path].type, - current_dir), - reason="not-a-directory") + raise VirtualDirectoryError( + error.format(path, current_dir.index[path].type, current_dir), reason="not-a-directory" + ) else: - if path == '.': + if path == ".": continue - elif path == '..': + elif path == "..": if current_dir.parent is not None: current_dir = current_dir.parent # In POSIX /.. == / so just stay at the root dir @@ -222,8 +230,7 @@ class CasBasedDirectory(Directory): current_dir = current_dir._add_directory(path) else: error = "'{}' not found in {}" - raise VirtualDirectoryError(error.format(path, str(current_dir)), - reason="directory-not-found") + raise VirtualDirectoryError(error.format(path, str(current_dir)), reason="directory-not-found") return current_dir @@ -299,12 +306,13 @@ class CasBasedDirectory(Directory): dest_subdir = self.descend(name, create=create_subdir) except VirtualDirectoryError: filetype = self.index[name].type - raise VirtualDirectoryError('Destination is a {}, not a directory: /{}' - .format(filetype, relative_pathname)) + raise VirtualDirectoryError( + "Destination is a {}, not a directory: /{}".format(filetype, relative_pathname) + ) - dest_subdir._partial_import_cas_into_cas(src_subdir, filter_callback, - path_prefix=relative_pathname, - origin=origin, result=result) + dest_subdir._partial_import_cas_into_cas( + src_subdir, filter_callback, path_prefix=relative_pathname, origin=origin, result=result + ) if filter_callback and not filter_callback(relative_pathname): if is_dir and create_subdir and dest_subdir.is_empty(): @@ -317,20 +325,22 @@ class CasBasedDirectory(Directory): if not is_dir: if self._check_replacement(name, relative_pathname, result): if entry.type == _FileType.REGULAR_FILE: - self.index[name] = IndexEntry(name, _FileType.REGULAR_FILE, - digest=entry.digest, - is_executable=entry.is_executable, - modified=True) + self.index[name] = IndexEntry( + name, + _FileType.REGULAR_FILE, + digest=entry.digest, + is_executable=entry.is_executable, + modified=True, + ) self.__invalidate_digest() else: assert entry.type == _FileType.SYMLINK self._add_new_link_direct(name=name, target=entry.target) result.files_written.append(relative_pathname) - def import_files(self, external_pathspec, *, - filter_callback=None, - report_written=True, update_mtime=False, - can_link=False): + def import_files( + self, external_pathspec, *, filter_callback=None, report_written=True, update_mtime=False, can_link=False + ): """ See superclass Directory for arguments """ result = FileListResult() @@ -358,13 +368,12 @@ class CasBasedDirectory(Directory): def import_single_file(self, external_pathspec): result = FileListResult() - if self._check_replacement(os.path.basename(external_pathspec), - os.path.dirname(external_pathspec), - result): - self._add_file(os.path.dirname(external_pathspec), - os.path.basename(external_pathspec), - modified=os.path.basename(external_pathspec) - in result.overwritten) + if self._check_replacement(os.path.basename(external_pathspec), os.path.dirname(external_pathspec), result): + self._add_file( + os.path.dirname(external_pathspec), + os.path.basename(external_pathspec), + modified=os.path.basename(external_pathspec) in result.overwritten, + ) result.files_written.append(external_pathspec) return result @@ -516,10 +525,8 @@ class CasBasedDirectory(Directory): """ - file_list = list(filter(lambda i: i[1].type != _FileType.DIRECTORY, - self.index.items())) - directory_list = filter(lambda i: i[1].type == _FileType.DIRECTORY, - self.index.items()) + file_list = list(filter(lambda i: i[1].type != _FileType.DIRECTORY, self.index.items())) + directory_list = filter(lambda i: i[1].type == _FileType.DIRECTORY, self.index.items()) if prefix != "": yield prefix @@ -553,10 +560,7 @@ class CasBasedDirectory(Directory): """ for leaf in sorted(self.index.keys()): entry = self.index[leaf] - info = { - "name": os.path.join(prefix, leaf), - "type": entry.type - } + info = {"name": os.path.join(prefix, leaf), "type": entry.type} if entry.type == _FileType.REGULAR_FILE: info["executable"] = entry.is_executable info["size"] = self.get_size() @@ -599,8 +603,9 @@ class CasBasedDirectory(Directory): def _get_underlying_directory(self): """ There is no underlying directory for a CAS-backed directory, so throw an exception. """ - raise VirtualDirectoryError("_get_underlying_directory was called on a CAS-backed directory," + - " which has no underlying directory.") + raise VirtualDirectoryError( + "_get_underlying_directory was called on a CAS-backed directory," + " which has no underlying directory." + ) # _get_digest(): # diff --git a/src/buildstream/storage/_filebaseddirectory.py b/src/buildstream/storage/_filebaseddirectory.py index 07c23c192..222b47979 100644 --- a/src/buildstream/storage/_filebaseddirectory.py +++ b/src/buildstream/storage/_filebaseddirectory.py @@ -65,23 +65,22 @@ class FileBasedDirectory(Directory): try: st = os.lstat(new_path) if not stat.S_ISDIR(st.st_mode): - raise VirtualDirectoryError("Cannot descend into '{}': '{}' is not a directory" - .format(path, new_path)) + raise VirtualDirectoryError( + "Cannot descend into '{}': '{}' is not a directory".format(path, new_path) + ) except FileNotFoundError: if create: os.mkdir(new_path) else: - raise VirtualDirectoryError("Cannot descend into '{}': '{}' does not exist" - .format(path, new_path)) + raise VirtualDirectoryError("Cannot descend into '{}': '{}' does not exist".format(path, new_path)) current_dir = FileBasedDirectory(new_path) return current_dir - def import_files(self, external_pathspec, *, - filter_callback=None, - report_written=True, update_mtime=False, - can_link=False): + def import_files( + self, external_pathspec, *, filter_callback=None, report_written=True, update_mtime=False, can_link=False + ): """ See superclass Directory for arguments """ from ._casbaseddirectory import CasBasedDirectory # pylint: disable=cyclic-import @@ -101,13 +100,21 @@ class FileBasedDirectory(Directory): source_directory = external_pathspec if can_link and not update_mtime: - import_result = link_files(source_directory, self.external_directory, - filter_callback=filter_callback, - ignore_missing=False, report_written=report_written) + import_result = link_files( + source_directory, + self.external_directory, + filter_callback=filter_callback, + ignore_missing=False, + report_written=report_written, + ) else: - import_result = copy_files(source_directory, self.external_directory, - filter_callback=filter_callback, - ignore_missing=False, report_written=report_written) + import_result = copy_files( + source_directory, + self.external_directory, + filter_callback=filter_callback, + ignore_missing=False, + report_written=report_written, + ) if update_mtime: cur_time = time.time() @@ -190,8 +197,11 @@ class FileBasedDirectory(Directory): Return value: List(str) - list of modified paths """ - return [f for f in list_relative_paths(self.external_directory) - if _get_link_mtime(os.path.join(self.external_directory, f)) != BST_ARBITRARY_TIMESTAMP] + return [ + f + for f in list_relative_paths(self.external_directory) + if _get_link_mtime(os.path.join(self.external_directory, f)) != BST_ARBITRARY_TIMESTAMP + ] def list_relative_paths(self): """Provide a list of all relative paths. @@ -251,11 +261,13 @@ class FileBasedDirectory(Directory): dest_subdir = self.descend(name, create=create_subdir) except VirtualDirectoryError: filetype = self._get_filetype(name) - raise VirtualDirectoryError('Destination is a {}, not a directory: /{}' - .format(filetype, relative_pathname)) + raise VirtualDirectoryError( + "Destination is a {}, not a directory: /{}".format(filetype, relative_pathname) + ) - dest_subdir._import_files_from_cas(src_subdir, actionfunc, filter_callback, - path_prefix=relative_pathname, result=result) + dest_subdir._import_files_from_cas( + src_subdir, actionfunc, filter_callback, path_prefix=relative_pathname, result=result + ) if filter_callback and not filter_callback(relative_pathname): if is_dir and create_subdir and dest_subdir.is_empty(): @@ -279,8 +291,16 @@ class FileBasedDirectory(Directory): src_path = source_directory.cas_cache.objpath(entry.digest) actionfunc(src_path, dest_path, result=result) if entry.is_executable: - os.chmod(dest_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | - stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) + os.chmod( + dest_path, + stat.S_IRUSR + | stat.S_IWUSR + | stat.S_IXUSR + | stat.S_IRGRP + | stat.S_IXGRP + | stat.S_IROTH + | stat.S_IXOTH, + ) else: assert entry.type == _FileType.SYMLINK os.symlink(entry.target, dest_path) diff --git a/src/buildstream/storage/directory.py b/src/buildstream/storage/directory.py index 29cbb53f2..89d20c433 100644 --- a/src/buildstream/storage/directory.py +++ b/src/buildstream/storage/directory.py @@ -46,11 +46,12 @@ class VirtualDirectoryError(BstError): or either of the :class:`.ElementError` or :class:`.SourceError` exceptions should be raised from this error. """ + def __init__(self, message, reason=None): super().__init__(message, domain=ErrorDomain.VIRTUAL_FS, reason=reason) -class Directory(): +class Directory: def __init__(self, external_directory=None): raise NotImplementedError() @@ -74,10 +75,15 @@ class Directory(): raise NotImplementedError() # Import and export of files and links - def import_files(self, external_pathspec: Union['Directory', str], *, - filter_callback: Optional[Callable[[str], bool]] = None, - report_written: bool = True, update_mtime: bool = False, - can_link: bool = False) -> FileListResult: + def import_files( + self, + external_pathspec: Union["Directory", str], + *, + filter_callback: Optional[Callable[[str], bool]] = None, + report_written: bool = True, + update_mtime: bool = False, + can_link: bool = False + ) -> FileListResult: """Imports some or all files from external_path into this directory. Args: @@ -214,4 +220,4 @@ class _FileType(FastEnum): def __str__(self): # https://github.com/PyCQA/pylint/issues/2062 - return self.name.lower().replace('_', ' ') # pylint: disable=no-member + return self.name.lower().replace("_", " ") # pylint: disable=no-member diff --git a/src/buildstream/testing/__init__.py b/src/buildstream/testing/__init__.py index 3926b4eab..67e96885a 100644 --- a/src/buildstream/testing/__init__.py +++ b/src/buildstream/testing/__init__.py @@ -31,9 +31,8 @@ from .integration import integration_cache try: import pytest except ImportError: - module_name = globals()['__name__'] - msg = "Could not import pytest:\n" \ - "To use the {} module, you must have pytest installed.".format(module_name) + module_name = globals()["__name__"] + msg = "Could not import pytest:\n" "To use the {} module, you must have pytest installed.".format(module_name) raise ImportError(msg) @@ -41,7 +40,7 @@ except ImportError: ALL_REPO_KINDS = OrderedDict() # type: OrderedDict[Repo, str] -def create_repo(kind, directory, subdir='repo'): +def create_repo(kind, directory, subdir="repo"): """Convenience method for creating a Repo Args: @@ -92,6 +91,7 @@ def sourcetests_collection_hook(session): Args: session (pytest.Session): The current pytest session """ + def should_collect_tests(config): args = config.args rootdir = config.rootdir @@ -112,6 +112,7 @@ def sourcetests_collection_hook(session): return True from . import _sourcetests + source_test_path = os.path.dirname(_sourcetests.__file__) # Add the location of the source tests to the session's # python_files config. Without this, pytest may filter out these diff --git a/src/buildstream/testing/_fixtures.py b/src/buildstream/testing/_fixtures.py index 2684782a1..5da51bb45 100644 --- a/src/buildstream/testing/_fixtures.py +++ b/src/buildstream/testing/_fixtures.py @@ -30,6 +30,7 @@ def thread_check(): yield assert utils._is_single_threaded() + # Reset global state in node.pyx to improve test isolation @pytest.fixture(autouse=True) def reset_global_node_state(): diff --git a/src/buildstream/testing/_sourcetests/build_checkout.py b/src/buildstream/testing/_sourcetests/build_checkout.py index 4d4bcf0e2..782d99814 100644 --- a/src/buildstream/testing/_sourcetests/build_checkout.py +++ b/src/buildstream/testing/_sourcetests/build_checkout.py @@ -29,23 +29,23 @@ from .utils import kind # pylint: disable=unused-import # Project directory TOP_DIR = os.path.dirname(os.path.realpath(__file__)) -DATA_DIR = os.path.join(TOP_DIR, 'project') +DATA_DIR = os.path.join(TOP_DIR, "project") def strict_args(args, strict): if strict != "strict": - return ['--no-strict', *args] + return ["--no-strict", *args] return args @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("strict", ["strict", "non-strict"]) def test_fetch_build_checkout(cli, tmpdir, datafiles, strict, kind): - checkout = os.path.join(cli.directory, 'checkout') + checkout = os.path.join(cli.directory, "checkout") project = str(datafiles) - dev_files_path = os.path.join(project, 'files', 'dev-files') - element_path = os.path.join(project, 'elements') - element_name = 'build-test-{}.bst'.format(kind) + dev_files_path = os.path.join(project, "files", "dev-files") + element_path = os.path.join(project, "elements") + element_name = "build-test-{}.bst".format(kind) # Create our repo object of the given source type with # the dev files, and then collect the initial ref. @@ -54,26 +54,20 @@ def test_fetch_build_checkout(cli, tmpdir, datafiles, strict, kind): ref = repo.create(dev_files_path) # Write out our test target - element = { - 'kind': 'import', - 'sources': [ - repo.source_config(ref=ref) - ] - } - _yaml.roundtrip_dump(element, - os.path.join(element_path, element_name)) + element = {"kind": "import", "sources": [repo.source_config(ref=ref)]} + _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) - assert cli.get_element_state(project, element_name) == 'fetch needed' - result = cli.run(project=project, args=strict_args(['build', element_name], strict)) + assert cli.get_element_state(project, element_name) == "fetch needed" + result = cli.run(project=project, args=strict_args(["build", element_name], strict)) result.assert_success() - assert cli.get_element_state(project, element_name) == 'cached' + assert cli.get_element_state(project, element_name) == "cached" # Now check it out - result = cli.run(project=project, args=strict_args([ - 'artifact', 'checkout', element_name, '--directory', checkout - ], strict)) + result = cli.run( + project=project, args=strict_args(["artifact", "checkout", element_name, "--directory", checkout], strict) + ) result.assert_success() # Check that the pony.h include from files/dev-files exists - filename = os.path.join(checkout, 'usr', 'include', 'pony.h') + filename = os.path.join(checkout, "usr", "include", "pony.h") assert os.path.exists(filename) diff --git a/src/buildstream/testing/_sourcetests/fetch.py b/src/buildstream/testing/_sourcetests/fetch.py index 897752297..05b43d793 100644 --- a/src/buildstream/testing/_sourcetests/fetch.py +++ b/src/buildstream/testing/_sourcetests/fetch.py @@ -32,15 +32,15 @@ from .utils import kind # pylint: disable=unused-import # Project directory TOP_DIR = os.path.dirname(os.path.realpath(__file__)) -DATA_DIR = os.path.join(TOP_DIR, 'project') +DATA_DIR = os.path.join(TOP_DIR, "project") @pytest.mark.datafiles(DATA_DIR) def test_fetch(cli, tmpdir, datafiles, kind): project = str(datafiles) - bin_files_path = os.path.join(project, 'files', 'bin-files') - element_path = os.path.join(project, 'elements') - element_name = 'fetch-test-{}.bst'.format(kind) + bin_files_path = os.path.join(project, "files", "bin-files") + element_path = os.path.join(project, "elements") + element_name = "fetch-test-{}.bst".format(kind) # Create our repo object of the given source type with # the bin files, and then collect the initial ref. @@ -49,59 +49,46 @@ def test_fetch(cli, tmpdir, datafiles, kind): ref = repo.create(bin_files_path) # Write out our test target - element = { - 'kind': 'import', - 'sources': [ - repo.source_config(ref=ref) - ] - } - _yaml.roundtrip_dump(element, - os.path.join(element_path, element_name)) + element = {"kind": "import", "sources": [repo.source_config(ref=ref)]} + _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) # Assert that a fetch is needed - assert cli.get_element_state(project, element_name) == 'fetch needed' + assert cli.get_element_state(project, element_name) == "fetch needed" # Now try to fetch it - result = cli.run(project=project, args=['source', 'fetch', element_name]) + result = cli.run(project=project, args=["source", "fetch", element_name]) result.assert_success() # Assert that we are now buildable because the source is # now cached. - assert cli.get_element_state(project, element_name) == 'buildable' + assert cli.get_element_state(project, element_name) == "buildable" @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("ref_storage", ['inline', 'project.refs']) +@pytest.mark.parametrize("ref_storage", ["inline", "project.refs"]) def test_fetch_cross_junction(cli, tmpdir, datafiles, ref_storage, kind): project = str(datafiles) - subproject_path = os.path.join(project, 'files', 'sub-project') - junction_path = os.path.join(project, 'elements', 'junction.bst') + subproject_path = os.path.join(project, "files", "sub-project") + junction_path = os.path.join(project, "elements", "junction.bst") - import_etc_path = os.path.join(subproject_path, 'elements', 'import-etc-repo.bst') - etc_files_path = os.path.join(subproject_path, 'files', 'etc-files') + import_etc_path = os.path.join(subproject_path, "elements", "import-etc-repo.bst") + etc_files_path = os.path.join(subproject_path, "files", "etc-files") - repo = create_repo(kind, str(tmpdir.join('import-etc'))) + repo = create_repo(kind, str(tmpdir.join("import-etc"))) ref = repo.create(etc_files_path) - element = { - 'kind': 'import', - 'sources': [ - repo.source_config(ref=(ref if ref_storage == 'inline' else None)) - ] - } + element = {"kind": "import", "sources": [repo.source_config(ref=(ref if ref_storage == "inline" else None))]} _yaml.roundtrip_dump(element, import_etc_path) - update_project_configuration(project, { - 'ref-storage': ref_storage - }) + update_project_configuration(project, {"ref-storage": ref_storage}) - generate_junction(tmpdir, subproject_path, junction_path, store_ref=(ref_storage == 'inline')) + generate_junction(tmpdir, subproject_path, junction_path, store_ref=(ref_storage == "inline")) - if ref_storage == 'project.refs': - result = cli.run(project=project, args=['source', 'track', 'junction.bst']) + if ref_storage == "project.refs": + result = cli.run(project=project, args=["source", "track", "junction.bst"]) result.assert_success() - result = cli.run(project=project, args=['source', 'track', 'junction.bst:import-etc.bst']) + result = cli.run(project=project, args=["source", "track", "junction.bst:import-etc.bst"]) result.assert_success() - result = cli.run(project=project, args=['source', 'fetch', 'junction.bst:import-etc.bst']) + result = cli.run(project=project, args=["source", "fetch", "junction.bst:import-etc.bst"]) result.assert_success() diff --git a/src/buildstream/testing/_sourcetests/mirror.py b/src/buildstream/testing/_sourcetests/mirror.py index b6316045d..3ff3fb981 100644 --- a/src/buildstream/testing/_sourcetests/mirror.py +++ b/src/buildstream/testing/_sourcetests/mirror.py @@ -31,25 +31,25 @@ from .utils import kind # pylint: disable=unused-import # Project directory TOP_DIR = os.path.dirname(os.path.realpath(__file__)) -DATA_DIR = os.path.join(TOP_DIR, 'project') +DATA_DIR = os.path.join(TOP_DIR, "project") def _set_project_mirrors_and_aliases(project_path, mirrors, aliases): - project_conf_path = os.path.join(project_path, 'project.conf') + project_conf_path = os.path.join(project_path, "project.conf") project_conf = _yaml.roundtrip_load(project_conf_path) - project_conf['mirrors'] = mirrors - project_conf['aliases'].update(aliases) + project_conf["mirrors"] = mirrors + project_conf["aliases"].update(aliases) _yaml.roundtrip_dump(project_conf, project_conf_path) def _set_project_includes_and_aliases(project_path, includes, aliases): - project_conf_path = os.path.join(project_path, 'project.conf') + project_conf_path = os.path.join(project_path, "project.conf") project_conf = _yaml.roundtrip_load(project_conf_path) - project_conf['aliases'].update(aliases) - project_conf['(@)'] = includes + project_conf["aliases"].update(aliases) + project_conf["(@)"] = includes _yaml.roundtrip_dump(project_conf, project_conf_path) @@ -57,11 +57,11 @@ def _set_project_includes_and_aliases(project_path, includes, aliases): @pytest.mark.datafiles(DATA_DIR) def test_mirror_fetch(cli, tmpdir, datafiles, kind): project_dir = str(datafiles) - bin_files_path = os.path.join(project_dir, 'files', 'bin-files', 'usr') - dev_files_path = os.path.join(project_dir, 'files', 'dev-files', 'usr') - upstream_repodir = os.path.join(str(tmpdir), 'upstream') - mirror_repodir = os.path.join(str(tmpdir), 'mirror') - element_dir = os.path.join(project_dir, 'elements') + bin_files_path = os.path.join(project_dir, "files", "bin-files", "usr") + dev_files_path = os.path.join(project_dir, "files", "dev-files", "usr") + upstream_repodir = os.path.join(str(tmpdir), "upstream") + mirror_repodir = os.path.join(str(tmpdir), "mirror") + element_dir = os.path.join(project_dir, "elements") # Create repo objects of the upstream and mirror upstream_repo = create_repo(kind, upstream_repodir) @@ -69,228 +69,162 @@ def test_mirror_fetch(cli, tmpdir, datafiles, kind): mirror_repo = upstream_repo.copy(mirror_repodir) upstream_ref = upstream_repo.create(dev_files_path) - element = { - 'kind': 'import', - 'sources': [ - upstream_repo.source_config(ref=upstream_ref) - ] - } - element_name = 'test.bst' + element = {"kind": "import", "sources": [upstream_repo.source_config(ref=upstream_ref)]} + element_name = "test.bst" element_path = os.path.join(element_dir, element_name) - full_repo = element['sources'][0]['url'] + full_repo = element["sources"][0]["url"] upstream_map, repo_name = os.path.split(full_repo) - alias = 'foo-' + kind - aliased_repo = alias + ':' + repo_name - element['sources'][0]['url'] = aliased_repo - full_mirror = mirror_repo.source_config()['url'] + alias = "foo-" + kind + aliased_repo = alias + ":" + repo_name + element["sources"][0]["url"] = aliased_repo + full_mirror = mirror_repo.source_config()["url"] mirror_map, _ = os.path.split(full_mirror) _yaml.roundtrip_dump(element, element_path) _set_project_mirrors_and_aliases( project_dir, - [ - { - 'name': 'middle-earth', - 'aliases': { - alias: [mirror_map + '/'], - }, - }, - ], - {alias: upstream_map + '/'}, + [{"name": "middle-earth", "aliases": {alias: [mirror_map + "/"],},},], + {alias: upstream_map + "/"}, ) # No obvious ways of checking that the mirror has been fetched # But at least we can be sure it succeeds - result = cli.run(project=project_dir, args=['source', 'fetch', element_name]) + result = cli.run(project=project_dir, args=["source", "fetch", element_name]) result.assert_success() @pytest.mark.datafiles(DATA_DIR) def test_mirror_fetch_upstream_absent(cli, tmpdir, datafiles, kind): project_dir = str(datafiles) - dev_files_path = os.path.join(project_dir, 'files', 'dev-files', 'usr') - upstream_repodir = os.path.join(project_dir, 'upstream') - mirror_repodir = os.path.join(str(tmpdir), 'mirror') - element_dir = os.path.join(project_dir, 'elements') + dev_files_path = os.path.join(project_dir, "files", "dev-files", "usr") + upstream_repodir = os.path.join(project_dir, "upstream") + mirror_repodir = os.path.join(str(tmpdir), "mirror") + element_dir = os.path.join(project_dir, "elements") # Create repo objects of the upstream and mirror upstream_repo = create_repo(kind, upstream_repodir) ref = upstream_repo.create(dev_files_path) mirror_repo = upstream_repo.copy(mirror_repodir) - element = { - 'kind': 'import', - 'sources': [ - upstream_repo.source_config(ref=ref) - ] - } + element = {"kind": "import", "sources": [upstream_repo.source_config(ref=ref)]} - element_name = 'test.bst' + element_name = "test.bst" element_path = os.path.join(element_dir, element_name) - full_repo = element['sources'][0]['url'] + full_repo = element["sources"][0]["url"] _, repo_name = os.path.split(full_repo) - alias = 'foo-' + kind - aliased_repo = alias + ':' + repo_name - element['sources'][0]['url'] = aliased_repo - full_mirror = mirror_repo.source_config()['url'] + alias = "foo-" + kind + aliased_repo = alias + ":" + repo_name + element["sources"][0]["url"] = aliased_repo + full_mirror = mirror_repo.source_config()["url"] mirror_map, _ = os.path.split(full_mirror) _yaml.roundtrip_dump(element, element_path) _set_project_mirrors_and_aliases( project_dir, - [ - { - 'name': 'middle-earth', - 'aliases': { - alias: [mirror_map + "/"] - }, - }, - ], - {alias: 'http://www.example.com'}, + [{"name": "middle-earth", "aliases": {alias: [mirror_map + "/"]},},], + {alias: "http://www.example.com"}, ) - result = cli.run(project=project_dir, args=['source', 'fetch', element_name]) + result = cli.run(project=project_dir, args=["source", "fetch", element_name]) result.assert_success() @pytest.mark.datafiles(DATA_DIR) def test_mirror_from_includes(cli, tmpdir, datafiles, kind): project_dir = str(datafiles) - bin_files_path = os.path.join(project_dir, 'files', 'bin-files', 'usr') - upstream_repodir = os.path.join(str(tmpdir), 'upstream') - mirror_repodir = os.path.join(str(tmpdir), 'mirror') - element_dir = os.path.join(project_dir, 'elements') + bin_files_path = os.path.join(project_dir, "files", "bin-files", "usr") + upstream_repodir = os.path.join(str(tmpdir), "upstream") + mirror_repodir = os.path.join(str(tmpdir), "mirror") + element_dir = os.path.join(project_dir, "elements") # Create repo objects of the upstream and mirror upstream_repo = create_repo(kind, upstream_repodir) upstream_ref = upstream_repo.create(bin_files_path) mirror_repo = upstream_repo.copy(mirror_repodir) - element = { - 'kind': 'import', - 'sources': [ - upstream_repo.source_config(ref=upstream_ref) - ] - } - element_name = 'test.bst' + element = {"kind": "import", "sources": [upstream_repo.source_config(ref=upstream_ref)]} + element_name = "test.bst" element_path = os.path.join(element_dir, element_name) - full_repo = element['sources'][0]['url'] + full_repo = element["sources"][0]["url"] upstream_map, repo_name = os.path.split(full_repo) - alias = 'foo-' + kind - aliased_repo = alias + ':' + repo_name - element['sources'][0]['url'] = aliased_repo - full_mirror = mirror_repo.source_config()['url'] + alias = "foo-" + kind + aliased_repo = alias + ":" + repo_name + element["sources"][0]["url"] = aliased_repo + full_mirror = mirror_repo.source_config()["url"] mirror_map, _ = os.path.split(full_mirror) _yaml.roundtrip_dump(element, element_path) - config_project_dir = str(tmpdir.join('config')) + config_project_dir = str(tmpdir.join("config")) os.makedirs(config_project_dir, exist_ok=True) - config_project = { - 'name': 'config' - } - _yaml.roundtrip_dump(config_project, os.path.join(config_project_dir, 'project.conf')) - extra_mirrors = { - 'mirrors': [ - { - 'name': 'middle-earth', - 'aliases': { - alias: [mirror_map + "/"], - } - } - ] - } - _yaml.roundtrip_dump(extra_mirrors, os.path.join(config_project_dir, 'mirrors.yml')) - generate_junction(str(tmpdir.join('config_repo')), - config_project_dir, - os.path.join(element_dir, 'config.bst')) + config_project = {"name": "config"} + _yaml.roundtrip_dump(config_project, os.path.join(config_project_dir, "project.conf")) + extra_mirrors = {"mirrors": [{"name": "middle-earth", "aliases": {alias: [mirror_map + "/"],}}]} + _yaml.roundtrip_dump(extra_mirrors, os.path.join(config_project_dir, "mirrors.yml")) + generate_junction(str(tmpdir.join("config_repo")), config_project_dir, os.path.join(element_dir, "config.bst")) _set_project_includes_and_aliases( - project_dir, - ['config.bst:mirrors.yml'], - {alias: upstream_map + '/'}, + project_dir, ["config.bst:mirrors.yml"], {alias: upstream_map + "/"}, ) # Now make the upstream unavailable. - os.rename(upstream_repo.repo, '{}.bak'.format(upstream_repo.repo)) - result = cli.run(project=project_dir, args=['source', 'fetch', element_name]) + os.rename(upstream_repo.repo, "{}.bak".format(upstream_repo.repo)) + result = cli.run(project=project_dir, args=["source", "fetch", element_name]) result.assert_success() @pytest.mark.datafiles(DATA_DIR) def test_mirror_junction_from_includes(cli, tmpdir, datafiles, kind): project_dir = str(datafiles) - bin_files_path = os.path.join(project_dir, 'files', 'bin-files', 'usr') - upstream_repodir = os.path.join(str(tmpdir), 'upstream') - mirror_repodir = os.path.join(str(tmpdir), 'mirror') - element_dir = os.path.join(project_dir, 'elements') + bin_files_path = os.path.join(project_dir, "files", "bin-files", "usr") + upstream_repodir = os.path.join(str(tmpdir), "upstream") + mirror_repodir = os.path.join(str(tmpdir), "mirror") + element_dir = os.path.join(project_dir, "elements") # Create repo objects of the upstream and mirror upstream_repo = create_repo(kind, upstream_repodir) upstream_ref = upstream_repo.create(bin_files_path) mirror_repo = upstream_repo.copy(mirror_repodir) - element = { - 'kind': 'junction', - 'sources': [ - upstream_repo.source_config(ref=upstream_ref) - ] - } - element_name = 'test.bst' + element = {"kind": "junction", "sources": [upstream_repo.source_config(ref=upstream_ref)]} + element_name = "test.bst" element_path = os.path.join(element_dir, element_name) - full_repo = element['sources'][0]['url'] + full_repo = element["sources"][0]["url"] upstream_map, repo_name = os.path.split(full_repo) - alias = 'foo-' + kind - aliased_repo = alias + ':' + repo_name - element['sources'][0]['url'] = aliased_repo - full_mirror = mirror_repo.source_config()['url'] + alias = "foo-" + kind + aliased_repo = alias + ":" + repo_name + element["sources"][0]["url"] = aliased_repo + full_mirror = mirror_repo.source_config()["url"] mirror_map, _ = os.path.split(full_mirror) _yaml.roundtrip_dump(element, element_path) - config_project_dir = str(tmpdir.join('config')) + config_project_dir = str(tmpdir.join("config")) os.makedirs(config_project_dir, exist_ok=True) - config_project = { - 'name': 'config' - } - _yaml.roundtrip_dump(config_project, os.path.join(config_project_dir, 'project.conf')) - extra_mirrors = { - 'mirrors': [ - { - 'name': 'middle-earth', - 'aliases': { - alias: [mirror_map + "/"], - } - } - ] - } - _yaml.roundtrip_dump(extra_mirrors, os.path.join(config_project_dir, 'mirrors.yml')) - generate_junction(str(tmpdir.join('config_repo')), - config_project_dir, - os.path.join(element_dir, 'config.bst')) + config_project = {"name": "config"} + _yaml.roundtrip_dump(config_project, os.path.join(config_project_dir, "project.conf")) + extra_mirrors = {"mirrors": [{"name": "middle-earth", "aliases": {alias: [mirror_map + "/"],}}]} + _yaml.roundtrip_dump(extra_mirrors, os.path.join(config_project_dir, "mirrors.yml")) + generate_junction(str(tmpdir.join("config_repo")), config_project_dir, os.path.join(element_dir, "config.bst")) - _set_project_includes_and_aliases( - project_dir, - ['config.bst:mirrors.yml'], - {alias: upstream_map + '/'} - ) + _set_project_includes_and_aliases(project_dir, ["config.bst:mirrors.yml"], {alias: upstream_map + "/"}) # Now make the upstream unavailable. - os.rename(upstream_repo.repo, '{}.bak'.format(upstream_repo.repo)) - result = cli.run(project=project_dir, args=['source', 'fetch', element_name]) + os.rename(upstream_repo.repo, "{}.bak".format(upstream_repo.repo)) + result = cli.run(project=project_dir, args=["source", "fetch", element_name]) result.assert_main_error(ErrorDomain.STREAM, None) # Now make the upstream available again. - os.rename('{}.bak'.format(upstream_repo.repo), upstream_repo.repo) - result = cli.run(project=project_dir, args=['source', 'fetch', element_name]) + os.rename("{}.bak".format(upstream_repo.repo), upstream_repo.repo) + result = cli.run(project=project_dir, args=["source", "fetch", element_name]) result.assert_success() @pytest.mark.datafiles(DATA_DIR) def test_mirror_track_upstream_present(cli, tmpdir, datafiles, kind): project_dir = str(datafiles) - bin_files_path = os.path.join(project_dir, 'files', 'bin-files', 'usr') - dev_files_path = os.path.join(project_dir, 'files', 'dev-files', 'usr') - upstream_repodir = os.path.join(str(tmpdir), 'upstream') - mirror_repodir = os.path.join(str(tmpdir), 'mirror') - element_dir = os.path.join(project_dir, 'elements') + bin_files_path = os.path.join(project_dir, "files", "bin-files", "usr") + dev_files_path = os.path.join(project_dir, "files", "dev-files", "usr") + upstream_repodir = os.path.join(str(tmpdir), "upstream") + mirror_repodir = os.path.join(str(tmpdir), "mirror") + element_dir = os.path.join(project_dir, "elements") # Create repo objects of the upstream and mirror upstream_repo = create_repo(kind, upstream_repodir) @@ -298,55 +232,43 @@ def test_mirror_track_upstream_present(cli, tmpdir, datafiles, kind): mirror_repo = upstream_repo.copy(mirror_repodir) upstream_ref = upstream_repo.create(dev_files_path) - element = { - 'kind': 'import', - 'sources': [ - upstream_repo.source_config(ref=upstream_ref) - ] - } + element = {"kind": "import", "sources": [upstream_repo.source_config(ref=upstream_ref)]} - element_name = 'test.bst' + element_name = "test.bst" element_path = os.path.join(element_dir, element_name) - full_repo = element['sources'][0]['url'] + full_repo = element["sources"][0]["url"] upstream_map, repo_name = os.path.split(full_repo) - alias = 'foo-' + kind - aliased_repo = alias + ':' + repo_name - element['sources'][0]['url'] = aliased_repo - full_mirror = mirror_repo.source_config()['url'] + alias = "foo-" + kind + aliased_repo = alias + ":" + repo_name + element["sources"][0]["url"] = aliased_repo + full_mirror = mirror_repo.source_config()["url"] mirror_map, _ = os.path.split(full_mirror) _yaml.roundtrip_dump(element, element_path) _set_project_mirrors_and_aliases( project_dir, - [ - { - 'name': 'middle-earth', - 'aliases': { - alias: [mirror_map + '/'], - }, - }, - ], - {alias: upstream_map + '/'}, + [{"name": "middle-earth", "aliases": {alias: [mirror_map + "/"],},},], + {alias: upstream_map + "/"}, ) - result = cli.run(project=project_dir, args=['source', 'track', element_name]) + result = cli.run(project=project_dir, args=["source", "track", element_name]) result.assert_success() # Tracking tries upstream first. Check the ref is from upstream. new_element = _yaml.load(element_path) - source = new_element.get_sequence('sources').mapping_at(0) - if 'ref' in source: - assert source.get_str('ref') == upstream_ref + source = new_element.get_sequence("sources").mapping_at(0) + if "ref" in source: + assert source.get_str("ref") == upstream_ref @pytest.mark.datafiles(DATA_DIR) def test_mirror_track_upstream_absent(cli, tmpdir, datafiles, kind): project_dir = str(datafiles) - bin_files_path = os.path.join(project_dir, 'files', 'bin-files', 'usr') - dev_files_path = os.path.join(project_dir, 'files', 'dev-files', 'usr') - upstream_repodir = os.path.join(str(tmpdir), 'upstream') - mirror_repodir = os.path.join(str(tmpdir), 'mirror') - element_dir = os.path.join(project_dir, 'elements') + bin_files_path = os.path.join(project_dir, "files", "bin-files", "usr") + dev_files_path = os.path.join(project_dir, "files", "dev-files", "usr") + upstream_repodir = os.path.join(str(tmpdir), "upstream") + mirror_repodir = os.path.join(str(tmpdir), "mirror") + element_dir = os.path.join(project_dir, "elements") # Create repo objects of the upstream and mirror upstream_repo = create_repo(kind, upstream_repodir) @@ -355,42 +277,30 @@ def test_mirror_track_upstream_absent(cli, tmpdir, datafiles, kind): mirror_ref = upstream_ref upstream_ref = upstream_repo.create(dev_files_path) - element = { - 'kind': 'import', - 'sources': [ - upstream_repo.source_config(ref=upstream_ref) - ] - } + element = {"kind": "import", "sources": [upstream_repo.source_config(ref=upstream_ref)]} - element_name = 'test.bst' + element_name = "test.bst" element_path = os.path.join(element_dir, element_name) - full_repo = element['sources'][0]['url'] + full_repo = element["sources"][0]["url"] _, repo_name = os.path.split(full_repo) - alias = 'foo-' + kind - aliased_repo = alias + ':' + repo_name - element['sources'][0]['url'] = aliased_repo - full_mirror = mirror_repo.source_config()['url'] + alias = "foo-" + kind + aliased_repo = alias + ":" + repo_name + element["sources"][0]["url"] = aliased_repo + full_mirror = mirror_repo.source_config()["url"] mirror_map, _ = os.path.split(full_mirror) _yaml.roundtrip_dump(element, element_path) _set_project_mirrors_and_aliases( project_dir, - [ - { - 'name': 'middle-earth', - 'aliases': { - alias: [mirror_map + '/'], - }, - }, - ], - {alias: 'http://www.example.com'}, + [{"name": "middle-earth", "aliases": {alias: [mirror_map + "/"],},},], + {alias: "http://www.example.com"}, ) - result = cli.run(project=project_dir, args=['source', 'track', element_name]) + result = cli.run(project=project_dir, args=["source", "track", element_name]) result.assert_success() # Check that tracking fell back to the mirror new_element = _yaml.load(element_path) - source = new_element.get_sequence('sources').mapping_at(0) - if 'ref' in source: - assert source.get_str('ref') == mirror_ref + source = new_element.get_sequence("sources").mapping_at(0) + if "ref" in source: + assert source.get_str("ref") == mirror_ref diff --git a/src/buildstream/testing/_sourcetests/source_determinism.py b/src/buildstream/testing/_sourcetests/source_determinism.py index fc0e4618c..27664e0c2 100644 --- a/src/buildstream/testing/_sourcetests/source_determinism.py +++ b/src/buildstream/testing/_sourcetests/source_determinism.py @@ -30,83 +30,72 @@ from .utils import kind # pylint: disable=unused-import # Project directory TOP_DIR = os.path.dirname(os.path.realpath(__file__)) -DATA_DIR = os.path.join(TOP_DIR, 'project') +DATA_DIR = os.path.join(TOP_DIR, "project") -def create_test_file(*path, mode=0o644, content='content\n'): +def create_test_file(*path, mode=0o644, content="content\n"): path = os.path.join(*path) os.makedirs(os.path.dirname(path), exist_ok=True) - with open(path, 'w') as f: + with open(path, "w") as f: f.write(content) os.fchmod(f.fileno(), mode) def create_test_directory(*path, mode=0o644): - create_test_file(*path, '.keep', content='') + create_test_file(*path, ".keep", content="") path = os.path.join(*path) os.chmod(path, mode) @pytest.mark.integration @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') -@pytest.mark.skipif(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox, Must Fix') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") +@pytest.mark.skipif(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox, Must Fix") def test_deterministic_source_umask(cli, tmpdir, datafiles, kind): project = str(datafiles) - element_name = 'list.bst' - element_path = os.path.join(project, 'elements', element_name) - repodir = os.path.join(str(tmpdir), 'repo') - sourcedir = os.path.join(project, 'source') - - create_test_file(sourcedir, 'a.txt', mode=0o700) - create_test_file(sourcedir, 'b.txt', mode=0o755) - create_test_file(sourcedir, 'c.txt', mode=0o600) - create_test_file(sourcedir, 'd.txt', mode=0o400) - create_test_file(sourcedir, 'e.txt', mode=0o644) - create_test_file(sourcedir, 'f.txt', mode=0o4755) - create_test_file(sourcedir, 'g.txt', mode=0o2755) - create_test_file(sourcedir, 'h.txt', mode=0o1755) - create_test_directory(sourcedir, 'dir-a', mode=0o0700) - create_test_directory(sourcedir, 'dir-c', mode=0o0755) - create_test_directory(sourcedir, 'dir-d', mode=0o4755) - create_test_directory(sourcedir, 'dir-e', mode=0o2755) - create_test_directory(sourcedir, 'dir-f', mode=0o1755) + element_name = "list.bst" + element_path = os.path.join(project, "elements", element_name) + repodir = os.path.join(str(tmpdir), "repo") + sourcedir = os.path.join(project, "source") + + create_test_file(sourcedir, "a.txt", mode=0o700) + create_test_file(sourcedir, "b.txt", mode=0o755) + create_test_file(sourcedir, "c.txt", mode=0o600) + create_test_file(sourcedir, "d.txt", mode=0o400) + create_test_file(sourcedir, "e.txt", mode=0o644) + create_test_file(sourcedir, "f.txt", mode=0o4755) + create_test_file(sourcedir, "g.txt", mode=0o2755) + create_test_file(sourcedir, "h.txt", mode=0o1755) + create_test_directory(sourcedir, "dir-a", mode=0o0700) + create_test_directory(sourcedir, "dir-c", mode=0o0755) + create_test_directory(sourcedir, "dir-d", mode=0o4755) + create_test_directory(sourcedir, "dir-e", mode=0o2755) + create_test_directory(sourcedir, "dir-f", mode=0o1755) repo = create_repo(kind, repodir) ref = repo.create(sourcedir) source = repo.source_config(ref=ref) element = { - 'kind': 'manual', - 'depends': [ - { - 'filename': 'base.bst', - 'type': 'build' - } - ], - 'sources': [ - source - ], - 'config': { - 'install-commands': [ - 'ls -l >"%{install-root}/ls-l"' - ] - } + "kind": "manual", + "depends": [{"filename": "base.bst", "type": "build"}], + "sources": [source], + "config": {"install-commands": ['ls -l >"%{install-root}/ls-l"']}, } _yaml.roundtrip_dump(element, element_path) def get_value_for_umask(umask): - checkoutdir = os.path.join(str(tmpdir), 'checkout-{}'.format(umask)) + checkoutdir = os.path.join(str(tmpdir), "checkout-{}".format(umask)) old_umask = os.umask(umask) try: - result = cli.run(project=project, args=['build', element_name]) + result = cli.run(project=project, args=["build", element_name]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkoutdir]) result.assert_success() - with open(os.path.join(checkoutdir, 'ls-l'), 'r') as f: + with open(os.path.join(checkoutdir, "ls-l"), "r") as f: return f.read() finally: os.umask(old_umask) diff --git a/src/buildstream/testing/_sourcetests/track.py b/src/buildstream/testing/_sourcetests/track.py index 48856b351..623045cd9 100644 --- a/src/buildstream/testing/_sourcetests/track.py +++ b/src/buildstream/testing/_sourcetests/track.py @@ -33,33 +33,26 @@ from .utils import kind # pylint: disable=unused-import # Project directory TOP_DIR = os.path.dirname(os.path.realpath(__file__)) -DATA_DIR = os.path.join(TOP_DIR, 'project') +DATA_DIR = os.path.join(TOP_DIR, "project") def generate_element(repo, element_path, dep_name=None): - element = { - 'kind': 'import', - 'sources': [ - repo.source_config() - ] - } + element = {"kind": "import", "sources": [repo.source_config()]} if dep_name: - element['depends'] = [dep_name] + element["depends"] = [dep_name] _yaml.roundtrip_dump(element, element_path) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("ref_storage", ['inline', 'project.refs']) +@pytest.mark.parametrize("ref_storage", ["inline", "project.refs"]) def test_track(cli, tmpdir, datafiles, ref_storage, kind): project = str(datafiles) - dev_files_path = os.path.join(project, 'files', 'dev-files') - element_path = os.path.join(project, 'elements') - element_name = 'track-test-{}.bst'.format(kind) + dev_files_path = os.path.join(project, "files", "dev-files") + element_path = os.path.join(project, "elements") + element_name = "track-test-{}.bst".format(kind) - update_project_configuration(project, { - 'ref-storage': ref_storage - }) + update_project_configuration(project, {"ref-storage": ref_storage}) # Create our repo object of the given source type with # the dev files, and then collect the initial ref. @@ -71,28 +64,28 @@ def test_track(cli, tmpdir, datafiles, ref_storage, kind): generate_element(repo, os.path.join(element_path, element_name)) # Assert that a fetch is needed - assert cli.get_element_state(project, element_name) == 'no reference' + assert cli.get_element_state(project, element_name) == "no reference" # Now first try to track it - result = cli.run(project=project, args=['source', 'track', element_name]) + result = cli.run(project=project, args=["source", "track", element_name]) result.assert_success() # And now fetch it: The Source has probably already cached the # latest ref locally, but it is not required to have cached # the associated content of the latest ref at track time, that # is the job of fetch. - result = cli.run(project=project, args=['source', 'fetch', element_name]) + result = cli.run(project=project, args=["source", "fetch", element_name]) result.assert_success() # Assert that we are now buildable because the source is # now cached. - assert cli.get_element_state(project, element_name) == 'buildable' + assert cli.get_element_state(project, element_name) == "buildable" # Assert there was a project.refs created, depending on the configuration - if ref_storage == 'project.refs': - assert os.path.exists(os.path.join(project, 'project.refs')) + if ref_storage == "project.refs": + assert os.path.exists(os.path.join(project, "project.refs")) else: - assert not os.path.exists(os.path.join(project, 'project.refs')) + assert not os.path.exists(os.path.join(project, "project.refs")) # NOTE: @@ -112,18 +105,14 @@ def test_track(cli, tmpdir, datafiles, ref_storage, kind): @pytest.mark.parametrize("amount", [1, 10]) def test_track_recurse(cli, tmpdir, datafiles, kind, amount): project = str(datafiles) - dev_files_path = os.path.join(project, 'files', 'dev-files') - element_path = os.path.join(project, 'elements') + dev_files_path = os.path.join(project, "files", "dev-files") + element_path = os.path.join(project, "elements") # Try to actually launch as many fetch jobs as possible at the same time # # This stresses the Source plugins and helps to ensure that # they handle concurrent access to the store correctly. - cli.configure({ - 'scheduler': { - 'fetchers': amount, - } - }) + cli.configure({"scheduler": {"fetchers": amount,}}) # Create our repo object of the given source type with # the dev files, and then collect the initial ref. @@ -135,7 +124,7 @@ def test_track_recurse(cli, tmpdir, datafiles, kind, amount): element_names = [] last_element_name = None for i in range(amount + 1): - element_name = 'track-test-{}-{}.bst'.format(kind, i + 1) + element_name = "track-test-{}-{}.bst".format(kind, i + 1) filename = os.path.join(element_path, element_name) element_names.append(element_name) @@ -146,39 +135,35 @@ def test_track_recurse(cli, tmpdir, datafiles, kind, amount): # Assert that a fetch is needed states = cli.get_element_states(project, [last_element_name]) for element_name in element_names: - assert states[element_name] == 'no reference' + assert states[element_name] == "no reference" # Now first try to track it - result = cli.run(project=project, args=[ - 'source', 'track', '--deps', 'all', - last_element_name]) + result = cli.run(project=project, args=["source", "track", "--deps", "all", last_element_name]) result.assert_success() # And now fetch it: The Source has probably already cached the # latest ref locally, but it is not required to have cached # the associated content of the latest ref at track time, that # is the job of fetch. - result = cli.run(project=project, args=[ - 'source', 'fetch', '--deps', 'all', - last_element_name]) + result = cli.run(project=project, args=["source", "fetch", "--deps", "all", last_element_name]) result.assert_success() # Assert that the base is buildable and the rest are waiting states = cli.get_element_states(project, [last_element_name]) for element_name in element_names: if element_name == element_names[0]: - assert states[element_name] == 'buildable' + assert states[element_name] == "buildable" else: - assert states[element_name] == 'waiting' + assert states[element_name] == "waiting" @pytest.mark.datafiles(DATA_DIR) def test_track_recurse_except(cli, tmpdir, datafiles, kind): project = str(datafiles) - dev_files_path = os.path.join(project, 'files', 'dev-files') - element_path = os.path.join(project, 'elements') - element_dep_name = 'track-test-dep-{}.bst'.format(kind) - element_target_name = 'track-test-target-{}.bst'.format(kind) + dev_files_path = os.path.join(project, "files", "dev-files") + element_path = os.path.join(project, "elements") + element_dep_name = "track-test-dep-{}.bst".format(kind) + element_target_name = "track-test-target-{}.bst".format(kind) # Create our repo object of the given source type with # the dev files, and then collect the initial ref. @@ -188,88 +173,79 @@ def test_track_recurse_except(cli, tmpdir, datafiles, kind): # Write out our test targets generate_element(repo, os.path.join(element_path, element_dep_name)) - generate_element(repo, os.path.join(element_path, element_target_name), - dep_name=element_dep_name) + generate_element(repo, os.path.join(element_path, element_target_name), dep_name=element_dep_name) # Assert that a fetch is needed states = cli.get_element_states(project, [element_target_name]) - assert states[element_dep_name] == 'no reference' - assert states[element_target_name] == 'no reference' + assert states[element_dep_name] == "no reference" + assert states[element_target_name] == "no reference" # Now first try to track it - result = cli.run(project=project, args=[ - 'source', 'track', '--deps', 'all', '--except', element_dep_name, - element_target_name]) + result = cli.run( + project=project, args=["source", "track", "--deps", "all", "--except", element_dep_name, element_target_name] + ) result.assert_success() # And now fetch it: The Source has probably already cached the # latest ref locally, but it is not required to have cached # the associated content of the latest ref at track time, that # is the job of fetch. - result = cli.run(project=project, args=[ - 'source', 'fetch', '--deps', 'none', - element_target_name]) + result = cli.run(project=project, args=["source", "fetch", "--deps", "none", element_target_name]) result.assert_success() # Assert that the dependency is buildable and the target is waiting states = cli.get_element_states(project, [element_target_name]) - assert states[element_dep_name] == 'no reference' - assert states[element_target_name] == 'waiting' + assert states[element_dep_name] == "no reference" + assert states[element_target_name] == "waiting" @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("ref_storage", ['inline', 'project.refs']) +@pytest.mark.parametrize("ref_storage", ["inline", "project.refs"]) def test_cross_junction(cli, tmpdir, datafiles, ref_storage, kind): project = str(datafiles) - subproject_path = os.path.join(project, 'files', 'sub-project') - junction_path = os.path.join(project, 'elements', 'junction.bst') - etc_files = os.path.join(subproject_path, 'files', 'etc-files') - repo_element_path = os.path.join(subproject_path, 'elements', - 'import-etc-repo.bst') + subproject_path = os.path.join(project, "files", "sub-project") + junction_path = os.path.join(project, "elements", "junction.bst") + etc_files = os.path.join(subproject_path, "files", "etc-files") + repo_element_path = os.path.join(subproject_path, "elements", "import-etc-repo.bst") - update_project_configuration(project, { - 'ref-storage': ref_storage - }) + update_project_configuration(project, {"ref-storage": ref_storage}) - repo = create_repo(kind, str(tmpdir.join('element_repo'))) + repo = create_repo(kind, str(tmpdir.join("element_repo"))) repo.create(etc_files) generate_element(repo, repo_element_path) - generate_junction(str(tmpdir.join('junction_repo')), - subproject_path, junction_path, store_ref=False) + generate_junction(str(tmpdir.join("junction_repo")), subproject_path, junction_path, store_ref=False) # Track the junction itself first. - result = cli.run(project=project, args=['source', 'track', 'junction.bst']) + result = cli.run(project=project, args=["source", "track", "junction.bst"]) result.assert_success() - assert cli.get_element_state(project, 'junction.bst:import-etc-repo.bst') == 'no reference' + assert cli.get_element_state(project, "junction.bst:import-etc-repo.bst") == "no reference" # Track the cross junction element. -J is not given, it is implied. - result = cli.run(project=project, args=['source', 'track', 'junction.bst:import-etc-repo.bst']) + result = cli.run(project=project, args=["source", "track", "junction.bst:import-etc-repo.bst"]) - if ref_storage == 'inline': + if ref_storage == "inline": # This is not allowed to track cross junction without project.refs. - result.assert_main_error(ErrorDomain.PIPELINE, 'untrackable-sources') + result.assert_main_error(ErrorDomain.PIPELINE, "untrackable-sources") else: result.assert_success() - assert cli.get_element_state(project, 'junction.bst:import-etc-repo.bst') == 'buildable' + assert cli.get_element_state(project, "junction.bst:import-etc-repo.bst") == "buildable" - assert os.path.exists(os.path.join(project, 'project.refs')) + assert os.path.exists(os.path.join(project, "project.refs")) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("ref_storage", ['inline', 'project.refs']) +@pytest.mark.parametrize("ref_storage", ["inline", "project.refs"]) def test_track_include(cli, tmpdir, datafiles, ref_storage, kind): project = str(datafiles) - dev_files_path = os.path.join(project, 'files', 'dev-files') - element_path = os.path.join(project, 'elements') - element_name = 'track-test-{}.bst'.format(kind) + dev_files_path = os.path.join(project, "files", "dev-files") + element_path = os.path.join(project, "elements") + element_name = "track-test-{}.bst".format(kind) - update_project_configuration(project, { - 'ref-storage': ref_storage - }) + update_project_configuration(project, {"ref-storage": ref_storage}) # Create our repo object of the given source type with # the dev files, and then collect the initial ref. @@ -278,139 +254,118 @@ def test_track_include(cli, tmpdir, datafiles, ref_storage, kind): ref = repo.create(dev_files_path) # Generate the element - element = { - 'kind': 'import', - '(@)': ['elements/sources.yml'] - } - sources = { - 'sources': [ - repo.source_config() - ] - } + element = {"kind": "import", "(@)": ["elements/sources.yml"]} + sources = {"sources": [repo.source_config()]} _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) - _yaml.roundtrip_dump(sources, os.path.join(element_path, 'sources.yml')) + _yaml.roundtrip_dump(sources, os.path.join(element_path, "sources.yml")) # Assert that a fetch is needed - assert cli.get_element_state(project, element_name) == 'no reference' + assert cli.get_element_state(project, element_name) == "no reference" # Now first try to track it - result = cli.run(project=project, args=['source', 'track', element_name]) + result = cli.run(project=project, args=["source", "track", element_name]) result.assert_success() # And now fetch it: The Source has probably already cached the # latest ref locally, but it is not required to have cached # the associated content of the latest ref at track time, that # is the job of fetch. - result = cli.run(project=project, args=['source', 'fetch', element_name]) + result = cli.run(project=project, args=["source", "fetch", element_name]) result.assert_success() # Assert that we are now buildable because the source is # now cached. - assert cli.get_element_state(project, element_name) == 'buildable' + assert cli.get_element_state(project, element_name) == "buildable" # Assert there was a project.refs created, depending on the configuration - if ref_storage == 'project.refs': - assert os.path.exists(os.path.join(project, 'project.refs')) + if ref_storage == "project.refs": + assert os.path.exists(os.path.join(project, "project.refs")) else: - assert not os.path.exists(os.path.join(project, 'project.refs')) + assert not os.path.exists(os.path.join(project, "project.refs")) - new_sources = _yaml.load(os.path.join(element_path, 'sources.yml')) + new_sources = _yaml.load(os.path.join(element_path, "sources.yml")) # Get all of the sources - assert 'sources' in new_sources - sources_list = new_sources.get_sequence('sources') + assert "sources" in new_sources + sources_list = new_sources.get_sequence("sources") assert len(sources_list) == 1 # Get the first source from the sources list new_source = sources_list.mapping_at(0) - assert 'ref' in new_source - assert ref == new_source.get_str('ref') + assert "ref" in new_source + assert ref == new_source.get_str("ref") @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("ref_storage", ['inline', 'project.refs']) +@pytest.mark.parametrize("ref_storage", ["inline", "project.refs"]) def test_track_include_junction(cli, tmpdir, datafiles, ref_storage, kind): project = str(datafiles) - dev_files_path = os.path.join(project, 'files', 'dev-files') - element_path = os.path.join(project, 'elements') - element_name = 'track-test-{}.bst'.format(kind) - subproject_path = os.path.join(project, 'files', 'sub-project') - sub_element_path = os.path.join(subproject_path, 'elements') - junction_path = os.path.join(element_path, 'junction.bst') + dev_files_path = os.path.join(project, "files", "dev-files") + element_path = os.path.join(project, "elements") + element_name = "track-test-{}.bst".format(kind) + subproject_path = os.path.join(project, "files", "sub-project") + sub_element_path = os.path.join(subproject_path, "elements") + junction_path = os.path.join(element_path, "junction.bst") - update_project_configuration(project, { - 'ref-storage': ref_storage - }) + update_project_configuration(project, {"ref-storage": ref_storage}) # Create our repo object of the given source type with # the dev files, and then collect the initial ref. # - repo = create_repo(kind, str(tmpdir.join('element_repo'))) + repo = create_repo(kind, str(tmpdir.join("element_repo"))) repo.create(dev_files_path) # Generate the element - element = { - 'kind': 'import', - '(@)': ['junction.bst:elements/sources.yml'] - } - sources = { - 'sources': [ - repo.source_config() - ] - } + element = {"kind": "import", "(@)": ["junction.bst:elements/sources.yml"]} + sources = {"sources": [repo.source_config()]} _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) - _yaml.roundtrip_dump(sources, os.path.join(sub_element_path, 'sources.yml')) + _yaml.roundtrip_dump(sources, os.path.join(sub_element_path, "sources.yml")) - generate_junction(str(tmpdir.join('junction_repo')), - subproject_path, junction_path, store_ref=True) + generate_junction(str(tmpdir.join("junction_repo")), subproject_path, junction_path, store_ref=True) - result = cli.run(project=project, args=['source', 'track', 'junction.bst']) + result = cli.run(project=project, args=["source", "track", "junction.bst"]) result.assert_success() # Assert that a fetch is needed - assert cli.get_element_state(project, element_name) == 'no reference' + assert cli.get_element_state(project, element_name) == "no reference" # Now first try to track it - result = cli.run(project=project, args=['source', 'track', element_name]) + result = cli.run(project=project, args=["source", "track", element_name]) # Assert there was a project.refs created, depending on the configuration - if ref_storage == 'inline': + if ref_storage == "inline": # FIXME: We should expect an error. But only a warning is emitted # result.assert_main_error(ErrorDomain.SOURCE, 'tracking-junction-fragment') - assert 'junction.bst:elements/sources.yml: Cannot track source in a fragment from a junction' in result.stderr + assert "junction.bst:elements/sources.yml: Cannot track source in a fragment from a junction" in result.stderr else: - assert os.path.exists(os.path.join(project, 'project.refs')) + assert os.path.exists(os.path.join(project, "project.refs")) # And now fetch it: The Source has probably already cached the # latest ref locally, but it is not required to have cached # the associated content of the latest ref at track time, that # is the job of fetch. - result = cli.run(project=project, args=['source', 'fetch', element_name]) + result = cli.run(project=project, args=["source", "fetch", element_name]) result.assert_success() # Assert that we are now buildable because the source is # now cached. - assert cli.get_element_state(project, element_name) == 'buildable' + assert cli.get_element_state(project, element_name) == "buildable" @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("ref_storage", ['inline', 'project.refs']) +@pytest.mark.parametrize("ref_storage", ["inline", "project.refs"]) def test_track_junction_included(cli, tmpdir, datafiles, ref_storage, kind): project = str(datafiles) - element_path = os.path.join(project, 'elements') - subproject_path = os.path.join(project, 'files', 'sub-project') - junction_path = os.path.join(element_path, 'junction.bst') + element_path = os.path.join(project, "elements") + subproject_path = os.path.join(project, "files", "sub-project") + junction_path = os.path.join(element_path, "junction.bst") - update_project_configuration(project, { - 'ref-storage': ref_storage, - '(@)': ['junction.bst:test.yml'] - }) + update_project_configuration(project, {"ref-storage": ref_storage, "(@)": ["junction.bst:test.yml"]}) - generate_junction(str(tmpdir.join('junction_repo')), - subproject_path, junction_path, store_ref=False) + generate_junction(str(tmpdir.join("junction_repo")), subproject_path, junction_path, store_ref=False) - result = cli.run(project=project, args=['source', 'track', 'junction.bst']) + result = cli.run(project=project, args=["source", "track", "junction.bst"]) result.assert_success() diff --git a/src/buildstream/testing/_sourcetests/track_cross_junction.py b/src/buildstream/testing/_sourcetests/track_cross_junction.py index 550f57faf..2477b37ee 100644 --- a/src/buildstream/testing/_sourcetests/track_cross_junction.py +++ b/src/buildstream/testing/_sourcetests/track_cross_junction.py @@ -32,32 +32,27 @@ from .utils import add_plugins_conf # Project directory TOP_DIR = os.path.dirname(os.path.realpath(__file__)) -DATA_DIR = os.path.join(TOP_DIR, 'project') +DATA_DIR = os.path.join(TOP_DIR, "project") def generate_element(repo, element_path, dep_name=None): - element = { - 'kind': 'import', - 'sources': [ - repo.source_config() - ] - } + element = {"kind": "import", "sources": [repo.source_config()]} if dep_name: - element['depends'] = [dep_name] + element["depends"] = [dep_name] _yaml.roundtrip_dump(element, element_path) def generate_import_element(tmpdir, kind, project, name): - element_name = 'import-{}.bst'.format(name) - repo_element_path = os.path.join(project, 'elements', element_name) + element_name = "import-{}.bst".format(name) + repo_element_path = os.path.join(project, "elements", element_name) files = str(tmpdir.join("imported_files_{}".format(name))) os.makedirs(files) - with open(os.path.join(files, '{}.txt'.format(name)), 'w') as f: + with open(os.path.join(files, "{}.txt".format(name)), "w") as f: f.write(name) - repo = create_repo(kind, str(tmpdir.join('element_{}_repo'.format(name)))) + repo = create_repo(kind, str(tmpdir.join("element_{}_repo".format(name)))) repo.create(files) generate_element(repo, repo_element_path) @@ -69,28 +64,22 @@ def generate_project(tmpdir, name, kind, config=None): if config is None: config = {} - project_name = 'project-{}'.format(name) + project_name = "project-{}".format(name) subproject_path = os.path.join(str(tmpdir.join(project_name))) - os.makedirs(os.path.join(subproject_path, 'elements')) + os.makedirs(os.path.join(subproject_path, "elements")) - project_conf = { - 'name': name, - 'element-path': 'elements' - } + project_conf = {"name": name, "element-path": "elements"} project_conf.update(config) - _yaml.roundtrip_dump(project_conf, os.path.join(subproject_path, 'project.conf')) + _yaml.roundtrip_dump(project_conf, os.path.join(subproject_path, "project.conf")) add_plugins_conf(subproject_path, kind) return project_name, subproject_path def generate_simple_stack(project, name, dependencies): - element_name = '{}.bst'.format(name) - element_path = os.path.join(project, 'elements', element_name) - element = { - 'kind': 'stack', - 'depends': dependencies - } + element_name = "{}.bst".format(name) + element_path = os.path.join(project, "elements", element_name) + element = {"kind": "stack", "depends": dependencies} _yaml.roundtrip_dump(element, element_path) return element_name @@ -98,11 +87,11 @@ def generate_simple_stack(project, name, dependencies): def generate_cross_element(project, subproject_name, import_name): basename, _ = os.path.splitext(import_name) - return generate_simple_stack(project, 'import-{}-{}'.format(subproject_name, basename), - [{ - 'junction': '{}.bst'.format(subproject_name), - 'filename': import_name - }]) + return generate_simple_stack( + project, + "import-{}-{}".format(subproject_name, basename), + [{"junction": "{}.bst".format(subproject_name), "filename": import_name}], + ) @pytest.mark.parametrize("kind", ALL_REPO_KINDS.keys()) @@ -110,30 +99,30 @@ def test_cross_junction_multiple_projects(cli, tmpdir, kind): tmpdir = tmpdir.join(kind) # Generate 3 projects: main, a, b - _, project = generate_project(tmpdir, 'main', kind, {'ref-storage': 'project.refs'}) - project_a, project_a_path = generate_project(tmpdir, 'a', kind) - project_b, project_b_path = generate_project(tmpdir, 'b', kind) + _, project = generate_project(tmpdir, "main", kind, {"ref-storage": "project.refs"}) + project_a, project_a_path = generate_project(tmpdir, "a", kind) + project_b, project_b_path = generate_project(tmpdir, "b", kind) # Generate an element with a trackable source for each project - element_a = generate_import_element(tmpdir, kind, project_a_path, 'a') - element_b = generate_import_element(tmpdir, kind, project_b_path, 'b') - element_c = generate_import_element(tmpdir, kind, project, 'c') + element_a = generate_import_element(tmpdir, kind, project_a_path, "a") + element_b = generate_import_element(tmpdir, kind, project_b_path, "b") + element_c = generate_import_element(tmpdir, kind, project, "c") # Create some indirections to the elements with dependencies to test --deps - stack_a = generate_simple_stack(project_a_path, 'stack-a', [element_a]) - stack_b = generate_simple_stack(project_b_path, 'stack-b', [element_b]) + stack_a = generate_simple_stack(project_a_path, "stack-a", [element_a]) + stack_b = generate_simple_stack(project_b_path, "stack-b", [element_b]) # Create junctions for projects a and b in main. - junction_a = '{}.bst'.format(project_a) - junction_a_path = os.path.join(project, 'elements', junction_a) - generate_junction(tmpdir.join('repo_a'), project_a_path, junction_a_path, store_ref=False) + junction_a = "{}.bst".format(project_a) + junction_a_path = os.path.join(project, "elements", junction_a) + generate_junction(tmpdir.join("repo_a"), project_a_path, junction_a_path, store_ref=False) - junction_b = '{}.bst'.format(project_b) - junction_b_path = os.path.join(project, 'elements', junction_b) - generate_junction(tmpdir.join('repo_b'), project_b_path, junction_b_path, store_ref=False) + junction_b = "{}.bst".format(project_b) + junction_b_path = os.path.join(project, "elements", junction_b) + generate_junction(tmpdir.join("repo_b"), project_b_path, junction_b_path, store_ref=False) # Track the junctions. - result = cli.run(project=project, args=['source', 'track', junction_a, junction_b]) + result = cli.run(project=project, args=["source", "track", junction_a, junction_b]) result.assert_success() # Import elements from a and b in to main. @@ -141,18 +130,16 @@ def test_cross_junction_multiple_projects(cli, tmpdir, kind): imported_b = generate_cross_element(project, project_b, stack_b) # Generate a top level stack depending on everything - all_bst = generate_simple_stack(project, 'all', [imported_a, imported_b, element_c]) + all_bst = generate_simple_stack(project, "all", [imported_a, imported_b, element_c]) # Track without following junctions. But explicitly also track the elements in project a. - result = cli.run(project=project, args=['source', 'track', - '--deps', 'all', - all_bst, - '{}:{}'.format(junction_a, stack_a)]) + result = cli.run( + project=project, args=["source", "track", "--deps", "all", all_bst, "{}:{}".format(junction_a, stack_a)] + ) result.assert_success() # Elements in project b should not be tracked. But elements in project a and main should. - expected = [element_c, - '{}:{}'.format(junction_a, element_a)] + expected = [element_c, "{}:{}".format(junction_a, element_a)] assert set(result.get_tracked_elements()) == set(expected) @@ -160,31 +147,38 @@ def test_cross_junction_multiple_projects(cli, tmpdir, kind): def test_track_exceptions(cli, tmpdir, kind): tmpdir = tmpdir.join(kind) - _, project = generate_project(tmpdir, 'main', kind, {'ref-storage': 'project.refs'}) - project_a, project_a_path = generate_project(tmpdir, 'a', kind) + _, project = generate_project(tmpdir, "main", kind, {"ref-storage": "project.refs"}) + project_a, project_a_path = generate_project(tmpdir, "a", kind) - element_a = generate_import_element(tmpdir, kind, project_a_path, 'a') - element_b = generate_import_element(tmpdir, kind, project_a_path, 'b') + element_a = generate_import_element(tmpdir, kind, project_a_path, "a") + element_b = generate_import_element(tmpdir, kind, project_a_path, "b") - all_bst = generate_simple_stack(project_a_path, 'all', [element_a, - element_b]) + all_bst = generate_simple_stack(project_a_path, "all", [element_a, element_b]) - junction_a = '{}.bst'.format(project_a) - junction_a_path = os.path.join(project, 'elements', junction_a) - generate_junction(tmpdir.join('repo_a'), project_a_path, junction_a_path, store_ref=False) + junction_a = "{}.bst".format(project_a) + junction_a_path = os.path.join(project, "elements", junction_a) + generate_junction(tmpdir.join("repo_a"), project_a_path, junction_a_path, store_ref=False) - result = cli.run(project=project, args=['source', 'track', junction_a]) + result = cli.run(project=project, args=["source", "track", junction_a]) result.assert_success() imported_b = generate_cross_element(project, project_a, element_b) - indirection = generate_simple_stack(project, 'indirection', [imported_b]) - - result = cli.run(project=project, - args=['source', 'track', '--deps', 'all', - '--except', indirection, - '{}:{}'.format(junction_a, all_bst), imported_b]) + indirection = generate_simple_stack(project, "indirection", [imported_b]) + + result = cli.run( + project=project, + args=[ + "source", + "track", + "--deps", + "all", + "--except", + indirection, + "{}:{}".format(junction_a, all_bst), + imported_b, + ], + ) result.assert_success() - expected = ['{}:{}'.format(junction_a, element_a), - '{}:{}'.format(junction_a, element_b)] + expected = ["{}:{}".format(junction_a, element_a), "{}:{}".format(junction_a, element_b)] assert set(result.get_tracked_elements()) == set(expected) diff --git a/src/buildstream/testing/_sourcetests/utils.py b/src/buildstream/testing/_sourcetests/utils.py index a0e65b4f4..116506807 100644 --- a/src/buildstream/testing/_sourcetests/utils.py +++ b/src/buildstream/testing/_sourcetests/utils.py @@ -27,9 +27,8 @@ import os try: import pytest except ImportError: - module_name = globals()['__name__'] - msg = "Could not import pytest:\n" \ - "To use the {} module, you must have pytest installed.".format(module_name) + module_name = globals()["__name__"] + msg = "Could not import pytest:\n" "To use the {} module, you must have pytest installed.".format(module_name) raise ImportError(msg) from buildstream import _yaml @@ -72,13 +71,7 @@ def add_plugins_conf(project, plugin_kind): if plugin_package is not None: project_conf["plugins"] = [ - { - "origin": "pip", - "package-name": plugin_package, - "sources": { - plugin_kind: 0, - }, - }, + {"origin": "pip", "package-name": plugin_package, "sources": {plugin_kind: 0,},}, ] _yaml.roundtrip_dump(project_conf, project_conf_file) @@ -96,7 +89,7 @@ def add_plugins_conf(project, plugin_kind): # updated_configuration (dict): configuration to merge into the existing one # def update_project_configuration(project_path, updated_configuration): - project_conf_path = os.path.join(project_path, 'project.conf') + project_conf_path = os.path.join(project_path, "project.conf") project_conf = _yaml.roundtrip_load(project_conf_path) project_conf.update(updated_configuration) diff --git a/src/buildstream/testing/_sourcetests/workspace.py b/src/buildstream/testing/_sourcetests/workspace.py index dd7977e76..34e2247ea 100644 --- a/src/buildstream/testing/_sourcetests/workspace.py +++ b/src/buildstream/testing/_sourcetests/workspace.py @@ -30,10 +30,10 @@ from .utils import kind # pylint: disable=unused-import # Project directory TOP_DIR = os.path.dirname(os.path.realpath(__file__)) -DATA_DIR = os.path.join(TOP_DIR, 'project') +DATA_DIR = os.path.join(TOP_DIR, "project") -class WorkspaceCreator(): +class WorkspaceCreator: def __init__(self, cli, tmpdir, datafiles, project_path=None): self.cli = cli self.tmpdir = tmpdir @@ -45,17 +45,16 @@ class WorkspaceCreator(): shutil.copytree(str(datafiles), project_path) self.project_path = project_path - self.bin_files_path = os.path.join(project_path, 'files', 'bin-files') + self.bin_files_path = os.path.join(project_path, "files", "bin-files") - self.workspace_cmd = os.path.join(self.project_path, 'workspace_cmd') + self.workspace_cmd = os.path.join(self.project_path, "workspace_cmd") - def create_workspace_element(self, kind, track, suffix='', workspace_dir=None, - element_attrs=None): - element_name = 'workspace-test-{}{}.bst'.format(kind, suffix) - element_path = os.path.join(self.project_path, 'elements') + def create_workspace_element(self, kind, track, suffix="", workspace_dir=None, element_attrs=None): + element_name = "workspace-test-{}{}.bst".format(kind, suffix) + element_path = os.path.join(self.project_path, "elements") if not workspace_dir: workspace_dir = os.path.join(self.workspace_cmd, element_name) - if workspace_dir[-4:] == '.bst': + if workspace_dir[-4:] == ".bst": workspace_dir = workspace_dir[:-4] # Create our repo object of the given source type with @@ -66,64 +65,53 @@ class WorkspaceCreator(): ref = None # Write out our test target - element = { - 'kind': 'import', - 'sources': [ - repo.source_config(ref=ref) - ] - } + element = {"kind": "import", "sources": [repo.source_config(ref=ref)]} if element_attrs: element = {**element, **element_attrs} - _yaml.roundtrip_dump(element, - os.path.join(element_path, element_name)) + _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) return element_name, element_path, workspace_dir - def create_workspace_elements(self, kinds, track, suffixs=None, workspace_dir_usr=None, - element_attrs=None): + def create_workspace_elements(self, kinds, track, suffixs=None, workspace_dir_usr=None, element_attrs=None): element_tuples = [] if suffixs is None: - suffixs = ['', ] * len(kinds) + suffixs = ["",] * len(kinds) else: if len(suffixs) != len(kinds): raise "terable error" for suffix, kind in zip(suffixs, kinds): - element_name, _, workspace_dir = \ - self.create_workspace_element(kind, track, suffix, workspace_dir_usr, - element_attrs) + element_name, _, workspace_dir = self.create_workspace_element( + kind, track, suffix, workspace_dir_usr, element_attrs + ) element_tuples.append((element_name, workspace_dir)) # Assert that there is no reference, a track & fetch is needed - states = self.cli.get_element_states(self.project_path, [ - e for e, _ in element_tuples - ]) + states = self.cli.get_element_states(self.project_path, [e for e, _ in element_tuples]) if track: - assert not any(states[e] != 'no reference' for e, _ in element_tuples) + assert not any(states[e] != "no reference" for e, _ in element_tuples) else: - assert not any(states[e] != 'fetch needed' for e, _ in element_tuples) + assert not any(states[e] != "fetch needed" for e, _ in element_tuples) return element_tuples - def open_workspaces(self, kinds, track, suffixs=None, workspace_dir=None, - element_attrs=None, no_checkout=False): + def open_workspaces(self, kinds, track, suffixs=None, workspace_dir=None, element_attrs=None, no_checkout=False): - element_tuples = self.create_workspace_elements(kinds, track, suffixs, workspace_dir, - element_attrs) + element_tuples = self.create_workspace_elements(kinds, track, suffixs, workspace_dir, element_attrs) os.makedirs(self.workspace_cmd, exist_ok=True) # Now open the workspace, this should have the effect of automatically # tracking & fetching the source from the repo. - args = ['workspace', 'open'] + args = ["workspace", "open"] if track: - args.append('--track') + args.append("--track") if no_checkout: - args.append('--no-checkout') + args.append("--no-checkout") if workspace_dir is not None: assert len(element_tuples) == 1, "test logic error" _, workspace_dir = element_tuples[0] - args.extend(['--directory', workspace_dir]) + args.extend(["--directory", workspace_dir]) args.extend([element_name for element_name, workspace_dir_suffix in element_tuples]) result = self.cli.run(cwd=self.workspace_cmd, project=self.project_path, args=args) @@ -132,24 +120,31 @@ class WorkspaceCreator(): if not no_checkout: # Assert that we are now buildable because the source is now cached. - states = self.cli.get_element_states(self.project_path, [ - e for e, _ in element_tuples - ]) - assert not any(states[e] != 'buildable' for e, _ in element_tuples) + states = self.cli.get_element_states(self.project_path, [e for e, _ in element_tuples]) + assert not any(states[e] != "buildable" for e, _ in element_tuples) # Check that the executable hello file is found in each workspace for _, workspace in element_tuples: - filename = os.path.join(workspace, 'usr', 'bin', 'hello') + filename = os.path.join(workspace, "usr", "bin", "hello") assert os.path.exists(filename) return element_tuples -def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', workspace_dir=None, - project_path=None, element_attrs=None, no_checkout=False): +def open_workspace( + cli, + tmpdir, + datafiles, + kind, + track, + suffix="", + workspace_dir=None, + project_path=None, + element_attrs=None, + no_checkout=False, +): workspace_object = WorkspaceCreator(cli, tmpdir, datafiles, project_path) - workspaces = workspace_object.open_workspaces((kind, ), track, (suffix, ), workspace_dir, - element_attrs, no_checkout) + workspaces = workspace_object.open_workspaces((kind,), track, (suffix,), workspace_dir, element_attrs, no_checkout) assert len(workspaces) == 1 element_name, workspace = workspaces[0] return element_name, workspace_object.project_path, workspace diff --git a/src/buildstream/testing/_utils/junction.py b/src/buildstream/testing/_utils/junction.py index 98d23b0a2..cfc5898a9 100644 --- a/src/buildstream/testing/_utils/junction.py +++ b/src/buildstream/testing/_utils/junction.py @@ -28,12 +28,7 @@ def generate_junction(tmpdir, subproject_path, junction_path, *, store_ref=True) if not store_ref: source_ref = None - element = { - 'kind': 'junction', - 'sources': [ - repo.source_config(ref=source_ref) - ] - } + element = {"kind": "junction", "sources": [repo.source_config(ref=source_ref)]} _yaml.roundtrip_dump(element, junction_path) return ref @@ -41,46 +36,38 @@ def generate_junction(tmpdir, subproject_path, junction_path, *, store_ref=True) # A barebones Git Repo class to use for generating junctions class _SimpleGit(Repo): - def __init__(self, directory, subdir='repo'): + def __init__(self, directory, subdir="repo"): if not HAVE_GIT: - pytest.skip('git is not available') + pytest.skip("git is not available") super().__init__(directory, subdir) def create(self, directory): self.copy_directory(directory, self.repo) - self._run_git('init', '.') - self._run_git('add', '.') - self._run_git('commit', '-m', 'Initial commit') + self._run_git("init", ".") + self._run_git("add", ".") + self._run_git("commit", "-m", "Initial commit") return self.latest_commit() def latest_commit(self): - return self._run_git( - 'rev-parse', 'HEAD', - stdout=subprocess.PIPE, - universal_newlines=True, - ).stdout.strip() + return self._run_git("rev-parse", "HEAD", stdout=subprocess.PIPE, universal_newlines=True,).stdout.strip() def source_config(self, ref=None): return self.source_config_extra(ref) def source_config_extra(self, ref=None, checkout_submodules=None): - config = { - 'kind': 'git', - 'url': 'file://' + self.repo, - 'track': 'master' - } + config = {"kind": "git", "url": "file://" + self.repo, "track": "master"} if ref is not None: - config['ref'] = ref + config["ref"] = ref if checkout_submodules is not None: - config['checkout-submodules'] = checkout_submodules + config["checkout-submodules"] = checkout_submodules return config def _run_git(self, *args, **kwargs): argv = [GIT] argv.extend(args) - if 'env' not in kwargs: - kwargs['env'] = dict(GIT_ENV, PWD=self.repo) - kwargs.setdefault('cwd', self.repo) - kwargs.setdefault('check', True) + if "env" not in kwargs: + kwargs["env"] = dict(GIT_ENV, PWD=self.repo) + kwargs.setdefault("cwd", self.repo) + kwargs.setdefault("check", True) return subprocess.run(argv, **kwargs) diff --git a/src/buildstream/testing/_utils/site.py b/src/buildstream/testing/_utils/site.py index ca74d9505..953d21607 100644 --- a/src/buildstream/testing/_utils/site.py +++ b/src/buildstream/testing/_utils/site.py @@ -5,29 +5,29 @@ import os import subprocess import sys import platform -from typing import Optional # pylint: disable=unused-import +from typing import Optional # pylint: disable=unused-import from buildstream import _site, utils, ProgramNotFoundError from buildstream._platform import Platform try: - GIT = utils.get_host_tool('git') # type: Optional[str] + GIT = utils.get_host_tool("git") # type: Optional[str] HAVE_GIT = True - out = str(subprocess.check_output(['git', '--version']), "utf-8") + out = str(subprocess.check_output(["git", "--version"]), "utf-8") # e.g. on Git for Windows we get "git version 2.21.0.windows.1". # e.g. on Mac via Homebrew we get "git version 2.19.0". - version = tuple(int(x) for x in out.split(' ')[2].split('.')[:3]) + version = tuple(int(x) for x in out.split(" ")[2].split(".")[:3]) HAVE_OLD_GIT = version < (1, 8, 5) GIT_ENV = { - 'GIT_AUTHOR_DATE': '1320966000 +0200', - 'GIT_AUTHOR_NAME': 'tomjon', - 'GIT_AUTHOR_EMAIL': 'tom@jon.com', - 'GIT_COMMITTER_DATE': '1320966000 +0200', - 'GIT_COMMITTER_NAME': 'tomjon', - 'GIT_COMMITTER_EMAIL': 'tom@jon.com' + "GIT_AUTHOR_DATE": "1320966000 +0200", + "GIT_AUTHOR_NAME": "tomjon", + "GIT_AUTHOR_EMAIL": "tom@jon.com", + "GIT_COMMITTER_DATE": "1320966000 +0200", + "GIT_COMMITTER_NAME": "tomjon", + "GIT_COMMITTER_EMAIL": "tom@jon.com", } except ProgramNotFoundError: GIT = None @@ -36,18 +36,16 @@ except ProgramNotFoundError: GIT_ENV = dict() try: - BZR = utils.get_host_tool('bzr') # type: Optional[str] + BZR = utils.get_host_tool("bzr") # type: Optional[str] HAVE_BZR = True - BZR_ENV = { - "BZR_EMAIL": "Testy McTesterson <testy.mctesterson@example.com>" - } + BZR_ENV = {"BZR_EMAIL": "Testy McTesterson <testy.mctesterson@example.com>"} except ProgramNotFoundError: BZR = None HAVE_BZR = False BZR_ENV = {} try: - utils.get_host_tool('bwrap') + utils.get_host_tool("bwrap") HAVE_BWRAP = True HAVE_BWRAP_JSON_STATUS = _site.get_bwrap_version() >= (0, 3, 2) except ProgramNotFoundError: @@ -55,32 +53,33 @@ except ProgramNotFoundError: HAVE_BWRAP_JSON_STATUS = False try: - utils.get_host_tool('lzip') + utils.get_host_tool("lzip") HAVE_LZIP = True except ProgramNotFoundError: HAVE_LZIP = False try: import arpy # pylint: disable=unused-import + HAVE_ARPY = True except ImportError: HAVE_ARPY = False try: - utils.get_host_tool('buildbox') + utils.get_host_tool("buildbox") HAVE_BUILDBOX = True except ProgramNotFoundError: HAVE_BUILDBOX = False -IS_LINUX = os.getenv('BST_FORCE_BACKEND', sys.platform).startswith('linux') -IS_WSL = (IS_LINUX and 'Microsoft' in platform.uname().release) -IS_WINDOWS = (os.name == 'nt') +IS_LINUX = os.getenv("BST_FORCE_BACKEND", sys.platform).startswith("linux") +IS_WSL = IS_LINUX and "Microsoft" in platform.uname().release +IS_WINDOWS = os.name == "nt" MACHINE_ARCH = Platform.get_host_arch() -HAVE_SANDBOX = os.getenv('BST_FORCE_SANDBOX') +HAVE_SANDBOX = os.getenv("BST_FORCE_SANDBOX") if HAVE_SANDBOX is not None: pass elif IS_LINUX and HAVE_BWRAP and (not IS_WSL): - HAVE_SANDBOX = 'bwrap' + HAVE_SANDBOX = "bwrap" diff --git a/src/buildstream/testing/integration.py b/src/buildstream/testing/integration.py index 01635de74..584d7da1b 100644 --- a/src/buildstream/testing/integration.py +++ b/src/buildstream/testing/integration.py @@ -39,11 +39,11 @@ def walk_dir(root): # print path to all subdirectories first. for subdirname in dirnames: - yield os.path.join(dirname, subdirname)[len(root):] + yield os.path.join(dirname, subdirname)[len(root) :] # print path to all filenames. for filename in filenames: - yield os.path.join(dirname, filename)[len(root):] + yield os.path.join(dirname, filename)[len(root) :] # Ensure that a directory contains the given filenames. @@ -51,35 +51,33 @@ def assert_contains(directory, expected): missing = set(expected) missing.difference_update(walk_dir(directory)) if missing: - raise AssertionError("Missing {} expected elements from list: {}" - .format(len(missing), missing)) + raise AssertionError("Missing {} expected elements from list: {}".format(len(missing), missing)) class IntegrationCache: - def __init__(self, cache): self.root = os.path.abspath(cache) os.makedirs(cache, exist_ok=True) # Use the same sources every time - self.sources = os.path.join(self.root, 'sources') + self.sources = os.path.join(self.root, "sources") # Create a temp directory for the duration of the test for # the artifacts directory try: - self.cachedir = tempfile.mkdtemp(dir=self.root, prefix='cache-') + self.cachedir = tempfile.mkdtemp(dir=self.root, prefix="cache-") except OSError as e: raise AssertionError("Unable to create test directory !") from e -@pytest.fixture(scope='session') +@pytest.fixture(scope="session") def integration_cache(request): # Set the cache dir to the INTEGRATION_CACHE variable, or the # default if that is not set. - if 'INTEGRATION_CACHE' in os.environ: - cache_dir = os.environ['INTEGRATION_CACHE'] + if "INTEGRATION_CACHE" in os.environ: + cache_dir = os.environ["INTEGRATION_CACHE"] else: - cache_dir = os.path.abspath('./integration-cache') + cache_dir = os.path.abspath("./integration-cache") cache = IntegrationCache(cache_dir) @@ -92,6 +90,6 @@ def integration_cache(request): except FileNotFoundError: pass try: - shutil.rmtree(os.path.join(cache.root, 'cas')) + shutil.rmtree(os.path.join(cache.root, "cas")) except FileNotFoundError: pass diff --git a/src/buildstream/testing/repo.py b/src/buildstream/testing/repo.py index c1538685d..1b46ec806 100644 --- a/src/buildstream/testing/repo.py +++ b/src/buildstream/testing/repo.py @@ -25,7 +25,7 @@ import os import shutil -class Repo(): +class Repo: """Repo() Abstract class providing scaffolding for generating data to be @@ -38,7 +38,8 @@ class Repo(): subdir (str): The subdir for the repo, in case there is more than one """ - def __init__(self, directory, subdir='repo'): + + def __init__(self, directory, subdir="repo"): # The working directory for the repo object # @@ -100,7 +101,7 @@ class Repo(): Returns: (Repo): A Repo object for the new repository. """ - subdir = self.repo[len(self.directory):].lstrip(os.sep) + subdir = self.repo[len(self.directory) :].lstrip(os.sep) new_dir = os.path.join(dest, subdir) os.makedirs(new_dir, exist_ok=True) self.copy_directory(self.repo, new_dir) diff --git a/src/buildstream/testing/runcli.py b/src/buildstream/testing/runcli.py index 36426c8af..d4f7e2ffd 100644 --- a/src/buildstream/testing/runcli.py +++ b/src/buildstream/testing/runcli.py @@ -61,14 +61,8 @@ from buildstream._protos.buildstream.v2 import artifact_pb2 # Wrapper for the click.testing result -class Result(): - - def __init__(self, - exit_code=None, - exception=None, - exc_info=None, - output=None, - stderr=None): +class Result: + def __init__(self, exit_code=None, exception=None, exc_info=None, output=None, stderr=None): self.exit_code = exit_code self.exc = exception self.exc_info = exc_info @@ -94,8 +88,7 @@ class Result(): self.unhandled_exception = True self.exception = get_last_exception() - self.task_error_domain, \ - self.task_error_reason = get_last_task_error() + self.task_error_domain, self.task_error_reason = get_last_task_error() else: self.exception = None self.task_error_domain = None @@ -111,7 +104,7 @@ class Result(): # Raises: # (AssertionError): If the session did not complete successfully # - def assert_success(self, fail_message=''): + def assert_success(self, fail_message=""): assert self.exit_code == 0, fail_message assert self.exc is None, fail_message assert self.exception is None, fail_message @@ -131,11 +124,7 @@ class Result(): # Raises: # (AssertionError): If any of the assertions fail # - def assert_main_error(self, - error_domain, - error_reason, - fail_message='', - *, debug=False): + def assert_main_error(self, error_domain, error_reason, fail_message="", *, debug=False): if debug: print( """ @@ -144,11 +133,9 @@ class Result(): Domain: {} Reason: {} """.format( - self.exit_code, - self.exception, - self.exception.domain, - self.exception.reason - )) + self.exit_code, self.exception, self.exception.domain, self.exception.reason + ) + ) assert self.exit_code == -1, fail_message assert self.exc is not None, fail_message assert self.exception is not None, fail_message @@ -172,10 +159,7 @@ class Result(): # Raises: # (AssertionError): If any of the assertions fail # - def assert_task_error(self, - error_domain, - error_reason, - fail_message=''): + def assert_task_error(self, error_domain, error_reason, fail_message=""): assert self.exit_code == -1, fail_message assert self.exc is not None, fail_message @@ -197,7 +181,7 @@ class Result(): # Raises: # (AssertionError): If any of the assertions fail # - def assert_shell_error(self, fail_message=''): + def assert_shell_error(self, fail_message=""): assert self.exit_code == 1, fail_message # get_start_order() @@ -212,7 +196,7 @@ class Result(): # (list): A list of element names in the order which they first appeared in the result # def get_start_order(self, activity): - results = re.findall(r'\[\s*{}:(\S+)\s*\]\s*START\s*.*\.log'.format(activity), self.stderr) + results = re.findall(r"\[\s*{}:(\S+)\s*\]\s*START\s*.*\.log".format(activity), self.stderr) if results is None: return [] return list(results) @@ -228,29 +212,28 @@ class Result(): # (list): A list of element names # def get_tracked_elements(self): - tracked = re.findall(r'\[\s*track:(\S+)\s*]', self.stderr) + tracked = re.findall(r"\[\s*track:(\S+)\s*]", self.stderr) if tracked is None: return [] return list(tracked) def get_pushed_elements(self): - pushed = re.findall(r'\[\s*push:(\S+)\s*\]\s*INFO\s*Pushed artifact', self.stderr) + pushed = re.findall(r"\[\s*push:(\S+)\s*\]\s*INFO\s*Pushed artifact", self.stderr) if pushed is None: return [] return list(pushed) def get_pulled_elements(self): - pulled = re.findall(r'\[\s*pull:(\S+)\s*\]\s*INFO\s*Pulled artifact', self.stderr) + pulled = re.findall(r"\[\s*pull:(\S+)\s*\]\s*INFO\s*Pulled artifact", self.stderr) if pulled is None: return [] return list(pulled) -class Cli(): - +class Cli: def __init__(self, directory, verbose=True, default_options=None): self.directory = directory self.config = None @@ -286,14 +269,13 @@ class Cli(): # element_name (str): The name of the element artifact # cache_dir (str): Specific cache dir to remove artifact from # - def remove_artifact_from_cache(self, project, element_name, - *, cache_dir=None): + def remove_artifact_from_cache(self, project, element_name, *, cache_dir=None): # Read configuration to figure out where artifacts are stored if not cache_dir: - default = os.path.join(project, 'cache') + default = os.path.join(project, "cache") if self.config is not None: - cache_dir = self.config.get('cachedir', default) + cache_dir = self.config.get("cachedir", default) else: cache_dir = default @@ -313,8 +295,17 @@ class Cli(): # args (list): A list of arguments to pass buildstream # binary_capture (bool): Whether to capture the stdout/stderr as binary # - def run(self, configure=True, project=None, silent=False, env=None, - cwd=None, options=None, args=None, binary_capture=False): + def run( + self, + configure=True, + project=None, + silent=False, + env=None, + cwd=None, + options=None, + args=None, + binary_capture=False, + ): # We don't want to carry the state of one bst invocation into another # bst invocation. Since node _FileInfo objects hold onto BuildStream @@ -335,22 +326,20 @@ class Cli(): options = self.default_options + options with ExitStack() as stack: - bst_args = ['--no-colors'] + bst_args = ["--no-colors"] if silent: - bst_args += ['--no-verbose'] + bst_args += ["--no-verbose"] if configure: - config_file = stack.enter_context( - configured(self.directory, self.config) - ) - bst_args += ['--config', config_file] + config_file = stack.enter_context(configured(self.directory, self.config)) + bst_args += ["--config", config_file] if project: - bst_args += ['--directory', str(project)] + bst_args += ["--directory", str(project)] for option, value in options: - bst_args += ['--option', option, value] + bst_args += ["--option", option, value] bst_args += args @@ -366,15 +355,14 @@ class Cli(): try: sys.__stdout__.fileno() except ValueError: - sys.__stdout__ = open('/dev/stdout', 'w') + sys.__stdout__ = open("/dev/stdout", "w") result = self._invoke(bst_cli, bst_args, binary_capture=binary_capture) # Some informative stdout we can observe when anything fails if self.verbose: command = "bst " + " ".join(bst_args) - print("BuildStream exited with code {} for invocation:\n\t{}" - .format(result.exit_code, command)) + print("BuildStream exited with code {} for invocation:\n\t{}".format(result.exit_code, command)) if result.output: print("Program output was:\n{}".format(result.output)) if result.stderr: @@ -409,9 +397,9 @@ class Cli(): exit_code = e.code if not isinstance(exit_code, int): - sys.stdout.write('Program exit code was not an integer: ') + sys.stdout.write("Program exit code was not an integer: ") sys.stdout.write(str(exit_code)) - sys.stdout.write('\n') + sys.stdout.write("\n") exit_code = 1 except Exception as e: # pylint: disable=broad-except exception = e @@ -424,11 +412,7 @@ class Cli(): out, err = capture.readouterr() capture.stop_capturing() - return Result(exit_code=exit_code, - exception=exception, - exc_info=exc_info, - output=out, - stderr=err) + return Result(exit_code=exit_code, exception=exception, exc_info=exc_info, output=out, stderr=err) # Fetch an element state by name by # invoking bst show on the project with the CLI @@ -437,12 +421,9 @@ class Cli(): # then use get_element_states(s) instead. # def get_element_state(self, project, element_name): - result = self.run(project=project, silent=True, args=[ - 'show', - '--deps', 'none', - '--format', '%{state}', - element_name - ]) + result = self.run( + project=project, silent=True, args=["show", "--deps", "none", "--format", "%{state}", element_name] + ) result.assert_success() return result.output.strip() @@ -450,18 +431,15 @@ class Cli(): # # Returns a dictionary with the element names as keys # - def get_element_states(self, project, targets, deps='all'): - result = self.run(project=project, silent=True, args=[ - 'show', - '--deps', deps, - '--format', '%{name}||%{state}', - *targets - ]) + def get_element_states(self, project, targets, deps="all"): + result = self.run( + project=project, silent=True, args=["show", "--deps", deps, "--format", "%{name}||%{state}", *targets] + ) result.assert_success() lines = result.output.splitlines() states = {} for line in lines: - split = line.split(sep='||') + split = line.split(sep="||") states[split[0]] = split[1] return states @@ -469,24 +447,18 @@ class Cli(): # on the project with the CLI # def get_element_key(self, project, element_name): - result = self.run(project=project, silent=True, args=[ - 'show', - '--deps', 'none', - '--format', '%{full-key}', - element_name - ]) + result = self.run( + project=project, silent=True, args=["show", "--deps", "none", "--format", "%{full-key}", element_name] + ) result.assert_success() return result.output.strip() # Get the decoded config of an element. # def get_element_config(self, project, element_name): - result = self.run(project=project, silent=True, args=[ - 'show', - '--deps', 'none', - '--format', '%{config}', - element_name - ]) + result = self.run( + project=project, silent=True, args=["show", "--deps", "none", "--format", "%{config}", element_name] + ) result.assert_success() return yaml.safe_load(result.output) @@ -494,12 +466,12 @@ class Cli(): # Fetch the elements that would be in the pipeline with the given # arguments. # - def get_pipeline(self, project, elements, except_=None, scope='plan'): + def get_pipeline(self, project, elements, except_=None, scope="plan"): if except_ is None: except_ = [] - args = ['show', '--deps', scope, '--format', '%{name}'] - args += list(itertools.chain.from_iterable(zip(itertools.repeat('--except'), except_))) + args = ["show", "--deps", scope, "--format", "%{name}"] + args += list(itertools.chain.from_iterable(zip(itertools.repeat("--except"), except_))) result = self.run(project=project, silent=True, args=args + elements) result.assert_success() @@ -523,11 +495,27 @@ class CliIntegration(Cli): # # This supports the same arguments as Cli.run(), see run_project_config(). # - def run(self, configure=True, project=None, silent=False, env=None, - cwd=None, options=None, args=None, binary_capture=False): + def run( + self, + configure=True, + project=None, + silent=False, + env=None, + cwd=None, + options=None, + args=None, + binary_capture=False, + ): return self.run_project_config( - configure=configure, project=project, silent=silent, env=env, - cwd=cwd, options=options, args=args, binary_capture=binary_capture) + configure=configure, + project=project, + silent=silent, + env=env, + cwd=cwd, + options=options, + args=args, + binary_capture=binary_capture, + ) # run_project_config() # @@ -549,9 +537,9 @@ class CliIntegration(Cli): # Save the original project.conf, because we will run more than # once in the same temp directory # - project_directory = kwargs['project'] - project_filename = os.path.join(project_directory, 'project.conf') - project_backup = os.path.join(project_directory, 'project.conf.backup') + project_directory = kwargs["project"] + project_filename = os.path.join(project_directory, "project.conf") + project_backup = os.path.join(project_directory, "project.conf.backup") project_load_filename = project_filename if not os.path.exists(project_backup): @@ -576,8 +564,8 @@ class CliIntegration(Cli): # with tempfile.TemporaryDirectory(dir=project_directory) as scratchdir: - temp_project = os.path.join(scratchdir, 'project.conf') - with open(temp_project, 'w') as f: + temp_project = os.path.join(scratchdir, "project.conf") + with open(temp_project, "w") as f: yaml.safe_dump(project_config, f) project_config = _yaml.load(temp_project) @@ -589,7 +577,7 @@ class CliIntegration(Cli): else: # Otherwise, just dump it as is - with open(project_filename, 'w') as f: + with open(project_filename, "w") as f: f.write(config) return super().run(**kwargs) @@ -611,50 +599,49 @@ class CliRemote(CliIntegration): # # Returns a list of configured services (by names). # - def ensure_services(self, actions=True, execution=True, storage=True, - artifacts=False, sources=False): + def ensure_services(self, actions=True, execution=True, storage=True, artifacts=False, sources=False): # Build a list of configured services by name: configured_services = [] if not self.config: return configured_services - if 'remote-execution' in self.config: - rexec_config = self.config['remote-execution'] + if "remote-execution" in self.config: + rexec_config = self.config["remote-execution"] - if 'action-cache-service' in rexec_config: + if "action-cache-service" in rexec_config: if actions: - configured_services.append('action-cache') + configured_services.append("action-cache") else: - rexec_config.pop('action-cache-service') + rexec_config.pop("action-cache-service") - if 'execution-service' in rexec_config: + if "execution-service" in rexec_config: if execution: - configured_services.append('execution') + configured_services.append("execution") else: - rexec_config.pop('execution-service') + rexec_config.pop("execution-service") - if 'storage-service' in rexec_config: + if "storage-service" in rexec_config: if storage: - configured_services.append('storage') + configured_services.append("storage") else: - rexec_config.pop('storage-service') + rexec_config.pop("storage-service") - if 'artifacts' in self.config: + if "artifacts" in self.config: if artifacts: - configured_services.append('artifact-cache') + configured_services.append("artifact-cache") else: - self.config.pop('artifacts') + self.config.pop("artifacts") - if 'source-caches' in self.config: + if "source-caches" in self.config: if sources: - configured_services.append('source-cache') + configured_services.append("source-cache") else: - self.config.pop('source-caches') + self.config.pop("source-caches") return configured_services -class TestArtifact(): +class TestArtifact: # remove_artifact_from_cache(): # @@ -666,10 +653,10 @@ class TestArtifact(): # def remove_artifact_from_cache(self, cache_dir, element_name): - cache_dir = os.path.join(cache_dir, 'artifacts', 'refs') + cache_dir = os.path.join(cache_dir, "artifacts", "refs") - normal_name = element_name.replace(os.sep, '-') - cache_dir = os.path.splitext(os.path.join(cache_dir, 'test', normal_name))[0] + normal_name = element_name.replace(os.sep, "-") + cache_dir = os.path.splitext(os.path.join(cache_dir, "test", normal_name))[0] shutil.rmtree(cache_dir) # is_cached(): @@ -688,7 +675,7 @@ class TestArtifact(): # cas = CASCache(str(cache_dir)) artifact_ref = element.get_artifact_name(element_key) - return os.path.exists(os.path.join(cache_dir, 'artifacts', 'refs', artifact_ref)) + return os.path.exists(os.path.join(cache_dir, "artifacts", "refs", artifact_ref)) # get_digest(): # @@ -705,9 +692,9 @@ class TestArtifact(): def get_digest(self, cache_dir, element, element_key): artifact_ref = element.get_artifact_name(element_key) - artifact_dir = os.path.join(cache_dir, 'artifacts', 'refs') + artifact_dir = os.path.join(cache_dir, "artifacts", "refs") artifact_proto = artifact_pb2.Artifact() - with open(os.path.join(artifact_dir, artifact_ref), 'rb') as f: + with open(os.path.join(artifact_dir, artifact_ref), "rb") as f: artifact_proto.ParseFromString(f.read()) return artifact_proto.files @@ -727,7 +714,7 @@ class TestArtifact(): def extract_buildtree(self, cache_dir, tmpdir, ref): artifact = artifact_pb2.Artifact() try: - with open(os.path.join(cache_dir, 'artifacts', 'refs', ref), 'rb') as f: + with open(os.path.join(cache_dir, "artifacts", "refs", ref), "rb") as f: artifact.ParseFromString(f.read()) except FileNotFoundError: yield None @@ -768,7 +755,7 @@ class TestArtifact(): # @pytest.fixture() def cli(tmpdir): - directory = os.path.join(str(tmpdir), 'cache') + directory = os.path.join(str(tmpdir), "cache") os.makedirs(directory) return Cli(directory) @@ -781,27 +768,26 @@ def cli(tmpdir): # stacktraces. @pytest.fixture() def cli_integration(tmpdir, integration_cache): - directory = os.path.join(str(tmpdir), 'cache') + directory = os.path.join(str(tmpdir), "cache") os.makedirs(directory) fixture = CliIntegration(directory) # We want to cache sources for integration tests more permanently, # to avoid downloading the huge base-sdk repeatedly - fixture.configure({ - 'cachedir': integration_cache.cachedir, - 'sourcedir': integration_cache.sources, - }) + fixture.configure( + {"cachedir": integration_cache.cachedir, "sourcedir": integration_cache.sources,} + ) yield fixture # remove following folders if necessary try: - shutil.rmtree(os.path.join(integration_cache.cachedir, 'build')) + shutil.rmtree(os.path.join(integration_cache.cachedir, "build")) except FileNotFoundError: pass try: - shutil.rmtree(os.path.join(integration_cache.cachedir, 'tmp')) + shutil.rmtree(os.path.join(integration_cache.cachedir, "tmp")) except FileNotFoundError: pass @@ -813,36 +799,32 @@ def cli_integration(tmpdir, integration_cache): # stacktraces. @pytest.fixture() def cli_remote_execution(tmpdir, remote_services): - directory = os.path.join(str(tmpdir), 'cache') + directory = os.path.join(str(tmpdir), "cache") os.makedirs(directory) fixture = CliRemote(directory) if remote_services.artifact_service: - fixture.configure({'artifacts': [{ - 'url': remote_services.artifact_service, - }]}) + fixture.configure({"artifacts": [{"url": remote_services.artifact_service,}]}) remote_execution = {} if remote_services.action_service: - remote_execution['action-cache-service'] = { - 'url': remote_services.action_service, + remote_execution["action-cache-service"] = { + "url": remote_services.action_service, } if remote_services.exec_service: - remote_execution['execution-service'] = { - 'url': remote_services.exec_service, + remote_execution["execution-service"] = { + "url": remote_services.exec_service, } if remote_services.storage_service: - remote_execution['storage-service'] = { - 'url': remote_services.storage_service, + remote_execution["storage-service"] = { + "url": remote_services.storage_service, } if remote_execution: - fixture.configure({'remote-execution': remote_execution}) + fixture.configure({"remote-execution": remote_execution}) if remote_services.source_service: - fixture.configure({'source-caches': [{ - 'url': remote_services.source_service, - }]}) + fixture.configure({"source-caches": [{"url": remote_services.source_service,}]}) return fixture @@ -882,12 +864,12 @@ def configured(directory, config=None): if not config: config = {} - if not config.get('sourcedir', False): - config['sourcedir'] = os.path.join(directory, 'sources') - if not config.get('cachedir', False): - config['cachedir'] = directory - if not config.get('logdir', False): - config['logdir'] = os.path.join(directory, 'logs') + if not config.get("sourcedir", False): + config["sourcedir"] = os.path.join(directory, "sources") + if not config.get("cachedir", False): + config["cachedir"] = directory + if not config.get("logdir", False): + config["logdir"] = os.path.join(directory, "logs") # Dump it and yield the filename for test scripts to feed it # to buildstream as an artument diff --git a/src/buildstream/types.py b/src/buildstream/types.py index 5688bf393..180044dbd 100644 --- a/src/buildstream/types.py +++ b/src/buildstream/types.py @@ -151,7 +151,7 @@ class Consistency(FastEnum): return self.value < other.value -class CoreWarnings(): +class CoreWarnings: """CoreWarnings() Some common warnings which are raised by core functionalities within BuildStream are found in this class. diff --git a/src/buildstream/utils.py b/src/buildstream/utils.py index de7c14b70..181ea1df9 100644 --- a/src/buildstream/utils.py +++ b/src/buildstream/utils.py @@ -52,7 +52,7 @@ from ._utils import url_directory_name # pylint: disable=unused-import BST_ARBITRARY_TIMESTAMP = calendar.timegm((2011, 11, 11, 11, 11, 11)) # The separator we use for user specified aliases -_ALIAS_SEPARATOR = ':' +_ALIAS_SEPARATOR = ":" _URI_SCHEMES = ["http", "https", "ftp", "file", "git", "sftp", "ssh"] # Main process pid @@ -74,6 +74,7 @@ class UtilError(BstError): or either of the :class:`.ElementError` or :class:`.SourceError` exceptions should be raised from this error. """ + def __init__(self, message, reason=None): super().__init__(message, domain=ErrorDomain.UTIL, reason=reason) @@ -83,6 +84,7 @@ class ProgramNotFoundError(BstError): It is normally unneeded to handle this exception from plugin code. """ + def __init__(self, message, reason=None): super().__init__(message, domain=ErrorDomain.PROG_NOT_FOUND, reason=reason) @@ -92,7 +94,7 @@ class DirectoryExistsError(OSError): """ -class FileListResult(): +class FileListResult: """An object which stores the result of one of the operations which run on a list of files. """ @@ -112,7 +114,7 @@ class FileListResult(): self.files_written = [] """List of files that were written.""" - def combine(self, other: 'FileListResult') -> 'FileListResult': + def combine(self, other: "FileListResult") -> "FileListResult": """Create a new FileListResult that contains the results of both. """ ret = FileListResult() @@ -165,10 +167,10 @@ def list_relative_paths(directory: str) -> Iterator[str]: # We don't want "./" pre-pended to all the entries in the root of # `directory`, prefer to have no prefix in that case. - basepath = relpath if relpath != '.' and dirpath != directory else '' + basepath = relpath if relpath != "." and dirpath != directory else "" # First yield the walked directory itself, except for the root - if basepath != '': + if basepath != "": yield basepath # List the filenames in the walked directory @@ -248,8 +250,7 @@ def sha256sum(filename: str) -> str: h.update(chunk) except OSError as e: - raise UtilError("Failed to get a checksum of file '{}': {}" - .format(filename, e)) from e + raise UtilError("Failed to get a checksum of file '{}': {}".format(filename, e)) from e return h.hexdigest() @@ -274,8 +275,7 @@ def safe_copy(src: str, dest: str, *, result: Optional[FileListResult] = None) - os.unlink(dest) except OSError as e: if e.errno != errno.ENOENT: - raise UtilError("Failed to remove destination file '{}': {}" - .format(dest, e)) from e + raise UtilError("Failed to remove destination file '{}': {}".format(dest, e)) from e shutil.copyfile(src, dest) try: @@ -291,8 +291,7 @@ def safe_copy(src: str, dest: str, *, result: Optional[FileListResult] = None) - result.failed_attributes.append(dest) except shutil.Error as e: - raise UtilError("Failed to copy '{} -> {}': {}" - .format(src, dest, e)) from e + raise UtilError("Failed to copy '{} -> {}': {}".format(src, dest, e)) from e def safe_link(src: str, dest: str, *, result: Optional[FileListResult] = None, _unlink=False) -> None: @@ -313,8 +312,7 @@ def safe_link(src: str, dest: str, *, result: Optional[FileListResult] = None, _ os.unlink(dest) except OSError as e: if e.errno != errno.ENOENT: - raise UtilError("Failed to remove destination file '{}': {}" - .format(dest, e)) from e + raise UtilError("Failed to remove destination file '{}': {}".format(dest, e)) from e # If we can't link it due to cross-device hardlink, copy try: @@ -326,8 +324,7 @@ def safe_link(src: str, dest: str, *, result: Optional[FileListResult] = None, _ elif e.errno == errno.EXDEV: safe_copy(src, dest) else: - raise UtilError("Failed to link '{} -> {}': {}" - .format(src, dest, e)) from e + raise UtilError("Failed to link '{} -> {}': {}".format(src, dest, e)) from e def safe_remove(path: str) -> bool: @@ -363,16 +360,17 @@ def safe_remove(path: str) -> bool: # Path does not exist return True - raise UtilError("Failed to remove '{}': {}" - .format(path, e)) + raise UtilError("Failed to remove '{}': {}".format(path, e)) -def copy_files(src: str, - dest: str, - *, - filter_callback: Optional[Callable[[str], bool]] = None, - ignore_missing: bool = False, - report_written: bool = False) -> FileListResult: +def copy_files( + src: str, + dest: str, + *, + filter_callback: Optional[Callable[[str], bool]] = None, + ignore_missing: bool = False, + report_written: bool = False +) -> FileListResult: """Copy files from source to destination. Args: @@ -401,22 +399,28 @@ def copy_files(src: str, """ result = FileListResult() try: - _process_list(src, dest, safe_copy, result, - filter_callback=filter_callback, - ignore_missing=ignore_missing, - report_written=report_written) + _process_list( + src, + dest, + safe_copy, + result, + filter_callback=filter_callback, + ignore_missing=ignore_missing, + report_written=report_written, + ) except OSError as e: - raise UtilError("Failed to copy '{} -> {}': {}" - .format(src, dest, e)) + raise UtilError("Failed to copy '{} -> {}': {}".format(src, dest, e)) return result -def link_files(src: str, - dest: str, - *, - filter_callback: Optional[Callable[[str], bool]] = None, - ignore_missing: bool = False, - report_written: bool = False) -> FileListResult: +def link_files( + src: str, + dest: str, + *, + filter_callback: Optional[Callable[[str], bool]] = None, + ignore_missing: bool = False, + report_written: bool = False +) -> FileListResult: """Hardlink files from source to destination. Args: @@ -450,13 +454,17 @@ def link_files(src: str, """ result = FileListResult() try: - _process_list(src, dest, safe_link, result, - filter_callback=filter_callback, - ignore_missing=ignore_missing, - report_written=report_written) + _process_list( + src, + dest, + safe_link, + result, + filter_callback=filter_callback, + ignore_missing=ignore_missing, + report_written=report_written, + ) except OSError as e: - raise UtilError("Failed to link '{} -> {}': {}" - .format(src, dest, e)) + raise UtilError("Failed to link '{} -> {}': {}".format(src, dest, e)) return result @@ -473,7 +481,7 @@ def get_host_tool(name: str) -> str: Raises: :class:`.ProgramNotFoundError` """ - search_path = os.environ.get('PATH') + search_path = os.environ.get("PATH") program_path = shutil.which(name, path=search_path) if not program_path: @@ -491,22 +499,27 @@ def get_bst_version() -> Tuple[int, int]: """ # Import this only conditionally, it's not resolved at bash complete time from . import __version__ # pylint: disable=cyclic-import - versions = __version__.split('.')[:2] - if versions[0] == '0+untagged': - raise UtilError("Your git repository has no tags - BuildStream can't " - "determine its version. Please run `git fetch --tags`.") + versions = __version__.split(".")[:2] + + if versions[0] == "0+untagged": + raise UtilError( + "Your git repository has no tags - BuildStream can't " + "determine its version. Please run `git fetch --tags`." + ) try: return (int(versions[0]), int(versions[1])) except IndexError: - raise UtilError("Cannot detect Major and Minor parts of the version\n" - "Version: {} not in XX.YY.whatever format" - .format(__version__)) + raise UtilError( + "Cannot detect Major and Minor parts of the version\n" + "Version: {} not in XX.YY.whatever format".format(__version__) + ) except ValueError: - raise UtilError("Cannot convert version to integer numbers\n" - "Version: {} not in Integer.Integer.whatever format" - .format(__version__)) + raise UtilError( + "Cannot convert version to integer numbers\n" + "Version: {} not in Integer.Integer.whatever format".format(__version__) + ) def move_atomic(source: Union[Path, str], destination: Union[Path, str], *, ensure_parents: bool = True) -> None: @@ -548,16 +561,18 @@ def move_atomic(source: Union[Path, str], destination: Union[Path, str], *, ensu @contextmanager -def save_file_atomic(filename: str, - mode: str = 'w', - *, - buffering: int = -1, - encoding: Optional[str] = None, - errors: Optional[str] = None, - newline: Optional[str] = None, - closefd: bool = True, - opener: Optional[Callable[[str, int], int]] = None, - tempdir: Optional[str] = None) -> Iterator[IO]: +def save_file_atomic( + filename: str, + mode: str = "w", + *, + buffering: int = -1, + encoding: Optional[str] = None, + errors: Optional[str] = None, + newline: Optional[str] = None, + closefd: bool = True, + opener: Optional[Callable[[str, int], int]] = None, + tempdir: Optional[str] = None +) -> Iterator[IO]: """Save a file with a temporary name and rename it into place when ready. This is a context manager which is meant for saving data to files. @@ -589,8 +604,16 @@ def save_file_atomic(filename: str, fd, tempname = tempfile.mkstemp(dir=tempdir) os.close(fd) - f = open(tempname, mode=mode, buffering=buffering, encoding=encoding, - errors=errors, newline=newline, closefd=closefd, opener=opener) + f = open( + tempname, + mode=mode, + buffering=buffering, + encoding=encoding, + errors=errors, + newline=newline, + closefd=closefd, + opener=opener, + ) def cleanup_tempfile(): f.close() @@ -604,7 +627,7 @@ def save_file_atomic(filename: str, try: with _signals.terminator(cleanup_tempfile): # Disable type-checking since "IO[Any]" has no attribute "real_filename" - f.real_filename = filename # type: ignore + f.real_filename = filename # type: ignore yield f f.close() # This operation is atomic, at least on platforms we care about: @@ -660,8 +683,7 @@ def _get_volume_size(path): try: usage = shutil.disk_usage(path) except OSError as e: - raise UtilError("Failed to retrieve stats on volume for path '{}': {}" - .format(path, e)) from e + raise UtilError("Failed to retrieve stats on volume for path '{}': {}".format(path, e)) from e return usage.total, usage.free @@ -685,16 +707,16 @@ def _get_volume_size(path): # UtilError if the string is not a valid data size. # def _parse_size(size, volume): - if size == 'infinity': + if size == "infinity": return None - matches = re.fullmatch(r'([0-9]+\.?[0-9]*)([KMGT%]?)', size) + matches = re.fullmatch(r"([0-9]+\.?[0-9]*)([KMGT%]?)", size) if matches is None: raise UtilError("{} is not a valid data size.".format(size)) num, unit = matches.groups() - if unit == '%': + if unit == "%": num = float(num) if num > 100: raise UtilError("{}% is not a valid percentage value.".format(num)) @@ -703,8 +725,8 @@ def _parse_size(size, volume): return disk_size * (num / 100) - units = ('', 'K', 'M', 'G', 'T') - return int(num) * 1024**units.index(unit) + units = ("", "K", "M", "G", "T") + return int(num) * 1024 ** units.index(unit) # _pretty_size() @@ -720,8 +742,8 @@ def _parse_size(size, volume): # (str): The string representation of the number of bytes in the largest def _pretty_size(size, dec_places=0): psize = size - unit = 'B' - units = ('B', 'K', 'M', 'G', 'T') + unit = "B" + units = ("B", "K", "M", "G", "T") for unit in units: if psize < 1024: break @@ -746,19 +768,17 @@ def _force_rmtree(rootpath, **kwargs): os.chmod(rootpath, 0o755) for root, dirs, _ in os.walk(rootpath): for d in dirs: - path = os.path.join(root, d.lstrip('/')) + path = os.path.join(root, d.lstrip("/")) if os.path.exists(path) and not os.path.islink(path): try: os.chmod(path, 0o755) except OSError as e: - raise UtilError("Failed to ensure write permission on file '{}': {}" - .format(path, e)) + raise UtilError("Failed to ensure write permission on file '{}': {}".format(path, e)) try: shutil.rmtree(rootpath, **kwargs) except OSError as e: - raise UtilError("Failed to remove cache directory '{}': {}" - .format(rootpath, e)) + raise UtilError("Failed to remove cache directory '{}': {}".format(rootpath, e)) # Recursively make directories in target area @@ -779,8 +799,7 @@ def _copy_directories(srcdir, destdir, target): os.makedirs(new_dir) yield (new_dir, mode) else: - raise UtilError('Source directory tree has file where ' - 'directory expected: {}'.format(old_dir)) + raise UtilError("Source directory tree has file where " "directory expected: {}".format(old_dir)) else: if not os.access(new_dir, os.W_OK): # If the destination directory is not writable, change permissions to make it @@ -806,16 +825,16 @@ def _ensure_real_directory(root, path): try: deststat = os.lstat(destpath) if not stat.S_ISDIR(deststat.st_mode): - relpath = destpath[len(root):] + relpath = destpath[len(root) :] if stat.S_ISLNK(deststat.st_mode): - filetype = 'symlink' + filetype = "symlink" elif stat.S_ISREG(deststat.st_mode): - filetype = 'regular file' + filetype = "regular file" else: - filetype = 'special file' + filetype = "special file" - raise UtilError('Destination is a {}, not a directory: {}'.format(filetype, relpath)) + raise UtilError("Destination is a {}, not a directory: {}".format(filetype, relpath)) except FileNotFoundError: os.makedirs(destpath) @@ -836,9 +855,9 @@ def _ensure_real_directory(root, path): # ignore_missing: Dont raise any error if a source file is missing # # -def _process_list(srcdir, destdir, actionfunc, result, - filter_callback=None, - ignore_missing=False, report_written=False): +def _process_list( + srcdir, destdir, actionfunc, result, filter_callback=None, ignore_missing=False, report_written=False +): # Keep track of directory permissions, since these need to be set # *after* files have been written. @@ -921,7 +940,7 @@ def _process_list(srcdir, destdir, actionfunc, result, else: # Unsupported type. - raise UtilError('Cannot extract {} into staging-area. Unsupported type.'.format(srcpath)) + raise UtilError("Cannot extract {} into staging-area. Unsupported type.".format(srcpath)) # Write directory permissions now that all files have been written for d, perms in permissions: @@ -1035,8 +1054,9 @@ def _tempnamedfile(suffix="", prefix="tmp", dir=None): # pylint: disable=redefi if temp is not None: temp.close() - with _signals.terminator(close_tempfile), \ - tempfile.NamedTemporaryFile(suffix=suffix, prefix=prefix, dir=dir) as temp: + with _signals.terminator(close_tempfile), tempfile.NamedTemporaryFile( + suffix=suffix, prefix=prefix, dir=dir + ) as temp: yield temp @@ -1145,13 +1165,13 @@ def _kill_process_tree(pid): # def _call(*popenargs, terminate=False, **kwargs): - kwargs['start_new_session'] = True + kwargs["start_new_session"] = True process = None - old_preexec_fn = kwargs.get('preexec_fn') - if 'preexec_fn' in kwargs: - del kwargs['preexec_fn'] + old_preexec_fn = kwargs.get("preexec_fn") + if "preexec_fn" in kwargs: + del kwargs["preexec_fn"] def preexec_fn(): os.umask(stat.S_IWGRP | stat.S_IWOTH) @@ -1203,7 +1223,8 @@ def _call(*popenargs, terminate=False, **kwargs): with _signals.suspendable(suspend_proc, resume_proc), _signals.terminator(kill_proc): process = subprocess.Popen( # pylint: disable=subprocess-popen-preexec-fn - *popenargs, preexec_fn=preexec_fn, universal_newlines=True, **kwargs) + *popenargs, preexec_fn=preexec_fn, universal_newlines=True, **kwargs + ) output, _ = process.communicate() exit_code = process.poll() @@ -1237,44 +1258,44 @@ def _call(*popenargs, terminate=False, **kwargs): # def _glob2re(pat): i, n = 0, len(pat) - res = '(?ms)' + res = "(?ms)" while i < n: c = pat[i] i = i + 1 - if c == '*': + if c == "*": # fnmatch.translate() simply uses the '.*' separator here, # we only want that for double asterisk (bash 'globstar' behavior) # - if i < n and pat[i] == '*': - res = res + '.*' + if i < n and pat[i] == "*": + res = res + ".*" i = i + 1 else: - res = res + '[^/]*' - elif c == '?': + res = res + "[^/]*" + elif c == "?": # fnmatch.translate() simply uses the '.' wildcard here, but # we dont want to match path separators here - res = res + '[^/]' - elif c == '[': + res = res + "[^/]" + elif c == "[": j = i - if j < n and pat[j] == '!': + if j < n and pat[j] == "!": j = j + 1 - if j < n and pat[j] == ']': + if j < n and pat[j] == "]": j = j + 1 - while j < n and pat[j] != ']': + while j < n and pat[j] != "]": j = j + 1 if j >= n: - res = res + '\\[' + res = res + "\\[" else: - stuff = pat[i:j].replace('\\', '\\\\') + stuff = pat[i:j].replace("\\", "\\\\") i = j + 1 - if stuff[0] == '!': - stuff = '^' + stuff[1:] - elif stuff[0] == '^': - stuff = '\\' + stuff - res = '{}[{}]'.format(res, stuff) + if stuff[0] == "!": + stuff = "^" + stuff[1:] + elif stuff[0] == "^": + stuff = "\\" + stuff + res = "{}[{}]".format(res, stuff) else: res = res + re.escape(c) - return res + r'\Z' + return res + r"\Z" # _deduplicate() @@ -1392,7 +1413,7 @@ def _deterministic_umask(): # # def _get_compression(tar): - mapped_extensions = {'.tar': '', '.gz': 'gz', '.xz': 'xz', '.bz2': 'bz2'} + mapped_extensions = {".tar": "", ".gz": "gz", ".xz": "xz", ".bz2": "bz2"} name, ext = os.path.splitext(tar) @@ -1403,12 +1424,14 @@ def _get_compression(tar): # If so, we assume we have been given an unsupported extension, # which expects compression. Raise an error _, suffix = os.path.splitext(name) - if suffix == '.tar': - raise UtilError("Expected compression with unknown file extension ('{}'), " - "supported extensions are ('.tar'), ('.gz'), ('.xz'), ('.bz2')".format(ext)) + if suffix == ".tar": + raise UtilError( + "Expected compression with unknown file extension ('{}'), " + "supported extensions are ('.tar'), ('.gz'), ('.xz'), ('.bz2')".format(ext) + ) # Assume just an unconventional name was provided, default to uncompressed - return '' + return "" # _is_single_threaded() diff --git a/tests/artifactcache/artifactservice.py b/tests/artifactcache/artifactservice.py index 51f128c32..c640665a3 100644 --- a/tests/artifactcache/artifactservice.py +++ b/tests/artifactcache/artifactservice.py @@ -22,11 +22,9 @@ from urllib.parse import urlparse import grpc import pytest -from buildstream._protos.buildstream.v2.artifact_pb2 \ - import Artifact, GetArtifactRequest, UpdateArtifactRequest +from buildstream._protos.buildstream.v2.artifact_pb2 import Artifact, GetArtifactRequest, UpdateArtifactRequest from buildstream._protos.buildstream.v2.artifact_pb2_grpc import ArtifactServiceStub -from buildstream._protos.build.bazel.remote.execution.v2 \ - import remote_execution_pb2 as re_pb2 +from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 as re_pb2 from buildstream import utils from tests.testutils.artifactshare import create_artifact_share diff --git a/tests/artifactcache/capabilities.py b/tests/artifactcache/capabilities.py index a28516aea..c8a49f9b5 100644 --- a/tests/artifactcache/capabilities.py +++ b/tests/artifactcache/capabilities.py @@ -13,10 +13,7 @@ from tests.testutils import dummy_context from tests.testutils.artifactshare import create_dummy_artifact_share -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project", -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",) @pytest.mark.datafiles(DATA_DIR) @@ -26,17 +23,12 @@ def test_artifact_cache_with_missing_capabilities_is_skipped(cli, tmpdir, datafi # Set up an artifact cache. with create_dummy_artifact_share() as share: # Configure artifact share - cache_dir = os.path.join(str(tmpdir), 'cache') - user_config_file = str(tmpdir.join('buildstream.conf')) + cache_dir = os.path.join(str(tmpdir), "cache") + user_config_file = str(tmpdir.join("buildstream.conf")) user_config = { - 'scheduler': { - 'pushers': 1 - }, - 'artifacts': { - 'url': share.repo, - 'push': True, - }, - 'cachedir': cache_dir + "scheduler": {"pushers": 1}, + "artifacts": {"url": share.repo, "push": True,}, + "cachedir": cache_dir, } _yaml.roundtrip_dump(user_config, file=user_config_file) @@ -51,5 +43,6 @@ def test_artifact_cache_with_missing_capabilities_is_skipped(cli, tmpdir, datafi # Manually setup the CAS remote artifactcache.setup_remotes(use_config=True) - assert not artifactcache.has_fetch_remotes(), \ - "System didn't realize the artifact cache didn't support BuildStream" + assert ( + not artifactcache.has_fetch_remotes() + ), "System didn't realize the artifact cache didn't support BuildStream" diff --git a/tests/artifactcache/config.py b/tests/artifactcache/config.py index 75cce267c..e5e2213eb 100644 --- a/tests/artifactcache/config.py +++ b/tests/artifactcache/config.py @@ -19,32 +19,20 @@ from tests.testutils import dummy_context DATA_DIR = os.path.dirname(os.path.realpath(__file__)) -cache1 = RemoteSpec(url='https://example.com/cache1', push=True) -cache2 = RemoteSpec(url='https://example.com/cache2', push=False) -cache3 = RemoteSpec(url='https://example.com/cache3', push=False) -cache4 = RemoteSpec(url='https://example.com/cache4', push=False) -cache5 = RemoteSpec(url='https://example.com/cache5', push=False) -cache6 = RemoteSpec(url='https://example.com/cache6', - push=True, - type=RemoteType.ALL) -cache7 = RemoteSpec(url='https://index.example.com/cache1', - push=True, - type=RemoteType.INDEX) -cache8 = RemoteSpec(url='https://storage.example.com/cache1', - push=True, - type=RemoteType.STORAGE) +cache1 = RemoteSpec(url="https://example.com/cache1", push=True) +cache2 = RemoteSpec(url="https://example.com/cache2", push=False) +cache3 = RemoteSpec(url="https://example.com/cache3", push=False) +cache4 = RemoteSpec(url="https://example.com/cache4", push=False) +cache5 = RemoteSpec(url="https://example.com/cache5", push=False) +cache6 = RemoteSpec(url="https://example.com/cache6", push=True, type=RemoteType.ALL) +cache7 = RemoteSpec(url="https://index.example.com/cache1", push=True, type=RemoteType.INDEX) +cache8 = RemoteSpec(url="https://storage.example.com/cache1", push=True, type=RemoteType.STORAGE) # Generate cache configuration fragments for the user config and project config files. # -def configure_remote_caches(override_caches, - project_caches=None, - user_caches=None): - type_strings = { - RemoteType.INDEX: 'index', - RemoteType.STORAGE: 'storage', - RemoteType.ALL: 'all' - } +def configure_remote_caches(override_caches, project_caches=None, user_caches=None): + type_strings = {RemoteType.INDEX: "index", RemoteType.STORAGE: "storage", RemoteType.ALL: "all"} if project_caches is None: project_caches = [] @@ -54,39 +42,32 @@ def configure_remote_caches(override_caches, user_config = {} if len(user_caches) == 1: - user_config['artifacts'] = { - 'url': user_caches[0].url, - 'push': user_caches[0].push, - 'type': type_strings[user_caches[0].type] + user_config["artifacts"] = { + "url": user_caches[0].url, + "push": user_caches[0].push, + "type": type_strings[user_caches[0].type], } elif len(user_caches) > 1: - user_config['artifacts'] = [ - { - 'url': cache.url, - 'push': cache.push, - 'type': type_strings[cache.type] - } for cache in user_caches + user_config["artifacts"] = [ + {"url": cache.url, "push": cache.push, "type": type_strings[cache.type]} for cache in user_caches ] if len(override_caches) == 1: - user_config['projects'] = { - 'test': { - 'artifacts': { - 'url': override_caches[0].url, - 'push': override_caches[0].push, - 'type': type_strings[override_caches[0].type] + user_config["projects"] = { + "test": { + "artifacts": { + "url": override_caches[0].url, + "push": override_caches[0].push, + "type": type_strings[override_caches[0].type], } } } elif len(override_caches) > 1: - user_config['projects'] = { - 'test': { - 'artifacts': [ - { - 'url': cache.url, - 'push': cache.push, - 'type': type_strings[cache.type] - } for cache in override_caches + user_config["projects"] = { + "test": { + "artifacts": [ + {"url": cache.url, "push": cache.push, "type": type_strings[cache.type]} + for cache in override_caches ] } } @@ -94,23 +75,24 @@ def configure_remote_caches(override_caches, project_config = {} if project_caches: if len(project_caches) == 1: - project_config.update({ - 'artifacts': { - 'url': project_caches[0].url, - 'push': project_caches[0].push, - 'type': type_strings[project_caches[0].type], + project_config.update( + { + "artifacts": { + "url": project_caches[0].url, + "push": project_caches[0].push, + "type": type_strings[project_caches[0].type], + } } - }) + ) elif len(project_caches) > 1: - project_config.update({ - 'artifacts': [ - { - 'url': cache.url, - 'push': cache.push, - 'type': type_strings[cache.type] - } for cache in project_caches - ] - }) + project_config.update( + { + "artifacts": [ + {"url": cache.url, "push": cache.push, "type": type_strings[cache.type]} + for cache in project_caches + ] + } + ) return user_config, project_config @@ -118,27 +100,28 @@ def configure_remote_caches(override_caches, # Test that parsing the remote artifact cache locations produces the # expected results. @pytest.mark.parametrize( - 'override_caches, project_caches, user_caches', + "override_caches, project_caches, user_caches", [ # The leftmost cache is the highest priority one in all cases here. - pytest.param([], [], [], id='empty-config'), - pytest.param([], [], [cache1, cache2], id='user-config'), - pytest.param([], [cache1, cache2], [cache3], id='project-config'), - pytest.param([cache1], [cache2], [cache3], id='project-override-in-user-config'), - pytest.param([cache1, cache2], [cache3, cache4], [cache5, cache6], id='list-order'), - pytest.param([cache1, cache2, cache1], [cache2], [cache2, cache1], id='duplicates'), - pytest.param([cache7, cache8], [], [cache1], id='split-caches') - ]) + pytest.param([], [], [], id="empty-config"), + pytest.param([], [], [cache1, cache2], id="user-config"), + pytest.param([], [cache1, cache2], [cache3], id="project-config"), + pytest.param([cache1], [cache2], [cache3], id="project-override-in-user-config"), + pytest.param([cache1, cache2], [cache3, cache4], [cache5, cache6], id="list-order"), + pytest.param([cache1, cache2, cache1], [cache2], [cache2, cache1], id="duplicates"), + pytest.param([cache7, cache8], [], [cache1], id="split-caches"), + ], +) def test_artifact_cache_precedence(tmpdir, override_caches, project_caches, user_caches): # Produce a fake user and project config with the cache configuration. user_config, project_config = configure_remote_caches(override_caches, project_caches, user_caches) - project_config['name'] = 'test' + project_config["name"] = "test" - user_config_file = str(tmpdir.join('buildstream.conf')) + user_config_file = str(tmpdir.join("buildstream.conf")) _yaml.roundtrip_dump(user_config, file=user_config_file) - project_dir = tmpdir.mkdir('project') - project_config_file = str(project_dir.join('project.conf')) + project_dir = tmpdir.mkdir("project") + project_config_file = str(project_dir.join("project.conf")) _yaml.roundtrip_dump(project_config, file=project_config_file) with dummy_context(config=user_config_file) as context: @@ -157,29 +140,21 @@ def test_artifact_cache_precedence(tmpdir, override_caches, project_caches, user # without specifying its counterpart, we get a comprehensive LoadError # instead of an unhandled exception. @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize('config_key, config_value', [ - ('client-cert', 'client.crt'), - ('client-key', 'client.key') -]) +@pytest.mark.parametrize("config_key, config_value", [("client-cert", "client.crt"), ("client-key", "client.key")]) def test_missing_certs(cli, datafiles, config_key, config_value): - project = os.path.join(datafiles.dirname, datafiles.basename, 'missing-certs') + project = os.path.join(datafiles.dirname, datafiles.basename, "missing-certs") project_conf = { - 'name': 'test', - - 'artifacts': { - 'url': 'https://cache.example.com:12345', - 'push': 'true', - config_key: config_value - } + "name": "test", + "artifacts": {"url": "https://cache.example.com:12345", "push": "true", config_key: config_value}, } - project_conf_file = os.path.join(project, 'project.conf') + project_conf_file = os.path.join(project, "project.conf") _yaml.roundtrip_dump(project_conf, project_conf_file) # Use `pull` here to ensure we try to initialize the remotes, triggering the error # # This does not happen for a simple `bst show`. - result = cli.run(project=project, args=['artifact', 'pull', 'element.bst']) + result = cli.run(project=project, args=["artifact", "pull", "element.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @@ -187,32 +162,33 @@ def test_missing_certs(cli, datafiles, config_key, config_value): # only one type of storage. @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( - 'override_caches, project_caches, user_caches', + "override_caches, project_caches, user_caches", [ # The leftmost cache is the highest priority one in all cases here. - pytest.param([], [], [cache7], id='index-user'), - pytest.param([], [], [cache8], id='storage-user'), - pytest.param([], [cache7], [], id='index-project'), - pytest.param([], [cache8], [], id='storage-project'), - pytest.param([cache7], [], [], id='index-override'), - pytest.param([cache8], [], [], id='storage-override'), - ]) + pytest.param([], [], [cache7], id="index-user"), + pytest.param([], [], [cache8], id="storage-user"), + pytest.param([], [cache7], [], id="index-project"), + pytest.param([], [cache8], [], id="storage-project"), + pytest.param([cache7], [], [], id="index-override"), + pytest.param([cache8], [], [], id="storage-override"), + ], +) def test_only_one(cli, datafiles, override_caches, project_caches, user_caches): - project = os.path.join(datafiles.dirname, datafiles.basename, 'only-one') + project = os.path.join(datafiles.dirname, datafiles.basename, "only-one") # Produce a fake user and project config with the cache configuration. user_config, project_config = configure_remote_caches(override_caches, project_caches, user_caches) - project_config['name'] = 'test' + project_config["name"] = "test" cli.configure(user_config) - project_config_file = os.path.join(project, 'project.conf') + project_config_file = os.path.join(project, "project.conf") _yaml.roundtrip_dump(project_config, file=project_config_file) # Use `pull` here to ensure we try to initialize the remotes, triggering the error # # This does not happen for a simple `bst show`. - result = cli.run(project=project, args=['artifact', 'pull', 'element.bst']) + result = cli.run(project=project, args=["artifact", "pull", "element.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) @@ -239,8 +215,8 @@ def test_only_one(cli, datafiles, override_caches, project_caches, user_caches): "client-cert": "~/client2.crt", "client-key": "~/client2.key", }, - ] - ) + ], + ), ) @pytest.mark.parametrize("in_user_config", [True, False]) def test_paths_for_artifact_config_are_expanded(tmpdir, monkeypatch, artifacts_config, in_user_config): @@ -260,11 +236,11 @@ def test_paths_for_artifact_config_are_expanded(tmpdir, monkeypatch, artifacts_c "artifacts": artifacts_config, } - user_config_file = str(tmpdir.join('buildstream.conf')) + user_config_file = str(tmpdir.join("buildstream.conf")) _yaml.roundtrip_dump(user_config, file=user_config_file) - project_dir = tmpdir.mkdir('project') - project_config_file = str(project_dir.join('project.conf')) + project_dir = tmpdir.mkdir("project") + project_config_file = str(project_dir.join("project.conf")) _yaml.roundtrip_dump(project_config, file=project_config_file) with dummy_context(config=user_config_file) as context: diff --git a/tests/artifactcache/expiry.py b/tests/artifactcache/expiry.py index 9ede1a8d3..83577f0c6 100644 --- a/tests/artifactcache/expiry.py +++ b/tests/artifactcache/expiry.py @@ -32,10 +32,7 @@ from buildstream.testing import cli # pylint: disable=unused-import from tests.testutils import create_element_size, wait_for_cache_granularity -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "expiry" -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "expiry") def get_cache_usage(directory): @@ -59,115 +56,109 @@ def get_cache_usage(directory): @pytest.mark.datafiles(DATA_DIR) def test_artifact_expires(cli, datafiles): project = str(datafiles) - element_path = 'elements' + element_path = "elements" - cli.configure({ - 'cache': { - 'quota': 10000000, - } - }) + cli.configure({"cache": {"quota": 10000000,}}) # Create an element that uses almost the entire cache (an empty # ostree cache starts at about ~10KiB, so we need a bit of a # buffer) - create_element_size('target.bst', project, element_path, [], 6000000) - res = cli.run(project=project, args=['build', 'target.bst']) + create_element_size("target.bst", project, element_path, [], 6000000) + res = cli.run(project=project, args=["build", "target.bst"]) res.assert_success() - assert cli.get_element_state(project, 'target.bst') == 'cached' + assert cli.get_element_state(project, "target.bst") == "cached" # Our cache should now be almost full. Let's create another # artifact and see if we can cause buildstream to delete the old # one. - create_element_size('target2.bst', project, element_path, [], 6000000) - res = cli.run(project=project, args=['build', 'target2.bst']) + create_element_size("target2.bst", project, element_path, [], 6000000) + res = cli.run(project=project, args=["build", "target2.bst"]) res.assert_success() # Check that the correct element remains in the cache - states = cli.get_element_states(project, ['target.bst', 'target2.bst']) - assert states['target.bst'] != 'cached' - assert states['target2.bst'] == 'cached' + states = cli.get_element_states(project, ["target.bst", "target2.bst"]) + assert states["target.bst"] != "cached" + assert states["target2.bst"] == "cached" # Ensure that we don't end up deleting the whole cache (or worse) if # we try to store an artifact that is too large to fit in the quota. -@pytest.mark.parametrize('size', [ - # Test an artifact that is obviously too large - (500000), - # Test an artifact that might be too large due to slight overhead - # of storing stuff in ostree - (399999) -]) +@pytest.mark.parametrize( + "size", + [ + # Test an artifact that is obviously too large + (500000), + # Test an artifact that might be too large due to slight overhead + # of storing stuff in ostree + (399999), + ], +) @pytest.mark.datafiles(DATA_DIR) def test_artifact_too_large(cli, datafiles, size): project = str(datafiles) - element_path = 'elements' + element_path = "elements" - cli.configure({ - 'cache': { - 'quota': 400000 - } - }) + cli.configure({"cache": {"quota": 400000}}) # Create an element whose artifact is too large - create_element_size('target.bst', project, element_path, [], size) - res = cli.run(project=project, args=['build', 'target.bst']) + create_element_size("target.bst", project, element_path, [], size) + res = cli.run(project=project, args=["build", "target.bst"]) res.assert_main_error(ErrorDomain.STREAM, None) - res.assert_task_error(ErrorDomain.CAS, 'cache-too-full') + res.assert_task_error(ErrorDomain.CAS, "cache-too-full") @pytest.mark.datafiles(DATA_DIR) def test_expiry_order(cli, datafiles): project = str(datafiles) - element_path = 'elements' - checkout = os.path.join(project, 'workspace') + element_path = "elements" + checkout = os.path.join(project, "workspace") - cli.configure({ - 'cache': { - 'quota': 9000000 - } - }) + cli.configure({"cache": {"quota": 9000000}}) # Create an artifact - create_element_size('dep.bst', project, element_path, [], 2000000) - res = cli.run(project=project, args=['build', 'dep.bst']) + create_element_size("dep.bst", project, element_path, [], 2000000) + res = cli.run(project=project, args=["build", "dep.bst"]) res.assert_success() # Create another artifact - create_element_size('unrelated.bst', project, element_path, [], 2000000) - res = cli.run(project=project, args=['build', 'unrelated.bst']) + create_element_size("unrelated.bst", project, element_path, [], 2000000) + res = cli.run(project=project, args=["build", "unrelated.bst"]) res.assert_success() # And build something else - create_element_size('target.bst', project, element_path, [], 2000000) - res = cli.run(project=project, args=['build', 'target.bst']) + create_element_size("target.bst", project, element_path, [], 2000000) + res = cli.run(project=project, args=["build", "target.bst"]) res.assert_success() - create_element_size('target2.bst', project, element_path, [], 2000000) - res = cli.run(project=project, args=['build', 'target2.bst']) + create_element_size("target2.bst", project, element_path, [], 2000000) + res = cli.run(project=project, args=["build", "target2.bst"]) res.assert_success() wait_for_cache_granularity() # Now extract dep.bst - res = cli.run(project=project, args=['artifact', 'checkout', 'dep.bst', '--directory', checkout]) + res = cli.run(project=project, args=["artifact", "checkout", "dep.bst", "--directory", checkout]) res.assert_success() # Finally, build something that will cause the cache to overflow - create_element_size('expire.bst', project, element_path, [], 2000000) - res = cli.run(project=project, args=['build', 'expire.bst']) + create_element_size("expire.bst", project, element_path, [], 2000000) + res = cli.run(project=project, args=["build", "expire.bst"]) res.assert_success() # While dep.bst was the first element to be created, it should not # have been removed. # Note that buildstream will reduce the cache to 50% of the # original size - we therefore remove multiple elements. - check_elements = [ - 'unrelated.bst', 'target.bst', 'target2.bst', 'dep.bst', 'expire.bst' - ] + check_elements = ["unrelated.bst", "target.bst", "target2.bst", "dep.bst", "expire.bst"] states = cli.get_element_states(project, check_elements) - assert (tuple(states[element] for element in check_elements) == - ('buildable', 'buildable', 'buildable', 'cached', 'cached', )) + assert tuple(states[element] for element in check_elements) == ( + "buildable", + "buildable", + "buildable", + "cached", + "cached", + ) # Ensure that we don't accidentally remove an artifact from something @@ -176,28 +167,24 @@ def test_expiry_order(cli, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_keep_dependencies(cli, datafiles): project = str(datafiles) - element_path = 'elements' + element_path = "elements" - cli.configure({ - 'cache': { - 'quota': 10000000 - } - }) + cli.configure({"cache": {"quota": 10000000}}) # Create a pretty big dependency - create_element_size('dependency.bst', project, element_path, [], 5000000) - res = cli.run(project=project, args=['build', 'dependency.bst']) + create_element_size("dependency.bst", project, element_path, [], 5000000) + res = cli.run(project=project, args=["build", "dependency.bst"]) res.assert_success() # Now create some other unrelated artifact - create_element_size('unrelated.bst', project, element_path, [], 4000000) - res = cli.run(project=project, args=['build', 'unrelated.bst']) + create_element_size("unrelated.bst", project, element_path, [], 4000000) + res = cli.run(project=project, args=["build", "unrelated.bst"]) res.assert_success() # Check that the correct element remains in the cache - states = cli.get_element_states(project, ['dependency.bst', 'unrelated.bst']) - assert states['dependency.bst'] == 'cached' - assert states['unrelated.bst'] == 'cached' + states = cli.get_element_states(project, ["dependency.bst", "unrelated.bst"]) + assert states["dependency.bst"] == "cached" + assert states["unrelated.bst"] == "cached" # We try to build an element which depends on the LRU artifact, # and could therefore fail if we didn't make sure dependencies @@ -207,54 +194,45 @@ def test_keep_dependencies(cli, datafiles): # duplicating artifacts (bad!) we need to make this equal in size # or smaller than half the size of its dependencies. # - create_element_size('target.bst', project, - element_path, ['dependency.bst'], 2000000) - res = cli.run(project=project, args=['build', 'target.bst']) + create_element_size("target.bst", project, element_path, ["dependency.bst"], 2000000) + res = cli.run(project=project, args=["build", "target.bst"]) res.assert_success() - states = cli.get_element_states(project, ['target.bst', 'unrelated.bst']) - assert states['target.bst'] == 'cached' - assert states['dependency.bst'] == 'cached' - assert states['unrelated.bst'] != 'cached' + states = cli.get_element_states(project, ["target.bst", "unrelated.bst"]) + assert states["target.bst"] == "cached" + assert states["dependency.bst"] == "cached" + assert states["unrelated.bst"] != "cached" # Assert that we never delete a dependency required for a build tree @pytest.mark.datafiles(DATA_DIR) def test_never_delete_required(cli, datafiles): project = str(datafiles) - element_path = 'elements' - - cli.configure({ - 'cache': { - 'quota': 10000000 - }, - 'scheduler': { - 'fetchers': 1, - 'builders': 1 - } - }) + element_path = "elements" + + cli.configure({"cache": {"quota": 10000000}, "scheduler": {"fetchers": 1, "builders": 1}}) # Create a linear build tree - create_element_size('dep1.bst', project, element_path, [], 8000000) - create_element_size('dep2.bst', project, element_path, ['dep1.bst'], 8000000) - create_element_size('dep3.bst', project, element_path, ['dep2.bst'], 8000000) - create_element_size('target.bst', project, element_path, ['dep3.bst'], 8000000) + create_element_size("dep1.bst", project, element_path, [], 8000000) + create_element_size("dep2.bst", project, element_path, ["dep1.bst"], 8000000) + create_element_size("dep3.bst", project, element_path, ["dep2.bst"], 8000000) + create_element_size("target.bst", project, element_path, ["dep3.bst"], 8000000) # Build dep1.bst, which should fit into the cache. - res = cli.run(project=project, args=['build', 'dep1.bst']) + res = cli.run(project=project, args=["build", "dep1.bst"]) res.assert_success() # We try to build this pipeline, but it's too big for the # cache. Since all elements are required, the build should fail. - res = cli.run(project=project, args=['build', 'target.bst']) + res = cli.run(project=project, args=["build", "target.bst"]) res.assert_main_error(ErrorDomain.STREAM, None) - res.assert_task_error(ErrorDomain.CAS, 'cache-too-full') + res.assert_task_error(ErrorDomain.CAS, "cache-too-full") - states = cli.get_element_states(project, ['target.bst']) - assert states['dep1.bst'] == 'cached' - assert states['dep2.bst'] != 'cached' - assert states['dep3.bst'] != 'cached' - assert states['target.bst'] != 'cached' + states = cli.get_element_states(project, ["target.bst"]) + assert states["dep1.bst"] == "cached" + assert states["dep2.bst"] != "cached" + assert states["dep3.bst"] != "cached" + assert states["target.bst"] != "cached" # Ensure that only valid cache quotas make it through the loading @@ -267,32 +245,33 @@ def test_never_delete_required(cli, datafiles): # # If err_domain is 'success', then err_reason is unused. # -@pytest.mark.parametrize("quota,err_domain,err_reason", [ - # Valid configurations - ("1", 'success', None), - ("1K", 'success', None), - ("50%", 'success', None), - ("infinity", 'success', None), - ("0", 'success', None), - # Invalid configurations - ("-1", ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA), - ("pony", ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA), - ("200%", ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA), -]) +@pytest.mark.parametrize( + "quota,err_domain,err_reason", + [ + # Valid configurations + ("1", "success", None), + ("1K", "success", None), + ("50%", "success", None), + ("infinity", "success", None), + ("0", "success", None), + # Invalid configurations + ("-1", ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA), + ("pony", ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA), + ("200%", ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA), + ], +) @pytest.mark.datafiles(DATA_DIR) def test_invalid_cache_quota(cli, datafiles, quota, err_domain, err_reason): project = str(datafiles) - os.makedirs(os.path.join(project, 'elements')) + os.makedirs(os.path.join(project, "elements")) - cli.configure({ - 'cache': { - 'quota': quota, - }, - }) + cli.configure( + {"cache": {"quota": quota,},} + ) - res = cli.run(project=project, args=['workspace', 'list']) + res = cli.run(project=project, args=["workspace", "list"]) - if err_domain == 'success': + if err_domain == "success": res.assert_success() else: res.assert_main_error(err_domain, err_reason) @@ -304,59 +283,47 @@ def test_invalid_cache_quota(cli, datafiles, quota, err_domain, err_reason): @pytest.mark.datafiles(DATA_DIR) def test_cleanup_first(cli, datafiles): project = str(datafiles) - element_path = 'elements' + element_path = "elements" - cli.configure({ - 'cache': { - 'quota': 10000000, - } - }) + cli.configure({"cache": {"quota": 10000000,}}) # Create an element that uses almost the entire cache (an empty # ostree cache starts at about ~10KiB, so we need a bit of a # buffer) - create_element_size('target.bst', project, element_path, [], 8000000) - res = cli.run(project=project, args=['build', 'target.bst']) + create_element_size("target.bst", project, element_path, [], 8000000) + res = cli.run(project=project, args=["build", "target.bst"]) res.assert_success() - assert cli.get_element_state(project, 'target.bst') == 'cached' + assert cli.get_element_state(project, "target.bst") == "cached" # Now configure with a smaller quota, create a situation # where the cache must be cleaned up before building anything else. # # Fix the fetchers and builders just to ensure a predictable # sequence of events (although it does not effect this test) - cli.configure({ - 'cache': { - 'quota': 5000000, - }, - 'scheduler': { - 'fetchers': 1, - 'builders': 1 - } - }) + cli.configure({"cache": {"quota": 5000000,}, "scheduler": {"fetchers": 1, "builders": 1}}) # Our cache is now more than full, BuildStream - create_element_size('target2.bst', project, element_path, [], 4000000) - res = cli.run(project=project, args=['build', 'target2.bst']) + create_element_size("target2.bst", project, element_path, [], 4000000) + res = cli.run(project=project, args=["build", "target2.bst"]) res.assert_success() # Check that the correct element remains in the cache - states = cli.get_element_states(project, ['target.bst', 'target2.bst']) - assert states['target.bst'] != 'cached' - assert states['target2.bst'] == 'cached' + states = cli.get_element_states(project, ["target.bst", "target2.bst"]) + assert states["target.bst"] != "cached" + assert states["target2.bst"] == "cached" @pytest.mark.datafiles(DATA_DIR) def test_cache_usage_monitor(cli, tmpdir, datafiles): project = str(datafiles) - element_path = 'elements' + element_path = "elements" assert get_cache_usage(cli.directory) == 0 ELEMENT_SIZE = 1000000 - create_element_size('target.bst', project, element_path, [], ELEMENT_SIZE) - res = cli.run(project=project, args=['build', 'target.bst']) + create_element_size("target.bst", project, element_path, [], ELEMENT_SIZE) + res = cli.run(project=project, args=["build", "target.bst"]) res.assert_success() assert get_cache_usage(cli.directory) >= ELEMENT_SIZE diff --git a/tests/artifactcache/junctions.py b/tests/artifactcache/junctions.py index dab69ea8d..76ba85fb5 100644 --- a/tests/artifactcache/junctions.py +++ b/tests/artifactcache/junctions.py @@ -11,187 +11,184 @@ from buildstream.testing import cli # pylint: disable=unused-import from tests.testutils import create_artifact_share, assert_shared, assert_not_shared -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "junctions", -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "junctions",) def project_set_artifacts(project, url): - project_conf_file = os.path.join(project, 'project.conf') + project_conf_file = os.path.join(project, "project.conf") project_config = _yaml.load(project_conf_file) - project_config['artifacts'] = { - 'url': url, - 'push': True - } + project_config["artifacts"] = {"url": url, "push": True} _yaml.roundtrip_dump(project_config.strip_node_info(), file=project_conf_file) @pytest.mark.datafiles(DATA_DIR) def test_push_pull(cli, tmpdir, datafiles): - project = os.path.join(str(datafiles), 'parent') - base_project = os.path.join(str(project), 'base') + project = os.path.join(str(datafiles), "parent") + base_project = os.path.join(str(project), "base") - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare-parent')) as share,\ - create_artifact_share(os.path.join(str(tmpdir), 'artifactshare-base')) as base_share: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare-parent")) as share, create_artifact_share( + os.path.join(str(tmpdir), "artifactshare-base") + ) as base_share: # First build it without the artifact cache configured - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) assert result.exit_code == 0 # Assert that we are now cached locally - state = cli.get_element_state(project, 'target.bst') - assert state == 'cached' - state = cli.get_element_state(base_project, 'base-element.bst') - assert state == 'cached' + state = cli.get_element_state(project, "target.bst") + assert state == "cached" + state = cli.get_element_state(base_project, "base-element.bst") + assert state == "cached" project_set_artifacts(project, share.repo) project_set_artifacts(base_project, base_share.repo) # Now try bst artifact push - result = cli.run(project=project, args=['artifact', 'push', '--deps', 'all', 'target.bst']) + result = cli.run(project=project, args=["artifact", "push", "--deps", "all", "target.bst"]) assert result.exit_code == 0 # And finally assert that the artifacts are in the right shares # # In the parent project's cache - assert_shared(cli, share, project, 'target.bst', project_name='parent') - assert_shared(cli, share, project, 'app.bst', project_name='parent') - assert_not_shared(cli, share, base_project, 'base-element.bst', project_name='base') + assert_shared(cli, share, project, "target.bst", project_name="parent") + assert_shared(cli, share, project, "app.bst", project_name="parent") + assert_not_shared(cli, share, base_project, "base-element.bst", project_name="base") # In the junction project's cache - assert_not_shared(cli, base_share, project, 'target.bst', project_name='parent') - assert_not_shared(cli, base_share, project, 'app.bst', project_name='parent') - assert_shared(cli, base_share, base_project, 'base-element.bst', project_name='base') + assert_not_shared(cli, base_share, project, "target.bst", project_name="parent") + assert_not_shared(cli, base_share, project, "app.bst", project_name="parent") + assert_shared(cli, base_share, base_project, "base-element.bst", project_name="base") # Now we've pushed, delete the user's local artifact cache # directory and try to redownload it from the share # - cas = os.path.join(cli.directory, 'cas') + cas = os.path.join(cli.directory, "cas") shutil.rmtree(cas) - artifact_dir = os.path.join(cli.directory, 'artifacts') + artifact_dir = os.path.join(cli.directory, "artifacts") shutil.rmtree(artifact_dir) # Assert that nothing is cached locally anymore - state = cli.get_element_state(project, 'target.bst') - assert state != 'cached' - state = cli.get_element_state(base_project, 'base-element.bst') - assert state != 'cached' + state = cli.get_element_state(project, "target.bst") + assert state != "cached" + state = cli.get_element_state(base_project, "base-element.bst") + assert state != "cached" # Now try bst artifact pull - result = cli.run(project=project, args=['artifact', 'pull', '--deps', 'all', 'target.bst']) + result = cli.run(project=project, args=["artifact", "pull", "--deps", "all", "target.bst"]) assert result.exit_code == 0 # And assert that they are again in the local cache, without having built - state = cli.get_element_state(project, 'target.bst') - assert state == 'cached' - state = cli.get_element_state(base_project, 'base-element.bst') - assert state == 'cached' + state = cli.get_element_state(project, "target.bst") + assert state == "cached" + state = cli.get_element_state(base_project, "base-element.bst") + assert state == "cached" @pytest.mark.datafiles(DATA_DIR) def test_caching_junction_elements(cli, tmpdir, datafiles): - project = os.path.join(str(datafiles), 'parent') - base_project = os.path.join(str(project), 'base') + project = os.path.join(str(datafiles), "parent") + base_project = os.path.join(str(project), "base") # Load the junction element - junction_element = os.path.join(project, 'base.bst') + junction_element = os.path.join(project, "base.bst") junction_data = _yaml.roundtrip_load(junction_element) # Add the "cache-junction-elements" boolean to the junction Element - junction_data['config'] = {"cache-junction-elements": True} + junction_data["config"] = {"cache-junction-elements": True} _yaml.roundtrip_dump(junction_data, junction_element) - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare-parent')) as share,\ - create_artifact_share(os.path.join(str(tmpdir), 'artifactshare-base')) as base_share: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare-parent")) as share, create_artifact_share( + os.path.join(str(tmpdir), "artifactshare-base") + ) as base_share: # First build it without the artifact cache configured - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) assert result.exit_code == 0 # Assert that we are now cached locally - state = cli.get_element_state(project, 'target.bst') - assert state == 'cached' - state = cli.get_element_state(base_project, 'base-element.bst') - assert state == 'cached' + state = cli.get_element_state(project, "target.bst") + assert state == "cached" + state = cli.get_element_state(base_project, "base-element.bst") + assert state == "cached" project_set_artifacts(project, share.repo) project_set_artifacts(base_project, base_share.repo) # Now try bst artifact push - result = cli.run(project=project, args=['artifact', 'push', '--deps', 'all', 'target.bst']) + result = cli.run(project=project, args=["artifact", "push", "--deps", "all", "target.bst"]) assert result.exit_code == 0 # And finally assert that the artifacts are in the right shares # # The parent project's cache should *also* contain elements from the junction - assert_shared(cli, share, project, 'target.bst', project_name='parent') - assert_shared(cli, share, project, 'app.bst', project_name='parent') - assert_shared(cli, share, base_project, 'base-element.bst', project_name='base') + assert_shared(cli, share, project, "target.bst", project_name="parent") + assert_shared(cli, share, project, "app.bst", project_name="parent") + assert_shared(cli, share, base_project, "base-element.bst", project_name="base") # The junction project's cache should only contain elements in the junction project - assert_not_shared(cli, base_share, project, 'target.bst', project_name='parent') - assert_not_shared(cli, base_share, project, 'app.bst', project_name='parent') - assert_shared(cli, base_share, base_project, 'base-element.bst', project_name='base') + assert_not_shared(cli, base_share, project, "target.bst", project_name="parent") + assert_not_shared(cli, base_share, project, "app.bst", project_name="parent") + assert_shared(cli, base_share, base_project, "base-element.bst", project_name="base") @pytest.mark.datafiles(DATA_DIR) def test_ignore_junction_remotes(cli, tmpdir, datafiles): - project = os.path.join(str(datafiles), 'parent') - base_project = os.path.join(str(project), 'base') + project = os.path.join(str(datafiles), "parent") + base_project = os.path.join(str(project), "base") # Load the junction element - junction_element = os.path.join(project, 'base.bst') + junction_element = os.path.join(project, "base.bst") junction_data = _yaml.roundtrip_load(junction_element) - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare-parent')) as share,\ - create_artifact_share(os.path.join(str(tmpdir), 'artifactshare-base')) as base_share: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare-parent")) as share, create_artifact_share( + os.path.join(str(tmpdir), "artifactshare-base") + ) as base_share: # Immediately declare the artifact caches in the appropriate project configs project_set_artifacts(project, share.repo) project_set_artifacts(base_project, base_share.repo) # Build and populate the project remotes with their respective elements - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) assert result.exit_code == 0 # And finally assert that the artifacts are in the right shares # # The parent project's cache should only contain project elements - assert_shared(cli, share, project, 'target.bst', project_name='parent') - assert_shared(cli, share, project, 'app.bst', project_name='parent') - assert_not_shared(cli, share, base_project, 'base-element.bst', project_name='base') + assert_shared(cli, share, project, "target.bst", project_name="parent") + assert_shared(cli, share, project, "app.bst", project_name="parent") + assert_not_shared(cli, share, base_project, "base-element.bst", project_name="base") # The junction project's cache should only contain elements in the junction project - assert_not_shared(cli, base_share, project, 'target.bst', project_name='parent') - assert_not_shared(cli, base_share, project, 'app.bst', project_name='parent') - assert_shared(cli, base_share, base_project, 'base-element.bst', project_name='base') + assert_not_shared(cli, base_share, project, "target.bst", project_name="parent") + assert_not_shared(cli, base_share, project, "app.bst", project_name="parent") + assert_shared(cli, base_share, base_project, "base-element.bst", project_name="base") # Ensure that, from now on, we ignore junction element remotes - junction_data['config'] = {"ignore-junction-remotes": True} + junction_data["config"] = {"ignore-junction-remotes": True} _yaml.roundtrip_dump(junction_data, junction_element) # Now delete everything from the local cache and try to # redownload from the shares. # - cas = os.path.join(cli.directory, 'cas') + cas = os.path.join(cli.directory, "cas") shutil.rmtree(cas) - artifact_dir = os.path.join(cli.directory, 'artifacts') + artifact_dir = os.path.join(cli.directory, "artifacts") shutil.rmtree(artifact_dir) # Assert that nothing is cached locally anymore - state = cli.get_element_state(project, 'target.bst') - assert state != 'cached' - state = cli.get_element_state(base_project, 'base-element.bst') - assert state != 'cached' + state = cli.get_element_state(project, "target.bst") + assert state != "cached" + state = cli.get_element_state(base_project, "base-element.bst") + assert state != "cached" # Now try bst artifact pull - result = cli.run(project=project, args=['artifact', 'pull', '--deps', 'all', 'target.bst']) + result = cli.run(project=project, args=["artifact", "pull", "--deps", "all", "target.bst"]) assert result.exit_code == 0 # And assert that they are again in the local cache, without having built - state = cli.get_element_state(project, 'target.bst') - assert state == 'cached' + state = cli.get_element_state(project, "target.bst") + assert state == "cached" # We shouldn't be able to download base-element! - state = cli.get_element_state(base_project, 'base-element.bst') - assert state != 'cached' + state = cli.get_element_state(base_project, "base-element.bst") + assert state != "cached" diff --git a/tests/artifactcache/pull.py b/tests/artifactcache/pull.py index a42a01bf7..e6eaec960 100644 --- a/tests/artifactcache/pull.py +++ b/tests/artifactcache/pull.py @@ -14,10 +14,7 @@ from tests.testutils import create_artifact_share, dummy_context # Project directory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project", -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",) def tree_maker(cas, tree, directory): @@ -27,7 +24,7 @@ def tree_maker(cas, tree, directory): for directory_node in directory.directories: child_directory = tree.children.add() - with open(cas.objpath(directory_node.digest), 'rb') as f: + with open(cas.objpath(directory_node.digest), "rb") as f: child_directory.ParseFromString(f.read()) tree_maker(cas, tree, child_directory) @@ -38,19 +35,14 @@ def test_pull(cli, tmpdir, datafiles): project_dir = str(datafiles) # Set up an artifact cache. - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # Configure artifact share - cache_dir = os.path.join(str(tmpdir), 'cache') - user_config_file = str(tmpdir.join('buildstream.conf')) + cache_dir = os.path.join(str(tmpdir), "cache") + user_config_file = str(tmpdir.join("buildstream.conf")) user_config = { - 'scheduler': { - 'pushers': 1 - }, - 'artifacts': { - 'url': share.repo, - 'push': True, - }, - 'cachedir': cache_dir + "scheduler": {"pushers": 1}, + "artifacts": {"url": share.repo, "push": True,}, + "cachedir": cache_dir, } # Write down the user configuration file @@ -59,19 +51,19 @@ def test_pull(cli, tmpdir, datafiles): cli.configure(user_config) # First build the project with the artifact cache configured - result = cli.run(project=project_dir, args=['build', 'target.bst']) + result = cli.run(project=project_dir, args=["build", "target.bst"]) result.assert_success() # Assert that we are now cached locally - assert cli.get_element_state(project_dir, 'target.bst') == 'cached' + assert cli.get_element_state(project_dir, "target.bst") == "cached" # Assert that we shared/pushed the cached artifact - assert share.get_artifact(cli.get_artifact_name(project_dir, 'test', 'target.bst')) + assert share.get_artifact(cli.get_artifact_name(project_dir, "test", "target.bst")) # Delete the artifact locally - cli.remove_artifact_from_cache(project_dir, 'target.bst') + cli.remove_artifact_from_cache(project_dir, "target.bst") # Assert that we are not cached locally anymore - assert cli.get_element_state(project_dir, 'target.bst') != 'cached' + assert cli.get_element_state(project_dir, "target.bst") != "cached" with dummy_context(config=user_config_file) as context: # Load the project @@ -79,13 +71,13 @@ def test_pull(cli, tmpdir, datafiles): project.ensure_fully_loaded() # Assert that the element's artifact is **not** cached - element = project.load_elements(['target.bst'])[0] - element_key = cli.get_element_key(project_dir, 'target.bst') + element = project.load_elements(["target.bst"])[0] + element_key = cli.get_element_key(project_dir, "target.bst") assert not cli.artifact.is_cached(cache_dir, element, element_key) context.cachedir = cache_dir - context.casdir = os.path.join(cache_dir, 'cas') - context.tmpdir = os.path.join(cache_dir, 'tmp') + context.casdir = os.path.join(cache_dir, "cas") + context.tmpdir = os.path.join(cache_dir, "tmp") # Load the project manually project = Project(project_dir, context) @@ -97,8 +89,7 @@ def test_pull(cli, tmpdir, datafiles): # Manually setup the CAS remote artifactcache.setup_remotes(use_config=True) - assert artifactcache.has_push_remotes(plugin=element), \ - "No remote configured for element target.bst" + assert artifactcache.has_push_remotes(plugin=element), "No remote configured for element target.bst" assert artifactcache.pull(element, element_key), "Pull operation failed" assert cli.artifact.is_cached(cache_dir, element, element_key) @@ -109,19 +100,14 @@ def test_pull_tree(cli, tmpdir, datafiles): project_dir = str(datafiles) # Set up an artifact cache. - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # Configure artifact share - rootcache_dir = os.path.join(str(tmpdir), 'cache') - user_config_file = str(tmpdir.join('buildstream.conf')) + rootcache_dir = os.path.join(str(tmpdir), "cache") + user_config_file = str(tmpdir.join("buildstream.conf")) user_config = { - 'scheduler': { - 'pushers': 1 - }, - 'artifacts': { - 'url': share.repo, - 'push': True, - }, - 'cachedir': rootcache_dir + "scheduler": {"pushers": 1}, + "artifacts": {"url": share.repo, "push": True,}, + "cachedir": rootcache_dir, } # Write down the user configuration file @@ -130,13 +116,13 @@ def test_pull_tree(cli, tmpdir, datafiles): cli.configure(user_config) # First build the project with the artifact cache configured - result = cli.run(project=project_dir, args=['build', 'target.bst']) + result = cli.run(project=project_dir, args=["build", "target.bst"]) result.assert_success() # Assert that we are now cached locally - assert cli.get_element_state(project_dir, 'target.bst') == 'cached' + assert cli.get_element_state(project_dir, "target.bst") == "cached" # Assert that we shared/pushed the cached artifact - assert share.get_artifact(cli.get_artifact_name(project_dir, 'test', 'target.bst')) + assert share.get_artifact(cli.get_artifact_name(project_dir, "test", "target.bst")) with dummy_context(config=user_config_file) as context: # Load the project and CAS cache @@ -145,8 +131,8 @@ def test_pull_tree(cli, tmpdir, datafiles): cas = context.get_cascache() # Assert that the element's artifact is cached - element = project.load_elements(['target.bst'])[0] - element_key = cli.get_element_key(project_dir, 'target.bst') + element = project.load_elements(["target.bst"])[0] + element_key = cli.get_element_key(project_dir, "target.bst") assert cli.artifact.is_cached(rootcache_dir, element, element_key) # Retrieve the Directory object from the cached artifact @@ -159,7 +145,7 @@ def test_pull_tree(cli, tmpdir, datafiles): directory = remote_execution_pb2.Directory() - with open(cas.objpath(artifact_digest), 'rb') as f: + with open(cas.objpath(artifact_digest), "rb") as f: directory.ParseFromString(f.read()) # Build the Tree object while we are still cached @@ -172,15 +158,14 @@ def test_pull_tree(cli, tmpdir, datafiles): assert tree_hash and tree_size # Now delete the artifact locally - cli.remove_artifact_from_cache(project_dir, 'target.bst') + cli.remove_artifact_from_cache(project_dir, "target.bst") # Assert that we are not cached locally anymore artifactcache.close_grpc_channels() cas.close_grpc_channels() - assert cli.get_element_state(project_dir, 'target.bst') != 'cached' + assert cli.get_element_state(project_dir, "target.bst") != "cached" - tree_digest = remote_execution_pb2.Digest(hash=tree_hash, - size_bytes=tree_size) + tree_digest = remote_execution_pb2.Digest(hash=tree_hash, size_bytes=tree_size) # Pull the artifact using the Tree object directory_digest = artifactcache.pull_tree(project, artifact_digest) @@ -189,8 +174,7 @@ def test_pull_tree(cli, tmpdir, datafiles): # Directory size now zero with AaaP and stack element commit #1cbc5e63dc assert directory_hash and not directory_size - directory_digest = remote_execution_pb2.Digest(hash=directory_hash, - size_bytes=directory_size) + directory_digest = remote_execution_pb2.Digest(hash=directory_hash, size_bytes=directory_size) # Ensure the entire Tree stucture has been pulled assert os.path.exists(cas.objpath(directory_digest)) diff --git a/tests/artifactcache/push.py b/tests/artifactcache/push.py index 62c443d61..238d5f7ef 100644 --- a/tests/artifactcache/push.py +++ b/tests/artifactcache/push.py @@ -14,10 +14,7 @@ from tests.testutils import create_artifact_share, create_split_share, dummy_con # Project directory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project", -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",) # Push the given element and return its artifact key for assertions. @@ -28,8 +25,8 @@ def _push(cli, cache_dir, project_dir, config_file, target): project.ensure_fully_loaded() # Assert that the element's artifact is cached - element = project.load_elements(['target.bst'])[0] - element_key = cli.get_element_key(project_dir, 'target.bst') + element = project.load_elements(["target.bst"])[0] + element_key = cli.get_element_key(project_dir, "target.bst") assert cli.artifact.is_cached(cache_dir, element, element_key) # Create a local artifact cache handle @@ -46,8 +43,7 @@ def _push(cli, cache_dir, project_dir, config_file, target): artifactcache.setup_remotes(use_config=True) artifactcache.initialize_remotes() - assert artifactcache.has_push_remotes(plugin=element), \ - "No remote configured for element target.bst" + assert artifactcache.has_push_remotes(plugin=element), "No remote configured for element target.bst" assert element._push(), "Push operation failed" return element_key @@ -58,32 +54,27 @@ def test_push(cli, tmpdir, datafiles): project_dir = str(datafiles) # First build the project without the artifact cache configured - result = cli.run(project=project_dir, args=['build', 'target.bst']) + result = cli.run(project=project_dir, args=["build", "target.bst"]) result.assert_success() # Assert that we are now cached locally - assert cli.get_element_state(project_dir, 'target.bst') == 'cached' + assert cli.get_element_state(project_dir, "target.bst") == "cached" # Set up an artifact cache. - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # Configure artifact share - rootcache_dir = os.path.join(str(tmpdir), 'cache') - user_config_file = str(tmpdir.join('buildstream.conf')) + rootcache_dir = os.path.join(str(tmpdir), "cache") + user_config_file = str(tmpdir.join("buildstream.conf")) user_config = { - 'scheduler': { - 'pushers': 1 - }, - 'artifacts': { - 'url': share.repo, - 'push': True, - }, - 'cachedir': rootcache_dir + "scheduler": {"pushers": 1}, + "artifacts": {"url": share.repo, "push": True,}, + "cachedir": rootcache_dir, } # Write down the user configuration file _yaml.roundtrip_dump(user_config, file=user_config_file) - element_key = _push(cli, rootcache_dir, project_dir, user_config_file, 'target.bst') - assert share.get_artifact(cli.get_artifact_name(project_dir, 'test', 'target.bst', cache_key=element_key)) + element_key = _push(cli, rootcache_dir, project_dir, user_config_file, "target.bst") + assert share.get_artifact(cli.get_artifact_name(project_dir, "test", "target.bst", cache_key=element_key)) @pytest.mark.datafiles(DATA_DIR) @@ -91,41 +82,33 @@ def test_push_split(cli, tmpdir, datafiles): project_dir = str(datafiles) # First build the project without the artifact cache configured - result = cli.run(project=project_dir, args=['build', 'target.bst']) + result = cli.run(project=project_dir, args=["build", "target.bst"]) result.assert_success() # Assert that we are now cached locally - assert cli.get_element_state(project_dir, 'target.bst') == 'cached' + assert cli.get_element_state(project_dir, "target.bst") == "cached" - indexshare = os.path.join(str(tmpdir), 'indexshare') - storageshare = os.path.join(str(tmpdir), 'storageshare') + indexshare = os.path.join(str(tmpdir), "indexshare") + storageshare = os.path.join(str(tmpdir), "storageshare") # Set up an artifact cache. with create_split_share(indexshare, storageshare) as (index, storage): - rootcache_dir = os.path.join(str(tmpdir), 'cache') + rootcache_dir = os.path.join(str(tmpdir), "cache") user_config = { - 'scheduler': { - 'pushers': 1 - }, - 'artifacts': [{ - 'url': index.repo, - 'push': True, - 'type': 'index' - }, { - 'url': storage.repo, - 'push': True, - 'type': 'storage' - }], - 'cachedir': rootcache_dir + "scheduler": {"pushers": 1}, + "artifacts": [ + {"url": index.repo, "push": True, "type": "index"}, + {"url": storage.repo, "push": True, "type": "storage"}, + ], + "cachedir": rootcache_dir, } - config_path = str(tmpdir.join('buildstream.conf')) + config_path = str(tmpdir.join("buildstream.conf")) _yaml.roundtrip_dump(user_config, file=config_path) - element_key = _push(cli, rootcache_dir, project_dir, config_path, 'target.bst') - proto = index.get_artifact_proto(cli.get_artifact_name(project_dir, - 'test', - 'target.bst', - cache_key=element_key)) + element_key = _push(cli, rootcache_dir, project_dir, config_path, "target.bst") + proto = index.get_artifact_proto( + cli.get_artifact_name(project_dir, "test", "target.bst", cache_key=element_key) + ) assert storage.get_cas_files(proto) is not None @@ -134,20 +117,15 @@ def test_push_message(tmpdir, datafiles): project_dir = str(datafiles) # Set up an artifact cache. - artifactshare = os.path.join(str(tmpdir), 'artifactshare') + artifactshare = os.path.join(str(tmpdir), "artifactshare") with create_artifact_share(artifactshare) as share: # Configure artifact share - rootcache_dir = os.path.join(str(tmpdir), 'cache') - user_config_file = str(tmpdir.join('buildstream.conf')) + rootcache_dir = os.path.join(str(tmpdir), "cache") + user_config_file = str(tmpdir.join("buildstream.conf")) user_config = { - 'scheduler': { - 'pushers': 1 - }, - 'artifacts': { - 'url': share.repo, - 'push': True, - }, - 'cachedir': rootcache_dir + "scheduler": {"pushers": 1}, + "artifacts": {"url": share.repo, "push": True,}, + "cachedir": rootcache_dir, } # Write down the user configuration file @@ -166,15 +144,16 @@ def test_push_message(tmpdir, datafiles): artifactcache.initialize_remotes() assert artifactcache.has_push_remotes() - command = remote_execution_pb2.Command(arguments=['/usr/bin/gcc', '--help'], - working_directory='/buildstream-build', - output_directories=['/buildstream-install']) + command = remote_execution_pb2.Command( + arguments=["/usr/bin/gcc", "--help"], + working_directory="/buildstream-build", + output_directories=["/buildstream-install"], + ) # Push the message object command_digest = artifactcache.push_message(project, command) message_hash, message_size = command_digest.hash, command_digest.size_bytes assert message_hash and message_size - message_digest = remote_execution_pb2.Digest(hash=message_hash, - size_bytes=message_size) + message_digest = remote_execution_pb2.Digest(hash=message_hash, size_bytes=message_size) assert share.has_object(message_digest) diff --git a/tests/cachekey/cachekey.py b/tests/cachekey/cachekey.py index fa93f5746..b669bacaa 100644 --- a/tests/cachekey/cachekey.py +++ b/tests/cachekey/cachekey.py @@ -62,7 +62,7 @@ def element_filename(project_dir, element_name, alt_suffix=None): if alt_suffix: # Just in case... - assert element_name.endswith('.bst') + assert element_name.endswith(".bst") # Chop off the 'bst' in '.bst' and add the new suffix element_name = element_name[:-3] @@ -93,18 +93,20 @@ def load_expected_keys(project_dir, actual_keys, raise_error=True): expected_keys = OrderedDict() for element_name in actual_keys: - expected = element_filename(project_dir, element_name, 'expected') + expected = element_filename(project_dir, element_name, "expected") try: - with open(expected, 'r') as f: + with open(expected, "r") as f: expected_key = f.read() expected_key = expected_key.strip() except FileNotFoundError: expected_key = None if raise_error: - raise Exception("Cache key test needs update, " + - "expected file {} not found.\n\n".format(expected) + - "Use tests/cachekey/update.py to automatically " + - "update this test case") + raise Exception( + "Cache key test needs update, " + + "expected file {} not found.\n\n".format(expected) + + "Use tests/cachekey/update.py to automatically " + + "update this test case" + ) expected_keys[element_name] = expected_key @@ -127,13 +129,17 @@ def assert_cache_keys(project_dir, output): if mismatches: info = "" for element_name in mismatches: - info += " Element: {}\n".format(element_name) + \ - " Expected: {}\n".format(expected_keys[element_name]) + \ - " Actual: {}\n".format(actual_keys[element_name]) + info += ( + " Element: {}\n".format(element_name) + + " Expected: {}\n".format(expected_keys[element_name]) + + " Actual: {}\n".format(actual_keys[element_name]) + ) - raise AssertionError("Cache key mismatches occurred:\n{}\n".format(info) + - "Use tests/cachekey/update.py to automatically " + - "update this test case") + raise AssertionError( + "Cache key mismatches occurred:\n{}\n".format(info) + + "Use tests/cachekey/update.py to automatically " + + "update this test case" + ) ############################################## @@ -141,18 +147,14 @@ def assert_cache_keys(project_dir, output): ############################################## # Project directory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project", -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",) # The cache key test uses a project which exercises all plugins, # so we cant run it at all if we dont have them installed. # -@pytest.mark.skipif(MACHINE_ARCH != 'x86-64', - reason='Cache keys depend on architecture') -@pytest.mark.skipif(not IS_LINUX, reason='Only available on linux') +@pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Cache keys depend on architecture") +@pytest.mark.skipif(not IS_LINUX, reason="Only available on linux") @pytest.mark.skipif(HAVE_BZR is False, reason="bzr is not available") @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") @pytest.mark.datafiles(DATA_DIR) @@ -163,55 +165,48 @@ def test_cache_key(datafiles, cli): # versions of setuptools fail to preserve symbolic links # when creating a source distribution, causing this test # to fail from a dist tarball. - goodbye_link = os.path.join(project, 'files', 'local', - 'usr', 'bin', 'goodbye') + goodbye_link = os.path.join(project, "files", "local", "usr", "bin", "goodbye") os.unlink(goodbye_link) - os.symlink('hello', goodbye_link) + os.symlink("hello", goodbye_link) # pytest-datafiles does not copy mode bits # https://github.com/omarkohl/pytest-datafiles/issues/11 os.chmod(goodbye_link, 0o755) - result = cli.run(project=project, silent=True, args=[ - 'show', - '--format', '%{name}::%{full-key}', - 'target.bst' - ]) + result = cli.run(project=project, silent=True, args=["show", "--format", "%{name}::%{full-key}", "target.bst"]) result.assert_success() assert_cache_keys(project, result.output) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("first_warnings, second_warnings, identical_keys", [ - [[], [], True], - [[], [CoreWarnings.REF_NOT_IN_TRACK], False], - [[CoreWarnings.REF_NOT_IN_TRACK], [], False], - [[CoreWarnings.REF_NOT_IN_TRACK], [CoreWarnings.REF_NOT_IN_TRACK], True], - [[CoreWarnings.REF_NOT_IN_TRACK, CoreWarnings.OVERLAPS], - [CoreWarnings.OVERLAPS, CoreWarnings.REF_NOT_IN_TRACK], True], -]) +@pytest.mark.parametrize( + "first_warnings, second_warnings, identical_keys", + [ + [[], [], True], + [[], [CoreWarnings.REF_NOT_IN_TRACK], False], + [[CoreWarnings.REF_NOT_IN_TRACK], [], False], + [[CoreWarnings.REF_NOT_IN_TRACK], [CoreWarnings.REF_NOT_IN_TRACK], True], + [ + [CoreWarnings.REF_NOT_IN_TRACK, CoreWarnings.OVERLAPS], + [CoreWarnings.OVERLAPS, CoreWarnings.REF_NOT_IN_TRACK], + True, + ], + ], +) def test_cache_key_fatal_warnings(cli, tmpdir, first_warnings, second_warnings, identical_keys): # Builds project, Runs bst show, gathers cache keys def run_get_cache_key(project_name, warnings): - config = { - 'name': project_name, - 'element-path': 'elements', - 'fatal-warnings': warnings - } + config = {"name": project_name, "element-path": "elements", "fatal-warnings": warnings} project_dir = tmpdir.mkdir(project_name) - project_config_file = str(project_dir.join('project.conf')) + project_config_file = str(project_dir.join("project.conf")) _yaml.roundtrip_dump(config, file=project_config_file) - elem_dir = project_dir.mkdir('elements') - element_file = str(elem_dir.join('stack.bst')) - _yaml.roundtrip_dump({'kind': 'stack'}, file=element_file) + elem_dir = project_dir.mkdir("elements") + element_file = str(elem_dir.join("stack.bst")) + _yaml.roundtrip_dump({"kind": "stack"}, file=element_file) - result = cli.run(project=str(project_dir), args=[ - 'show', - '--format', '%{name}::%{full-key}', - 'stack.bst' - ]) + result = cli.run(project=str(project_dir), args=["show", "--format", "%{name}::%{full-key}", "stack.bst"]) return result.output # Returns true if all keys are identical @@ -226,34 +221,20 @@ def test_cache_key_fatal_warnings(cli, tmpdir, first_warnings, second_warnings, @pytest.mark.datafiles(DATA_DIR) def test_keys_stable_over_targets(cli, datafiles): - root_element = 'elements/key-stability/top-level.bst' - target1 = 'elements/key-stability/t1.bst' - target2 = 'elements/key-stability/t2.bst' + root_element = "elements/key-stability/top-level.bst" + target1 = "elements/key-stability/t1.bst" + target2 = "elements/key-stability/t2.bst" project = str(datafiles) - full_graph_result = cli.run(project=project, args=[ - 'show', - '--format', '%{name}::%{full-key}', - root_element - ]) + full_graph_result = cli.run(project=project, args=["show", "--format", "%{name}::%{full-key}", root_element]) full_graph_result.assert_success() all_cache_keys = parse_output_keys(full_graph_result.output) - ordering1_result = cli.run(project=project, args=[ - 'show', - '--format', '%{name}::%{full-key}', - target1, - target2 - ]) + ordering1_result = cli.run(project=project, args=["show", "--format", "%{name}::%{full-key}", target1, target2]) ordering1_result.assert_success() ordering1_cache_keys = parse_output_keys(ordering1_result.output) - ordering2_result = cli.run(project=project, args=[ - 'show', - '--format', '%{name}::%{full-key}', - target2, - target1 - ]) + ordering2_result = cli.run(project=project, args=["show", "--format", "%{name}::%{full-key}", target2, target1]) ordering2_result.assert_success() ordering2_cache_keys = parse_output_keys(ordering2_result.output) diff --git a/tests/cachekey/update.py b/tests/cachekey/update.py index feda5dbde..49af39fe1 100755 --- a/tests/cachekey/update.py +++ b/tests/cachekey/update.py @@ -25,31 +25,28 @@ except ImportError: from .cachekey import element_filename, parse_output_keys, load_expected_keys # Project directory -PROJECT_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project", -) +PROJECT_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",) def write_expected_key(element_name, actual_key): - expected_file = element_filename(PROJECT_DIR, element_name, 'expected') - with open(expected_file, 'w') as f: + expected_file = element_filename(PROJECT_DIR, element_name, "expected") + with open(expected_file, "w") as f: f.write(actual_key) def update_keys(): with tempfile.TemporaryDirectory(dir=PROJECT_DIR) as tmpdir: - directory = os.path.join(tmpdir, 'cache') + directory = os.path.join(tmpdir, "cache") os.makedirs(directory) cli = Cli(directory, verbose=True) # Run bst show - result = cli.run(project=PROJECT_DIR, silent=True, args=[ - '--no-colors', - 'show', '--format', '%{name}::%{full-key}', - 'target.bst' - ]) + result = cli.run( + project=PROJECT_DIR, + silent=True, + args=["--no-colors", "show", "--format", "%{name}::%{full-key}", "target.bst"], + ) # Load the actual keys, and the expected ones if they exist if not result.output: @@ -59,7 +56,7 @@ def update_keys(): expected_keys = load_expected_keys(PROJECT_DIR, actual_keys, raise_error=False) for element_name in actual_keys: - expected = element_filename(PROJECT_DIR, element_name, 'expected') + expected = element_filename(PROJECT_DIR, element_name, "expected") if actual_keys[element_name] != expected_keys[element_name]: if not expected_keys[element_name]: @@ -70,10 +67,10 @@ def update_keys(): write_expected_key(element_name, actual_keys[element_name]) -if __name__ == '__main__': +if __name__ == "__main__": # patch the environment BST_TEST_SUITE value to something if it's not # present. This avoids an exception thrown at the cli level - bst = 'BST_TEST_SUITE' - mock_bst = os.environ.get(bst, 'True') + bst = "BST_TEST_SUITE" + mock_bst = os.environ.get(bst, "True") with mock.patch.dict(os.environ, {**os.environ, bst: mock_bst}): update_keys() diff --git a/tests/conftest.py b/tests/conftest.py index 216c83893..cfca4ad06 100755 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -45,28 +45,26 @@ from tests.testutils.repo.zip import Zip # Implement pytest option # ################################################# def pytest_addoption(parser): - parser.addoption('--integration', action='store_true', default=False, - help='Run integration tests') + parser.addoption("--integration", action="store_true", default=False, help="Run integration tests") - parser.addoption('--remote-execution', action='store_true', default=False, - help='Run remote-execution tests only') + parser.addoption("--remote-execution", action="store_true", default=False, help="Run remote-execution tests only") def pytest_runtest_setup(item): # Without --integration: skip tests not marked with 'integration' - if not item.config.getvalue('integration'): - if item.get_closest_marker('integration'): - pytest.skip('skipping integration test') + if not item.config.getvalue("integration"): + if item.get_closest_marker("integration"): + pytest.skip("skipping integration test") # With --remote-execution: only run tests marked with 'remoteexecution' - if item.config.getvalue('remote_execution'): - if not item.get_closest_marker('remoteexecution'): - pytest.skip('skipping non remote-execution test') + if item.config.getvalue("remote_execution"): + if not item.get_closest_marker("remoteexecution"): + pytest.skip("skipping non remote-execution test") # Without --remote-execution: skip tests marked with 'remoteexecution' else: - if item.get_closest_marker('remoteexecution'): - pytest.skip('skipping remote-execution test') + if item.get_closest_marker("remoteexecution"): + pytest.skip("skipping remote-execution test") ################################################# @@ -75,30 +73,29 @@ def pytest_runtest_setup(item): # # This is returned by the `remote_services` fixture # -class RemoteServices(): - +class RemoteServices: def __init__(self, **kwargs): - self.action_service = kwargs.get('action_service') - self.artifact_service = kwargs.get('artifact_service') - self.exec_service = kwargs.get('exec_service') - self.source_service = kwargs.get('source_service') - self.storage_service = kwargs.get('storage_service') + self.action_service = kwargs.get("action_service") + self.artifact_service = kwargs.get("artifact_service") + self.exec_service = kwargs.get("exec_service") + self.source_service = kwargs.get("source_service") + self.storage_service = kwargs.get("storage_service") -@pytest.fixture(scope='session') +@pytest.fixture(scope="session") def remote_services(request): kwargs = {} # Look for remote services configuration in environment. - if 'ARTIFACT_CACHE_SERVICE' in os.environ: - kwargs['artifact_service'] = os.environ.get('ARTIFACT_CACHE_SERVICE') + if "ARTIFACT_CACHE_SERVICE" in os.environ: + kwargs["artifact_service"] = os.environ.get("ARTIFACT_CACHE_SERVICE") - if 'REMOTE_EXECUTION_SERVICE' in os.environ: - kwargs['action_service'] = os.environ.get('REMOTE_EXECUTION_SERVICE') - kwargs['exec_service'] = os.environ.get('REMOTE_EXECUTION_SERVICE') - kwargs['storage_service'] = os.environ.get('REMOTE_EXECUTION_SERVICE') + if "REMOTE_EXECUTION_SERVICE" in os.environ: + kwargs["action_service"] = os.environ.get("REMOTE_EXECUTION_SERVICE") + kwargs["exec_service"] = os.environ.get("REMOTE_EXECUTION_SERVICE") + kwargs["storage_service"] = os.environ.get("REMOTE_EXECUTION_SERVICE") - if 'SOURCE_CACHE_SERVICE' in os.environ: - kwargs['source_service'] = os.environ.get('SOURCE_CACHE_SERVICE') + if "SOURCE_CACHE_SERVICE" in os.environ: + kwargs["source_service"] = os.environ.get("SOURCE_CACHE_SERVICE") return RemoteServices(**kwargs) @@ -106,10 +103,10 @@ def remote_services(request): ################################################# # Setup for templated source tests # ################################################# -register_repo_kind('git', Git, None) -register_repo_kind('bzr', Bzr, None) -register_repo_kind('tar', Tar, None) -register_repo_kind('zip', Zip, None) +register_repo_kind("git", Git, None) +register_repo_kind("bzr", Bzr, None) +register_repo_kind("tar", Tar, None) +register_repo_kind("zip", Zip, None) # This hook enables pytest to collect the templated source tests from @@ -124,10 +121,10 @@ def pytest_sessionstart(session): @pytest.fixture(scope="session", autouse=True) def set_xdg_paths(pytestconfig): for env_var, default in [ - ("HOME", "tmp"), - ("XDG_CACHE_HOME", "tmp/cache"), - ("XDG_CONFIG_HOME", "tmp/config"), - ("XDG_DATA_HOME", "tmp/share"), + ("HOME", "tmp"), + ("XDG_CACHE_HOME", "tmp/cache"), + ("XDG_CONFIG_HOME", "tmp/config"), + ("XDG_DATA_HOME", "tmp/share"), ]: value = os.environ.get("BST_TEST_{}".format(env_var)) if value is None: @@ -141,10 +138,9 @@ def pytest_configure(config): # possible. Note that some tests implicitly set the start method by using # multiprocessing. If we wait for bst to do it, it will already be too # late. - if 'BST_FORCE_START_METHOD' in os.environ: - start_method = os.environ['BST_FORCE_START_METHOD'] + if "BST_FORCE_START_METHOD" in os.environ: + start_method = os.environ["BST_FORCE_START_METHOD"] multiprocessing.set_start_method(start_method) print( - "Multiprocessing method set to:", - start_method, + "Multiprocessing method set to:", start_method, ) diff --git a/tests/elements/filter.py b/tests/elements/filter.py index 99370052e..0befa46f8 100644 --- a/tests/elements/filter.py +++ b/tests/elements/filter.py @@ -12,188 +12,186 @@ from buildstream.testing._utils.site import HAVE_SANDBOX from buildstream._exceptions import ErrorDomain from buildstream import _yaml -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - 'filter', -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "filter",) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_include(datafiles, cli, tmpdir): project = str(datafiles) - result = cli.run(project=project, args=['build', 'output-include.bst']) + result = cli.run(project=project, args=["build", "output-include.bst"]) result.assert_success() - checkout = os.path.join(tmpdir.dirname, tmpdir.basename, 'checkout') - result = cli.run(project=project, args=['artifact', 'checkout', 'output-include.bst', '--directory', checkout]) + checkout = os.path.join(tmpdir.dirname, tmpdir.basename, "checkout") + result = cli.run(project=project, args=["artifact", "checkout", "output-include.bst", "--directory", checkout]) result.assert_success() assert os.path.exists(os.path.join(checkout, "foo")) assert not os.path.exists(os.path.join(checkout, "bar")) -@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox') -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic')) +@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox") +@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_include_dynamic(datafiles, cli, tmpdir): project = str(datafiles) - result = cli.run(project=project, args=['build', 'output-dynamic-include.bst']) + result = cli.run(project=project, args=["build", "output-dynamic-include.bst"]) result.assert_success() - checkout = os.path.join(tmpdir.dirname, tmpdir.basename, 'checkout') - result = cli.run(project=project, args=['artifact', 'checkout', 'output-dynamic-include.bst', - '--directory', checkout]) + checkout = os.path.join(tmpdir.dirname, tmpdir.basename, "checkout") + result = cli.run( + project=project, args=["artifact", "checkout", "output-dynamic-include.bst", "--directory", checkout] + ) result.assert_success() assert os.path.exists(os.path.join(checkout, "foo")) assert not os.path.exists(os.path.join(checkout, "bar")) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_exclude(datafiles, cli, tmpdir): project = str(datafiles) - result = cli.run(project=project, args=['build', 'output-exclude.bst']) + result = cli.run(project=project, args=["build", "output-exclude.bst"]) result.assert_success() - checkout = os.path.join(tmpdir.dirname, tmpdir.basename, 'checkout') - result = cli.run(project=project, args=['artifact', 'checkout', 'output-exclude.bst', '--directory', checkout]) + checkout = os.path.join(tmpdir.dirname, tmpdir.basename, "checkout") + result = cli.run(project=project, args=["artifact", "checkout", "output-exclude.bst", "--directory", checkout]) result.assert_success() assert not os.path.exists(os.path.join(checkout, "foo")) assert os.path.exists(os.path.join(checkout, "bar")) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_orphans(datafiles, cli, tmpdir): project = str(datafiles) - result = cli.run(project=project, args=['build', 'output-orphans.bst']) + result = cli.run(project=project, args=["build", "output-orphans.bst"]) result.assert_success() - checkout = os.path.join(tmpdir.dirname, tmpdir.basename, 'checkout') - result = cli.run(project=project, args=['artifact', 'checkout', 'output-orphans.bst', '--directory', checkout]) + checkout = os.path.join(tmpdir.dirname, tmpdir.basename, "checkout") + result = cli.run(project=project, args=["artifact", "checkout", "output-orphans.bst", "--directory", checkout]) result.assert_success() assert os.path.exists(os.path.join(checkout, "baz")) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_deps_ok(datafiles, cli): project = str(datafiles) - result = cli.run(project=project, args=['build', 'deps-permitted.bst']) + result = cli.run(project=project, args=["build", "deps-permitted.bst"]) result.assert_success() - result = cli.run(project=project, - args=['show', '--deps=run', "--format='%{name}'", 'deps-permitted.bst']) + result = cli.run(project=project, args=["show", "--deps=run", "--format='%{name}'", "deps-permitted.bst"]) result.assert_success() - assert 'output-exclude.bst' in result.output - assert 'output-orphans.bst' in result.output + assert "output-exclude.bst" in result.output + assert "output-orphans.bst" in result.output -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_forbid_sources(datafiles, cli): project = str(datafiles) - result = cli.run(project=project, args=['build', 'forbidden-source.bst']) - result.assert_main_error(ErrorDomain.ELEMENT, 'element-forbidden-sources') + result = cli.run(project=project, args=["build", "forbidden-source.bst"]) + result.assert_main_error(ErrorDomain.ELEMENT, "element-forbidden-sources") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_forbid_multi_bdep(datafiles, cli): project = str(datafiles) - result = cli.run(project=project, args=['build', 'forbidden-multi-bdep.bst']) - result.assert_main_error(ErrorDomain.ELEMENT, 'filter-bdepend-wrong-count') + result = cli.run(project=project, args=["build", "forbidden-multi-bdep.bst"]) + result.assert_main_error(ErrorDomain.ELEMENT, "filter-bdepend-wrong-count") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_forbid_no_bdep(datafiles, cli): project = str(datafiles) - result = cli.run(project=project, args=['build', 'forbidden-no-bdep.bst']) - result.assert_main_error(ErrorDomain.ELEMENT, 'filter-bdepend-wrong-count') + result = cli.run(project=project, args=["build", "forbidden-no-bdep.bst"]) + result.assert_main_error(ErrorDomain.ELEMENT, "filter-bdepend-wrong-count") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_forbid_also_rdep(datafiles, cli): project = str(datafiles) - result = cli.run(project=project, args=['build', 'forbidden-also-rdep.bst']) - result.assert_main_error(ErrorDomain.ELEMENT, 'filter-bdepend-also-rdepend') + result = cli.run(project=project, args=["build", "forbidden-also-rdep.bst"]) + result.assert_main_error(ErrorDomain.ELEMENT, "filter-bdepend-also-rdepend") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_workspace_open(datafiles, cli, tmpdir): project = str(datafiles) workspace_dir = os.path.join(tmpdir.dirname, tmpdir.basename, "workspace") - result = cli.run(project=project, args=['workspace', 'open', '--directory', workspace_dir, 'deps-permitted.bst']) + result = cli.run(project=project, args=["workspace", "open", "--directory", workspace_dir, "deps-permitted.bst"]) result.assert_success() assert os.path.exists(os.path.join(workspace_dir, "foo")) assert os.path.exists(os.path.join(workspace_dir, "bar")) assert os.path.exists(os.path.join(workspace_dir, "baz")) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_workspace_open_multi(datafiles, cli): project = str(datafiles) - result = cli.run(cwd=project, project=project, args=['workspace', 'open', 'deps-permitted.bst', - 'output-orphans.bst']) + result = cli.run( + cwd=project, project=project, args=["workspace", "open", "deps-permitted.bst", "output-orphans.bst"] + ) result.assert_success() assert os.path.exists(os.path.join(project, "input")) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_workspace_build(datafiles, cli, tmpdir): project = str(datafiles) tempdir = os.path.join(tmpdir.dirname, tmpdir.basename) workspace_dir = os.path.join(tempdir, "workspace") - result = cli.run(project=project, args=['workspace', 'open', '--directory', workspace_dir, 'output-orphans.bst']) + result = cli.run(project=project, args=["workspace", "open", "--directory", workspace_dir, "output-orphans.bst"]) result.assert_success() src = os.path.join(workspace_dir, "foo") dst = os.path.join(workspace_dir, "quux") shutil.copyfile(src, dst) - result = cli.run(project=project, args=['build', 'output-orphans.bst']) + result = cli.run(project=project, args=["build", "output-orphans.bst"]) result.assert_success() checkout_dir = os.path.join(tempdir, "checkout") - result = cli.run(project=project, args=['artifact', 'checkout', 'output-orphans.bst', '--directory', checkout_dir]) + result = cli.run(project=project, args=["artifact", "checkout", "output-orphans.bst", "--directory", checkout_dir]) result.assert_success() assert os.path.exists(os.path.join(checkout_dir, "quux")) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_workspace_close(datafiles, cli, tmpdir): project = str(datafiles) tempdir = os.path.join(tmpdir.dirname, tmpdir.basename) workspace_dir = os.path.join(tempdir, "workspace") - result = cli.run(project=project, args=['workspace', 'open', '--directory', workspace_dir, 'output-orphans.bst']) + result = cli.run(project=project, args=["workspace", "open", "--directory", workspace_dir, "output-orphans.bst"]) result.assert_success() src = os.path.join(workspace_dir, "foo") dst = os.path.join(workspace_dir, "quux") shutil.copyfile(src, dst) - result = cli.run(project=project, args=['workspace', 'close', 'deps-permitted.bst']) + result = cli.run(project=project, args=["workspace", "close", "deps-permitted.bst"]) result.assert_success() - result = cli.run(project=project, args=['build', 'output-orphans.bst']) + result = cli.run(project=project, args=["build", "output-orphans.bst"]) result.assert_success() checkout_dir = os.path.join(tempdir, "checkout") - result = cli.run(project=project, args=['artifact', 'checkout', 'output-orphans.bst', '--directory', checkout_dir]) + result = cli.run(project=project, args=["artifact", "checkout", "output-orphans.bst", "--directory", checkout_dir]) result.assert_success() assert not os.path.exists(os.path.join(checkout_dir, "quux")) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_workspace_reset(datafiles, cli, tmpdir): project = str(datafiles) tempdir = os.path.join(tmpdir.dirname, tmpdir.basename) workspace_dir = os.path.join(tempdir, "workspace") - result = cli.run(project=project, args=['workspace', 'open', '--directory', workspace_dir, 'output-orphans.bst']) + result = cli.run(project=project, args=["workspace", "open", "--directory", workspace_dir, "output-orphans.bst"]) result.assert_success() src = os.path.join(workspace_dir, "foo") dst = os.path.join(workspace_dir, "quux") shutil.copyfile(src, dst) - result = cli.run(project=project, args=['workspace', 'reset', 'deps-permitted.bst']) + result = cli.run(project=project, args=["workspace", "reset", "deps-permitted.bst"]) result.assert_success() - result = cli.run(project=project, args=['build', 'output-orphans.bst']) + result = cli.run(project=project, args=["build", "output-orphans.bst"]) result.assert_success() checkout_dir = os.path.join(tempdir, "checkout") - result = cli.run(project=project, args=['artifact', 'checkout', 'output-orphans.bst', '--directory', checkout_dir]) + result = cli.run(project=project, args=["artifact", "checkout", "output-orphans.bst", "--directory", checkout_dir]) result.assert_success() assert not os.path.exists(os.path.join(checkout_dir, "quux")) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_track(datafiles, cli, tmpdir): - repo = create_repo('git', str(tmpdir)) + repo = create_repo("git", str(tmpdir)) ref = repo.create(os.path.join(str(datafiles), "files")) elements_dir = os.path.join(str(tmpdir), "elements") project = str(tmpdir) @@ -214,26 +212,16 @@ def test_filter_track(datafiles, cli, tmpdir): input_file = os.path.join(elements_dir, input_name) _yaml.roundtrip_dump(input_config, input_file) - filter1_config = { - "kind": "filter", - "depends": [ - {"filename": input_name, "type": "build"} - ] - } + filter1_config = {"kind": "filter", "depends": [{"filename": input_name, "type": "build"}]} filter1_file = os.path.join(elements_dir, "filter1.bst") _yaml.roundtrip_dump(filter1_config, filter1_file) - filter2_config = { - "kind": "filter", - "depends": [ - {"filename": "filter1.bst", "type": "build"} - ] - } + filter2_config = {"kind": "filter", "depends": [{"filename": "filter1.bst", "type": "build"}]} filter2_file = os.path.join(elements_dir, "filter2.bst") _yaml.roundtrip_dump(filter2_config, filter2_file) # Assert that a fetch is needed - assert cli.get_element_state(project, input_name) == 'no reference' + assert cli.get_element_state(project, input_name) == "no reference" # Now try to track it result = cli.run(project=project, args=["source", "track", "filter2.bst"]) @@ -241,14 +229,14 @@ def test_filter_track(datafiles, cli, tmpdir): # Now check that a ref field exists new_input = _yaml.load(input_file) - source_node = new_input.get_sequence('sources').mapping_at(0) - new_input_ref = source_node.get_str('ref') + source_node = new_input.get_sequence("sources").mapping_at(0) + new_input_ref = source_node.get_str("ref") assert new_input_ref == ref -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_track_excepted(datafiles, cli, tmpdir): - repo = create_repo('git', str(tmpdir)) + repo = create_repo("git", str(tmpdir)) repo.create(os.path.join(str(datafiles), "files")) elements_dir = os.path.join(str(tmpdir), "elements") project = str(tmpdir) @@ -269,26 +257,16 @@ def test_filter_track_excepted(datafiles, cli, tmpdir): input_file = os.path.join(elements_dir, input_name) _yaml.roundtrip_dump(input_config, input_file) - filter1_config = { - "kind": "filter", - "depends": [ - {"filename": input_name, "type": "build"} - ] - } + filter1_config = {"kind": "filter", "depends": [{"filename": input_name, "type": "build"}]} filter1_file = os.path.join(elements_dir, "filter1.bst") _yaml.roundtrip_dump(filter1_config, filter1_file) - filter2_config = { - "kind": "filter", - "depends": [ - {"filename": "filter1.bst", "type": "build"} - ] - } + filter2_config = {"kind": "filter", "depends": [{"filename": "filter1.bst", "type": "build"}]} filter2_file = os.path.join(elements_dir, "filter2.bst") _yaml.roundtrip_dump(filter2_config, filter2_file) # Assert that a fetch is needed - assert cli.get_element_state(project, input_name) == 'no reference' + assert cli.get_element_state(project, input_name) == "no reference" # Now try to track it result = cli.run(project=project, args=["source", "track", "filter2.bst", "--except", "input.bst"]) @@ -296,13 +274,13 @@ def test_filter_track_excepted(datafiles, cli, tmpdir): # Now check that a ref field exists new_input = _yaml.load(input_file) - source_node = new_input.get_sequence('sources').mapping_at(0) - assert 'ref' not in source_node + source_node = new_input.get_sequence("sources").mapping_at(0) + assert "ref" not in source_node -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_track_multi_to_one(datafiles, cli, tmpdir): - repo = create_repo('git', str(tmpdir)) + repo = create_repo("git", str(tmpdir)) ref = repo.create(os.path.join(str(datafiles), "files")) elements_dir = os.path.join(str(tmpdir), "elements") project = str(tmpdir) @@ -323,26 +301,16 @@ def test_filter_track_multi_to_one(datafiles, cli, tmpdir): input_file = os.path.join(elements_dir, input_name) _yaml.roundtrip_dump(input_config, input_file) - filter1_config = { - "kind": "filter", - "depends": [ - {"filename": input_name, "type": "build"} - ] - } + filter1_config = {"kind": "filter", "depends": [{"filename": input_name, "type": "build"}]} filter1_file = os.path.join(elements_dir, "filter1.bst") _yaml.roundtrip_dump(filter1_config, filter1_file) - filter2_config = { - "kind": "filter", - "depends": [ - {"filename": input_name, "type": "build"} - ] - } + filter2_config = {"kind": "filter", "depends": [{"filename": input_name, "type": "build"}]} filter2_file = os.path.join(elements_dir, "filter2.bst") _yaml.roundtrip_dump(filter2_config, filter2_file) # Assert that a fetch is needed - assert cli.get_element_state(project, input_name) == 'no reference' + assert cli.get_element_state(project, input_name) == "no reference" # Now try to track it result = cli.run(project=project, args=["source", "track", "filter1.bst", "filter2.bst"]) @@ -350,14 +318,14 @@ def test_filter_track_multi_to_one(datafiles, cli, tmpdir): # Now check that a ref field exists new_input = _yaml.load(input_file) - source_node = new_input.get_sequence('sources').mapping_at(0) - new_ref = source_node.get_str('ref') + source_node = new_input.get_sequence("sources").mapping_at(0) + new_ref = source_node.get_str("ref") assert new_ref == ref -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_track_multi(datafiles, cli, tmpdir): - repo = create_repo('git', str(tmpdir)) + repo = create_repo("git", str(tmpdir)) ref = repo.create(os.path.join(str(datafiles), "files")) elements_dir = os.path.join(str(tmpdir), "elements") project = str(tmpdir) @@ -383,21 +351,11 @@ def test_filter_track_multi(datafiles, cli, tmpdir): input2_file = os.path.join(elements_dir, input2_name) _yaml.roundtrip_dump(input2_config, input2_file) - filter1_config = { - "kind": "filter", - "depends": [ - {"filename": input_name, "type": "build"} - ] - } + filter1_config = {"kind": "filter", "depends": [{"filename": input_name, "type": "build"}]} filter1_file = os.path.join(elements_dir, "filter1.bst") _yaml.roundtrip_dump(filter1_config, filter1_file) - filter2_config = { - "kind": "filter", - "depends": [ - {"filename": input2_name, "type": "build"} - ] - } + filter2_config = {"kind": "filter", "depends": [{"filename": input2_name, "type": "build"}]} filter2_file = os.path.join(elements_dir, "filter2.bst") _yaml.roundtrip_dump(filter2_config, filter2_file) @@ -415,19 +373,19 @@ def test_filter_track_multi(datafiles, cli, tmpdir): # Now check that a ref field exists new_input = _yaml.load(input_file) - source_node = new_input.get_sequence('sources').mapping_at(0) - new_ref = source_node.get_str('ref') + source_node = new_input.get_sequence("sources").mapping_at(0) + new_ref = source_node.get_str("ref") assert new_ref == ref new_input2 = _yaml.load(input2_file) - source_node2 = new_input2.get_sequence('sources').mapping_at(0) - new_ref2 = source_node2.get_str('ref') + source_node2 = new_input2.get_sequence("sources").mapping_at(0) + new_ref2 = source_node2.get_str("ref") assert new_ref2 == ref -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_track_multi_exclude(datafiles, cli, tmpdir): - repo = create_repo('git', str(tmpdir)) + repo = create_repo("git", str(tmpdir)) ref = repo.create(os.path.join(str(datafiles), "files")) elements_dir = os.path.join(str(tmpdir), "elements") project = str(tmpdir) @@ -453,21 +411,11 @@ def test_filter_track_multi_exclude(datafiles, cli, tmpdir): input2_file = os.path.join(elements_dir, input2_name) _yaml.roundtrip_dump(input2_config, input2_file) - filter1_config = { - "kind": "filter", - "depends": [ - {"filename": input_name, "type": "build"} - ] - } + filter1_config = {"kind": "filter", "depends": [{"filename": input_name, "type": "build"}]} filter1_file = os.path.join(elements_dir, "filter1.bst") _yaml.roundtrip_dump(filter1_config, filter1_file) - filter2_config = { - "kind": "filter", - "depends": [ - {"filename": input2_name, "type": "build"} - ] - } + filter2_config = {"kind": "filter", "depends": [{"filename": input2_name, "type": "build"}]} filter2_file = os.path.join(elements_dir, "filter2.bst") _yaml.roundtrip_dump(filter2_config, filter2_file) @@ -484,25 +432,26 @@ def test_filter_track_multi_exclude(datafiles, cli, tmpdir): # Now check that a ref field exists new_input = _yaml.load(input_file) - source_node = new_input.get_sequence('sources').mapping_at(0) - assert 'ref' not in source_node + source_node = new_input.get_sequence("sources").mapping_at(0) + assert "ref" not in source_node new_input2 = _yaml.load(input2_file) - source_node2 = new_input2.get_sequence('sources').mapping_at(0) - new_ref2 = source_node2.get_str('ref') + source_node2 = new_input2.get_sequence("sources").mapping_at(0) + new_ref2 = source_node2.get_str("ref") assert new_ref2 == ref -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_include_with_indirect_deps(datafiles, cli, tmpdir): project = str(datafiles) - result = cli.run(project=project, args=[ - 'build', 'output-include-with-indirect-deps.bst']) + result = cli.run(project=project, args=["build", "output-include-with-indirect-deps.bst"]) result.assert_success() - checkout = os.path.join(tmpdir.dirname, tmpdir.basename, 'checkout') - result = cli.run(project=project, args=[ - 'artifact', 'checkout', 'output-include-with-indirect-deps.bst', '--directory', checkout]) + checkout = os.path.join(tmpdir.dirname, tmpdir.basename, "checkout") + result = cli.run( + project=project, + args=["artifact", "checkout", "output-include-with-indirect-deps.bst", "--directory", checkout], + ) result.assert_success() # direct dependencies should be staged and filtered @@ -513,46 +462,50 @@ def test_filter_include_with_indirect_deps(datafiles, cli, tmpdir): assert not os.path.exists(os.path.join(checkout, "bar")) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_fails_for_nonexisting_domain(datafiles, cli): project = str(datafiles) - result = cli.run(project=project, args=['build', 'output-include-nonexistent-domain.bst']) + result = cli.run(project=project, args=["build", "output-include-nonexistent-domain.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) error = "Unknown domains were used in output-include-nonexistent-domain.bst [line 7 column 2]" assert error in result.stderr - assert '- unknown_file' in result.stderr + assert "- unknown_file" in result.stderr -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_pass_integration(datafiles, cli): project = str(datafiles) # Explicitly not passing integration commands should be fine - result = cli.run(project=project, args=['build', 'no-pass-integration.bst']) + result = cli.run(project=project, args=["build", "no-pass-integration.bst"]) result.assert_success() # Passing integration commands should build nicely - result = cli.run(project=project, args=['build', 'pass-integration.bst']) + result = cli.run(project=project, args=["build", "pass-integration.bst"]) result.assert_success() # Checking out elements which don't pass integration commands should still work - checkout_dir = os.path.join(project, 'no-pass') - result = cli.run(project=project, args=['artifact', 'checkout', '--integrate', - '--directory', checkout_dir, 'no-pass-integration.bst']) + checkout_dir = os.path.join(project, "no-pass") + result = cli.run( + project=project, + args=["artifact", "checkout", "--integrate", "--directory", checkout_dir, "no-pass-integration.bst"], + ) result.assert_success() # Checking out the artifact should fail if we run integration commands, as # the staged artifacts don't have a shell - checkout_dir = os.path.join(project, 'pass') - result = cli.run(project=project, args=['artifact', 'checkout', '--integrate', - '--directory', checkout_dir, 'pass-integration.bst']) + checkout_dir = os.path.join(project, "pass") + result = cli.run( + project=project, + args=["artifact", "checkout", "--integrate", "--directory", checkout_dir, "pass-integration.bst"], + ) result.assert_main_error(ErrorDomain.STREAM, "missing-command") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_stack_depend_failure(datafiles, cli): project = str(datafiles) - result = cli.run(project=project, args=['build', 'forbidden-stack-dep.bst']) + result = cli.run(project=project, args=["build", "forbidden-stack-dep.bst"]) result.assert_main_error(ErrorDomain.ELEMENT, "filter-bdepend-no-artifact") diff --git a/tests/elements/filter/basic/element_plugins/dynamic.py b/tests/elements/filter/basic/element_plugins/dynamic.py index fe83d7295..bf079111f 100644 --- a/tests/elements/filter/basic/element_plugins/dynamic.py +++ b/tests/elements/filter/basic/element_plugins/dynamic.py @@ -4,14 +4,14 @@ from buildstream import Element, Scope # Copies files from the dependent element but inserts split-rules using dynamic data class DynamicElement(Element): def configure(self, node): - node.validate_keys(['split-rules']) - self.split_rules = {key: value.as_str_list() for key, value in node.get_mapping('split-rules').items()} + node.validate_keys(["split-rules"]) + self.split_rules = {key: value.as_str_list() for key, value in node.get_mapping("split-rules").items()} def preflight(self): pass def get_unique_key(self): - return {'split-rules': self.split_rules} + return {"split-rules": self.split_rules} def configure_sandbox(self, sandbox): pass diff --git a/tests/examples/autotools.py b/tests/examples/autotools.py index 45783b30c..d4c12ecac 100644 --- a/tests/examples/autotools.py +++ b/tests/examples/autotools.py @@ -10,48 +10,52 @@ from buildstream.testing._utils.site import IS_LINUX, MACHINE_ARCH, HAVE_SANDBOX pytestmark = pytest.mark.integration -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), '..', '..', 'doc', 'examples', 'autotools' -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "doc", "examples", "autotools") # Tests a build of the autotools amhello project on a alpine-linux base runtime -@pytest.mark.skipif(MACHINE_ARCH != 'x86-64', - reason='Examples are written for x86-64') -@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox') -@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox') +@pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64") +@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with sandbox") +@pytest.mark.skipif(HAVE_SANDBOX == "chroot", reason="This test is not meant to work with chroot sandbox") @pytest.mark.datafiles(DATA_DIR) def test_autotools_build(cli, datafiles): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') + checkout = os.path.join(cli.directory, "checkout") # Check that the project can be built correctly. - result = cli.run(project=project, args=['build', 'hello.bst']) + result = cli.run(project=project, args=["build", "hello.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'hello.bst', '--directory', checkout]) + result = cli.run(project=project, args=["artifact", "checkout", "hello.bst", "--directory", checkout]) result.assert_success() - assert_contains(checkout, ['/usr', '/usr/lib', '/usr/bin', - '/usr/share', - '/usr/bin/hello', - '/usr/share/doc', '/usr/share/doc/amhello', - '/usr/share/doc/amhello/README']) + assert_contains( + checkout, + [ + "/usr", + "/usr/lib", + "/usr/bin", + "/usr/share", + "/usr/bin/hello", + "/usr/share/doc", + "/usr/share/doc/amhello", + "/usr/share/doc/amhello/README", + ], + ) # Test running an executable built with autotools. -@pytest.mark.skipif(MACHINE_ARCH != 'x86-64', - reason='Examples are written for x86-64') -@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox') -@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox') -@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox') +@pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64") +@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with sandbox") +@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox") +@pytest.mark.skipif(HAVE_SANDBOX == "chroot", reason="This test is not meant to work with chroot sandbox") @pytest.mark.datafiles(DATA_DIR) def test_autotools_run(cli, datafiles): project = str(datafiles) - result = cli.run(project=project, args=['build', 'hello.bst']) + result = cli.run(project=project, args=["build", "hello.bst"]) result.assert_success() - result = cli.run(project=project, args=['shell', 'hello.bst', 'hello']) + result = cli.run(project=project, args=["shell", "hello.bst", "hello"]) result.assert_success() - assert result.output == 'Hello World!\nThis is amhello 1.0.\n' + assert result.output == "Hello World!\nThis is amhello 1.0.\n" diff --git a/tests/examples/developing.py b/tests/examples/developing.py index 53a554b86..d33a26425 100644 --- a/tests/examples/developing.py +++ b/tests/examples/developing.py @@ -11,93 +11,86 @@ import tests.testutils.patch as patch pytestmark = pytest.mark.integration -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), '..', '..', 'doc', 'examples', 'developing' -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "doc", "examples", "developing") # Test that the project builds successfully -@pytest.mark.skipif(MACHINE_ARCH != 'x86-64', - reason='Examples are written for x86-64') -@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with SANDBOX') -@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This is not meant to work with chroot') +@pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64") +@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with SANDBOX") +@pytest.mark.skipif(HAVE_SANDBOX == "chroot", reason="This is not meant to work with chroot") @pytest.mark.datafiles(DATA_DIR) def test_autotools_build(cli, datafiles): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') + checkout = os.path.join(cli.directory, "checkout") # Check that the project can be built correctly. - result = cli.run(project=project, args=['build', 'hello.bst']) + result = cli.run(project=project, args=["build", "hello.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'hello.bst', '--directory', checkout]) + result = cli.run(project=project, args=["artifact", "checkout", "hello.bst", "--directory", checkout]) result.assert_success() - assert_contains(checkout, ['/usr', '/usr/lib', '/usr/bin', - '/usr/share', - '/usr/bin/hello']) + assert_contains(checkout, ["/usr", "/usr/lib", "/usr/bin", "/usr/share", "/usr/bin/hello"]) # Test the unmodified hello command works as expected. -@pytest.mark.skipif(MACHINE_ARCH != 'x86-64', - reason='Examples are written for x86-64') -@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with SANDBOX') -@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This is not meant to work with chroot') +@pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64") +@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with SANDBOX") +@pytest.mark.skipif(HAVE_SANDBOX == "chroot", reason="This is not meant to work with chroot") @pytest.mark.datafiles(DATA_DIR) def test_run_unmodified_hello(cli, datafiles): project = str(datafiles) - result = cli.run(project=project, args=['build', 'hello.bst']) + result = cli.run(project=project, args=["build", "hello.bst"]) result.assert_success() - result = cli.run(project=project, args=['shell', 'hello.bst', 'hello']) + result = cli.run(project=project, args=["shell", "hello.bst", "hello"]) result.assert_success() - assert result.output == 'Hello World\n' + assert result.output == "Hello World\n" # Test opening a workspace -@pytest.mark.skipif(not IS_LINUX, reason='Only available on linux') +@pytest.mark.skipif(not IS_LINUX, reason="Only available on linux") @pytest.mark.datafiles(DATA_DIR) def test_open_workspace(cli, tmpdir, datafiles): project = str(datafiles) workspace_dir = os.path.join(str(tmpdir), "workspace_hello") - result = cli.run(project=project, args=['workspace', 'open', '-f', '--directory', workspace_dir, 'hello.bst', ]) + result = cli.run(project=project, args=["workspace", "open", "-f", "--directory", workspace_dir, "hello.bst",]) result.assert_success() - result = cli.run(project=project, args=['workspace', 'list']) + result = cli.run(project=project, args=["workspace", "list"]) result.assert_success() - result = cli.run(project=project, args=['workspace', 'close', '--remove-dir', 'hello.bst']) + result = cli.run(project=project, args=["workspace", "close", "--remove-dir", "hello.bst"]) result.assert_success() # Test making a change using the workspace -@pytest.mark.skipif(MACHINE_ARCH != 'x86-64', - reason='Examples are written for x86-64') -@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with SANDBOX') -@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This is not meant to work with chroot') +@pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64") +@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with SANDBOX") +@pytest.mark.skipif(HAVE_SANDBOX == "chroot", reason="This is not meant to work with chroot") @pytest.mark.datafiles(DATA_DIR) def test_make_change_in_workspace(cli, tmpdir, datafiles): project = str(datafiles) workspace_dir = os.path.join(str(tmpdir), "workspace_hello") - result = cli.run(project=project, args=['workspace', 'open', '-f', '--directory', workspace_dir, 'hello.bst']) + result = cli.run(project=project, args=["workspace", "open", "-f", "--directory", workspace_dir, "hello.bst"]) result.assert_success() - result = cli.run(project=project, args=['workspace', 'list']) + result = cli.run(project=project, args=["workspace", "list"]) result.assert_success() patch_target = os.path.join(workspace_dir, "hello.c") patch_source = os.path.join(project, "update.patch") patch.apply(patch_target, patch_source) - result = cli.run(project=project, args=['build', 'hello.bst']) + result = cli.run(project=project, args=["build", "hello.bst"]) result.assert_success() - result = cli.run(project=project, args=['shell', 'hello.bst', '--', 'hello']) + result = cli.run(project=project, args=["shell", "hello.bst", "--", "hello"]) result.assert_success() - assert result.output == 'Hello World\nWe can use workspaces!\n' + assert result.output == "Hello World\nWe can use workspaces!\n" - result = cli.run(project=project, args=['workspace', 'close', '--remove-dir', 'hello.bst']) + result = cli.run(project=project, args=["workspace", "close", "--remove-dir", "hello.bst"]) result.assert_success() diff --git a/tests/examples/first-project.py b/tests/examples/first-project.py index 84ab7aa61..8783c4ae3 100644 --- a/tests/examples/first-project.py +++ b/tests/examples/first-project.py @@ -12,21 +12,19 @@ from buildstream.testing._utils.site import IS_LINUX pytestmark = pytest.mark.integration -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), '..', '..', 'doc', 'examples', 'first-project' -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "doc", "examples", "first-project") -@pytest.mark.skipif(not IS_LINUX, reason='Only available on linux') +@pytest.mark.skipif(not IS_LINUX, reason="Only available on linux") @pytest.mark.datafiles(DATA_DIR) def test_first_project_build_checkout(cli, datafiles): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') + checkout = os.path.join(cli.directory, "checkout") - result = cli.run(project=project, args=['build', 'hello.bst']) + result = cli.run(project=project, args=["build", "hello.bst"]) assert result.exit_code == 0 - result = cli.run(project=project, args=['artifact', 'checkout', 'hello.bst', '--directory', checkout]) + result = cli.run(project=project, args=["artifact", "checkout", "hello.bst", "--directory", checkout]) assert result.exit_code == 0 - assert_contains(checkout, ['/hello.world']) + assert_contains(checkout, ["/hello.world"]) diff --git a/tests/examples/flatpak-autotools.py b/tests/examples/flatpak-autotools.py index 2418807c0..d20c7e136 100644 --- a/tests/examples/flatpak-autotools.py +++ b/tests/examples/flatpak-autotools.py @@ -13,12 +13,13 @@ pytestmark = pytest.mark.integration DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), '..', '..', 'doc', 'examples', 'flatpak-autotools' + os.path.dirname(os.path.realpath(__file__)), "..", "..", "doc", "examples", "flatpak-autotools" ) try: from bst_plugins_experimental.sources import _ostree # pylint: disable=unused-import + # Even when we have the plugin, it might be missing dependencies. This requires # bst_plugins_experimantal to be fully installed, with host ostree dependencies HAVE_OSTREE_PLUGIN = True @@ -44,40 +45,46 @@ def workaround_setuptools_bug(project): # Test that a build upon flatpak runtime 'works' - we use the autotools sample # amhello project for this. -@pytest.mark.skipif(MACHINE_ARCH != 'x86-64', - reason='Examples are written for x86-64') -@pytest.mark.skipif(not IS_LINUX or not HAVE_OSTREE_PLUGIN, reason='Only available on linux with ostree') +@pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64") +@pytest.mark.skipif(not IS_LINUX or not HAVE_OSTREE_PLUGIN, reason="Only available on linux with ostree") @pytest.mark.datafiles(DATA_DIR) def test_autotools_build(cli, datafiles): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') + checkout = os.path.join(cli.directory, "checkout") workaround_setuptools_bug(project) - result = cli.run(project=project, args=['build', 'hello.bst']) + result = cli.run(project=project, args=["build", "hello.bst"]) assert result.exit_code == 0 - result = cli.run(project=project, args=['artifact', 'checkout', 'hello.bst', '--directory', checkout]) + result = cli.run(project=project, args=["artifact", "checkout", "hello.bst", "--directory", checkout]) assert result.exit_code == 0 - assert_contains(checkout, ['/usr', '/usr/lib', '/usr/bin', - '/usr/share', - '/usr/bin/hello', '/usr/share/doc', - '/usr/share/doc/amhello', - '/usr/share/doc/amhello/README']) + assert_contains( + checkout, + [ + "/usr", + "/usr/lib", + "/usr/bin", + "/usr/share", + "/usr/bin/hello", + "/usr/share/doc", + "/usr/share/doc/amhello", + "/usr/share/doc/amhello/README", + ], + ) # Test running an executable built with autotools -@pytest.mark.skipif(MACHINE_ARCH != 'x86-64', - reason='Examples are written for x86-64') -@pytest.mark.skipif(not IS_LINUX or not HAVE_OSTREE_PLUGIN, reason='Only available on linux with ostree') +@pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64") +@pytest.mark.skipif(not IS_LINUX or not HAVE_OSTREE_PLUGIN, reason="Only available on linux with ostree") @pytest.mark.datafiles(DATA_DIR) def test_autotools_run(cli, datafiles): project = str(datafiles) workaround_setuptools_bug(project) - result = cli.run(project=project, args=['build', 'hello.bst']) + result = cli.run(project=project, args=["build", "hello.bst"]) assert result.exit_code == 0 - result = cli.run(project=project, args=['shell', 'hello.bst', '/usr/bin/hello']) + result = cli.run(project=project, args=["shell", "hello.bst", "/usr/bin/hello"]) assert result.exit_code == 0 - assert result.output == 'Hello World!\nThis is amhello 1.0.\n' + assert result.output == "Hello World!\nThis is amhello 1.0.\n" diff --git a/tests/examples/integration-commands.py b/tests/examples/integration-commands.py index 1ed888b5d..2bdbfb3ec 100644 --- a/tests/examples/integration-commands.py +++ b/tests/examples/integration-commands.py @@ -10,34 +10,32 @@ from buildstream.testing._utils.site import IS_LINUX, MACHINE_ARCH, HAVE_SANDBOX pytestmark = pytest.mark.integration DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), '..', '..', 'doc', 'examples', 'integration-commands' + os.path.dirname(os.path.realpath(__file__)), "..", "..", "doc", "examples", "integration-commands" ) -@pytest.mark.skipif(MACHINE_ARCH != 'x86-64', - reason='Examples are written for x86-64') -@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox') -@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox') +@pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64") +@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with sandbox") +@pytest.mark.skipif(HAVE_SANDBOX == "chroot", reason="This test is not meant to work with chroot sandbox") @pytest.mark.datafiles(DATA_DIR) def test_integration_commands_build(cli, datafiles): project = str(datafiles) - result = cli.run(project=project, args=['build', 'hello.bst']) + result = cli.run(project=project, args=["build", "hello.bst"]) assert result.exit_code == 0 # Test running the executable -@pytest.mark.skipif(MACHINE_ARCH != 'x86-64', - reason='Examples are written for x86-64') -@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox') -@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox') +@pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64") +@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with sandbox") +@pytest.mark.skipif(HAVE_SANDBOX == "chroot", reason="This test is not meant to work with chroot sandbox") @pytest.mark.datafiles(DATA_DIR) def test_integration_commands_run(cli, datafiles): project = str(datafiles) - result = cli.run(project=project, args=['build', 'hello.bst']) + result = cli.run(project=project, args=["build", "hello.bst"]) assert result.exit_code == 0 - result = cli.run(project=project, args=['shell', 'hello.bst', '--', 'hello', 'pony']) + result = cli.run(project=project, args=["shell", "hello.bst", "--", "hello", "pony"]) assert result.exit_code == 0 - assert result.output == 'Hello pony\n' + assert result.output == "Hello pony\n" diff --git a/tests/examples/junctions.py b/tests/examples/junctions.py index 18bf4da4f..28b7a6108 100644 --- a/tests/examples/junctions.py +++ b/tests/examples/junctions.py @@ -9,52 +9,48 @@ from buildstream.testing._utils.site import IS_LINUX, MACHINE_ARCH, HAVE_SANDBOX pytestmark = pytest.mark.integration -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), '..', '..', 'doc', 'examples', 'junctions' -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "doc", "examples", "junctions") # Test that the project builds successfully -@pytest.mark.skipif(MACHINE_ARCH != 'x86-64', - reason='Examples are written for x86-64') -@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with bubblewrap') -@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox') +@pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64") +@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with bubblewrap") +@pytest.mark.skipif(HAVE_SANDBOX == "chroot", reason="This test is not meant to work with chroot sandbox") @pytest.mark.datafiles(DATA_DIR) def test_build(cli, datafiles): project = str(datafiles) - result = cli.run(project=project, args=['build', 'callHello.bst']) + result = cli.run(project=project, args=["build", "callHello.bst"]) result.assert_success() # Test the callHello script works as expected. -@pytest.mark.skipif(MACHINE_ARCH != 'x86-64', - reason='Examples are written for x86-64') -@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with bubblewrap') -@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox') +@pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64") +@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with bubblewrap") +@pytest.mark.skipif(HAVE_SANDBOX == "chroot", reason="This test is not meant to work with chroot sandbox") @pytest.mark.datafiles(DATA_DIR) def test_shell_call_hello(cli, datafiles): project = str(datafiles) - result = cli.run(project=project, args=['build', 'callHello.bst']) + result = cli.run(project=project, args=["build", "callHello.bst"]) result.assert_success() - result = cli.run(project=project, args=['shell', 'callHello.bst', '--', '/bin/sh', 'callHello.sh']) + result = cli.run(project=project, args=["shell", "callHello.bst", "--", "/bin/sh", "callHello.sh"]) result.assert_success() - assert result.output == 'Calling hello:\nHello World!\nThis is amhello 1.0.\n' + assert result.output == "Calling hello:\nHello World!\nThis is amhello 1.0.\n" # Test opening a cross-junction workspace -@pytest.mark.skipif(not IS_LINUX, reason='Only available on linux') +@pytest.mark.skipif(not IS_LINUX, reason="Only available on linux") @pytest.mark.datafiles(DATA_DIR) def test_open_cross_junction_workspace(cli, tmpdir, datafiles): project = str(datafiles) workspace_dir = os.path.join(str(tmpdir), "workspace_hello_junction") - result = cli.run(project=project, - args=['workspace', 'open', '--directory', workspace_dir, 'hello-junction.bst:hello.bst']) + result = cli.run( + project=project, args=["workspace", "open", "--directory", workspace_dir, "hello-junction.bst:hello.bst"] + ) result.assert_success() - result = cli.run(project=project, - args=['workspace', 'close', '--remove-dir', 'hello-junction.bst:hello.bst']) + result = cli.run(project=project, args=["workspace", "close", "--remove-dir", "hello-junction.bst:hello.bst"]) result.assert_success() diff --git a/tests/examples/running-commands.py b/tests/examples/running-commands.py index 1c419d524..bf8a9c002 100644 --- a/tests/examples/running-commands.py +++ b/tests/examples/running-commands.py @@ -9,35 +9,31 @@ from buildstream.testing._utils.site import IS_LINUX, MACHINE_ARCH, HAVE_SANDBOX pytestmark = pytest.mark.integration -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), '..', '..', 'doc', 'examples', 'running-commands' -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "doc", "examples", "running-commands") -@pytest.mark.skipif(MACHINE_ARCH != 'x86-64', - reason='Examples are written for x86-64') +@pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64") @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox') -@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox') +@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with sandbox") +@pytest.mark.skipif(HAVE_SANDBOX == "chroot", reason="This test is not meant to work with chroot sandbox") def test_running_commands_build(cli, datafiles): project = str(datafiles) - result = cli.run(project=project, args=['build', 'hello.bst']) + result = cli.run(project=project, args=["build", "hello.bst"]) assert result.exit_code == 0 # Test running the executable -@pytest.mark.skipif(MACHINE_ARCH != 'x86-64', - reason='Examples are written for x86-64') -@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason='Only available on linux with sandbox') -@pytest.mark.skipif(HAVE_SANDBOX == 'chroot', reason='This test is not meant to work with chroot sandbox') +@pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64") +@pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with sandbox") +@pytest.mark.skipif(HAVE_SANDBOX == "chroot", reason="This test is not meant to work with chroot sandbox") @pytest.mark.datafiles(DATA_DIR) def test_running_commands_run(cli, datafiles): project = str(datafiles) - result = cli.run(project=project, args=['build', 'hello.bst']) + result = cli.run(project=project, args=["build", "hello.bst"]) assert result.exit_code == 0 - result = cli.run(project=project, args=['shell', 'hello.bst', '--', 'hello']) + result = cli.run(project=project, args=["shell", "hello.bst", "--", "hello"]) assert result.exit_code == 0 - assert result.output == 'Hello World\n' + assert result.output == "Hello World\n" diff --git a/tests/external_plugins.py b/tests/external_plugins.py index f6f67707b..2123b846b 100644 --- a/tests/external_plugins.py +++ b/tests/external_plugins.py @@ -16,7 +16,7 @@ import pytest # test_match_patterns (list[str]): A list of shell style globs which may be # used to specify a subset of test files from the repository to run. # These must be specified relative to the root of the repository. -class ExternalPluginRepo(): +class ExternalPluginRepo: def __init__(self, name, url, ref, test_match_patterns=None): self.name = name self.url = url @@ -30,17 +30,13 @@ class ExternalPluginRepo(): def clone(self, location): self._clone_location = os.path.join(location, self.name) - subprocess.run(['git', 'clone', - '--single-branch', - '--branch', self.ref, - '--depth', '1', - self.url, - self._clone_location, - ]) + subprocess.run( + ["git", "clone", "--single-branch", "--branch", self.ref, "--depth", "1", self.url, self._clone_location,] + ) return self._clone_location def install(self): - subprocess.run(['pip3', 'install', self._clone_location]) + subprocess.run(["pip3", "install", self._clone_location]) def test(self, args): test_files = self._match_test_patterns() diff --git a/tests/format/assertion.py b/tests/format/assertion.py index 7e87977cb..67436250a 100644 --- a/tests/format/assertion.py +++ b/tests/format/assertion.py @@ -7,30 +7,41 @@ from buildstream._exceptions import ErrorDomain, LoadErrorReason from buildstream.testing.runcli import cli # pylint: disable=unused-import # Project directory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - 'assertion' -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "assertion") @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("target,opt_pony,opt_horsy,assertion", [ - # Test an unconditional (!) directly in the element - ('raw-assertion.bst', 'False', 'False', 'Raw assertion boogey'), - # Test an assertion in a conditional - ('conditional-assertion.bst', 'True', 'False', "It's not pony time yet"), - # Test that we get the first composited assertion - ('ordered-assertion.bst', 'True', 'True', "It's not horsy time yet"), -]) +@pytest.mark.parametrize( + "target,opt_pony,opt_horsy,assertion", + [ + # Test an unconditional (!) directly in the element + ("raw-assertion.bst", "False", "False", "Raw assertion boogey"), + # Test an assertion in a conditional + ("conditional-assertion.bst", "True", "False", "It's not pony time yet"), + # Test that we get the first composited assertion + ("ordered-assertion.bst", "True", "True", "It's not horsy time yet"), + ], +) def test_assertion_cli(cli, datafiles, target, opt_pony, opt_horsy, assertion): project = str(datafiles) - result = cli.run(project=project, silent=True, args=[ - '--option', 'pony', opt_pony, - '--option', 'horsy', opt_horsy, - 'show', - '--deps', 'none', - '--format', '%{vars}', - target]) + result = cli.run( + project=project, + silent=True, + args=[ + "--option", + "pony", + opt_pony, + "--option", + "horsy", + opt_horsy, + "show", + "--deps", + "none", + "--format", + "%{vars}", + target, + ], + ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.USER_ASSERTION) # Assert that the assertion text provided by the user diff --git a/tests/format/dependencies.py b/tests/format/dependencies.py index f92b89afa..e54b9b2d5 100644 --- a/tests/format/dependencies.py +++ b/tests/format/dependencies.py @@ -15,125 +15,125 @@ DATA_DIR = os.path.dirname(os.path.realpath(__file__)) # @pytest.mark.datafiles(DATA_DIR) def test_two_files(cli, datafiles): - project = os.path.join(str(datafiles), 'dependencies1') + project = os.path.join(str(datafiles), "dependencies1") - elements = cli.get_pipeline(project, ['target.bst']) - assert elements == ['firstdep.bst', 'target.bst'] + elements = cli.get_pipeline(project, ["target.bst"]) + assert elements == ["firstdep.bst", "target.bst"] @pytest.mark.datafiles(DATA_DIR) def test_shared_dependency(cli, datafiles): - project = os.path.join(str(datafiles), 'dependencies1') + project = os.path.join(str(datafiles), "dependencies1") - elements = cli.get_pipeline(project, ['shareddeptarget.bst']) - assert elements == ['firstdep.bst', 'shareddep.bst', 'shareddeptarget.bst'] + elements = cli.get_pipeline(project, ["shareddeptarget.bst"]) + assert elements == ["firstdep.bst", "shareddep.bst", "shareddeptarget.bst"] @pytest.mark.datafiles(DATA_DIR) def test_dependency_dict(cli, datafiles): - project = os.path.join(str(datafiles), 'dependencies1') - elements = cli.get_pipeline(project, ['target-depdict.bst']) - assert elements == ['firstdep.bst', 'target-depdict.bst'] + project = os.path.join(str(datafiles), "dependencies1") + elements = cli.get_pipeline(project, ["target-depdict.bst"]) + assert elements == ["firstdep.bst", "target-depdict.bst"] @pytest.mark.datafiles(DATA_DIR) def test_invalid_dependency_declaration(cli, datafiles): - project = os.path.join(str(datafiles), 'dependencies1') - result = cli.run(project=project, args=['show', 'invaliddep.bst']) + project = os.path.join(str(datafiles), "dependencies1") + result = cli.run(project=project, args=["show", "invaliddep.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @pytest.mark.datafiles(DATA_DIR) def test_invalid_dependency_type(cli, datafiles): - project = os.path.join(str(datafiles), 'dependencies1') - result = cli.run(project=project, args=['show', 'invaliddeptype.bst']) + project = os.path.join(str(datafiles), "dependencies1") + result = cli.run(project=project, args=["show", "invaliddeptype.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @pytest.mark.datafiles(DATA_DIR) def test_invalid_strict_dependency(cli, datafiles): - project = os.path.join(str(datafiles), 'dependencies1') - result = cli.run(project=project, args=['show', 'invalidstrict.bst']) + project = os.path.join(str(datafiles), "dependencies1") + result = cli.run(project=project, args=["show", "invalidstrict.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @pytest.mark.datafiles(DATA_DIR) def test_invalid_non_strict_dependency(cli, datafiles): - project = os.path.join(str(datafiles), 'dependencies1') - result = cli.run(project=project, args=['show', 'invalidnonstrict.bst']) + project = os.path.join(str(datafiles), "dependencies1") + result = cli.run(project=project, args=["show", "invalidnonstrict.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @pytest.mark.datafiles(DATA_DIR) def test_circular_dependency(cli, datafiles): - project = os.path.join(str(datafiles), 'dependencies1') - result = cli.run(project=project, args=['show', 'circulartarget.bst']) + project = os.path.join(str(datafiles), "dependencies1") + result = cli.run(project=project, args=["show", "circulartarget.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.CIRCULAR_DEPENDENCY) @pytest.mark.datafiles(DATA_DIR) def test_build_dependency(cli, datafiles): - project = os.path.join(str(datafiles), 'dependencies1') + project = os.path.join(str(datafiles), "dependencies1") - elements = cli.get_pipeline(project, ['builddep.bst'], scope='run') - assert elements == ['builddep.bst'] + elements = cli.get_pipeline(project, ["builddep.bst"], scope="run") + assert elements == ["builddep.bst"] - elements = cli.get_pipeline(project, ['builddep.bst'], scope='build') - assert elements == ['firstdep.bst'] + elements = cli.get_pipeline(project, ["builddep.bst"], scope="build") + assert elements == ["firstdep.bst"] @pytest.mark.datafiles(DATA_DIR) def test_runtime_dependency(cli, datafiles): - project = os.path.join(str(datafiles), 'dependencies1') - elements = cli.get_pipeline(project, ['runtimedep.bst'], scope='build') + project = os.path.join(str(datafiles), "dependencies1") + elements = cli.get_pipeline(project, ["runtimedep.bst"], scope="build") # FIXME: The empty line should probably never happen here when there are no results. - assert elements == [''] - elements = cli.get_pipeline(project, ['runtimedep.bst'], scope='run') - assert elements == ['firstdep.bst', 'runtimedep.bst'] + assert elements == [""] + elements = cli.get_pipeline(project, ["runtimedep.bst"], scope="run") + assert elements == ["firstdep.bst", "runtimedep.bst"] @pytest.mark.datafiles(DATA_DIR) def test_all_dependency(cli, datafiles): - project = os.path.join(str(datafiles), 'dependencies1') + project = os.path.join(str(datafiles), "dependencies1") - elements = cli.get_pipeline(project, ['alldep.bst'], scope='build') - assert elements == ['firstdep.bst'] + elements = cli.get_pipeline(project, ["alldep.bst"], scope="build") + assert elements == ["firstdep.bst"] - elements = cli.get_pipeline(project, ['alldep.bst'], scope='run') - assert elements == ['firstdep.bst', 'alldep.bst'] + elements = cli.get_pipeline(project, ["alldep.bst"], scope="run") + assert elements == ["firstdep.bst", "alldep.bst"] @pytest.mark.datafiles(DATA_DIR) def test_list_build_dependency(cli, datafiles): - project = os.path.join(str(datafiles), 'dependencies1') + project = os.path.join(str(datafiles), "dependencies1") # Check that the pipeline includes the build dependency - deps = cli.get_pipeline(project, ['builddep-list.bst'], scope="build") + deps = cli.get_pipeline(project, ["builddep-list.bst"], scope="build") assert "firstdep.bst" in deps @pytest.mark.datafiles(DATA_DIR) def test_list_runtime_dependency(cli, datafiles): - project = os.path.join(str(datafiles), 'dependencies1') + project = os.path.join(str(datafiles), "dependencies1") # Check that the pipeline includes the runtime dependency - deps = cli.get_pipeline(project, ['runtimedep-list.bst'], scope="run") + deps = cli.get_pipeline(project, ["runtimedep-list.bst"], scope="run") assert "firstdep.bst" in deps @pytest.mark.datafiles(DATA_DIR) def test_list_dependencies_combined(cli, datafiles): - project = os.path.join(str(datafiles), 'dependencies1') + project = os.path.join(str(datafiles), "dependencies1") # Check that runtime deps get combined - rundeps = cli.get_pipeline(project, ['list-combine.bst'], scope="run") + rundeps = cli.get_pipeline(project, ["list-combine.bst"], scope="run") assert "firstdep.bst" not in rundeps assert "seconddep.bst" in rundeps assert "thirddep.bst" in rundeps # Check that build deps get combined - builddeps = cli.get_pipeline(project, ['list-combine.bst'], scope="build") + builddeps = cli.get_pipeline(project, ["list-combine.bst"], scope="build") assert "firstdep.bst" in builddeps assert "seconddep.bst" not in builddeps assert "thirddep.bst" in builddeps @@ -141,12 +141,12 @@ def test_list_dependencies_combined(cli, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_list_overlap(cli, datafiles): - project = os.path.join(str(datafiles), 'dependencies1') + project = os.path.join(str(datafiles), "dependencies1") # Check that dependencies get merged - rundeps = cli.get_pipeline(project, ['list-overlap.bst'], scope="run") + rundeps = cli.get_pipeline(project, ["list-overlap.bst"], scope="run") assert "firstdep.bst" in rundeps - builddeps = cli.get_pipeline(project, ['list-overlap.bst'], scope="build") + builddeps = cli.get_pipeline(project, ["list-overlap.bst"], scope="build") assert "firstdep.bst" in builddeps @@ -156,10 +156,10 @@ def test_list_overlap(cli, datafiles): # @pytest.mark.datafiles(DATA_DIR) def test_scope_all(cli, datafiles): - project = os.path.join(str(datafiles), 'dependencies2') - elements = ['target.bst'] + project = os.path.join(str(datafiles), "dependencies2") + elements = ["target.bst"] - element_list = cli.get_pipeline(project, elements, scope='all') + element_list = cli.get_pipeline(project, elements, scope="all") assert element_list == [ "build-build.bst", @@ -174,10 +174,10 @@ def test_scope_all(cli, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_scope_run(cli, datafiles): - project = os.path.join(str(datafiles), 'dependencies2') - elements = ['target.bst'] + project = os.path.join(str(datafiles), "dependencies2") + elements = ["target.bst"] - element_list = cli.get_pipeline(project, elements, scope='run') + element_list = cli.get_pipeline(project, elements, scope="run") assert element_list == [ "dep-one.bst", @@ -189,69 +189,71 @@ def test_scope_run(cli, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_scope_build(cli, datafiles): - project = os.path.join(str(datafiles), 'dependencies2') - elements = ['target.bst'] + project = os.path.join(str(datafiles), "dependencies2") + elements = ["target.bst"] - element_list = cli.get_pipeline(project, elements, scope='build') + element_list = cli.get_pipeline(project, elements, scope="build") assert element_list == ["dep-one.bst", "run.bst", "dep-two.bst"] @pytest.mark.datafiles(DATA_DIR) def test_scope_build_of_child(cli, datafiles): - project = os.path.join(str(datafiles), 'dependencies2') - elements = ['target.bst'] + project = os.path.join(str(datafiles), "dependencies2") + elements = ["target.bst"] - element_list = cli.get_pipeline(project, elements, scope='build') + element_list = cli.get_pipeline(project, elements, scope="build") # First pass, lets check dep-two element = element_list[2] # Pass two, let's look at these - element_list = cli.get_pipeline(project, [element], scope='build') + element_list = cli.get_pipeline(project, [element], scope="build") assert element_list == ["run-build.bst", "build.bst"] @pytest.mark.datafiles(DATA_DIR) def test_no_recurse(cli, datafiles): - project = os.path.join(str(datafiles), 'dependencies2') - elements = ['target.bst'] + project = os.path.join(str(datafiles), "dependencies2") + elements = ["target.bst"] # We abuse the 'plan' scope here to ensure that we call # element.dependencies() with recurse=False - currently, no `bst # show` option does this directly. - element_list = cli.get_pipeline(project, elements, scope='plan') + element_list = cli.get_pipeline(project, elements, scope="plan") assert element_list == [ - 'build-build.bst', - 'run-build.bst', - 'build.bst', - 'dep-one.bst', - 'run.bst', - 'dep-two.bst', - 'target.bst', + "build-build.bst", + "run-build.bst", + "build.bst", + "dep-one.bst", + "run.bst", + "dep-two.bst", + "target.bst", ] @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize(("element", "asserts"), [ - ('build-runtime', False), - ('build-build', True), - ('build-all', True), - ('runtime-runtime', True), - ('runtime-all', True), - ('all-all', True), -]) +@pytest.mark.parametrize( + ("element", "asserts"), + [ + ("build-runtime", False), + ("build-build", True), + ("build-all", True), + ("runtime-runtime", True), + ("runtime-all", True), + ("all-all", True), + ], +) def test_duplicate_deps(cli, datafiles, element, asserts): - project = os.path.join(str(datafiles), 'dependencies3') + project = os.path.join(str(datafiles), "dependencies3") - result = cli.run(project=project, args=['show', '{}.bst'.format(element)]) + result = cli.run(project=project, args=["show", "{}.bst".format(element)]) if asserts: - result.assert_main_error(ErrorDomain.LOAD, - LoadErrorReason.DUPLICATE_DEPENDENCY) - assert '[line 10 column 2]' in result.stderr - assert '[line 8 column 2]' in result.stderr + result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.DUPLICATE_DEPENDENCY) + assert "[line 10 column 2]" in result.stderr + assert "[line 8 column 2]" in result.stderr else: result.assert_success() diff --git a/tests/format/include.py b/tests/format/include.py index 434a94d1f..5efbc62a9 100644 --- a/tests/format/include.py +++ b/tests/format/include.py @@ -12,97 +12,90 @@ from tests.testutils import generate_junction # Project directory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - 'include' -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "include") @pytest.mark.datafiles(DATA_DIR) def test_include_project_file(cli, datafiles): - project = os.path.join(str(datafiles), 'file') - result = cli.run(project=project, args=[ - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst']) + project = os.path.join(str(datafiles), "file") + result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]) result.assert_success() loaded = _yaml.load_data(result.output) - assert loaded.get_bool('included') + assert loaded.get_bool("included") def test_include_missing_file(cli, tmpdir): - tmpdir.join('project.conf').write('{"name": "test"}') - element = tmpdir.join('include_missing_file.bst') + tmpdir.join("project.conf").write('{"name": "test"}') + element = tmpdir.join("include_missing_file.bst") # Normally we would use dicts and _yaml.roundtrip_dump to write such things, but here # we want to be sure of a stable line and column number. - element.write(textwrap.dedent(""" + element.write( + textwrap.dedent( + """ kind: manual "(@)": - nosuch.yaml - """).strip()) + """ + ).strip() + ) - result = cli.run(project=str(tmpdir), args=['show', str(element.basename)]) + result = cli.run(project=str(tmpdir), args=["show", str(element.basename)]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE) # Make sure the root cause provenance is in the output. - assert 'line 4 column 2' in result.stderr + assert "line 4 column 2" in result.stderr def test_include_dir(cli, tmpdir): - tmpdir.join('project.conf').write('{"name": "test"}') - tmpdir.mkdir('subdir') - element = tmpdir.join('include_dir.bst') + tmpdir.join("project.conf").write('{"name": "test"}') + tmpdir.mkdir("subdir") + element = tmpdir.join("include_dir.bst") # Normally we would use dicts and _yaml.roundtrip_dump to write such things, but here # we want to be sure of a stable line and column number. - element.write(textwrap.dedent(""" + element.write( + textwrap.dedent( + """ kind: manual "(@)": - subdir/ - """).strip()) + """ + ).strip() + ) - result = cli.run(project=str(tmpdir), args=['show', str(element.basename)]) - result.assert_main_error( - ErrorDomain.LOAD, LoadErrorReason.LOADING_DIRECTORY) + result = cli.run(project=str(tmpdir), args=["show", str(element.basename)]) + result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.LOADING_DIRECTORY) # Make sure the root cause provenance is in the output. - assert 'line 4 column 2' in result.stderr + assert "line 4 column 2" in result.stderr @pytest.mark.datafiles(DATA_DIR) def test_include_junction_file(cli, tmpdir, datafiles): - project = os.path.join(str(datafiles), 'junction') - - generate_junction(tmpdir, - os.path.join(project, 'subproject'), - os.path.join(project, 'junction.bst'), - store_ref=True) - - result = cli.run(project=project, args=[ - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst']) + project = os.path.join(str(datafiles), "junction") + + generate_junction( + tmpdir, os.path.join(project, "subproject"), os.path.join(project, "junction.bst"), store_ref=True + ) + + result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]) result.assert_success() loaded = _yaml.load_data(result.output) - assert loaded.get_bool('included') + assert loaded.get_bool("included") @pytest.mark.datafiles(DATA_DIR) def test_include_junction_options(cli, datafiles): - project = os.path.join(str(datafiles), 'options') - - result = cli.run(project=project, args=[ - '-o', 'build_arch', 'x86_64', - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst']) + project = os.path.join(str(datafiles), "options") + + result = cli.run( + project=project, + args=["-o", "build_arch", "x86_64", "show", "--deps", "none", "--format", "%{vars}", "element.bst"], + ) result.assert_success() loaded = _yaml.load_data(result.output) - assert loaded.get_str('build_arch') == 'x86_64' + assert loaded.get_str("build_arch") == "x86_64" @pytest.mark.datafiles(DATA_DIR) @@ -111,31 +104,22 @@ def test_junction_element_partial_project_project(cli, tmpdir, datafiles): Junction elements never depend on fully include processed project. """ - project = os.path.join(str(datafiles), 'junction') + project = os.path.join(str(datafiles), "junction") - subproject_path = os.path.join(project, 'subproject') - junction_path = os.path.join(project, 'junction.bst') + subproject_path = os.path.join(project, "subproject") + junction_path = os.path.join(project, "junction.bst") - repo = create_repo('git', str(tmpdir)) + repo = create_repo("git", str(tmpdir)) ref = repo.create(subproject_path) - element = { - 'kind': 'junction', - 'sources': [ - repo.source_config(ref=ref) - ] - } + element = {"kind": "junction", "sources": [repo.source_config(ref=ref)]} _yaml.roundtrip_dump(element, junction_path) - result = cli.run(project=project, args=[ - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'junction.bst']) + result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "junction.bst"]) result.assert_success() loaded = _yaml.load_data(result.output) - assert loaded.get_str('included', default=None) is None + assert loaded.get_str("included", default=None) is None @pytest.mark.datafiles(DATA_DIR) @@ -144,170 +128,127 @@ def test_junction_element_not_partial_project_file(cli, tmpdir, datafiles): Junction elements never depend on fully include processed project. """ - project = os.path.join(str(datafiles), 'file_with_subproject') + project = os.path.join(str(datafiles), "file_with_subproject") - subproject_path = os.path.join(project, 'subproject') - junction_path = os.path.join(project, 'junction.bst') + subproject_path = os.path.join(project, "subproject") + junction_path = os.path.join(project, "junction.bst") - repo = create_repo('git', str(tmpdir)) + repo = create_repo("git", str(tmpdir)) ref = repo.create(subproject_path) - element = { - 'kind': 'junction', - 'sources': [ - repo.source_config(ref=ref) - ] - } + element = {"kind": "junction", "sources": [repo.source_config(ref=ref)]} _yaml.roundtrip_dump(element, junction_path) - result = cli.run(project=project, args=[ - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'junction.bst']) + result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "junction.bst"]) result.assert_success() loaded = _yaml.load_data(result.output) - assert loaded.get_str('included', default=None) is not None + assert loaded.get_str("included", default=None) is not None @pytest.mark.datafiles(DATA_DIR) def test_include_element_overrides(cli, datafiles): - project = os.path.join(str(datafiles), 'overrides') + project = os.path.join(str(datafiles), "overrides") - result = cli.run(project=project, args=[ - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst']) + result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]) result.assert_success() loaded = _yaml.load_data(result.output) - assert loaded.get_str('manual_main_override', default=None) is not None - assert loaded.get_str('manual_included_override', default=None) is not None + assert loaded.get_str("manual_main_override", default=None) is not None + assert loaded.get_str("manual_included_override", default=None) is not None @pytest.mark.datafiles(DATA_DIR) def test_include_element_overrides_composition(cli, datafiles): - project = os.path.join(str(datafiles), 'overrides') + project = os.path.join(str(datafiles), "overrides") - result = cli.run(project=project, args=[ - 'show', - '--deps', 'none', - '--format', '%{config}', - 'element.bst']) + result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{config}", "element.bst"]) result.assert_success() loaded = _yaml.load_data(result.output) - assert loaded.get_str_list('build-commands') == ['first', 'second'] + assert loaded.get_str_list("build-commands") == ["first", "second"] @pytest.mark.datafiles(DATA_DIR) def test_list_overide_does_not_fail_upon_first_composition(cli, datafiles): - project = os.path.join(str(datafiles), 'eventual_overrides') + project = os.path.join(str(datafiles), "eventual_overrides") - result = cli.run(project=project, args=[ - 'show', - '--deps', 'none', - '--format', '%{public}', - 'element.bst']) + result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{public}", "element.bst"]) result.assert_success() loaded = _yaml.load_data(result.output) # Assert that the explicitly overwritten public data is present - bst = loaded.get_mapping('bst') - assert 'foo-commands' in bst - assert bst.get_str_list('foo-commands') == ['need', 'this'] + bst = loaded.get_mapping("bst") + assert "foo-commands" in bst + assert bst.get_str_list("foo-commands") == ["need", "this"] @pytest.mark.datafiles(DATA_DIR) def test_include_element_overrides_sub_include(cli, datafiles): - project = os.path.join(str(datafiles), 'sub-include') + project = os.path.join(str(datafiles), "sub-include") - result = cli.run(project=project, args=[ - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst']) + result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]) result.assert_success() loaded = _yaml.load_data(result.output) - assert loaded.get_str('included', default=None) is not None + assert loaded.get_str("included", default=None) is not None @pytest.mark.datafiles(DATA_DIR) def test_junction_do_not_use_included_overrides(cli, tmpdir, datafiles): - project = os.path.join(str(datafiles), 'overrides-junction') - - generate_junction(tmpdir, - os.path.join(project, 'subproject'), - os.path.join(project, 'junction.bst'), - store_ref=True) - - result = cli.run(project=project, args=[ - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'junction.bst']) + project = os.path.join(str(datafiles), "overrides-junction") + + generate_junction( + tmpdir, os.path.join(project, "subproject"), os.path.join(project, "junction.bst"), store_ref=True + ) + + result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "junction.bst"]) result.assert_success() loaded = _yaml.load_data(result.output) - assert loaded.get_str('main_override', default=None) is not None - assert loaded.get_str('included_override', default=None) is None + assert loaded.get_str("main_override", default=None) is not None + assert loaded.get_str("included_override", default=None) is None @pytest.mark.datafiles(DATA_DIR) def test_conditional_in_fragment(cli, datafiles): - project = os.path.join(str(datafiles), 'conditional') - - result = cli.run(project=project, args=[ - '-o', 'build_arch', 'x86_64', - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst']) + project = os.path.join(str(datafiles), "conditional") + + result = cli.run( + project=project, + args=["-o", "build_arch", "x86_64", "show", "--deps", "none", "--format", "%{vars}", "element.bst"], + ) result.assert_success() loaded = _yaml.load_data(result.output) - assert loaded.get_str('size') == '8' + assert loaded.get_str("size") == "8" @pytest.mark.datafiles(DATA_DIR) def test_inner(cli, datafiles): - project = os.path.join(str(datafiles), 'inner') - result = cli.run(project=project, args=[ - '-o', 'build_arch', 'x86_64', - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst']) + project = os.path.join(str(datafiles), "inner") + result = cli.run( + project=project, + args=["-o", "build_arch", "x86_64", "show", "--deps", "none", "--format", "%{vars}", "element.bst"], + ) result.assert_success() loaded = _yaml.load_data(result.output) - assert loaded.get_str('build_arch') == 'x86_64' + assert loaded.get_str("build_arch") == "x86_64" @pytest.mark.datafiles(DATA_DIR) def test_recursive_include(cli, datafiles): - project = os.path.join(str(datafiles), 'recursive') + project = os.path.join(str(datafiles), "recursive") - result = cli.run(project=project, args=[ - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst']) + result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.RECURSIVE_INCLUDE) - assert 'line 2 column 2' in result.stderr + assert "line 2 column 2" in result.stderr @pytest.mark.datafiles(DATA_DIR) def test_local_to_junction(cli, tmpdir, datafiles): - project = os.path.join(str(datafiles), 'local_to_junction') - - generate_junction(tmpdir, - os.path.join(project, 'subproject'), - os.path.join(project, 'junction.bst'), - store_ref=True) - - result = cli.run(project=project, args=[ - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst']) + project = os.path.join(str(datafiles), "local_to_junction") + + generate_junction( + tmpdir, os.path.join(project, "subproject"), os.path.join(project, "junction.bst"), store_ref=True + ) + + result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]) result.assert_success() loaded = _yaml.load_data(result.output) - assert loaded.get_bool('included') + assert loaded.get_bool("included") diff --git a/tests/format/include_composition.py b/tests/format/include_composition.py index e10e28bc0..4af6862fa 100644 --- a/tests/format/include_composition.py +++ b/tests/format/include_composition.py @@ -11,7 +11,7 @@ from tests.testutils import dummy_context @contextmanager def make_includes(basedir): - _yaml.roundtrip_dump({'name': 'test'}, os.path.join(basedir, 'project.conf')) + _yaml.roundtrip_dump({"name": "test"}, os.path.join(basedir, "project.conf")) with dummy_context() as context: project = Project(basedir, context) loader = project.loader @@ -21,131 +21,118 @@ def make_includes(basedir): def test_main_has_priority(tmpdir): with make_includes(str(tmpdir)) as includes: - _yaml.roundtrip_dump({'(@)': ['a.yml'], 'test': ['main']}, - str(tmpdir.join('main.yml'))) + _yaml.roundtrip_dump({"(@)": ["a.yml"], "test": ["main"]}, str(tmpdir.join("main.yml"))) - main = _yaml.load(str(tmpdir.join('main.yml'))) + main = _yaml.load(str(tmpdir.join("main.yml"))) - _yaml.roundtrip_dump({'test': ['a']}, str(tmpdir.join('a.yml'))) + _yaml.roundtrip_dump({"test": ["a"]}, str(tmpdir.join("a.yml"))) includes.process(main) - assert main.get_str_list('test') == ['main'] + assert main.get_str_list("test") == ["main"] def test_include_cannot_append(tmpdir): with make_includes(str(tmpdir)) as includes: - _yaml.roundtrip_dump({'(@)': ['a.yml'], 'test': ['main']}, - str(tmpdir.join('main.yml'))) - main = _yaml.load(str(tmpdir.join('main.yml'))) + _yaml.roundtrip_dump({"(@)": ["a.yml"], "test": ["main"]}, str(tmpdir.join("main.yml"))) + main = _yaml.load(str(tmpdir.join("main.yml"))) - _yaml.roundtrip_dump({'test': {'(>)': ['a']}}, - str(tmpdir.join('a.yml'))) + _yaml.roundtrip_dump({"test": {"(>)": ["a"]}}, str(tmpdir.join("a.yml"))) includes.process(main) - assert main.get_str_list('test') == ['main'] + assert main.get_str_list("test") == ["main"] def test_main_can_append(tmpdir): with make_includes(str(tmpdir)) as includes: - _yaml.roundtrip_dump({'(@)': ['a.yml'], 'test': {'(>)': ['main']}}, - str(tmpdir.join('main.yml'))) - main = _yaml.load(str(tmpdir.join('main.yml'))) + _yaml.roundtrip_dump({"(@)": ["a.yml"], "test": {"(>)": ["main"]}}, str(tmpdir.join("main.yml"))) + main = _yaml.load(str(tmpdir.join("main.yml"))) - _yaml.roundtrip_dump({'test': ['a']}, str(tmpdir.join('a.yml'))) + _yaml.roundtrip_dump({"test": ["a"]}, str(tmpdir.join("a.yml"))) includes.process(main) - assert main.get_str_list('test') == ['a', 'main'] + assert main.get_str_list("test") == ["a", "main"] def test_sibling_cannot_append_backward(tmpdir): with make_includes(str(tmpdir)) as includes: - _yaml.roundtrip_dump({'(@)': ['a.yml', 'b.yml']}, - str(tmpdir.join('main.yml'))) - main = _yaml.load(str(tmpdir.join('main.yml'))) + _yaml.roundtrip_dump({"(@)": ["a.yml", "b.yml"]}, str(tmpdir.join("main.yml"))) + main = _yaml.load(str(tmpdir.join("main.yml"))) - _yaml.roundtrip_dump({'test': {'(>)': ['a']}}, - str(tmpdir.join('a.yml'))) - _yaml.roundtrip_dump({'test': ['b']}, - str(tmpdir.join('b.yml'))) + _yaml.roundtrip_dump({"test": {"(>)": ["a"]}}, str(tmpdir.join("a.yml"))) + _yaml.roundtrip_dump({"test": ["b"]}, str(tmpdir.join("b.yml"))) includes.process(main) - assert main.get_str_list('test') == ['b'] + assert main.get_str_list("test") == ["b"] def test_sibling_can_append_forward(tmpdir): with make_includes(str(tmpdir)) as includes: - _yaml.roundtrip_dump({'(@)': ['a.yml', 'b.yml']}, - str(tmpdir.join('main.yml'))) - main = _yaml.load(str(tmpdir.join('main.yml'))) + _yaml.roundtrip_dump({"(@)": ["a.yml", "b.yml"]}, str(tmpdir.join("main.yml"))) + main = _yaml.load(str(tmpdir.join("main.yml"))) - _yaml.roundtrip_dump({'test': ['a']}, - str(tmpdir.join('a.yml'))) - _yaml.roundtrip_dump({'test': {'(>)': ['b']}}, - str(tmpdir.join('b.yml'))) + _yaml.roundtrip_dump({"test": ["a"]}, str(tmpdir.join("a.yml"))) + _yaml.roundtrip_dump({"test": {"(>)": ["b"]}}, str(tmpdir.join("b.yml"))) includes.process(main) - assert main.get_str_list('test') == ['a', 'b'] + assert main.get_str_list("test") == ["a", "b"] def test_lastest_sibling_has_priority(tmpdir): with make_includes(str(tmpdir)) as includes: - _yaml.roundtrip_dump({'(@)': ['a.yml', 'b.yml']}, - str(tmpdir.join('main.yml'))) - main = _yaml.load(str(tmpdir.join('main.yml'))) + _yaml.roundtrip_dump({"(@)": ["a.yml", "b.yml"]}, str(tmpdir.join("main.yml"))) + main = _yaml.load(str(tmpdir.join("main.yml"))) - _yaml.roundtrip_dump({'test': ['a']}, - str(tmpdir.join('a.yml'))) - _yaml.roundtrip_dump({'test': ['b']}, - str(tmpdir.join('b.yml'))) + _yaml.roundtrip_dump({"test": ["a"]}, str(tmpdir.join("a.yml"))) + _yaml.roundtrip_dump({"test": ["b"]}, str(tmpdir.join("b.yml"))) includes.process(main) - assert main.get_str_list('test') == ['b'] + assert main.get_str_list("test") == ["b"] def test_main_keeps_keys(tmpdir): with make_includes(str(tmpdir)) as includes: - _yaml.roundtrip_dump({'(@)': ['a.yml'], 'something': 'else'}, - str(tmpdir.join('main.yml'))) - main = _yaml.load(str(tmpdir.join('main.yml'))) + _yaml.roundtrip_dump({"(@)": ["a.yml"], "something": "else"}, str(tmpdir.join("main.yml"))) + main = _yaml.load(str(tmpdir.join("main.yml"))) - _yaml.roundtrip_dump({'test': ['a']}, str(tmpdir.join('a.yml'))) + _yaml.roundtrip_dump({"test": ["a"]}, str(tmpdir.join("a.yml"))) includes.process(main) - assert main.get_str_list('test') == ['a'] - assert main.get_str('something') == 'else' + assert main.get_str_list("test") == ["a"] + assert main.get_str("something") == "else" def test_overwrite_directive_on_later_composite(tmpdir): with make_includes(str(tmpdir)) as includes: - _yaml.roundtrip_dump({'(@)': ['a.yml', 'b.yml'], 'test': {'(=)': ['Overwritten']}}, - str(tmpdir.join('main.yml'))) + _yaml.roundtrip_dump( + {"(@)": ["a.yml", "b.yml"], "test": {"(=)": ["Overwritten"]}}, str(tmpdir.join("main.yml")) + ) - main = _yaml.load(str(tmpdir.join('main.yml'))) + main = _yaml.load(str(tmpdir.join("main.yml"))) # a.yml - _yaml.roundtrip_dump({'test': ['some useless', 'list', 'to be overwritten'], - 'foo': 'should not be present'}, - str(tmpdir.join('a.yml'))) + _yaml.roundtrip_dump( + {"test": ["some useless", "list", "to be overwritten"], "foo": "should not be present"}, + str(tmpdir.join("a.yml")), + ) # b.yaml isn't going to have a 'test' node to overwrite - _yaml.roundtrip_dump({'foo': 'should be present'}, - str(tmpdir.join('b.yml'))) + _yaml.roundtrip_dump({"foo": "should be present"}, str(tmpdir.join("b.yml"))) includes.process(main) - assert main.get_str_list('test') == ['Overwritten'] - assert main.get_str('foo') == 'should be present' + assert main.get_str_list("test") == ["Overwritten"] + assert main.get_str("foo") == "should be present" diff --git a/tests/format/invalid_keys.py b/tests/format/invalid_keys.py index 861cfeabd..ce1e2e487 100644 --- a/tests/format/invalid_keys.py +++ b/tests/format/invalid_keys.py @@ -7,21 +7,21 @@ from buildstream._exceptions import ErrorDomain, LoadErrorReason from buildstream.testing.runcli import cli # pylint: disable=unused-import # Project directory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - 'invalid-keys' -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "invalid-keys") @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize(("element", "location"), [ - ("no-path-specified.bst", "line 4 column 4"), - ("optional-source.bst", "line 6 column 10"), - ("included-source.bst", "line 4 column 4"), -]) +@pytest.mark.parametrize( + ("element", "location"), + [ + ("no-path-specified.bst", "line 4 column 4"), + ("optional-source.bst", "line 6 column 10"), + ("included-source.bst", "line 4 column 4"), + ], +) def test_compositied_node_fails_usefully(cli, datafiles, element, location): project = str(datafiles) - result = cli.run(project=project, args=['show', element]) + result = cli.run(project=project, args=["show", element]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) assert "synthetic node" not in result.stderr diff --git a/tests/format/junctions.py b/tests/format/junctions.py index a0af521a2..45ccff222 100644 --- a/tests/format/junctions.py +++ b/tests/format/junctions.py @@ -13,10 +13,7 @@ from buildstream.testing import create_repo from buildstream.testing._utils.site import HAVE_GIT -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - 'junctions', -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "junctions",) def copy_subprojects(project, datafiles, subprojects): @@ -26,41 +23,41 @@ def copy_subprojects(project, datafiles, subprojects): @pytest.mark.datafiles(DATA_DIR) def test_simple_pipeline(cli, datafiles): - project = os.path.join(str(datafiles), 'foo') - copy_subprojects(project, datafiles, ['base']) + project = os.path.join(str(datafiles), "foo") + copy_subprojects(project, datafiles, ["base"]) # Check that the pipeline includes the subproject element - element_list = cli.get_pipeline(project, ['target.bst']) - assert 'base.bst:target.bst' in element_list + element_list = cli.get_pipeline(project, ["target.bst"]) + assert "base.bst:target.bst" in element_list @pytest.mark.datafiles(DATA_DIR) def test_simple_build(cli, tmpdir, datafiles): - project = os.path.join(str(datafiles), 'foo') - copy_subprojects(project, datafiles, ['base']) + project = os.path.join(str(datafiles), "foo") + copy_subprojects(project, datafiles, ["base"]) checkoutdir = os.path.join(str(tmpdir), "checkout") # Build, checkout - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the checkout contains the expected files from both projects - assert os.path.exists(os.path.join(checkoutdir, 'base.txt')) - assert os.path.exists(os.path.join(checkoutdir, 'foo.txt')) + assert os.path.exists(os.path.join(checkoutdir, "base.txt")) + assert os.path.exists(os.path.join(checkoutdir, "foo.txt")) @pytest.mark.datafiles(DATA_DIR) def test_junction_missing_project_conf(cli, datafiles): - project = datafiles / 'foo' - copy_subprojects(project, datafiles, ['base']) + project = datafiles / "foo" + copy_subprojects(project, datafiles, ["base"]) # TODO: see if datafiles can tidy this concat up # py3.5 requires this str conversion. - os.remove(str(project / 'base' / 'project.conf')) + os.remove(str(project / "base" / "project.conf")) # Note that both 'foo' and 'base' projects have a 'target.bst'. The # 'app.bst' in 'foo' depends on the 'target.bst' in 'base', i.e.: @@ -78,7 +75,7 @@ def test_junction_missing_project_conf(cli, datafiles): # That would lead to a 'circular dependency error' in this setup, when we # expect an 'invalid junction'. # - result = cli.run(project=project, args=['build', 'app.bst']) + result = cli.run(project=project, args=["build", "app.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_JUNCTION) # Assert that we have the expected provenance encoded into the error @@ -89,20 +86,18 @@ def test_junction_missing_project_conf(cli, datafiles): def test_workspaced_junction_missing_project_conf(cli, datafiles): # See test_junction_missing_project_conf for some more background. - project = datafiles / 'foo' - workspace_dir = project / 'base_workspace' - copy_subprojects(project, datafiles, ['base']) + project = datafiles / "foo" + workspace_dir = project / "base_workspace" + copy_subprojects(project, datafiles, ["base"]) - result = cli.run( - project=project, - args=['workspace', 'open', 'base.bst', '--directory', workspace_dir]) + result = cli.run(project=project, args=["workspace", "open", "base.bst", "--directory", workspace_dir]) print(result) result.assert_success() # py3.5 requires this str conversion. - os.remove(str(workspace_dir / 'project.conf')) + os.remove(str(workspace_dir / "project.conf")) - result = cli.run(project=project, args=['build', 'app.bst']) + result = cli.run(project=project, args=["build", "app.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_JUNCTION) # Assert that we have the expected provenance encoded into the error @@ -111,18 +106,18 @@ def test_workspaced_junction_missing_project_conf(cli, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_build_of_same_junction_used_twice(cli, datafiles): - project = os.path.join(str(datafiles), 'inconsistent-names') + project = os.path.join(str(datafiles), "inconsistent-names") # Check we can build a project that contains the same junction # that is used twice, but named differently - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() @pytest.mark.datafiles(DATA_DIR) def test_missing_file_in_subproject(cli, datafiles): - project = os.path.join(str(datafiles), 'missing-element') - result = cli.run(project=project, args=['show', 'target.bst']) + project = os.path.join(str(datafiles), "missing-element") + result = cli.run(project=project, args=["show", "target.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE) # Assert that we have the expected provenance encoded into the error @@ -131,8 +126,8 @@ def test_missing_file_in_subproject(cli, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_missing_file_in_subsubproject(cli, datafiles): - project = os.path.join(str(datafiles), 'missing-element') - result = cli.run(project=project, args=['show', 'sub-target.bst']) + project = os.path.join(str(datafiles), "missing-element") + result = cli.run(project=project, args=["show", "sub-target.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE) # Assert that we have the expected provenance encoded into the error @@ -141,8 +136,8 @@ def test_missing_file_in_subsubproject(cli, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_missing_junction_in_subproject(cli, datafiles): - project = os.path.join(str(datafiles), 'missing-element') - result = cli.run(project=project, args=['show', 'sub-target-bad-junction.bst']) + project = os.path.join(str(datafiles), "missing-element") + result = cli.run(project=project, args=["show", "sub-target-bad-junction.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE) # Assert that we have the expected provenance encoded into the error @@ -151,62 +146,62 @@ def test_missing_junction_in_subproject(cli, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_nested_simple(cli, tmpdir, datafiles): - project = os.path.join(str(datafiles), 'foo') - copy_subprojects(project, datafiles, ['base']) + project = os.path.join(str(datafiles), "foo") + copy_subprojects(project, datafiles, ["base"]) - project = os.path.join(str(datafiles), 'nested') - copy_subprojects(project, datafiles, ['foo']) + project = os.path.join(str(datafiles), "nested") + copy_subprojects(project, datafiles, ["foo"]) checkoutdir = os.path.join(str(tmpdir), "checkout") # Build, checkout - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the checkout contains the expected files from all subprojects - assert os.path.exists(os.path.join(checkoutdir, 'base.txt')) - assert os.path.exists(os.path.join(checkoutdir, 'foo.txt')) + assert os.path.exists(os.path.join(checkoutdir, "base.txt")) + assert os.path.exists(os.path.join(checkoutdir, "foo.txt")) @pytest.mark.datafiles(DATA_DIR) def test_nested_double(cli, tmpdir, datafiles): - project_foo = os.path.join(str(datafiles), 'foo') - copy_subprojects(project_foo, datafiles, ['base']) + project_foo = os.path.join(str(datafiles), "foo") + copy_subprojects(project_foo, datafiles, ["base"]) - project_bar = os.path.join(str(datafiles), 'bar') - copy_subprojects(project_bar, datafiles, ['base']) + project_bar = os.path.join(str(datafiles), "bar") + copy_subprojects(project_bar, datafiles, ["base"]) - project = os.path.join(str(datafiles), 'toplevel') - copy_subprojects(project, datafiles, ['base', 'foo', 'bar']) + project = os.path.join(str(datafiles), "toplevel") + copy_subprojects(project, datafiles, ["base", "foo", "bar"]) checkoutdir = os.path.join(str(tmpdir), "checkout") # Build, checkout - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the checkout contains the expected files from all subprojects - assert os.path.exists(os.path.join(checkoutdir, 'base.txt')) - assert os.path.exists(os.path.join(checkoutdir, 'foo.txt')) - assert os.path.exists(os.path.join(checkoutdir, 'bar.txt')) + assert os.path.exists(os.path.join(checkoutdir, "base.txt")) + assert os.path.exists(os.path.join(checkoutdir, "foo.txt")) + assert os.path.exists(os.path.join(checkoutdir, "bar.txt")) @pytest.mark.datafiles(DATA_DIR) def test_nested_conflict(cli, datafiles): - project_foo = os.path.join(str(datafiles), 'foo') - copy_subprojects(project_foo, datafiles, ['base']) + project_foo = os.path.join(str(datafiles), "foo") + copy_subprojects(project_foo, datafiles, ["base"]) - project_bar = os.path.join(str(datafiles), 'bar') - copy_subprojects(project_bar, datafiles, ['base']) + project_bar = os.path.join(str(datafiles), "bar") + copy_subprojects(project_bar, datafiles, ["base"]) - project = os.path.join(str(datafiles), 'conflict') - copy_subprojects(project, datafiles, ['foo', 'bar']) + project = os.path.join(str(datafiles), "conflict") + copy_subprojects(project, datafiles, ["foo", "bar"]) - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.CONFLICTING_JUNCTION) assert "bar.bst:target.bst [line 3 column 2]" in result.stderr @@ -215,49 +210,49 @@ def test_nested_conflict(cli, datafiles): # Test that we error correctly when the junction element itself is missing @pytest.mark.datafiles(DATA_DIR) def test_missing_junction(cli, datafiles): - project = os.path.join(str(datafiles), 'invalid') + project = os.path.join(str(datafiles), "invalid") - result = cli.run(project=project, args=['build', 'missing.bst']) + result = cli.run(project=project, args=["build", "missing.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE) # Test that we error correctly when an element is not found in the subproject @pytest.mark.datafiles(DATA_DIR) def test_missing_subproject_element(cli, datafiles): - project = os.path.join(str(datafiles), 'invalid') - copy_subprojects(project, datafiles, ['base']) + project = os.path.join(str(datafiles), "invalid") + copy_subprojects(project, datafiles, ["base"]) - result = cli.run(project=project, args=['build', 'missing-element.bst']) + result = cli.run(project=project, args=["build", "missing-element.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE) # Test that we error correctly when a junction itself has dependencies @pytest.mark.datafiles(DATA_DIR) def test_invalid_with_deps(cli, datafiles): - project = os.path.join(str(datafiles), 'invalid') - copy_subprojects(project, datafiles, ['base']) + project = os.path.join(str(datafiles), "invalid") + copy_subprojects(project, datafiles, ["base"]) - result = cli.run(project=project, args=['build', 'junction-with-deps.bst']) + result = cli.run(project=project, args=["build", "junction-with-deps.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_JUNCTION) # Test that we error correctly when a junction is directly depended on @pytest.mark.datafiles(DATA_DIR) def test_invalid_junction_dep(cli, datafiles): - project = os.path.join(str(datafiles), 'invalid') - copy_subprojects(project, datafiles, ['base']) + project = os.path.join(str(datafiles), "invalid") + copy_subprojects(project, datafiles, ["base"]) - result = cli.run(project=project, args=['build', 'junction-dep.bst']) + result = cli.run(project=project, args=["build", "junction-dep.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) # Test that we error correctly when we junction-depend on a non-junction @pytest.mark.datafiles(DATA_DIR) def test_invalid_junctiondep_not_a_junction(cli, datafiles): - project = os.path.join(str(datafiles), 'invalid') - copy_subprojects(project, datafiles, ['base']) + project = os.path.join(str(datafiles), "invalid") + copy_subprojects(project, datafiles, ["base"]) - result = cli.run(project=project, args=['build', 'junctiondep-not-a-junction.bst']) + result = cli.run(project=project, args=["build", "junctiondep-not-a-junction.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) # Assert that we have the expected provenance encoded into the error @@ -266,132 +261,117 @@ def test_invalid_junctiondep_not_a_junction(cli, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_options_default(cli, tmpdir, datafiles): - project = os.path.join(str(datafiles), 'options-default') - copy_subprojects(project, datafiles, ['options-base']) + project = os.path.join(str(datafiles), "options-default") + copy_subprojects(project, datafiles, ["options-base"]) checkoutdir = os.path.join(str(tmpdir), "checkout") # Build, checkout - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() - assert os.path.exists(os.path.join(checkoutdir, 'pony.txt')) - assert not os.path.exists(os.path.join(checkoutdir, 'horsy.txt')) + assert os.path.exists(os.path.join(checkoutdir, "pony.txt")) + assert not os.path.exists(os.path.join(checkoutdir, "horsy.txt")) @pytest.mark.datafiles(DATA_DIR) def test_options(cli, tmpdir, datafiles): - project = os.path.join(str(datafiles), 'options') - copy_subprojects(project, datafiles, ['options-base']) + project = os.path.join(str(datafiles), "options") + copy_subprojects(project, datafiles, ["options-base"]) checkoutdir = os.path.join(str(tmpdir), "checkout") # Build, checkout - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() - assert not os.path.exists(os.path.join(checkoutdir, 'pony.txt')) - assert os.path.exists(os.path.join(checkoutdir, 'horsy.txt')) + assert not os.path.exists(os.path.join(checkoutdir, "pony.txt")) + assert os.path.exists(os.path.join(checkoutdir, "horsy.txt")) @pytest.mark.datafiles(DATA_DIR) def test_options_inherit(cli, tmpdir, datafiles): - project = os.path.join(str(datafiles), 'options-inherit') - copy_subprojects(project, datafiles, ['options-base']) + project = os.path.join(str(datafiles), "options-inherit") + copy_subprojects(project, datafiles, ["options-base"]) checkoutdir = os.path.join(str(tmpdir), "checkout") # Build, checkout - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() - assert not os.path.exists(os.path.join(checkoutdir, 'pony.txt')) - assert os.path.exists(os.path.join(checkoutdir, 'horsy.txt')) + assert not os.path.exists(os.path.join(checkoutdir, "pony.txt")) + assert os.path.exists(os.path.join(checkoutdir, "horsy.txt")) @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") @pytest.mark.datafiles(DATA_DIR) def test_git_show(cli, tmpdir, datafiles): - project = os.path.join(str(datafiles), 'foo') + project = os.path.join(str(datafiles), "foo") # Create the repo from 'base' subdir - repo = create_repo('git', str(tmpdir)) - ref = repo.create(os.path.join(str(datafiles), 'base')) + repo = create_repo("git", str(tmpdir)) + ref = repo.create(os.path.join(str(datafiles), "base")) # Write out junction element with git source - element = { - 'kind': 'junction', - 'sources': [ - repo.source_config(ref=ref) - ] - } - _yaml.roundtrip_dump(element, os.path.join(project, 'base.bst')) + element = {"kind": "junction", "sources": [repo.source_config(ref=ref)]} + _yaml.roundtrip_dump(element, os.path.join(project, "base.bst")) # Check that bst show succeeds with implicit subproject fetching and the # pipeline includes the subproject element - element_list = cli.get_pipeline(project, ['target.bst']) - assert 'base.bst:target.bst' in element_list + element_list = cli.get_pipeline(project, ["target.bst"]) + assert "base.bst:target.bst" in element_list @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") @pytest.mark.datafiles(DATA_DIR) def test_git_build(cli, tmpdir, datafiles): - project = os.path.join(str(datafiles), 'foo') + project = os.path.join(str(datafiles), "foo") checkoutdir = os.path.join(str(tmpdir), "checkout") # Create the repo from 'base' subdir - repo = create_repo('git', str(tmpdir)) - ref = repo.create(os.path.join(str(datafiles), 'base')) + repo = create_repo("git", str(tmpdir)) + ref = repo.create(os.path.join(str(datafiles), "base")) # Write out junction element with git source - element = { - 'kind': 'junction', - 'sources': [ - repo.source_config(ref=ref) - ] - } - _yaml.roundtrip_dump(element, os.path.join(project, 'base.bst')) + element = {"kind": "junction", "sources": [repo.source_config(ref=ref)]} + _yaml.roundtrip_dump(element, os.path.join(project, "base.bst")) # Build (with implicit fetch of subproject), checkout - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the checkout contains the expected files from both projects - assert os.path.exists(os.path.join(checkoutdir, 'base.txt')) - assert os.path.exists(os.path.join(checkoutdir, 'foo.txt')) + assert os.path.exists(os.path.join(checkoutdir, "base.txt")) + assert os.path.exists(os.path.join(checkoutdir, "foo.txt")) @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") @pytest.mark.datafiles(DATA_DIR) def test_git_missing_project_conf(cli, tmpdir, datafiles): - project = datafiles / 'foo' + project = datafiles / "foo" # See test_junction_missing_project_conf for some more background. # py3.5 requires this str conversion. - os.remove(str(datafiles / 'base' / 'project.conf')) + os.remove(str(datafiles / "base" / "project.conf")) # Create the repo from 'base' subdir - repo = create_repo('git', str(tmpdir)) - ref = repo.create(os.path.join(str(datafiles), 'base')) + repo = create_repo("git", str(tmpdir)) + ref = repo.create(os.path.join(str(datafiles), "base")) # Write out junction element with git source - element = { - 'kind': 'junction', - 'sources': [ - repo.source_config(ref=ref) - ] - } - _yaml.roundtrip_dump(element, str(project / 'base.bst')) - - result = cli.run(project=project, args=['build', 'app.bst']) + element = {"kind": "junction", "sources": [repo.source_config(ref=ref)]} + _yaml.roundtrip_dump(element, str(project / "base.bst")) + + result = cli.run(project=project, args=["build", "app.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_JUNCTION) # Assert that we have the expected provenance encoded into the error @@ -400,64 +380,59 @@ def test_git_missing_project_conf(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_cross_junction_names(cli, datafiles): - project = os.path.join(str(datafiles), 'foo') - copy_subprojects(project, datafiles, ['base']) + project = os.path.join(str(datafiles), "foo") + copy_subprojects(project, datafiles, ["base"]) - element_list = cli.get_pipeline(project, ['base.bst:target.bst']) - assert 'base.bst:target.bst' in element_list + element_list = cli.get_pipeline(project, ["base.bst:target.bst"]) + assert "base.bst:target.bst" in element_list @pytest.mark.datafiles(DATA_DIR) def test_build_git_cross_junction_names(cli, tmpdir, datafiles): - project = os.path.join(str(datafiles), 'foo') + project = os.path.join(str(datafiles), "foo") checkoutdir = os.path.join(str(tmpdir), "checkout") # Create the repo from 'base' subdir - repo = create_repo('git', str(tmpdir)) - ref = repo.create(os.path.join(str(datafiles), 'base')) + repo = create_repo("git", str(tmpdir)) + ref = repo.create(os.path.join(str(datafiles), "base")) # Write out junction element with git source - element = { - 'kind': 'junction', - 'sources': [ - repo.source_config(ref=ref) - ] - } - _yaml.roundtrip_dump(element, os.path.join(project, 'base.bst')) + element = {"kind": "junction", "sources": [repo.source_config(ref=ref)]} + _yaml.roundtrip_dump(element, os.path.join(project, "base.bst")) print(element) - print(cli.get_pipeline(project, ['base.bst'])) + print(cli.get_pipeline(project, ["base.bst"])) # Build (with implicit fetch of subproject), checkout - result = cli.run(project=project, args=['build', 'base.bst:target.bst']) + result = cli.run(project=project, args=["build", "base.bst:target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'base.bst:target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "base.bst:target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the checkout contains the expected files from both projects - assert os.path.exists(os.path.join(checkoutdir, 'base.txt')) + assert os.path.exists(os.path.join(checkoutdir, "base.txt")) @pytest.mark.datafiles(DATA_DIR) def test_config_target(cli, tmpdir, datafiles): - project = os.path.join(str(datafiles), 'config-target') - checkoutdir = os.path.join(str(tmpdir), 'checkout') + project = os.path.join(str(datafiles), "config-target") + checkoutdir = os.path.join(str(tmpdir), "checkout") # Build, checkout - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the checkout contains the expected files from sub-sub-project - assert os.path.exists(os.path.join(checkoutdir, 'hello.txt')) + assert os.path.exists(os.path.join(checkoutdir, "hello.txt")) @pytest.mark.datafiles(DATA_DIR) def test_invalid_sources_and_target(cli, tmpdir, datafiles): - project = os.path.join(str(datafiles), 'config-target') + project = os.path.join(str(datafiles), "config-target") - result = cli.run(project=project, args=['show', 'invalid-source-target.bst']) + result = cli.run(project=project, args=["show", "invalid-source-target.bst"]) result.assert_main_error(ErrorDomain.ELEMENT, None) assert "junction elements cannot define both 'sources' and 'target' config option" in result.stderr @@ -465,15 +440,15 @@ def test_invalid_sources_and_target(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_invalid_target_name(cli, tmpdir, datafiles): - project = os.path.join(str(datafiles), 'config-target') + project = os.path.join(str(datafiles), "config-target") # Rename our junction element to the same name as its target - old_path = os.path.join(project, 'elements/subsubproject.bst') - new_path = os.path.join(project, 'elements/subsubproject-junction.bst') + old_path = os.path.join(project, "elements/subsubproject.bst") + new_path = os.path.join(project, "elements/subsubproject-junction.bst") os.rename(old_path, new_path) # This should fail now - result = cli.run(project=project, args=['show', 'subsubproject-junction.bst']) + result = cli.run(project=project, args=["show", "subsubproject-junction.bst"]) result.assert_main_error(ErrorDomain.ELEMENT, None) assert "junction elements cannot target an element with the same name" in result.stderr @@ -481,12 +456,12 @@ def test_invalid_target_name(cli, tmpdir, datafiles): # We cannot exhaustively test all possible ways in which this can go wrong, so # test a couple of common ways in which we expect this to go wrong. -@pytest.mark.parametrize('target', ['no-junction.bst', 'nested-junction-target.bst']) +@pytest.mark.parametrize("target", ["no-junction.bst", "nested-junction-target.bst"]) @pytest.mark.datafiles(DATA_DIR) def test_invalid_target_format(cli, tmpdir, datafiles, target): - project = os.path.join(str(datafiles), 'config-target') + project = os.path.join(str(datafiles), "config-target") - result = cli.run(project=project, args=['show', target]) + result = cli.run(project=project, args=["show", target]) result.assert_main_error(ErrorDomain.ELEMENT, None) assert "'target' option must be in format '{junction-name}:{element-name}'" in result.stderr diff --git a/tests/format/listdirectiveerrors.py b/tests/format/listdirectiveerrors.py index 269b521a4..79102cb02 100644 --- a/tests/format/listdirectiveerrors.py +++ b/tests/format/listdirectiveerrors.py @@ -12,38 +12,28 @@ DATA_DIR = os.path.dirname(os.path.realpath(__file__)) @pytest.mark.datafiles(DATA_DIR) def test_project_error(cli, datafiles): - project = os.path.join(datafiles.dirname, datafiles.basename, 'list-directive-error-project') - result = cli.run(project=project, silent=True, args=[ - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst']) + project = os.path.join(datafiles.dirname, datafiles.basename, "list-directive-error-project") + result = cli.run( + project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"] + ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.TRAILING_LIST_DIRECTIVE) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("target", [ - ('variables.bst'), ('environment.bst'), ('config.bst'), ('public.bst') -]) +@pytest.mark.parametrize("target", [("variables.bst"), ("environment.bst"), ("config.bst"), ("public.bst")]) def test_element_error(cli, datafiles, target): - project = os.path.join(datafiles.dirname, datafiles.basename, 'list-directive-error-element') - result = cli.run(project=project, silent=True, args=[ - 'show', - '--deps', 'none', - '--format', '%{vars}', - target]) + project = os.path.join(datafiles.dirname, datafiles.basename, "list-directive-error-element") + result = cli.run(project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", target]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.TRAILING_LIST_DIRECTIVE) @pytest.mark.datafiles(DATA_DIR) def test_project_composite_error(cli, datafiles): - project = os.path.join(datafiles.dirname, datafiles.basename, 'list-directive-type-error') - result = cli.run(project=project, silent=True, args=[ - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst']) + project = os.path.join(datafiles.dirname, datafiles.basename, "list-directive-type-error") + result = cli.run( + project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"] + ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.ILLEGAL_COMPOSITE) diff --git a/tests/format/optionarch.py b/tests/format/optionarch.py index f347e27ae..75cae9abe 100644 --- a/tests/format/optionarch.py +++ b/tests/format/optionarch.py @@ -16,52 +16,45 @@ DATA_DIR = os.path.dirname(os.path.realpath(__file__)) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("machine,value,expected", [ - # Test explicitly provided arches - ('arm', 'aarch32', 'Army'), - ('arm', 'aarch64', 'Aarchy'), - - # Test automatically derived arches - ('arm', None, 'Army'), - ('aarch64', None, 'Aarchy'), - - # Test that explicitly provided arches dont error out - # when the `uname` reported arch is not supported - ('i386', 'aarch32', 'Army'), - ('x86_64', 'aarch64', 'Aarchy'), -]) +@pytest.mark.parametrize( + "machine,value,expected", + [ + # Test explicitly provided arches + ("arm", "aarch32", "Army"), + ("arm", "aarch64", "Aarchy"), + # Test automatically derived arches + ("arm", None, "Army"), + ("aarch64", None, "Aarchy"), + # Test that explicitly provided arches dont error out + # when the `uname` reported arch is not supported + ("i386", "aarch32", "Army"), + ("x86_64", "aarch64", "Aarchy"), + ], +) def test_conditional(cli, datafiles, machine, value, expected): with override_platform_uname(machine=machine): - project = os.path.join(datafiles.dirname, datafiles.basename, 'option-arch') + project = os.path.join(datafiles.dirname, datafiles.basename, "option-arch") bst_args = [] if value is not None: - bst_args += ['--option', 'machine_arch', value] - - bst_args += [ - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst' - ] + bst_args += ["--option", "machine_arch", value] + + bst_args += ["show", "--deps", "none", "--format", "%{vars}", "element.bst"] result = cli.run(project=project, silent=True, args=bst_args) result.assert_success() loaded = _yaml.load_data(result.output) - assert loaded.get_str('result') == expected + assert loaded.get_str("result") == expected @pytest.mark.datafiles(DATA_DIR) def test_unsupported_arch(cli, datafiles): with override_platform_uname(machine="x86_64"): - project = os.path.join(datafiles.dirname, datafiles.basename, 'option-arch') - result = cli.run(project=project, silent=True, args=[ - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst' - ]) + project = os.path.join(datafiles.dirname, datafiles.basename, "option-arch") + result = cli.run( + project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"] + ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @@ -70,13 +63,10 @@ def test_unsupported_arch(cli, datafiles): def test_alias(cli, datafiles): with override_platform_uname(machine="arm"): - project = os.path.join(datafiles.dirname, datafiles.basename, 'option-arch-alias') - result = cli.run(project=project, silent=True, args=[ - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst' - ]) + project = os.path.join(datafiles.dirname, datafiles.basename, "option-arch-alias") + result = cli.run( + project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"] + ) result.assert_success() @@ -85,13 +75,10 @@ def test_alias(cli, datafiles): def test_unknown_host_arch(cli, datafiles): with override_platform_uname(machine="x86_128"): - project = os.path.join(datafiles.dirname, datafiles.basename, 'option-arch') - result = cli.run(project=project, silent=True, args=[ - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst' - ]) + project = os.path.join(datafiles.dirname, datafiles.basename, "option-arch") + result = cli.run( + project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"] + ) result.assert_main_error(ErrorDomain.PLATFORM, None) @@ -99,12 +86,9 @@ def test_unknown_host_arch(cli, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_unknown_project_arch(cli, datafiles): - project = os.path.join(datafiles.dirname, datafiles.basename, 'option-arch-unknown') - result = cli.run(project=project, silent=True, args=[ - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst' - ]) + project = os.path.join(datafiles.dirname, datafiles.basename, "option-arch-unknown") + result = cli.run( + project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"] + ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) diff --git a/tests/format/optionbool.py b/tests/format/optionbool.py index d772b483c..3b05aafa1 100644 --- a/tests/format/optionbool.py +++ b/tests/format/optionbool.py @@ -12,101 +12,72 @@ DATA_DIR = os.path.dirname(os.path.realpath(__file__)) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("target,option,expected", [ - # Test 'foo' syntax, and valid values of 'True' / 'False' - ('element.bst', 'True', 'a pony'), - ('element.bst', 'true', 'a pony'), - ('element.bst', 'False', 'not pony'), - ('element.bst', 'false', 'not pony'), - - # Test 'not foo' syntax - ('element-not.bst', 'False', 'not pony'), - ('element-not.bst', 'True', 'a pony'), - - # Test 'foo == True' syntax - ('element-equals.bst', 'False', 'not pony'), - ('element-equals.bst', 'True', 'a pony'), - - # Test 'foo != True' syntax - ('element-not-equals.bst', 'False', 'not pony'), - ('element-not-equals.bst', 'True', 'a pony'), -]) +@pytest.mark.parametrize( + "target,option,expected", + [ + # Test 'foo' syntax, and valid values of 'True' / 'False' + ("element.bst", "True", "a pony"), + ("element.bst", "true", "a pony"), + ("element.bst", "False", "not pony"), + ("element.bst", "false", "not pony"), + # Test 'not foo' syntax + ("element-not.bst", "False", "not pony"), + ("element-not.bst", "True", "a pony"), + # Test 'foo == True' syntax + ("element-equals.bst", "False", "not pony"), + ("element-equals.bst", "True", "a pony"), + # Test 'foo != True' syntax + ("element-not-equals.bst", "False", "not pony"), + ("element-not-equals.bst", "True", "a pony"), + ], +) def test_conditional_cli(cli, datafiles, target, option, expected): - project = os.path.join(datafiles.dirname, datafiles.basename, 'option-bool') - result = cli.run(project=project, silent=True, args=[ - '--option', 'pony', option, - 'show', - '--deps', 'none', - '--format', '%{vars}', - target]) + project = os.path.join(datafiles.dirname, datafiles.basename, "option-bool") + result = cli.run( + project=project, + silent=True, + args=["--option", "pony", option, "show", "--deps", "none", "--format", "%{vars}", target], + ) result.assert_success() loaded = _yaml.load_data(result.output) - assert loaded.get_str('thepony') == expected + assert loaded.get_str("thepony") == expected # Test configuration of boolean option in the config file # @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("target,option,expected", [ - ('element.bst', True, 'a pony'), - ('element.bst', False, 'not pony'), -]) +@pytest.mark.parametrize( + "target,option,expected", [("element.bst", True, "a pony"), ("element.bst", False, "not pony"),] +) def test_conditional_config(cli, datafiles, target, option, expected): - project = os.path.join(datafiles.dirname, datafiles.basename, 'option-bool') - cli.configure({ - 'projects': { - 'test': { - 'options': { - 'pony': option - } - } - } - }) - result = cli.run(project=project, silent=True, args=[ - 'show', - '--deps', 'none', - '--format', '%{vars}', - target]) + project = os.path.join(datafiles.dirname, datafiles.basename, "option-bool") + cli.configure({"projects": {"test": {"options": {"pony": option}}}}) + result = cli.run(project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", target]) result.assert_success() loaded = _yaml.load_data(result.output) - assert loaded.get_str('thepony') == expected + assert loaded.get_str("thepony") == expected @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("cli_option", [ - ('falsey'), ('pony'), ('trUE') -]) +@pytest.mark.parametrize("cli_option", [("falsey"), ("pony"), ("trUE")]) def test_invalid_value_cli(cli, datafiles, cli_option): - project = os.path.join(datafiles.dirname, datafiles.basename, 'option-bool') - result = cli.run(project=project, silent=True, args=[ - '--option', 'pony', cli_option, - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst']) + project = os.path.join(datafiles.dirname, datafiles.basename, "option-bool") + result = cli.run( + project=project, + silent=True, + args=["--option", "pony", cli_option, "show", "--deps", "none", "--format", "%{vars}", "element.bst"], + ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("config_option", [ - ('pony'), (['its', 'a', 'list']), ({'dic': 'tionary'}) -]) +@pytest.mark.parametrize("config_option", [("pony"), (["its", "a", "list"]), ({"dic": "tionary"})]) def test_invalid_value_config(cli, datafiles, config_option): - project = os.path.join(datafiles.dirname, datafiles.basename, 'option-bool') - cli.configure({ - 'projects': { - 'test': { - 'options': { - 'pony': config_option - } - } - } - }) - result = cli.run(project=project, silent=True, args=[ - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst']) + project = os.path.join(datafiles.dirname, datafiles.basename, "option-bool") + cli.configure({"projects": {"test": {"options": {"pony": config_option}}}}) + result = cli.run( + project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"] + ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) diff --git a/tests/format/optioneltmask.py b/tests/format/optioneltmask.py index 75265fdd7..399b37b97 100644 --- a/tests/format/optioneltmask.py +++ b/tests/format/optioneltmask.py @@ -12,73 +12,61 @@ DATA_DIR = os.path.dirname(os.path.realpath(__file__)) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("target,value,expected", [ - ('pony.bst', 'pony.bst', 'True'), - ('horsy.bst', 'pony.bst, horsy.bst', 'True'), - ('zebry.bst', 'pony.bst, horsy.bst', 'False'), -]) +@pytest.mark.parametrize( + "target,value,expected", + [ + ("pony.bst", "pony.bst", "True"), + ("horsy.bst", "pony.bst, horsy.bst", "True"), + ("zebry.bst", "pony.bst, horsy.bst", "False"), + ], +) def test_conditional_cli(cli, datafiles, target, value, expected): - project = os.path.join(datafiles.dirname, datafiles.basename, 'option-element-mask') - result = cli.run(project=project, silent=True, args=[ - '--option', 'debug_elements', value, - 'show', - '--deps', 'none', - '--format', '%{vars}', - target]) + project = os.path.join(datafiles.dirname, datafiles.basename, "option-element-mask") + result = cli.run( + project=project, + silent=True, + args=["--option", "debug_elements", value, "show", "--deps", "none", "--format", "%{vars}", target], + ) result.assert_success() loaded = _yaml.load_data(result.output) - assert loaded.get_str('debug') == expected + assert loaded.get_str("debug") == expected @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("target,value,expected", [ - ('pony.bst', ['pony.bst'], 'True'), - ('horsy.bst', ['pony.bst', 'horsy.bst'], 'True'), - ('zebry.bst', ['pony.bst', 'horsy.bst'], 'False'), -]) +@pytest.mark.parametrize( + "target,value,expected", + [ + ("pony.bst", ["pony.bst"], "True"), + ("horsy.bst", ["pony.bst", "horsy.bst"], "True"), + ("zebry.bst", ["pony.bst", "horsy.bst"], "False"), + ], +) def test_conditional_config(cli, datafiles, target, value, expected): - project = os.path.join(datafiles.dirname, datafiles.basename, 'option-element-mask') - cli.configure({ - 'projects': { - 'test': { - 'options': { - 'debug_elements': value - } - } - } - }) - result = cli.run(project=project, silent=True, args=[ - 'show', - '--deps', 'none', - '--format', '%{vars}', - target]) + project = os.path.join(datafiles.dirname, datafiles.basename, "option-element-mask") + cli.configure({"projects": {"test": {"options": {"debug_elements": value}}}}) + result = cli.run(project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", target]) result.assert_success() loaded = _yaml.load_data(result.output) - assert loaded.get_str('debug') == expected + assert loaded.get_str("debug") == expected @pytest.mark.datafiles(DATA_DIR) def test_invalid_declaration(cli, datafiles): - project = os.path.join(datafiles.dirname, datafiles.basename, 'option-element-mask-invalid') - result = cli.run(project=project, silent=True, args=[ - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'pony.bst']) + project = os.path.join(datafiles.dirname, datafiles.basename, "option-element-mask-invalid") + result = cli.run(project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "pony.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @pytest.mark.datafiles(DATA_DIR) def test_invalid_value(cli, datafiles): - project = os.path.join(datafiles.dirname, datafiles.basename, 'option-element-mask') - result = cli.run(project=project, silent=True, args=[ - '--option', 'debug_elements', 'kitten.bst', - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'pony.bst']) + project = os.path.join(datafiles.dirname, datafiles.basename, "option-element-mask") + result = cli.run( + project=project, + silent=True, + args=["--option", "debug_elements", "kitten.bst", "show", "--deps", "none", "--format", "%{vars}", "pony.bst"], + ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) diff --git a/tests/format/optionenum.py b/tests/format/optionenum.py index f9aff503f..ba79bea79 100644 --- a/tests/format/optionenum.py +++ b/tests/format/optionenum.py @@ -12,104 +12,82 @@ DATA_DIR = os.path.dirname(os.path.realpath(__file__)) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("target,option,value,expected", [ - # Test 'var == "foo"' syntax - ('element.bst', 'brother', 'pony', 'a pony'), - ('element.bst', 'brother', 'zebry', 'a zebry'), - ('element.bst', 'brother', 'horsy', 'a horsy'), - # Test 'var1 == var2' syntax - ('element-compare.bst', 'brother', 'horsy', 'different'), - ('element-compare.bst', 'brother', 'zebry', 'same'), - ('element-compare.bst', 'sister', 'pony', 'same'), -]) +@pytest.mark.parametrize( + "target,option,value,expected", + [ + # Test 'var == "foo"' syntax + ("element.bst", "brother", "pony", "a pony"), + ("element.bst", "brother", "zebry", "a zebry"), + ("element.bst", "brother", "horsy", "a horsy"), + # Test 'var1 == var2' syntax + ("element-compare.bst", "brother", "horsy", "different"), + ("element-compare.bst", "brother", "zebry", "same"), + ("element-compare.bst", "sister", "pony", "same"), + ], +) def test_conditional_cli(cli, datafiles, target, option, value, expected): - project = os.path.join(datafiles.dirname, datafiles.basename, 'option-enum') - result = cli.run(project=project, silent=True, args=[ - '--option', option, value, - 'show', - '--deps', 'none', - '--format', '%{vars}', - target]) + project = os.path.join(datafiles.dirname, datafiles.basename, "option-enum") + result = cli.run( + project=project, + silent=True, + args=["--option", option, value, "show", "--deps", "none", "--format", "%{vars}", target], + ) result.assert_success() loaded = _yaml.load_data(result.output) - assert loaded.get_str('result') == expected + assert loaded.get_str("result") == expected @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("target,option,value,expected", [ - # Test 'var == "foo"' syntax - ('element.bst', 'brother', 'pony', 'a pony'), - ('element.bst', 'brother', 'zebry', 'a zebry'), - ('element.bst', 'brother', 'horsy', 'a horsy'), - # Test 'var1 == var2' syntax - ('element-compare.bst', 'brother', 'horsy', 'different'), - ('element-compare.bst', 'brother', 'zebry', 'same'), - ('element-compare.bst', 'sister', 'pony', 'same'), -]) +@pytest.mark.parametrize( + "target,option,value,expected", + [ + # Test 'var == "foo"' syntax + ("element.bst", "brother", "pony", "a pony"), + ("element.bst", "brother", "zebry", "a zebry"), + ("element.bst", "brother", "horsy", "a horsy"), + # Test 'var1 == var2' syntax + ("element-compare.bst", "brother", "horsy", "different"), + ("element-compare.bst", "brother", "zebry", "same"), + ("element-compare.bst", "sister", "pony", "same"), + ], +) def test_conditional_config(cli, datafiles, target, option, value, expected): - project = os.path.join(datafiles.dirname, datafiles.basename, 'option-enum') - cli.configure({ - 'projects': { - 'test': { - 'options': { - option: value - } - } - } - }) - result = cli.run(project=project, silent=True, args=[ - 'show', - '--deps', 'none', - '--format', '%{vars}', - target]) + project = os.path.join(datafiles.dirname, datafiles.basename, "option-enum") + cli.configure({"projects": {"test": {"options": {option: value}}}}) + result = cli.run(project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", target]) result.assert_success() loaded = _yaml.load_data(result.output) - assert loaded.get_str('result') == expected + assert loaded.get_str("result") == expected @pytest.mark.datafiles(DATA_DIR) def test_invalid_value_cli(cli, datafiles): - project = os.path.join(datafiles.dirname, datafiles.basename, 'option-enum') - result = cli.run(project=project, silent=True, args=[ - '--option', 'brother', 'giraffy', - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst']) + project = os.path.join(datafiles.dirname, datafiles.basename, "option-enum") + result = cli.run( + project=project, + silent=True, + args=["--option", "brother", "giraffy", "show", "--deps", "none", "--format", "%{vars}", "element.bst"], + ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("config_option", [ - ('giraffy'), (['its', 'a', 'list']), ({'dic': 'tionary'}) -]) +@pytest.mark.parametrize("config_option", [("giraffy"), (["its", "a", "list"]), ({"dic": "tionary"})]) def test_invalid_value_config(cli, datafiles, config_option): - project = os.path.join(datafiles.dirname, datafiles.basename, 'option-enum') - cli.configure({ - 'projects': { - 'test': { - 'options': { - 'brother': config_option - } - } - } - }) - result = cli.run(project=project, silent=True, args=[ - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst']) + project = os.path.join(datafiles.dirname, datafiles.basename, "option-enum") + cli.configure({"projects": {"test": {"options": {"brother": config_option}}}}) + result = cli.run( + project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"] + ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @pytest.mark.datafiles(DATA_DIR) def test_missing_values(cli, datafiles): - project = os.path.join(datafiles.dirname, datafiles.basename, 'option-enum-missing') - result = cli.run(project=project, silent=True, args=[ - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst']) + project = os.path.join(datafiles.dirname, datafiles.basename, "option-enum-missing") + result = cli.run( + project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"] + ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) diff --git a/tests/format/optionexports.py b/tests/format/optionexports.py index 104abcf83..4f95dc109 100644 --- a/tests/format/optionexports.py +++ b/tests/format/optionexports.py @@ -11,28 +11,27 @@ DATA_DIR = os.path.dirname(os.path.realpath(__file__)) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("option_name,option_value,var_name,var_value", [ - - # Test boolean - ('bool_export', 'False', 'exported-bool', '0'), - ('bool_export', 'True', 'exported-bool', '1'), - - # Enum - ('enum_export', 'pony', 'exported-enum', 'pony'), - ('enum_export', 'horsy', 'exported-enum', 'horsy'), - - # Flags - ('flags_export', 'pony', 'exported-flags', 'pony'), - ('flags_export', 'pony, horsy', 'exported-flags', 'horsy,pony'), -]) +@pytest.mark.parametrize( + "option_name,option_value,var_name,var_value", + [ + # Test boolean + ("bool_export", "False", "exported-bool", "0"), + ("bool_export", "True", "exported-bool", "1"), + # Enum + ("enum_export", "pony", "exported-enum", "pony"), + ("enum_export", "horsy", "exported-enum", "horsy"), + # Flags + ("flags_export", "pony", "exported-flags", "pony"), + ("flags_export", "pony, horsy", "exported-flags", "horsy,pony"), + ], +) def test_export(cli, datafiles, option_name, option_value, var_name, var_value): - project = os.path.join(datafiles.dirname, datafiles.basename, 'option-exports') - result = cli.run(project=project, silent=True, args=[ - '--option', option_name, option_value, - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst']) + project = os.path.join(datafiles.dirname, datafiles.basename, "option-exports") + result = cli.run( + project=project, + silent=True, + args=["--option", option_name, option_value, "show", "--deps", "none", "--format", "%{vars}", "element.bst"], + ) result.assert_success() loaded = _yaml.load_data(result.output) diff --git a/tests/format/optionflags.py b/tests/format/optionflags.py index 29bb7ec2c..53e674919 100644 --- a/tests/format/optionflags.py +++ b/tests/format/optionflags.py @@ -12,113 +12,93 @@ DATA_DIR = os.path.dirname(os.path.realpath(__file__)) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("target,option,value,expected", [ - # Test (var == [ "foo" ]) syntax - ('element.bst', 'farm', 'pony', 'a pony'), - ('element.bst', 'farm', 'zebry', 'a zebry'), - ('element.bst', 'farm', 'pony, horsy', 'a pony and a horsy'), - ('element.bst', 'farm', 'zebry,horsy , pony', 'all the animals'), - - # Test ("literal" in var) syntax - ('element-in.bst', 'farm', 'zebry, horsy, pony', 'a zebry'), - - # Test ("literal" not in var) syntax - ('element-in.bst', 'farm', 'zebry, horsy', 'no pony'), - - # Test (var1 not in var2) syntax (where var1 is enum and var2 is flags) - ('element-in.bst', 'farm', 'zebry, pony', 'no horsy'), -]) +@pytest.mark.parametrize( + "target,option,value,expected", + [ + # Test (var == [ "foo" ]) syntax + ("element.bst", "farm", "pony", "a pony"), + ("element.bst", "farm", "zebry", "a zebry"), + ("element.bst", "farm", "pony, horsy", "a pony and a horsy"), + ("element.bst", "farm", "zebry,horsy , pony", "all the animals"), + # Test ("literal" in var) syntax + ("element-in.bst", "farm", "zebry, horsy, pony", "a zebry"), + # Test ("literal" not in var) syntax + ("element-in.bst", "farm", "zebry, horsy", "no pony"), + # Test (var1 not in var2) syntax (where var1 is enum and var2 is flags) + ("element-in.bst", "farm", "zebry, pony", "no horsy"), + ], +) def test_conditional_cli(cli, datafiles, target, option, value, expected): - project = os.path.join(datafiles.dirname, datafiles.basename, 'option-flags') - result = cli.run(project=project, silent=True, args=[ - '--option', option, value, - 'show', - '--deps', 'none', - '--format', '%{vars}', - target]) + project = os.path.join(datafiles.dirname, datafiles.basename, "option-flags") + result = cli.run( + project=project, + silent=True, + args=["--option", option, value, "show", "--deps", "none", "--format", "%{vars}", target], + ) result.assert_success() loaded = _yaml.load_data(result.output) - assert loaded.get_str('result') == expected + assert loaded.get_str("result") == expected @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("target,option,value,expected", [ - # Test 'var == [ "foo" ]' syntax - ('element.bst', 'farm', ['pony'], 'a pony'), - ('element.bst', 'farm', ['zebry'], 'a zebry'), - ('element.bst', 'farm', ['pony', 'horsy'], 'a pony and a horsy'), - ('element.bst', 'farm', ['zebry', 'horsy', 'pony'], 'all the animals'), -]) +@pytest.mark.parametrize( + "target,option,value,expected", + [ + # Test 'var == [ "foo" ]' syntax + ("element.bst", "farm", ["pony"], "a pony"), + ("element.bst", "farm", ["zebry"], "a zebry"), + ("element.bst", "farm", ["pony", "horsy"], "a pony and a horsy"), + ("element.bst", "farm", ["zebry", "horsy", "pony"], "all the animals"), + ], +) def test_conditional_config(cli, datafiles, target, option, value, expected): - project = os.path.join(datafiles.dirname, datafiles.basename, 'option-flags') - cli.configure({ - 'projects': { - 'test': { - 'options': { - option: value - } - } - } - }) - result = cli.run(project=project, silent=True, args=[ - 'show', - '--deps', 'none', - '--format', '%{vars}', - target]) + project = os.path.join(datafiles.dirname, datafiles.basename, "option-flags") + cli.configure({"projects": {"test": {"options": {option: value}}}}) + result = cli.run(project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", target]) result.assert_success() loaded = _yaml.load_data(result.output) - assert loaded.get_str('result') == expected + assert loaded.get_str("result") == expected @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("cli_option", [ - ('giraffy'), # Not a valid animal for the farm option - ('horsy pony') # Does not include comma separators -]) +@pytest.mark.parametrize( + "cli_option", + [("giraffy"), ("horsy pony")], # Not a valid animal for the farm option # Does not include comma separators +) def test_invalid_value_cli(cli, datafiles, cli_option): - project = os.path.join(datafiles.dirname, datafiles.basename, 'option-flags') - result = cli.run(project=project, silent=True, args=[ - '--option', 'farm', cli_option, - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst']) + project = os.path.join(datafiles.dirname, datafiles.basename, "option-flags") + result = cli.run( + project=project, + silent=True, + args=["--option", "farm", cli_option, "show", "--deps", "none", "--format", "%{vars}", "element.bst"], + ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("config_option", [ - ('pony'), # Not specified as a list - (['horsy', 'pony', 'giraffy']), # Invalid giraffy animal for farm option - ({'dic': 'tionary'}) # Dicts also dont make sense in the config for flags -]) +@pytest.mark.parametrize( + "config_option", + [ + ("pony"), # Not specified as a list + (["horsy", "pony", "giraffy"]), # Invalid giraffy animal for farm option + ({"dic": "tionary"}), # Dicts also dont make sense in the config for flags + ], +) def test_invalid_value_config(cli, datafiles, config_option): - project = os.path.join(datafiles.dirname, datafiles.basename, 'option-flags') - cli.configure({ - 'projects': { - 'test': { - 'options': { - 'farm': config_option - } - } - } - }) - result = cli.run(project=project, silent=True, args=[ - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst']) + project = os.path.join(datafiles.dirname, datafiles.basename, "option-flags") + cli.configure({"projects": {"test": {"options": {"farm": config_option}}}}) + result = cli.run( + project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"] + ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @pytest.mark.datafiles(DATA_DIR) def test_missing_values(cli, datafiles): - project = os.path.join(datafiles.dirname, datafiles.basename, 'option-flags-missing') - result = cli.run(project=project, silent=True, args=[ - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst']) + project = os.path.join(datafiles.dirname, datafiles.basename, "option-flags-missing") + result = cli.run( + project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"] + ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) diff --git a/tests/format/optionos.py b/tests/format/optionos.py index f915d889e..92486104e 100644 --- a/tests/format/optionos.py +++ b/tests/format/optionos.py @@ -15,51 +15,44 @@ DATA_DIR = os.path.dirname(os.path.realpath(__file__)) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("system,value,expected", [ - # Test explicitly provided arches - ('Darwin', 'Linux', 'Linuxy'), - ('SunOS', 'FreeBSD', 'FreeBSDy'), - - # Test automatically derived arches - ('Linux', None, 'Linuxy'), - ('Darwin', None, 'Darwiny'), - - # Test that explicitly provided arches dont error out - # when the `uname` reported arch is not supported - ('AIX', 'Linux', 'Linuxy'), - ('HaikuOS', 'SunOS', 'SunOSy'), -]) +@pytest.mark.parametrize( + "system,value,expected", + [ + # Test explicitly provided arches + ("Darwin", "Linux", "Linuxy"), + ("SunOS", "FreeBSD", "FreeBSDy"), + # Test automatically derived arches + ("Linux", None, "Linuxy"), + ("Darwin", None, "Darwiny"), + # Test that explicitly provided arches dont error out + # when the `uname` reported arch is not supported + ("AIX", "Linux", "Linuxy"), + ("HaikuOS", "SunOS", "SunOSy"), + ], +) def test_conditionals(cli, datafiles, system, value, expected): with override_platform_uname(system=system): - project = os.path.join(datafiles.dirname, datafiles.basename, 'option-os') + project = os.path.join(datafiles.dirname, datafiles.basename, "option-os") bst_args = [] if value is not None: - bst_args += ['--option', 'machine_os', value] + bst_args += ["--option", "machine_os", value] - bst_args += [ - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst' - ] + bst_args += ["show", "--deps", "none", "--format", "%{vars}", "element.bst"] result = cli.run(project=project, silent=True, args=bst_args) result.assert_success() loaded = _yaml.load_data(result.output) - assert loaded.get_str('result') == expected + assert loaded.get_str("result") == expected @pytest.mark.datafiles(DATA_DIR) def test_unsupported_arch(cli, datafiles): with override_platform_uname(system="AIX"): - project = os.path.join(datafiles.dirname, datafiles.basename, 'option-os') - result = cli.run(project=project, silent=True, args=[ - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst' - ]) + project = os.path.join(datafiles.dirname, datafiles.basename, "option-os") + result = cli.run( + project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"] + ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) diff --git a/tests/format/optionoverrides.py b/tests/format/optionoverrides.py index d4ed257dd..ba12e751f 100644 --- a/tests/format/optionoverrides.py +++ b/tests/format/optionoverrides.py @@ -11,22 +11,17 @@ DATA_DIR = os.path.dirname(os.path.realpath(__file__)) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("arch", [('i686'), ('x86_64')]) +@pytest.mark.parametrize("arch", [("i686"), ("x86_64")]) def test_override(cli, datafiles, arch): - project = os.path.join(datafiles.dirname, datafiles.basename, 'option-overrides') + project = os.path.join(datafiles.dirname, datafiles.basename, "option-overrides") - bst_args = ['--option', 'arch', arch] - bst_args += [ - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst' - ] + bst_args = ["--option", "arch", arch] + bst_args += ["show", "--deps", "none", "--format", "%{vars}", "element.bst"] result = cli.run(project=project, silent=True, args=bst_args) result.assert_success() # See the associated project.conf for the expected values - expected_value = '--host={}-unknown-linux-gnu'.format(arch) + expected_value = "--host={}-unknown-linux-gnu".format(arch) loaded = _yaml.load_data(result.output) - assert loaded.get_str('conf-global') == expected_value + assert loaded.get_str("conf-global") == expected_value diff --git a/tests/format/options.py b/tests/format/options.py index 9c0e043f9..aa0854e96 100644 --- a/tests/format/options.py +++ b/tests/format/options.py @@ -8,244 +8,233 @@ from buildstream._exceptions import ErrorDomain, LoadErrorReason from buildstream.testing.runcli import cli # pylint: disable=unused-import # Project directory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - 'options' -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "options") @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("project_dir", [ - ('invalid-name-spaces'), - ('invalid-name-dashes'), - ('invalid-name-plus'), - ('invalid-name-leading-number'), -]) +@pytest.mark.parametrize( + "project_dir", + [("invalid-name-spaces"), ("invalid-name-dashes"), ("invalid-name-plus"), ("invalid-name-leading-number"),], +) def test_invalid_option_name(cli, datafiles, project_dir): project = os.path.join(datafiles.dirname, datafiles.basename, project_dir) - result = cli.run(project=project, silent=True, args=['show', 'element.bst']) + result = cli.run(project=project, silent=True, args=["show", "element.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_SYMBOL_NAME) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("project_dir", [ - ('invalid-variable-name-spaces'), - ('invalid-variable-name-plus'), -]) +@pytest.mark.parametrize("project_dir", [("invalid-variable-name-spaces"), ("invalid-variable-name-plus"),]) def test_invalid_variable_name(cli, datafiles, project_dir): project = os.path.join(datafiles.dirname, datafiles.basename, project_dir) - result = cli.run(project=project, silent=True, args=['show', 'element.bst']) + result = cli.run(project=project, silent=True, args=["show", "element.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_SYMBOL_NAME) @pytest.mark.datafiles(DATA_DIR) def test_invalid_option_type(cli, datafiles): - project = os.path.join(datafiles.dirname, datafiles.basename, 'invalid-type') + project = os.path.join(datafiles.dirname, datafiles.basename, "invalid-type") # Test with the opt option set - result = cli.run(project=project, silent=True, args=[ - '--option', 'opt', 'funny', - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst']) + result = cli.run( + project=project, + silent=True, + args=["--option", "opt", "funny", "show", "--deps", "none", "--format", "%{vars}", "element.bst"], + ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @pytest.mark.datafiles(DATA_DIR) def test_invalid_option_cli(cli, datafiles): - project = os.path.join(datafiles.dirname, datafiles.basename, 'simple-condition') + project = os.path.join(datafiles.dirname, datafiles.basename, "simple-condition") # Test with the opt option set - result = cli.run(project=project, silent=True, args=[ - '--option', 'fart', 'funny', - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst']) + result = cli.run( + project=project, + silent=True, + args=["--option", "fart", "funny", "show", "--deps", "none", "--format", "%{vars}", "element.bst"], + ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @pytest.mark.datafiles(DATA_DIR) def test_invalid_option_config(cli, datafiles): - project = os.path.join(datafiles.dirname, datafiles.basename, 'simple-condition') - cli.configure({ - 'projects': { - 'test': { - 'options': { - 'fart': 'Hello' - } - } - } - }) - result = cli.run(project=project, silent=True, args=[ - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst']) + project = os.path.join(datafiles.dirname, datafiles.basename, "simple-condition") + cli.configure({"projects": {"test": {"options": {"fart": "Hello"}}}}) + result = cli.run( + project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"] + ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @pytest.mark.datafiles(DATA_DIR) def test_invalid_expression(cli, datafiles): - project = os.path.join(datafiles.dirname, datafiles.basename, 'invalid-expression') - result = cli.run(project=project, silent=True, args=[ - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst']) + project = os.path.join(datafiles.dirname, datafiles.basename, "invalid-expression") + result = cli.run( + project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"] + ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.EXPRESSION_FAILED) @pytest.mark.datafiles(DATA_DIR) def test_undefined(cli, datafiles): - project = os.path.join(datafiles.dirname, datafiles.basename, 'undefined-variable') - result = cli.run(project=project, silent=True, args=[ - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst']) + project = os.path.join(datafiles.dirname, datafiles.basename, "undefined-variable") + result = cli.run( + project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"] + ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.EXPRESSION_FAILED) @pytest.mark.datafiles(DATA_DIR) def test_invalid_condition(cli, datafiles): - project = os.path.join(datafiles.dirname, datafiles.basename, 'invalid-condition') - result = cli.run(project=project, silent=True, args=[ - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst']) + project = os.path.join(datafiles.dirname, datafiles.basename, "invalid-condition") + result = cli.run( + project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"] + ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("opt_option,expected_prefix", [ - ('False', '/usr'), - ('True', '/opt'), -]) +@pytest.mark.parametrize("opt_option,expected_prefix", [("False", "/usr"), ("True", "/opt"),]) def test_simple_conditional(cli, datafiles, opt_option, expected_prefix): - project = os.path.join(datafiles.dirname, datafiles.basename, 'simple-condition') + project = os.path.join(datafiles.dirname, datafiles.basename, "simple-condition") # Test with the opt option set - result = cli.run(project=project, silent=True, args=[ - '--option', 'opt', opt_option, - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst']) + result = cli.run( + project=project, + silent=True, + args=["--option", "opt", opt_option, "show", "--deps", "none", "--format", "%{vars}", "element.bst"], + ) result.assert_success() loaded = _yaml.load_data(result.output) - assert loaded.get_str('prefix') == expected_prefix + assert loaded.get_str("prefix") == expected_prefix @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("debug,logging,expected", [ - ('False', 'False', 'False'), - ('True', 'False', 'False'), - ('False', 'True', 'False'), - ('True', 'True', 'True'), -]) +@pytest.mark.parametrize( + "debug,logging,expected", + [("False", "False", "False"), ("True", "False", "False"), ("False", "True", "False"), ("True", "True", "True"),], +) def test_nested_conditional(cli, datafiles, debug, logging, expected): - project = os.path.join(datafiles.dirname, datafiles.basename, 'nested-condition') + project = os.path.join(datafiles.dirname, datafiles.basename, "nested-condition") # Test with the opt option set - result = cli.run(project=project, silent=True, args=[ - '--option', 'debug', debug, - '--option', 'logging', logging, - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst']) + result = cli.run( + project=project, + silent=True, + args=[ + "--option", + "debug", + debug, + "--option", + "logging", + logging, + "show", + "--deps", + "none", + "--format", + "%{vars}", + "element.bst", + ], + ) result.assert_success() loaded = _yaml.load_data(result.output) - assert loaded.get_str('debug') == expected + assert loaded.get_str("debug") == expected @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("debug,logging,expected", [ - ('False', 'False', 'False'), - ('True', 'False', 'False'), - ('False', 'True', 'False'), - ('True', 'True', 'True'), -]) +@pytest.mark.parametrize( + "debug,logging,expected", + [("False", "False", "False"), ("True", "False", "False"), ("False", "True", "False"), ("True", "True", "True"),], +) def test_compound_and_conditional(cli, datafiles, debug, logging, expected): - project = os.path.join(datafiles.dirname, datafiles.basename, 'compound-and-condition') + project = os.path.join(datafiles.dirname, datafiles.basename, "compound-and-condition") # Test with the opt option set - result = cli.run(project=project, silent=True, args=[ - '--option', 'debug', debug, - '--option', 'logging', logging, - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst']) + result = cli.run( + project=project, + silent=True, + args=[ + "--option", + "debug", + debug, + "--option", + "logging", + logging, + "show", + "--deps", + "none", + "--format", + "%{vars}", + "element.bst", + ], + ) result.assert_success() loaded = _yaml.load_data(result.output) - assert loaded.get_str('debug') == expected + assert loaded.get_str("debug") == expected @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("debug,logging,expected", [ - ('False', 'False', 'False'), - ('True', 'False', 'True'), - ('False', 'True', 'True'), - ('True', 'True', 'True'), -]) +@pytest.mark.parametrize( + "debug,logging,expected", + [("False", "False", "False"), ("True", "False", "True"), ("False", "True", "True"), ("True", "True", "True"),], +) def test_compound_or_conditional(cli, datafiles, debug, logging, expected): - project = os.path.join(datafiles.dirname, datafiles.basename, 'compound-or-condition') + project = os.path.join(datafiles.dirname, datafiles.basename, "compound-or-condition") # Test with the opt option set - result = cli.run(project=project, silent=True, args=[ - '--option', 'debug', debug, - '--option', 'logging', logging, - 'show', - '--deps', 'none', - '--format', '%{vars}', - 'element.bst']) + result = cli.run( + project=project, + silent=True, + args=[ + "--option", + "debug", + debug, + "--option", + "logging", + logging, + "show", + "--deps", + "none", + "--format", + "%{vars}", + "element.bst", + ], + ) result.assert_success() loaded = _yaml.load_data(result.output) - assert loaded.get_str('logging') == expected + assert loaded.get_str("logging") == expected @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("option,expected", [ - ('False', 'horsy'), - ('True', 'pony'), -]) +@pytest.mark.parametrize("option,expected", [("False", "horsy"), ("True", "pony"),]) def test_deep_nesting_level1(cli, datafiles, option, expected): - project = os.path.join(datafiles.dirname, datafiles.basename, 'deep-nesting') - result = cli.run(project=project, silent=True, args=[ - '--option', 'pony', option, - 'show', - '--deps', 'none', - '--format', '%{public}', - 'element.bst']) + project = os.path.join(datafiles.dirname, datafiles.basename, "deep-nesting") + result = cli.run( + project=project, + silent=True, + args=["--option", "pony", option, "show", "--deps", "none", "--format", "%{public}", "element.bst"], + ) result.assert_success() loaded = _yaml.load_data(result.output) - shallow_list = loaded.get_sequence('shallow-nest') + shallow_list = loaded.get_sequence("shallow-nest") first_dict = shallow_list.mapping_at(0) - assert first_dict.get_str('animal') == expected + assert first_dict.get_str("animal") == expected @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("option,expected", [ - ('False', 'horsy'), - ('True', 'pony'), -]) +@pytest.mark.parametrize("option,expected", [("False", "horsy"), ("True", "pony"),]) def test_deep_nesting_level2(cli, datafiles, option, expected): - project = os.path.join(datafiles.dirname, datafiles.basename, 'deep-nesting') - result = cli.run(project=project, silent=True, args=[ - '--option', 'pony', option, - 'show', - '--deps', 'none', - '--format', '%{public}', - 'element-deeper.bst']) + project = os.path.join(datafiles.dirname, datafiles.basename, "deep-nesting") + result = cli.run( + project=project, + silent=True, + args=["--option", "pony", option, "show", "--deps", "none", "--format", "%{public}", "element-deeper.bst"], + ) result.assert_success() loaded = _yaml.load_data(result.output) - shallow_list = loaded.get_sequence('deep-nest') + shallow_list = loaded.get_sequence("deep-nest") deeper_list = shallow_list.sequence_at(0) first_dict = deeper_list.mapping_at(0) - assert first_dict.get_str('animal') == expected + assert first_dict.get_str("animal") == expected diff --git a/tests/format/project.py b/tests/format/project.py index 2e0a729dc..e6bc6a5cd 100644 --- a/tests/format/project.py +++ b/tests/format/project.py @@ -11,30 +11,27 @@ from tests.testutils import filetypegenerator # Project directory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project" -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_missing_project_conf(cli, datafiles): project = str(datafiles) - result = cli.run(project=project, args=['workspace', 'list']) + result = cli.run(project=project, args=["workspace", "list"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_PROJECT_CONF) @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_missing_project_name(cli, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename, "missingname") - result = cli.run(project=project, args=['workspace', 'list']) + result = cli.run(project=project, args=["workspace", "list"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_missing_element(cli, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename, "missing-element") - result = cli.run(project=project, args=['show', 'manual.bst']) + result = cli.run(project=project, args=["show", "manual.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE) # Assert that we have the expected provenance encoded into the error @@ -44,7 +41,7 @@ def test_missing_element(cli, datafiles): @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_missing_junction(cli, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename, "missing-junction") - result = cli.run(project=project, args=['show', 'manual.bst']) + result = cli.run(project=project, args=["show", "manual.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE) # Assert that we have the expected provenance encoded into the error @@ -54,176 +51,156 @@ def test_missing_junction(cli, datafiles): @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_empty_project_name(cli, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename, "emptyname") - result = cli.run(project=project, args=['workspace', 'list']) + result = cli.run(project=project, args=["workspace", "list"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_SYMBOL_NAME) @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_invalid_project_name(cli, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename, "invalidname") - result = cli.run(project=project, args=['workspace', 'list']) + result = cli.run(project=project, args=["workspace", "list"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_SYMBOL_NAME) @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_invalid_yaml(cli, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename, "invalid-yaml") - result = cli.run(project=project, args=['workspace', 'list']) + result = cli.run(project=project, args=["workspace", "list"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_YAML) @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_load_default_project(cli, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename, "default") - result = cli.run(project=project, args=[ - 'show', '--format', '%{env}', 'manual.bst' - ]) + result = cli.run(project=project, args=["show", "--format", "%{env}", "manual.bst"]) result.assert_success() # Read back some of our project defaults from the env env = _yaml.load_data(result.output) - assert env.get_str('USER') == "tomjon" - assert env.get_str('TERM') == "dumb" + assert env.get_str("USER") == "tomjon" + assert env.get_str("TERM") == "dumb" @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_load_project_from_subdir(cli, datafiles): - project = os.path.join(datafiles.dirname, datafiles.basename, 'project-from-subdir') + project = os.path.join(datafiles.dirname, datafiles.basename, "project-from-subdir") result = cli.run( - project=project, - cwd=os.path.join(project, 'subdirectory'), - args=['show', '--format', '%{env}', 'manual.bst']) + project=project, cwd=os.path.join(project, "subdirectory"), args=["show", "--format", "%{env}", "manual.bst"] + ) result.assert_success() # Read back some of our project defaults from the env env = _yaml.load_data(result.output) - assert env.get_str('USER') == "tomjon" - assert env.get_str('TERM') == "dumb" + assert env.get_str("USER") == "tomjon" + assert env.get_str("TERM") == "dumb" @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_override_project_path(cli, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename, "overridepath") - result = cli.run(project=project, args=[ - 'show', '--format', '%{env}', 'manual.bst' - ]) + result = cli.run(project=project, args=["show", "--format", "%{env}", "manual.bst"]) result.assert_success() # Read back the overridden path env = _yaml.load_data(result.output) - assert env.get_str('PATH') == "/bin:/sbin" + assert env.get_str("PATH") == "/bin:/sbin" @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_project_unsupported(cli, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename, "unsupported") - result = cli.run(project=project, args=['workspace', 'list']) + result = cli.run(project=project, args=["workspace", "list"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.UNSUPPORTED_PROJECT) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'element-path')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "element-path")) def test_missing_element_path_directory(cli, datafiles): project = str(datafiles) - result = cli.run(project=project, args=['workspace', 'list']) - result.assert_main_error(ErrorDomain.LOAD, - LoadErrorReason.MISSING_FILE) + result = cli.run(project=project, args=["workspace", "list"]) + result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'element-path')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "element-path")) def test_element_path_not_a_directory(cli, datafiles): project = str(datafiles) - path = os.path.join(project, 'elements') + path = os.path.join(project, "elements") for _file_type in filetypegenerator.generate_file_types(path): - result = cli.run(project=project, args=['workspace', 'list']) + result = cli.run(project=project, args=["workspace", "list"]) if not os.path.isdir(path): - result.assert_main_error(ErrorDomain.LOAD, - LoadErrorReason.PROJ_PATH_INVALID_KIND) + result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.PROJ_PATH_INVALID_KIND) else: result.assert_success() -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'local-plugin')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "local-plugin")) def test_missing_local_plugin_directory(cli, datafiles): project = str(datafiles) - result = cli.run(project=project, args=['workspace', 'list']) - result.assert_main_error(ErrorDomain.LOAD, - LoadErrorReason.MISSING_FILE) + result = cli.run(project=project, args=["workspace", "list"]) + result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'local-plugin')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "local-plugin")) def test_local_plugin_not_directory(cli, datafiles): project = str(datafiles) - path = os.path.join(project, 'plugins') + path = os.path.join(project, "plugins") for _file_type in filetypegenerator.generate_file_types(path): - result = cli.run(project=project, args=['workspace', 'list']) + result = cli.run(project=project, args=["workspace", "list"]) if not os.path.isdir(path): - result.assert_main_error(ErrorDomain.LOAD, - LoadErrorReason.PROJ_PATH_INVALID_KIND) + result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.PROJ_PATH_INVALID_KIND) else: result.assert_success() @pytest.mark.datafiles(DATA_DIR) def test_plugin_load_allowed(cli, datafiles): - project = os.path.join(datafiles.dirname, datafiles.basename, 'plugin-allowed') - result = cli.run(project=project, silent=True, args=[ - 'show', 'element.bst']) + project = os.path.join(datafiles.dirname, datafiles.basename, "plugin-allowed") + result = cli.run(project=project, silent=True, args=["show", "element.bst"]) result.assert_success() @pytest.mark.datafiles(DATA_DIR) def test_plugin_load_forbidden(cli, datafiles): - project = os.path.join(datafiles.dirname, datafiles.basename, 'plugin-forbidden') - result = cli.run(project=project, silent=True, args=[ - 'show', 'element.bst']) + project = os.path.join(datafiles.dirname, datafiles.basename, "plugin-forbidden") + result = cli.run(project=project, silent=True, args=["show", "element.bst"]) result.assert_main_error(ErrorDomain.PLUGIN, None) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')]) +@pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")]) def test_plugin_no_load_ref(cli, datafiles, ref_storage): - project = os.path.join(datafiles.dirname, datafiles.basename, 'plugin-no-load-ref') + project = os.path.join(datafiles.dirname, datafiles.basename, "plugin-no-load-ref") # Generate project with access to the noloadref plugin and project.refs enabled # config = { - 'name': 'test', - 'ref-storage': ref_storage, - 'plugins': [ - { - 'origin': 'local', - 'path': 'plugins', - 'sources': { - 'noloadref': 0 - } - } - ] + "name": "test", + "ref-storage": ref_storage, + "plugins": [{"origin": "local", "path": "plugins", "sources": {"noloadref": 0}}], } - _yaml.roundtrip_dump(config, os.path.join(project, 'project.conf')) + _yaml.roundtrip_dump(config, os.path.join(project, "project.conf")) - result = cli.run(project=project, silent=True, args=['show', 'noloadref.bst']) + result = cli.run(project=project, silent=True, args=["show", "noloadref.bst"]) # There is no error if project.refs is not in use, otherwise we # assert our graceful failure - if ref_storage == 'inline': + if ref_storage == "inline": result.assert_success() else: - result.assert_main_error(ErrorDomain.SOURCE, 'unsupported-load-ref') + result.assert_main_error(ErrorDomain.SOURCE, "unsupported-load-ref") @pytest.mark.datafiles(DATA_DIR) def test_plugin_preflight_error(cli, datafiles): - project = os.path.join(datafiles.dirname, datafiles.basename, 'plugin-preflight-error') - result = cli.run(project=project, args=['source', 'fetch', 'error.bst']) + project = os.path.join(datafiles.dirname, datafiles.basename, "plugin-preflight-error") + result = cli.run(project=project, args=["source", "fetch", "error.bst"]) result.assert_main_error(ErrorDomain.SOURCE, "the-preflight-error") @pytest.mark.datafiles(DATA_DIR) def test_duplicate_plugins(cli, datafiles): - project = os.path.join(datafiles.dirname, datafiles.basename, 'duplicate-plugins') - result = cli.run(project=project, silent=True, args=[ - 'show', 'element.bst']) + project = os.path.join(datafiles.dirname, datafiles.basename, "duplicate-plugins") + result = cli.run(project=project, silent=True, args=["show", "element.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_YAML) @@ -232,42 +209,40 @@ def test_duplicate_plugins(cli, datafiles): # @pytest.mark.datafiles(DATA_DIR) def test_project_refs_options(cli, datafiles): - project = os.path.join(datafiles.dirname, datafiles.basename, 'refs-options') - - result1 = cli.run(project=project, silent=True, args=[ - '--option', 'test', 'True', - 'show', - '--deps', 'none', - '--format', '%{key}', - 'target.bst']) + project = os.path.join(datafiles.dirname, datafiles.basename, "refs-options") + + result1 = cli.run( + project=project, + silent=True, + args=["--option", "test", "True", "show", "--deps", "none", "--format", "%{key}", "target.bst"], + ) result1.assert_success() - result2 = cli.run(project=project, silent=True, args=[ - '--option', 'test', 'False', - 'show', - '--deps', 'none', - '--format', '%{key}', - 'target.bst']) + result2 = cli.run( + project=project, + silent=True, + args=["--option", "test", "False", "show", "--deps", "none", "--format", "%{key}", "target.bst"], + ) result2.assert_success() # Assert that the cache keys are different assert result1.output != result2.output -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'element-path')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "element-path")) def test_element_path_project_path_contains_symlinks(cli, datafiles, tmpdir): real_project = str(datafiles) - linked_project = os.path.join(str(tmpdir), 'linked') + linked_project = os.path.join(str(tmpdir), "linked") os.symlink(real_project, linked_project) - os.makedirs(os.path.join(real_project, 'elements'), exist_ok=True) - with open(os.path.join(real_project, 'elements', 'element.bst'), 'w') as f: + os.makedirs(os.path.join(real_project, "elements"), exist_ok=True) + with open(os.path.join(real_project, "elements", "element.bst"), "w") as f: f.write("kind: manual\n") - result = cli.run(project=linked_project, args=['show', 'element.bst']) + result = cli.run(project=linked_project, args=["show", "element.bst"]) result.assert_success() @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_empty_depends(cli, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename, "empty-depends") - result = cli.run(project=project, args=['show', 'manual.bst']) + result = cli.run(project=project, args=["show", "manual.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) diff --git a/tests/format/project/plugin-no-load-ref/plugins/noloadref.py b/tests/format/project/plugin-no-load-ref/plugins/noloadref.py index 0cc457f07..5d44cd5c7 100644 --- a/tests/format/project/plugin-no-load-ref/plugins/noloadref.py +++ b/tests/format/project/plugin-no-load-ref/plugins/noloadref.py @@ -6,7 +6,6 @@ from buildstream import Source, Consistency # Use this to test that the core behaves as expected with such plugins. # class NoLoadRefSource(Source): - def configure(self, node): pass diff --git a/tests/format/project/plugin-preflight-error/errorplugin/preflighterror.py b/tests/format/project/plugin-preflight-error/errorplugin/preflighterror.py index 0eee463ad..3fa8afb93 100644 --- a/tests/format/project/plugin-preflight-error/errorplugin/preflighterror.py +++ b/tests/format/project/plugin-preflight-error/errorplugin/preflighterror.py @@ -2,15 +2,13 @@ from buildstream import Source, SourceError, Consistency class PreflightErrorSource(Source): - def configure(self, node): pass def preflight(self): # Raise a preflight error unconditionally - raise SourceError("Unsatisfied requirements in preflight, raising this error", - reason="the-preflight-error") + raise SourceError("Unsatisfied requirements in preflight, raising this error", reason="the-preflight-error") def get_unique_key(self): return {} diff --git a/tests/format/projectoverrides.py b/tests/format/projectoverrides.py index 7932ffb4a..16aad2cf8 100644 --- a/tests/format/projectoverrides.py +++ b/tests/format/projectoverrides.py @@ -7,23 +7,18 @@ from buildstream import _yaml from buildstream.testing.runcli import cli # pylint: disable=unused-import # Project directory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project-overrides" -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project-overrides") @pytest.mark.datafiles(DATA_DIR) def test_prepend_configure_commands(cli, datafiles): - project = os.path.join(datafiles.dirname, datafiles.basename, 'prepend-configure-commands') - result = cli.run(project=project, silent=True, args=[ - 'show', - '--deps', 'none', - '--format', '%{config}', - 'element.bst']) + project = os.path.join(datafiles.dirname, datafiles.basename, "prepend-configure-commands") + result = cli.run( + project=project, silent=True, args=["show", "--deps", "none", "--format", "%{config}", "element.bst"] + ) result.assert_success() loaded = _yaml.load_data(result.output) - config_commands = loaded.get_str_list('configure-commands') + config_commands = loaded.get_str_list("configure-commands") assert len(config_commands) == 3 assert config_commands[0] == 'echo "Hello World!"' diff --git a/tests/format/userconfig.py b/tests/format/userconfig.py index 72172a24d..9b514cc3d 100644 --- a/tests/format/userconfig.py +++ b/tests/format/userconfig.py @@ -14,11 +14,7 @@ DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") @pytest.mark.datafiles(DATA_DIR) def test_ensure_misformed_project_overrides_give_sensible_errors(cli, datafiles): - userconfig = { - "projects": { - "test": [] - } - } + userconfig = {"projects": {"test": []}} cli.configure(userconfig) result = cli.run(project=datafiles, args=["show"]) diff --git a/tests/format/variables.py b/tests/format/variables.py index 4610e039f..35b105f66 100644 --- a/tests/format/variables.py +++ b/tests/format/variables.py @@ -12,13 +12,10 @@ from buildstream.testing.runcli import cli # pylint: disable=unused-import # Project directory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "variables" -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "variables") # List of BuildStream protected variables -PROTECTED_VARIABLES = [('project-name'), ('element-name'), ('max-jobs')] +PROTECTED_VARIABLES = [("project-name"), ("element-name"), ("max-jobs")] def print_warning(msg): @@ -29,24 +26,31 @@ def print_warning(msg): ############################################################### # Test proper loading of some default commands from plugins # ############################################################### -@pytest.mark.parametrize("target,varname,expected", [ - ('autotools.bst', 'make-install', "make -j1 DESTDIR=\"/buildstream-install\" install"), - ('cmake.bst', 'cmake', - "cmake -B_builddir -H\".\" -G\"Unix Makefiles\" " + "-DCMAKE_INSTALL_PREFIX:PATH=\"/usr\" \\\n" + - "-DCMAKE_INSTALL_LIBDIR:PATH=\"lib\""), - ('distutils.bst', 'python-install', - "python3 ./setup.py install --prefix \"/usr\" \\\n" + - "--root \"/buildstream-install\""), - ('makemaker.bst', 'configure', "perl Makefile.PL PREFIX=/buildstream-install/usr"), - ('modulebuild.bst', 'configure', "perl Build.PL --prefix \"/buildstream-install/usr\""), - ('qmake.bst', 'make-install', "make -j1 INSTALL_ROOT=\"/buildstream-install\" install"), -]) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'defaults')) +@pytest.mark.parametrize( + "target,varname,expected", + [ + ("autotools.bst", "make-install", 'make -j1 DESTDIR="/buildstream-install" install'), + ( + "cmake.bst", + "cmake", + 'cmake -B_builddir -H"." -G"Unix Makefiles" ' + + '-DCMAKE_INSTALL_PREFIX:PATH="/usr" \\\n' + + '-DCMAKE_INSTALL_LIBDIR:PATH="lib"', + ), + ( + "distutils.bst", + "python-install", + 'python3 ./setup.py install --prefix "/usr" \\\n' + '--root "/buildstream-install"', + ), + ("makemaker.bst", "configure", "perl Makefile.PL PREFIX=/buildstream-install/usr"), + ("modulebuild.bst", "configure", 'perl Build.PL --prefix "/buildstream-install/usr"'), + ("qmake.bst", "make-install", 'make -j1 INSTALL_ROOT="/buildstream-install" install'), + ], +) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "defaults")) def test_defaults(cli, datafiles, target, varname, expected): project = str(datafiles) - result = cli.run(project=project, silent=True, args=[ - 'show', '--deps', 'none', '--format', '%{vars}', target - ]) + result = cli.run(project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", target]) result.assert_success() result_vars = _yaml.load_data(result.output) assert result_vars.get_str(varname) == expected @@ -55,130 +59,97 @@ def test_defaults(cli, datafiles, target, varname, expected): ################################################################ # Test overriding of variables to produce different commands # ################################################################ -@pytest.mark.parametrize("target,varname,expected", [ - ('autotools.bst', 'make-install', "make -j1 DESTDIR=\"/custom/install/root\" install"), - ('cmake.bst', 'cmake', - "cmake -B_builddir -H\".\" -G\"Ninja\" " + "-DCMAKE_INSTALL_PREFIX:PATH=\"/opt\" \\\n" + - "-DCMAKE_INSTALL_LIBDIR:PATH=\"lib\""), - ('distutils.bst', 'python-install', - "python3 ./setup.py install --prefix \"/opt\" \\\n" + - "--root \"/custom/install/root\""), - ('makemaker.bst', 'configure', "perl Makefile.PL PREFIX=/custom/install/root/opt"), - ('modulebuild.bst', 'configure', "perl Build.PL --prefix \"/custom/install/root/opt\""), - ('qmake.bst', 'make-install', "make -j1 INSTALL_ROOT=\"/custom/install/root\" install"), -]) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'overrides')) +@pytest.mark.parametrize( + "target,varname,expected", + [ + ("autotools.bst", "make-install", 'make -j1 DESTDIR="/custom/install/root" install'), + ( + "cmake.bst", + "cmake", + 'cmake -B_builddir -H"." -G"Ninja" ' + + '-DCMAKE_INSTALL_PREFIX:PATH="/opt" \\\n' + + '-DCMAKE_INSTALL_LIBDIR:PATH="lib"', + ), + ( + "distutils.bst", + "python-install", + 'python3 ./setup.py install --prefix "/opt" \\\n' + '--root "/custom/install/root"', + ), + ("makemaker.bst", "configure", "perl Makefile.PL PREFIX=/custom/install/root/opt"), + ("modulebuild.bst", "configure", 'perl Build.PL --prefix "/custom/install/root/opt"'), + ("qmake.bst", "make-install", 'make -j1 INSTALL_ROOT="/custom/install/root" install'), + ], +) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "overrides")) def test_overrides(cli, datafiles, target, varname, expected): project = str(datafiles) - result = cli.run(project=project, silent=True, args=[ - 'show', '--deps', 'none', '--format', '%{vars}', target - ]) + result = cli.run(project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", target]) result.assert_success() result_vars = _yaml.load_data(result.output) assert result_vars.get_str(varname) == expected @pytest.mark.parametrize("element", ["manual.bst", "manual2.bst"]) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'missing_variables')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "missing_variables")) def test_missing_variable(cli, datafiles, element): project = str(datafiles) - result = cli.run(project=project, silent=True, args=[ - 'show', '--deps', 'none', '--format', '%{config}', element - ]) - result.assert_main_error(ErrorDomain.LOAD, - LoadErrorReason.UNRESOLVED_VARIABLE) + result = cli.run(project=project, silent=True, args=["show", "--deps", "none", "--format", "%{config}", element]) + result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.UNRESOLVED_VARIABLE) @pytest.mark.timeout(3, method="signal") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'cyclic_variables')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "cyclic_variables")) def test_cyclic_variables(cli, datafiles): - print_warning("Performing cyclic test, if this test times out it will " + - "exit the test sequence") + print_warning("Performing cyclic test, if this test times out it will " + "exit the test sequence") project = str(datafiles) - result = cli.run(project=project, silent=True, args=[ - "build", "cyclic.bst" - ]) + result = cli.run(project=project, silent=True, args=["build", "cyclic.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.RECURSIVE_VARIABLE) -@pytest.mark.parametrize('protected_var', PROTECTED_VARIABLES) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'protected-vars')) +@pytest.mark.parametrize("protected_var", PROTECTED_VARIABLES) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "protected-vars")) def test_use_of_protected_var_project_conf(cli, datafiles, protected_var): project = str(datafiles) - conf = { - 'name': 'test', - 'variables': { - protected_var: 'some-value' - } - } - _yaml.roundtrip_dump(conf, os.path.join(project, 'project.conf')) + conf = {"name": "test", "variables": {protected_var: "some-value"}} + _yaml.roundtrip_dump(conf, os.path.join(project, "project.conf")) element = { - 'kind': 'import', - 'sources': [ - { - 'kind': 'local', - 'path': 'foo.txt' - } - ], + "kind": "import", + "sources": [{"kind": "local", "path": "foo.txt"}], } - _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) + _yaml.roundtrip_dump(element, os.path.join(project, "target.bst")) - result = cli.run(project=project, args=['build', 'target.bst']) - result.assert_main_error(ErrorDomain.LOAD, - LoadErrorReason.PROTECTED_VARIABLE_REDEFINED) + result = cli.run(project=project, args=["build", "target.bst"]) + result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.PROTECTED_VARIABLE_REDEFINED) -@pytest.mark.parametrize('protected_var', PROTECTED_VARIABLES) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'protected-vars')) +@pytest.mark.parametrize("protected_var", PROTECTED_VARIABLES) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "protected-vars")) def test_use_of_protected_var_element_overrides(cli, datafiles, protected_var): project = str(datafiles) - conf = { - 'name': 'test', - 'elements': { - 'manual': { - 'variables': { - protected_var: 'some-value' - } - } - } - } - _yaml.roundtrip_dump(conf, os.path.join(project, 'project.conf')) + conf = {"name": "test", "elements": {"manual": {"variables": {protected_var: "some-value"}}}} + _yaml.roundtrip_dump(conf, os.path.join(project, "project.conf")) element = { - 'kind': 'manual', - 'sources': [ - { - 'kind': 'local', - 'path': 'foo.txt' - } - ], + "kind": "manual", + "sources": [{"kind": "local", "path": "foo.txt"}], } - _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) + _yaml.roundtrip_dump(element, os.path.join(project, "target.bst")) - result = cli.run(project=project, args=['build', 'target.bst']) - result.assert_main_error(ErrorDomain.LOAD, - LoadErrorReason.PROTECTED_VARIABLE_REDEFINED) + result = cli.run(project=project, args=["build", "target.bst"]) + result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.PROTECTED_VARIABLE_REDEFINED) -@pytest.mark.parametrize('protected_var', PROTECTED_VARIABLES) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'protected-vars')) +@pytest.mark.parametrize("protected_var", PROTECTED_VARIABLES) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "protected-vars")) def test_use_of_protected_var_in_element(cli, datafiles, protected_var): project = str(datafiles) element = { - 'kind': 'import', - 'sources': [ - { - 'kind': 'local', - 'path': 'foo.txt' - } - ], - 'variables': { - protected_var: 'some-value' - } + "kind": "import", + "sources": [{"kind": "local", "path": "foo.txt"}], + "variables": {protected_var: "some-value"}, } - _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) + _yaml.roundtrip_dump(element, os.path.join(project, "target.bst")) - result = cli.run(project=project, args=['build', 'target.bst']) - result.assert_main_error(ErrorDomain.LOAD, - LoadErrorReason.PROTECTED_VARIABLE_REDEFINED) + result = cli.run(project=project, args=["build", "target.bst"]) + result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.PROTECTED_VARIABLE_REDEFINED) diff --git a/tests/frontend/__init__.py b/tests/frontend/__init__.py index f1c8c41b8..ad1f9ea92 100644 --- a/tests/frontend/__init__.py +++ b/tests/frontend/__init__.py @@ -5,6 +5,6 @@ from buildstream import _yaml # Shared function to configure the project.conf inline # def configure_project(path, config): - config['name'] = 'test' - config['element-path'] = 'elements' - _yaml.roundtrip_dump(config, os.path.join(path, 'project.conf')) + config["name"] = "test" + config["element-path"] = "elements" + _yaml.roundtrip_dump(config, os.path.join(path, "project.conf")) diff --git a/tests/frontend/artifact_delete.py b/tests/frontend/artifact_delete.py index a9f5ec6da..a93d99ef6 100644 --- a/tests/frontend/artifact_delete.py +++ b/tests/frontend/artifact_delete.py @@ -28,10 +28,7 @@ from tests.testutils import create_artifact_share # Project directory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project", -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",) # Test that we can delete the artifact of the element which corresponds @@ -39,60 +36,60 @@ DATA_DIR = os.path.join( @pytest.mark.datafiles(DATA_DIR) def test_artifact_delete_element(cli, tmpdir, datafiles): project = str(datafiles) - element = 'target.bst' + element = "target.bst" # Build the element and ensure it's cached - result = cli.run(project=project, args=['build', element]) + result = cli.run(project=project, args=["build", element]) result.assert_success() - assert cli.get_element_state(project, element) == 'cached' + assert cli.get_element_state(project, element) == "cached" - result = cli.run(project=project, args=['artifact', 'delete', element]) + result = cli.run(project=project, args=["artifact", "delete", element]) result.assert_success() - assert cli.get_element_state(project, element) != 'cached' + assert cli.get_element_state(project, element) != "cached" # Test that we can delete an artifact by specifying its ref. @pytest.mark.datafiles(DATA_DIR) def test_artifact_delete_artifact(cli, tmpdir, datafiles): project = str(datafiles) - element = 'target.bst' + element = "target.bst" # Configure a local cache - local_cache = os.path.join(str(tmpdir), 'cache') - cli.configure({'cachedir': local_cache}) + local_cache = os.path.join(str(tmpdir), "cache") + cli.configure({"cachedir": local_cache}) # First build an element so that we can find its artifact - result = cli.run(project=project, args=['build', element]) + result = cli.run(project=project, args=["build", element]) result.assert_success() # Obtain the artifact ref cache_key = cli.get_element_key(project, element) - artifact = os.path.join('test', os.path.splitext(element)[0], cache_key) + artifact = os.path.join("test", os.path.splitext(element)[0], cache_key) # Explicitly check that the ARTIFACT exists in the cache - assert os.path.exists(os.path.join(local_cache, 'artifacts', 'refs', artifact)) + assert os.path.exists(os.path.join(local_cache, "artifacts", "refs", artifact)) # Delete the artifact - result = cli.run(project=project, args=['artifact', 'delete', artifact]) + result = cli.run(project=project, args=["artifact", "delete", artifact]) result.assert_success() # Check that the ARTIFACT is no longer in the cache - assert not os.path.exists(os.path.join(local_cache, 'cas', 'refs', 'heads', artifact)) + assert not os.path.exists(os.path.join(local_cache, "cas", "refs", "heads", artifact)) # Test the `bst artifact delete` command with multiple, different arguments. @pytest.mark.datafiles(DATA_DIR) def test_artifact_delete_element_and_artifact(cli, tmpdir, datafiles): project = str(datafiles) - element = 'target.bst' - dep = 'compose-all.bst' + element = "target.bst" + dep = "compose-all.bst" # Configure a local cache - local_cache = os.path.join(str(tmpdir), 'cache') - cli.configure({'cachedir': local_cache}) + local_cache = os.path.join(str(tmpdir), "cache") + cli.configure({"cachedir": local_cache}) # First build an element so that we can find its artifact - result = cli.run(project=project, args=['build', element]) + result = cli.run(project=project, args=["build", element]) result.assert_success() assert cli.get_element_states(project, [element, dep], deps="none") == { element: "cached", @@ -101,20 +98,20 @@ def test_artifact_delete_element_and_artifact(cli, tmpdir, datafiles): # Obtain the artifact ref cache_key = cli.get_element_key(project, element) - artifact = os.path.join('test', os.path.splitext(element)[0], cache_key) + artifact = os.path.join("test", os.path.splitext(element)[0], cache_key) # Explicitly check that the ARTIFACT exists in the cache - assert os.path.exists(os.path.join(local_cache, 'artifacts', 'refs', artifact)) + assert os.path.exists(os.path.join(local_cache, "artifacts", "refs", artifact)) # Delete the artifact - result = cli.run(project=project, args=['artifact', 'delete', artifact, dep]) + result = cli.run(project=project, args=["artifact", "delete", artifact, dep]) result.assert_success() # Check that the ARTIFACT is no longer in the cache - assert not os.path.exists(os.path.join(local_cache, 'artifacts', artifact)) + assert not os.path.exists(os.path.join(local_cache, "artifacts", artifact)) # Check that the dependency ELEMENT is no longer cached - assert cli.get_element_state(project, dep) != 'cached' + assert cli.get_element_state(project, dep) != "cached" # Test that we receive the appropriate stderr when we try to delete an artifact @@ -122,19 +119,19 @@ def test_artifact_delete_element_and_artifact(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_artifact_delete_unbuilt_artifact(cli, tmpdir, datafiles): project = str(datafiles) - element = 'target.bst' + element = "target.bst" # delete it, just in case it's there - _ = cli.run(project=project, args=['artifact', 'delete', element]) + _ = cli.run(project=project, args=["artifact", "delete", element]) # Ensure the element is not cached - assert cli.get_element_state(project, element) != 'cached' + assert cli.get_element_state(project, element) != "cached" # Now try and remove it again (now we know its not there) - result = cli.run(project=project, args=['artifact', 'delete', element]) + result = cli.run(project=project, args=["artifact", "delete", element]) cache_key = cli.get_element_key(project, element) - artifact = os.path.join('test', os.path.splitext(element)[0], cache_key) + artifact = os.path.join("test", os.path.splitext(element)[0], cache_key) expected_err = "WARNING Could not find ref '{}'".format(artifact) assert expected_err in result.stderr @@ -144,122 +141,121 @@ def test_artifact_delete_unbuilt_artifact(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_artifact_delete_pulled_artifact_without_buildtree(cli, tmpdir, datafiles): project = str(datafiles) - element = 'target.bst' + element = "target.bst" # Set up remote and local shares - local_cache = os.path.join(str(tmpdir), 'artifacts') - with create_artifact_share(os.path.join(str(tmpdir), 'remote')) as remote: - cli.configure({ - 'artifacts': {'url': remote.repo, 'push': True}, - 'cachedir': local_cache, - }) + local_cache = os.path.join(str(tmpdir), "artifacts") + with create_artifact_share(os.path.join(str(tmpdir), "remote")) as remote: + cli.configure( + {"artifacts": {"url": remote.repo, "push": True}, "cachedir": local_cache,} + ) # Build the element - result = cli.run(project=project, args=['build', element]) + result = cli.run(project=project, args=["build", element]) result.assert_success() # Make sure it's in the share - assert remote.get_artifact(cli.get_artifact_name(project, 'test', element)) + assert remote.get_artifact(cli.get_artifact_name(project, "test", element)) # Delete and then pull the artifact (without its buildtree) - result = cli.run(project=project, args=['artifact', 'delete', element]) + result = cli.run(project=project, args=["artifact", "delete", element]) result.assert_success() - assert cli.get_element_state(project, element) != 'cached' - result = cli.run(project=project, args=['artifact', 'pull', element]) + assert cli.get_element_state(project, element) != "cached" + result = cli.run(project=project, args=["artifact", "pull", element]) result.assert_success() - assert cli.get_element_state(project, element) == 'cached' + assert cli.get_element_state(project, element) == "cached" # Now delete it again (it should have been pulled without the buildtree, but # a digest of the buildtree is pointed to in the artifact's metadata - result = cli.run(project=project, args=['artifact', 'delete', element]) + result = cli.run(project=project, args=["artifact", "delete", element]) result.assert_success() - assert cli.get_element_state(project, element) != 'cached' + assert cli.get_element_state(project, element) != "cached" # Test that we can delete the build deps of an element @pytest.mark.datafiles(DATA_DIR) def test_artifact_delete_elements_build_deps(cli, tmpdir, datafiles): project = str(datafiles) - element = 'target.bst' + element = "target.bst" # Build the element and ensure it's cached - result = cli.run(project=project, args=['build', element]) + result = cli.run(project=project, args=["build", element]) result.assert_success() # Assert element and build deps are cached - assert cli.get_element_state(project, element) == 'cached' - bdep_states = cli.get_element_states(project, [element], deps='build') + assert cli.get_element_state(project, element) == "cached" + bdep_states = cli.get_element_states(project, [element], deps="build") for state in bdep_states.values(): - assert state == 'cached' + assert state == "cached" - result = cli.run(project=project, args=['artifact', 'delete', '--deps', 'build', element]) + result = cli.run(project=project, args=["artifact", "delete", "--deps", "build", element]) result.assert_success() # Assert that the build deps have been deleted and that the artifact remains cached - assert cli.get_element_state(project, element) == 'cached' - bdep_states = cli.get_element_states(project, [element], deps='build') + assert cli.get_element_state(project, element) == "cached" + bdep_states = cli.get_element_states(project, [element], deps="build") for state in bdep_states.values(): - assert state != 'cached' + assert state != "cached" # Test that we can delete the build deps of an artifact by providing an artifact ref @pytest.mark.datafiles(DATA_DIR) def test_artifact_delete_artifacts_build_deps(cli, tmpdir, datafiles): project = str(datafiles) - element = 'target.bst' + element = "target.bst" # Configure a local cache - local_cache = os.path.join(str(tmpdir), 'cache') - cli.configure({'cachedir': local_cache}) + local_cache = os.path.join(str(tmpdir), "cache") + cli.configure({"cachedir": local_cache}) # First build an element so that we can find its artifact - result = cli.run(project=project, args=['build', element]) + result = cli.run(project=project, args=["build", element]) result.assert_success() # Obtain the artifact ref cache_key = cli.get_element_key(project, element) - artifact = os.path.join('test', os.path.splitext(element)[0], cache_key) + artifact = os.path.join("test", os.path.splitext(element)[0], cache_key) # Explicitly check that the ARTIFACT exists in the cache - assert os.path.exists(os.path.join(local_cache, 'artifacts', 'refs', artifact)) + assert os.path.exists(os.path.join(local_cache, "artifacts", "refs", artifact)) # get the artifact refs of the build dependencies bdep_refs = [] - bdep_states = cli.get_element_states(project, [element], deps='build') + bdep_states = cli.get_element_states(project, [element], deps="build") for bdep in bdep_states.keys(): - bdep_refs.append(os.path.join('test', _get_normal_name(bdep), cli.get_element_key(project, bdep))) + bdep_refs.append(os.path.join("test", _get_normal_name(bdep), cli.get_element_key(project, bdep))) # Assert build dependencies are cached for ref in bdep_refs: - assert os.path.exists(os.path.join(local_cache, 'artifacts', 'refs', ref)) + assert os.path.exists(os.path.join(local_cache, "artifacts", "refs", ref)) # Delete the artifact - result = cli.run(project=project, args=['artifact', 'delete', '--deps', 'build', artifact]) + result = cli.run(project=project, args=["artifact", "delete", "--deps", "build", artifact]) result.assert_success() # Check that the artifact's build deps are no longer in the cache # Assert build dependencies have been deleted and that the artifact remains for ref in bdep_refs: - assert not os.path.exists(os.path.join(local_cache, 'artifacts', 'refs', ref)) - assert os.path.exists(os.path.join(local_cache, 'artifacts', 'refs', artifact)) + assert not os.path.exists(os.path.join(local_cache, "artifacts", "refs", ref)) + assert os.path.exists(os.path.join(local_cache, "artifacts", "refs", artifact)) # Test that `--deps all` option fails if an artifact ref is specified @pytest.mark.datafiles(DATA_DIR) def test_artifact_delete_artifact_with_deps_all_fails(cli, tmpdir, datafiles): project = str(datafiles) - element = 'target.bst' + element = "target.bst" # First build an element so that we can find its artifact - result = cli.run(project=project, args=['build', element]) + result = cli.run(project=project, args=["build", element]) result.assert_success() # Obtain the artifact ref cache_key = cli.get_element_key(project, element) - artifact = os.path.join('test', os.path.splitext(element)[0], cache_key) + artifact = os.path.join("test", os.path.splitext(element)[0], cache_key) # Try to delete the artifact with all of its dependencies - result = cli.run(project=project, args=['artifact', 'delete', '--deps', 'all', artifact]) + result = cli.run(project=project, args=["artifact", "delete", "--deps", "all", artifact]) result.assert_main_error(ErrorDomain.STREAM, None) assert "Error: '--deps all' is not supported for artifact refs" in result.stderr diff --git a/tests/frontend/artifact_list_contents.py b/tests/frontend/artifact_list_contents.py index 626eb3fa7..a1473c806 100644 --- a/tests/frontend/artifact_list_contents.py +++ b/tests/frontend/artifact_list_contents.py @@ -25,10 +25,7 @@ from buildstream.testing import cli # pylint: disable=unused-import # Project directory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project", -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",) @pytest.mark.datafiles(DATA_DIR) @@ -36,16 +33,13 @@ def test_artifact_list_exact_contents_element(cli, datafiles): project = str(datafiles) # Ensure we have an artifact to read - result = cli.run(project=project, args=['build', 'import-bin.bst']) + result = cli.run(project=project, args=["build", "import-bin.bst"]) assert result.exit_code == 0 # List the contents via the element name - result = cli.run(project=project, args=['artifact', 'list-contents', 'import-bin.bst']) + result = cli.run(project=project, args=["artifact", "list-contents", "import-bin.bst"]) assert result.exit_code == 0 - expected_output = ("import-bin.bst:\n" - "\tusr\n" - "\tusr/bin\n" - "\tusr/bin/hello\n\n") + expected_output = "import-bin.bst:\n" "\tusr\n" "\tusr/bin\n" "\tusr/bin/hello\n\n" assert expected_output in result.output @@ -54,20 +48,17 @@ def test_artifact_list_exact_contents_ref(cli, datafiles): project = str(datafiles) # Get the cache key of our test element - key = cli.get_element_key(project, 'import-bin.bst') + key = cli.get_element_key(project, "import-bin.bst") # Ensure we have an artifact to read - result = cli.run(project=project, args=['build', 'import-bin.bst']) + result = cli.run(project=project, args=["build", "import-bin.bst"]) assert result.exit_code == 0 # List the contents via the key - result = cli.run(project=project, args=['artifact', 'list-contents', 'test/import-bin/' + key]) + result = cli.run(project=project, args=["artifact", "list-contents", "test/import-bin/" + key]) assert result.exit_code == 0 - expected_output = ("test/import-bin/" + key + ":\n" - "\tusr\n" - "\tusr/bin\n" - "\tusr/bin/hello\n\n") + expected_output = "test/import-bin/" + key + ":\n" "\tusr\n" "\tusr/bin\n" "\tusr/bin/hello\n\n" assert expected_output in result.output @@ -76,23 +67,25 @@ def test_artifact_list_exact_contents_glob(cli, datafiles): project = str(datafiles) # Ensure we have an artifact to read - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) assert result.exit_code == 0 # List the contents via glob - result = cli.run(project=project, args=['artifact', 'list-contents', 'test/*']) + result = cli.run(project=project, args=["artifact", "list-contents", "test/*"]) assert result.exit_code == 0 # get the cahe keys for each element in the glob - import_bin_key = cli.get_element_key(project, 'import-bin.bst') - import_dev_key = cli.get_element_key(project, 'import-dev.bst') - compose_all_key = cli.get_element_key(project, 'compose-all.bst') - target_key = cli.get_element_key(project, 'target.bst') - - expected_artifacts = ["test/import-bin/" + import_bin_key, - "test/import-dev/" + import_dev_key, - "test/compose-all/" + compose_all_key, - "test/target/" + target_key] + import_bin_key = cli.get_element_key(project, "import-bin.bst") + import_dev_key = cli.get_element_key(project, "import-dev.bst") + compose_all_key = cli.get_element_key(project, "compose-all.bst") + target_key = cli.get_element_key(project, "target.bst") + + expected_artifacts = [ + "test/import-bin/" + import_bin_key, + "test/import-dev/" + import_dev_key, + "test/compose-all/" + compose_all_key, + "test/target/" + target_key, + ] for artifact in expected_artifacts: assert artifact in result.output @@ -103,16 +96,18 @@ def test_artifact_list_exact_contents_element_long(cli, datafiles): project = str(datafiles) # Ensure we have an artifact to read - result = cli.run(project=project, args=['build', 'import-bin.bst']) + result = cli.run(project=project, args=["build", "import-bin.bst"]) assert result.exit_code == 0 # List the contents via the element name - result = cli.run(project=project, args=['artifact', 'list-contents', '--long', 'import-bin.bst']) + result = cli.run(project=project, args=["artifact", "list-contents", "--long", "import-bin.bst"]) assert result.exit_code == 0 - expected_output = ("import-bin.bst:\n" - "\tdrwxr-xr-x dir 1 usr\n" - "\tdrwxr-xr-x dir 1 usr/bin\n" - "\t-rw-r--r-- reg 107 usr/bin/hello\n\n") + expected_output = ( + "import-bin.bst:\n" + "\tdrwxr-xr-x dir 1 usr\n" + "\tdrwxr-xr-x dir 1 usr/bin\n" + "\t-rw-r--r-- reg 107 usr/bin/hello\n\n" + ) assert expected_output in result.output @@ -122,19 +117,21 @@ def test_artifact_list_exact_contents_ref_long(cli, datafiles): project = str(datafiles) # Get the cache key of our test element - key = cli.get_element_key(project, 'import-bin.bst') + key = cli.get_element_key(project, "import-bin.bst") # Ensure we have an artifact to read - result = cli.run(project=project, args=['build', 'import-bin.bst']) + result = cli.run(project=project, args=["build", "import-bin.bst"]) assert result.exit_code == 0 # List the contents via the key - result = cli.run(project=project, args=['artifact', 'list-contents', '-l', 'test/import-bin/' + key]) + result = cli.run(project=project, args=["artifact", "list-contents", "-l", "test/import-bin/" + key]) assert result.exit_code == 0 - expected_output = (" test/import-bin/" + key + ":\n" - "\tdrwxr-xr-x dir 1 usr\n" - "\tdrwxr-xr-x dir 1 usr/bin\n" - "\t-rw-r--r-- reg 107 usr/bin/hello\n\n") + expected_output = ( + " test/import-bin/" + key + ":\n" + "\tdrwxr-xr-x dir 1 usr\n" + "\tdrwxr-xr-x dir 1 usr/bin\n" + "\t-rw-r--r-- reg 107 usr/bin/hello\n\n" + ) assert expected_output in result.output diff --git a/tests/frontend/artifact_log.py b/tests/frontend/artifact_log.py index 39c9458ad..6ea610a25 100644 --- a/tests/frontend/artifact_log.py +++ b/tests/frontend/artifact_log.py @@ -25,10 +25,7 @@ from buildstream.testing import cli # pylint: disable=unused-import # Project directory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project", -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",) @pytest.mark.datafiles(DATA_DIR) @@ -36,32 +33,32 @@ def test_artifact_log(cli, datafiles): project = str(datafiles) # Get the cache key of our test element - result = cli.run(project=project, silent=True, args=[ - '--no-colors', - 'show', '--deps', 'none', '--format', '%{full-key}', - 'target.bst' - ]) + result = cli.run( + project=project, + silent=True, + args=["--no-colors", "show", "--deps", "none", "--format", "%{full-key}", "target.bst"], + ) key = result.output.strip() # Ensure we have an artifact to read - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) assert result.exit_code == 0 # Read the log via the element name - result = cli.run(project=project, args=['artifact', 'log', 'target.bst']) + result = cli.run(project=project, args=["artifact", "log", "target.bst"]) assert result.exit_code == 0 log = result.output # Assert that there actually was a log file - assert log != '' + assert log != "" # Read the log via the key - result = cli.run(project=project, args=['artifact', 'log', 'test/target/' + key]) + result = cli.run(project=project, args=["artifact", "log", "test/target/" + key]) assert result.exit_code == 0 assert log == result.output # Read the log via glob - result = cli.run(project=project, args=['artifact', 'log', 'test/target/*']) + result = cli.run(project=project, args=["artifact", "log", "test/target/*"]) assert result.exit_code == 0 # The artifact is cached under both a strong key and a weak key assert log == result.output @@ -72,7 +69,7 @@ def test_artifact_log_files(cli, datafiles): project = str(datafiles) # Ensure we have an artifact to read - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) assert result.exit_code == 0 logfiles = os.path.join(project, "logfiles") @@ -84,16 +81,16 @@ def test_artifact_log_files(cli, datafiles): assert not os.path.exists(import_bin) # Run the command and ensure the file now exists - result = cli.run(project=project, args=['artifact', 'log', '--out', logfiles, 'target.bst', 'import-bin.bst']) + result = cli.run(project=project, args=["artifact", "log", "--out", logfiles, "target.bst", "import-bin.bst"]) assert result.exit_code == 0 assert os.path.exists(logfiles) assert os.path.exists(target) assert os.path.exists(import_bin) # Ensure the file contains the logs by checking for the LOG line - with open(target, 'r') as f: + with open(target, "r") as f: data = f.read() assert "LOG target.bst" in data - with open(import_bin, 'r') as f: + with open(import_bin, "r") as f: data = f.read() assert "LOG import-bin.bst" in data diff --git a/tests/frontend/artifact_show.py b/tests/frontend/artifact_show.py index 76ea93d63..6f824c0e4 100644 --- a/tests/frontend/artifact_show.py +++ b/tests/frontend/artifact_show.py @@ -27,110 +27,106 @@ from tests.testutils import create_artifact_share # Project directory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project", -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",) # Test artifact show @pytest.mark.datafiles(DATA_DIR) def test_artifact_show_element_name(cli, tmpdir, datafiles): project = str(datafiles) - element = 'target.bst' + element = "target.bst" - result = cli.run(project=project, args=['artifact', 'show', element]) + result = cli.run(project=project, args=["artifact", "show", element]) result.assert_success() - assert 'not cached {}'.format(element) in result.output + assert "not cached {}".format(element) in result.output - result = cli.run(project=project, args=['build', element]) + result = cli.run(project=project, args=["build", element]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'show', element]) + result = cli.run(project=project, args=["artifact", "show", element]) result.assert_success() - assert 'cached {}'.format(element) in result.output + assert "cached {}".format(element) in result.output # Test artifact show on a failed element @pytest.mark.datafiles(DATA_DIR) def test_artifact_show_failed_element(cli, tmpdir, datafiles): project = str(datafiles) - element = 'manual.bst' + element = "manual.bst" - result = cli.run(project=project, args=['artifact', 'show', element]) + result = cli.run(project=project, args=["artifact", "show", element]) result.assert_success() - assert 'not cached {}'.format(element) in result.output + assert "not cached {}".format(element) in result.output - result = cli.run(project=project, args=['build', element]) - result.assert_task_error(ErrorDomain.SANDBOX, 'missing-command') + result = cli.run(project=project, args=["build", element]) + result.assert_task_error(ErrorDomain.SANDBOX, "missing-command") - result = cli.run(project=project, args=['artifact', 'show', element]) + result = cli.run(project=project, args=["artifact", "show", element]) result.assert_success() - assert 'failed {}'.format(element) in result.output + assert "failed {}".format(element) in result.output # Test artifact show with a deleted dependency @pytest.mark.datafiles(DATA_DIR) def test_artifact_show_element_missing_deps(cli, tmpdir, datafiles): project = str(datafiles) - element = 'target.bst' - dependency = 'import-bin.bst' + element = "target.bst" + dependency = "import-bin.bst" - result = cli.run(project=project, args=['build', element]) + result = cli.run(project=project, args=["build", element]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'delete', dependency]) + result = cli.run(project=project, args=["artifact", "delete", dependency]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'show', '--deps', 'all', element]) + result = cli.run(project=project, args=["artifact", "show", "--deps", "all", element]) result.assert_success() - assert 'not cached {}'.format(dependency) in result.output - assert 'cached {}'.format(element) in result.output + assert "not cached {}".format(dependency) in result.output + assert "cached {}".format(element) in result.output # Test artifact show with artifact ref @pytest.mark.datafiles(DATA_DIR) def test_artifact_show_artifact_ref(cli, tmpdir, datafiles): project = str(datafiles) - element = 'target.bst' + element = "target.bst" - result = cli.run(project=project, args=['build', element]) + result = cli.run(project=project, args=["build", element]) result.assert_success() cache_key = cli.get_element_key(project, element) - artifact_ref = 'test/target/' + cache_key + artifact_ref = "test/target/" + cache_key - result = cli.run(project=project, args=['artifact', 'show', artifact_ref]) + result = cli.run(project=project, args=["artifact", "show", artifact_ref]) result.assert_success() - assert 'cached {}'.format(artifact_ref) in result.output + assert "cached {}".format(artifact_ref) in result.output # Test artifact show artifact in remote @pytest.mark.datafiles(DATA_DIR) def test_artifact_show_element_available_remotely(cli, tmpdir, datafiles): project = str(datafiles) - element = 'target.bst' + element = "target.bst" # Set up remote and local shares - local_cache = os.path.join(str(tmpdir), 'artifacts') - with create_artifact_share(os.path.join(str(tmpdir), 'remote')) as remote: - cli.configure({ - 'artifacts': {'url': remote.repo, 'push': True}, - 'cachedir': local_cache, - }) + local_cache = os.path.join(str(tmpdir), "artifacts") + with create_artifact_share(os.path.join(str(tmpdir), "remote")) as remote: + cli.configure( + {"artifacts": {"url": remote.repo, "push": True}, "cachedir": local_cache,} + ) # Build the element - result = cli.run(project=project, args=['build', element]) + result = cli.run(project=project, args=["build", element]) result.assert_success() # Make sure it's in the share - assert remote.get_artifact(cli.get_artifact_name(project, 'test', element)) + assert remote.get_artifact(cli.get_artifact_name(project, "test", element)) # Delete the artifact from the local cache - result = cli.run(project=project, args=['artifact', 'delete', element]) + result = cli.run(project=project, args=["artifact", "delete", element]) result.assert_success() - assert cli.get_element_state(project, element) != 'cached' + assert cli.get_element_state(project, element) != "cached" - result = cli.run(project=project, args=['artifact', 'show', element]) + result = cli.run(project=project, args=["artifact", "show", element]) result.assert_success() - assert 'available {}'.format(element) in result.output + assert "available {}".format(element) in result.output diff --git a/tests/frontend/buildcheckout.py b/tests/frontend/buildcheckout.py index a24446d61..364ff98d1 100644 --- a/tests/frontend/buildcheckout.py +++ b/tests/frontend/buildcheckout.py @@ -21,69 +21,64 @@ from tests.testutils import generate_junction, create_artifact_share from . import configure_project # Project directory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project", -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",) def strict_args(args, strict): if strict != "strict": - return ['--no-strict', *args] + return ["--no-strict", *args] return args @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("strict,hardlinks", [ - ("strict", "copies"), - ("strict", "hardlinks"), - ("non-strict", "copies"), - ("non-strict", "hardlinks"), -]) +@pytest.mark.parametrize( + "strict,hardlinks", + [("strict", "copies"), ("strict", "hardlinks"), ("non-strict", "copies"), ("non-strict", "hardlinks"),], +) def test_build_checkout(datafiles, cli, strict, hardlinks): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') + checkout = os.path.join(cli.directory, "checkout") # First build it - result = cli.run(project=project, args=strict_args(['build', 'target.bst'], strict)) + result = cli.run(project=project, args=strict_args(["build", "target.bst"], strict)) result.assert_success() # Assert that after a successful build, the builddir is empty - builddir = os.path.join(cli.directory, 'build') + builddir = os.path.join(cli.directory, "build") assert os.path.isdir(builddir) assert not os.listdir(builddir) # Prepare checkout args - checkout_args = strict_args(['artifact', 'checkout'], strict) + checkout_args = strict_args(["artifact", "checkout"], strict) if hardlinks == "hardlinks": - checkout_args += ['--hardlinks'] - checkout_args += ['target.bst', '--directory', checkout] + checkout_args += ["--hardlinks"] + checkout_args += ["target.bst", "--directory", checkout] # Now check it out result = cli.run(project=project, args=checkout_args) result.assert_success() # Check that the executable hello file is found in the checkout - filename = os.path.join(checkout, 'usr', 'bin', 'hello') + filename = os.path.join(checkout, "usr", "bin", "hello") assert os.path.exists(filename) - filename = os.path.join(checkout, 'usr', 'include', 'pony.h') + filename = os.path.join(checkout, "usr", "include", "pony.h") assert os.path.exists(filename) @pytest.mark.datafiles(DATA_DIR + "_world") def test_build_default_all(datafiles, cli): project = str(datafiles) - result = cli.run(project=project, silent=True, args=['build']) + result = cli.run(project=project, silent=True, args=["build"]) result.assert_success() target_dir = os.path.join(cli.directory, DATA_DIR + "_world", "elements") output_dir = os.path.join(cli.directory, "logs", "test") - expected = subprocess.Popen(('ls', target_dir), stdout=subprocess.PIPE) + expected = subprocess.Popen(("ls", target_dir), stdout=subprocess.PIPE) expected = subprocess.check_output(("wc", "-w"), stdin=expected.stdout) - results = subprocess.Popen(('ls', output_dir), stdout=subprocess.PIPE) + results = subprocess.Popen(("ls", output_dir), stdout=subprocess.PIPE) results = subprocess.check_output(("wc", "-w"), stdin=results.stdout) assert results == expected @@ -92,7 +87,7 @@ def test_build_default_all(datafiles, cli): @pytest.mark.datafiles(DATA_DIR + "_default") def test_build_default(cli, datafiles): project = str(datafiles) - result = cli.run(project=project, silent=True, args=['build']) + result = cli.run(project=project, silent=True, args=["build"]) result.assert_success() results = cli.get_element_state(project, "target2.bst") @@ -101,60 +96,56 @@ def test_build_default(cli, datafiles): @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("strict,hardlinks", [ - ("non-strict", "hardlinks"), -]) +@pytest.mark.parametrize("strict,hardlinks", [("non-strict", "hardlinks"),]) def test_build_invalid_suffix(datafiles, cli, strict, hardlinks): project = str(datafiles) - result = cli.run(project=project, args=strict_args(['build', 'target.foo'], strict)) + result = cli.run(project=project, args=strict_args(["build", "target.foo"], strict)) result.assert_main_error(ErrorDomain.LOAD, "bad-element-suffix") @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("strict,hardlinks", [ - ("non-strict", "hardlinks"), -]) +@pytest.mark.parametrize("strict,hardlinks", [("non-strict", "hardlinks"),]) def test_build_invalid_suffix_dep(datafiles, cli, strict, hardlinks): project = str(datafiles) # target2.bst depends on an element called target.foo - result = cli.run(project=project, args=strict_args(['build', 'target2.bst'], strict)) + result = cli.run(project=project, args=strict_args(["build", "target2.bst"], strict)) result.assert_main_error(ErrorDomain.LOAD, "bad-element-suffix") -@pytest.mark.skipif(IS_WINDOWS, reason='Not available on Windows') +@pytest.mark.skipif(IS_WINDOWS, reason="Not available on Windows") @pytest.mark.datafiles(DATA_DIR) def test_build_invalid_filename_chars(datafiles, cli): project = str(datafiles) - element_name = 'invalid-chars|<>-in-name.bst' + element_name = "invalid-chars|<>-in-name.bst" # The name of this file contains characters that are not allowed by # BuildStream, using it should raise a warning. element = { - 'kind': 'stack', + "kind": "stack", } - _yaml.roundtrip_dump(element, os.path.join(project, 'elements', element_name)) + _yaml.roundtrip_dump(element, os.path.join(project, "elements", element_name)) - result = cli.run(project=project, args=strict_args(['build', element_name], 'non-strict')) + result = cli.run(project=project, args=strict_args(["build", element_name], "non-strict")) result.assert_main_error(ErrorDomain.LOAD, "bad-characters-in-name") -@pytest.mark.skipif(IS_WINDOWS, reason='Not available on Windows') +@pytest.mark.skipif(IS_WINDOWS, reason="Not available on Windows") @pytest.mark.datafiles(DATA_DIR) def test_build_invalid_filename_chars_dep(datafiles, cli): project = str(datafiles) - element_name = 'invalid-chars|<>-in-name.bst' + element_name = "invalid-chars|<>-in-name.bst" # The name of this file contains characters that are not allowed by # BuildStream, and is listed as a dependency of 'invalid-chars-in-dep.bst'. # This should also raise a warning. element = { - 'kind': 'stack', + "kind": "stack", } - _yaml.roundtrip_dump(element, os.path.join(project, 'elements', element_name)) + _yaml.roundtrip_dump(element, os.path.join(project, "elements", element_name)) - result = cli.run(project=project, args=strict_args(['build', 'invalid-chars-in-dep.bst'], 'non-strict')) + result = cli.run(project=project, args=strict_args(["build", "invalid-chars-in-dep.bst"], "non-strict")) result.assert_main_error(ErrorDomain.LOAD, "bad-characters-in-name") @@ -162,39 +153,40 @@ def test_build_invalid_filename_chars_dep(datafiles, cli): @pytest.mark.parametrize("deps", [("run"), ("none"), ("build"), ("all")]) def test_build_checkout_deps(datafiles, cli, deps): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') + checkout = os.path.join(cli.directory, "checkout") element_name = "checkout-deps.bst" # First build it - result = cli.run(project=project, args=['build', element_name]) + result = cli.run(project=project, args=["build", element_name]) result.assert_success() # Assert that after a successful build, the builddir is empty - builddir = os.path.join(cli.directory, 'build') + builddir = os.path.join(cli.directory, "build") assert os.path.isdir(builddir) assert not os.listdir(builddir) # Now check it out - result = cli.run(project=project, args=['artifact', 'checkout', element_name, - '--deps', deps, '--directory', checkout]) + result = cli.run( + project=project, args=["artifact", "checkout", element_name, "--deps", deps, "--directory", checkout] + ) result.assert_success() # Verify output of this element - filename = os.path.join(checkout, 'etc', 'buildstream', 'config') + filename = os.path.join(checkout, "etc", "buildstream", "config") if deps == "build": assert not os.path.exists(filename) else: assert os.path.exists(filename) # Verify output of this element's build dependencies - filename = os.path.join(checkout, 'usr', 'include', 'pony.h') + filename = os.path.join(checkout, "usr", "include", "pony.h") if deps in ["build", "all"]: assert os.path.exists(filename) else: assert not os.path.exists(filename) # Verify output of this element's runtime dependencies - filename = os.path.join(checkout, 'usr', 'bin', 'hello') + filename = os.path.join(checkout, "usr", "bin", "hello") if deps in ["run", "all"]: assert os.path.exists(filename) else: @@ -204,162 +196,165 @@ def test_build_checkout_deps(datafiles, cli, deps): @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_unbuilt(datafiles, cli): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') + checkout = os.path.join(cli.directory, "checkout") # Check that checking out an unbuilt element fails nicely - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkout]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkout]) result.assert_main_error(ErrorDomain.STREAM, "uncached-checkout-attempt") @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_compression_no_tar(datafiles, cli): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout.tar') + checkout = os.path.join(cli.directory, "checkout.tar") - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - checkout_args = ['artifact', 'checkout', '--directory', checkout, '--compression', 'gz', 'target.bst'] + checkout_args = ["artifact", "checkout", "--directory", checkout, "--compression", "gz", "target.bst"] result = cli.run(project=project, args=checkout_args) assert "ERROR: --compression can only be provided if --tar is provided" in result.stderr assert result.exit_code != 0 + # If we don't support the extension, we default to an uncompressed tarball @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_tar_with_unconventional_name(datafiles, cli): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout.foo') + checkout = os.path.join(cli.directory, "checkout.foo") - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - checkout_args = ['artifact', 'checkout', '--tar', checkout, 'target.bst'] + checkout_args = ["artifact", "checkout", "--tar", checkout, "target.bst"] result = cli.run(project=project, args=checkout_args) result.assert_success() - tar = tarfile.open(name=checkout, mode='r') - assert os.path.join('.', 'usr', 'bin', 'hello') in tar.getnames() - assert os.path.join('.', 'usr', 'include', 'pony.h') in tar.getnames() + tar = tarfile.open(name=checkout, mode="r") + assert os.path.join(".", "usr", "bin", "hello") in tar.getnames() + assert os.path.join(".", "usr", "include", "pony.h") in tar.getnames() @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_tar_with_unsupported_ext(datafiles, cli): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout.tar.foo') + checkout = os.path.join(cli.directory, "checkout.tar.foo") - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - checkout_args = ['artifact', 'checkout', '--tar', checkout, 'target.bst'] + checkout_args = ["artifact", "checkout", "--tar", checkout, "target.bst"] result = cli.run(project=project, args=checkout_args) - assert "Invalid file extension given with '--tar': Expected compression with unknown file extension ('.foo'), " \ - "supported extensions are ('.tar'), ('.gz'), ('.xz'), ('.bz2')" in result.stderr + assert ( + "Invalid file extension given with '--tar': Expected compression with unknown file extension ('.foo'), " + "supported extensions are ('.tar'), ('.gz'), ('.xz'), ('.bz2')" in result.stderr + ) @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_tar_no_compression(datafiles, cli): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout.tar.gz') + checkout = os.path.join(cli.directory, "checkout.tar.gz") - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - builddir = os.path.join(cli.directory, 'build') + builddir = os.path.join(cli.directory, "build") assert os.path.isdir(builddir) assert not os.listdir(builddir) - checkout_args = ['artifact', 'checkout', '--tar', checkout, 'target.bst'] + checkout_args = ["artifact", "checkout", "--tar", checkout, "target.bst"] result = cli.run(project=project, args=checkout_args) result.assert_success() - tar = tarfile.open(name=checkout, mode='r:gz') - assert os.path.join('.', 'usr', 'bin', 'hello') in tar.getnames() - assert os.path.join('.', 'usr', 'include', 'pony.h') in tar.getnames() + tar = tarfile.open(name=checkout, mode="r:gz") + assert os.path.join(".", "usr", "bin", "hello") in tar.getnames() + assert os.path.join(".", "usr", "include", "pony.h") in tar.getnames() @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_tarball(datafiles, cli): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout.tar') + checkout = os.path.join(cli.directory, "checkout.tar") - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - builddir = os.path.join(cli.directory, 'build') + builddir = os.path.join(cli.directory, "build") assert os.path.isdir(builddir) assert not os.listdir(builddir) - checkout_args = ['artifact', 'checkout', '--tar', checkout, 'target.bst'] + checkout_args = ["artifact", "checkout", "--tar", checkout, "target.bst"] result = cli.run(project=project, args=checkout_args) result.assert_success() tar = tarfile.TarFile(checkout) - assert os.path.join('.', 'usr', 'bin', 'hello') in tar.getnames() - assert os.path.join('.', 'usr', 'include', 'pony.h') in tar.getnames() + assert os.path.join(".", "usr", "bin", "hello") in tar.getnames() + assert os.path.join(".", "usr", "include", "pony.h") in tar.getnames() @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_using_ref(datafiles, cli): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') + checkout = os.path.join(cli.directory, "checkout") - result = cli.run(project=project, args=['build', 'checkout-deps.bst']) + result = cli.run(project=project, args=["build", "checkout-deps.bst"]) result.assert_success() - key = cli.get_element_key(project, 'checkout-deps.bst') - checkout_args = ['artifact', 'checkout', '--directory', checkout, '--deps', 'none', 'test/checkout-deps/' + key] + key = cli.get_element_key(project, "checkout-deps.bst") + checkout_args = ["artifact", "checkout", "--directory", checkout, "--deps", "none", "test/checkout-deps/" + key] result = cli.run(project=project, args=checkout_args) result.assert_success() - filename = os.path.join(checkout, 'etc', 'buildstream', 'config') + filename = os.path.join(checkout, "etc", "buildstream", "config") assert os.path.exists(filename) @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_tarball_using_ref(datafiles, cli): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout.tar') + checkout = os.path.join(cli.directory, "checkout.tar") - result = cli.run(project=project, args=['build', 'checkout-deps.bst']) + result = cli.run(project=project, args=["build", "checkout-deps.bst"]) result.assert_success() - builddir = os.path.join(cli.directory, 'build') + builddir = os.path.join(cli.directory, "build") assert os.path.isdir(builddir) assert not os.listdir(builddir) - key = cli.get_element_key(project, 'checkout-deps.bst') - checkout_args = ['artifact', 'checkout', '--deps', 'none', '--tar', checkout, 'test/checkout-deps/' + key] + key = cli.get_element_key(project, "checkout-deps.bst") + checkout_args = ["artifact", "checkout", "--deps", "none", "--tar", checkout, "test/checkout-deps/" + key] result = cli.run(project=project, args=checkout_args) result.assert_success() tar = tarfile.TarFile(checkout) - assert os.path.join('.', 'etc', 'buildstream', 'config') in tar.getnames() + assert os.path.join(".", "etc", "buildstream", "config") in tar.getnames() @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_build_deps_using_ref(datafiles, cli): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') + checkout = os.path.join(cli.directory, "checkout") - result = cli.run(project=project, args=['build', 'checkout-deps.bst']) + result = cli.run(project=project, args=["build", "checkout-deps.bst"]) result.assert_success() - key = cli.get_element_key(project, 'checkout-deps.bst') - checkout_args = ['artifact', 'checkout', '--directory', checkout, '--deps', 'build', 'test/checkout-deps/' + key] + key = cli.get_element_key(project, "checkout-deps.bst") + checkout_args = ["artifact", "checkout", "--directory", checkout, "--deps", "build", "test/checkout-deps/" + key] result = cli.run(project=project, args=checkout_args) result.assert_success() - build_dep_files = os.path.join(checkout, 'usr', 'include', 'pony.h') - runtime_dep_files = os.path.join(checkout, 'usr', 'bin', 'hello') - target_files = os.path.join(checkout, 'etc', 'buildstream', 'config') + build_dep_files = os.path.join(checkout, "usr", "include", "pony.h") + runtime_dep_files = os.path.join(checkout, "usr", "bin", "hello") + target_files = os.path.join(checkout, "etc", "buildstream", "config") assert os.path.exists(build_dep_files) assert not os.path.exists(runtime_dep_files) assert not os.path.exists(target_files) @@ -368,13 +363,13 @@ def test_build_checkout_build_deps_using_ref(datafiles, cli): @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_runtime_deps_using_ref_fails(datafiles, cli): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') + checkout = os.path.join(cli.directory, "checkout") - result = cli.run(project=project, args=['build', 'checkout-deps.bst']) + result = cli.run(project=project, args=["build", "checkout-deps.bst"]) result.assert_success() - key = cli.get_element_key(project, 'checkout-deps.bst') - checkout_args = ['artifact', 'checkout', '--directory', checkout, '--deps', 'run', 'test/checkout-deps/' + key] + key = cli.get_element_key(project, "checkout-deps.bst") + checkout_args = ["artifact", "checkout", "--directory", checkout, "--deps", "run", "test/checkout-deps/" + key] result = cli.run(project=project, args=checkout_args) result.assert_main_error(ErrorDomain.STREAM, None) @@ -383,17 +378,17 @@ def test_build_checkout_runtime_deps_using_ref_fails(datafiles, cli): @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_invalid_ref(datafiles, cli): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout.tar') + checkout = os.path.join(cli.directory, "checkout.tar") - result = cli.run(project=project, args=['build', 'checkout-deps.bst']) + result = cli.run(project=project, args=["build", "checkout-deps.bst"]) result.assert_success() - builddir = os.path.join(cli.directory, 'build') + builddir = os.path.join(cli.directory, "build") assert os.path.isdir(builddir) assert not os.listdir(builddir) - non_existent_artifact = 'test/checkout-deps/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' - checkout_args = ['artifact', 'checkout', '--deps', 'none', '--tar', checkout, non_existent_artifact] + non_existent_artifact = "test/checkout-deps/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + checkout_args = ["artifact", "checkout", "--deps", "none", "--tar", checkout, non_existent_artifact] result = cli.run(project=project, args=checkout_args) assert "Error while staging dependencies into a sandbox: 'No artifacts to stage'" in result.stderr @@ -404,16 +399,16 @@ def test_build_checkout_no_tar_no_directory(datafiles, cli, tmpdir): project = str(datafiles) runtestdir = str(tmpdir) - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - checkout_args = ['artifact', 'checkout', 'target.bst'] + checkout_args = ["artifact", "checkout", "target.bst"] result = cli.run(cwd=runtestdir, project=project, args=checkout_args) result.assert_success() - filename = os.path.join(runtestdir, 'target', 'usr', 'bin', 'hello') + filename = os.path.join(runtestdir, "target", "usr", "bin", "hello") assert os.path.exists(filename) - filename = os.path.join(runtestdir, 'target', 'usr', 'include', 'pony.h') + filename = os.path.join(runtestdir, "target", "usr", "include", "pony.h") assert os.path.exists(filename) @@ -421,61 +416,61 @@ def test_build_checkout_no_tar_no_directory(datafiles, cli, tmpdir): @pytest.mark.parametrize("compression", [("gz"), ("xz"), ("bz2")]) def test_build_checkout_tarball_compression(datafiles, cli, compression): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout.tar') + checkout = os.path.join(cli.directory, "checkout.tar") - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - builddir = os.path.join(cli.directory, 'build') + builddir = os.path.join(cli.directory, "build") assert os.path.isdir(builddir) assert not os.listdir(builddir) - checkout_args = ['artifact', 'checkout', '--tar', checkout, '--compression', compression, 'target.bst'] + checkout_args = ["artifact", "checkout", "--tar", checkout, "--compression", compression, "target.bst"] result = cli.run(project=project, args=checkout_args) result.assert_success() # Docs say not to use TarFile class directly, using .open seems to work. # https://docs.python.org/3/library/tarfile.html#tarfile.TarFile - tar = tarfile.open(name=checkout, mode='r:' + compression) - assert os.path.join('.', 'usr', 'bin', 'hello') in tar.getnames() - assert os.path.join('.', 'usr', 'include', 'pony.h') in tar.getnames() + tar = tarfile.open(name=checkout, mode="r:" + compression) + assert os.path.join(".", "usr", "bin", "hello") in tar.getnames() + assert os.path.join(".", "usr", "include", "pony.h") in tar.getnames() @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_tarball_stdout(datafiles, cli): project = str(datafiles) - tarball = os.path.join(cli.directory, 'tarball.tar') + tarball = os.path.join(cli.directory, "tarball.tar") - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - builddir = os.path.join(cli.directory, 'build') + builddir = os.path.join(cli.directory, "build") assert os.path.isdir(builddir) assert not os.listdir(builddir) - checkout_args = ['artifact', 'checkout', '--tar', '-', 'target.bst'] + checkout_args = ["artifact", "checkout", "--tar", "-", "target.bst"] result = cli.run(project=project, args=checkout_args, binary_capture=True) result.assert_success() - with open(tarball, 'wb') as f: + with open(tarball, "wb") as f: f.write(result.output) tar = tarfile.TarFile(tarball) - assert os.path.join('.', 'usr', 'bin', 'hello') in tar.getnames() - assert os.path.join('.', 'usr', 'include', 'pony.h') in tar.getnames() + assert os.path.join(".", "usr", "bin", "hello") in tar.getnames() + assert os.path.join(".", "usr", "include", "pony.h") in tar.getnames() @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_tarball_mtime_nonzero(datafiles, cli): project = str(datafiles) - tarpath = os.path.join(cli.directory, 'mtime_tar.tar') + tarpath = os.path.join(cli.directory, "mtime_tar.tar") - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - checkout_args = ['artifact', 'checkout', '--tar', tarpath, 'target.bst'] + checkout_args = ["artifact", "checkout", "--tar", tarpath, "target.bst"] result = cli.run(project=project, args=checkout_args) result.assert_success() @@ -490,31 +485,31 @@ def test_build_checkout_tarball_mtime_nonzero(datafiles, cli): @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_tarball_is_deterministic(datafiles, cli): project = str(datafiles) - tarball1 = os.path.join(cli.directory, 'tarball1.tar') - tarball2 = os.path.join(cli.directory, 'tarball2.tar') + tarball1 = os.path.join(cli.directory, "tarball1.tar") + tarball2 = os.path.join(cli.directory, "tarball2.tar") - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - builddir = os.path.join(cli.directory, 'build') + builddir = os.path.join(cli.directory, "build") assert os.path.isdir(builddir) assert not os.listdir(builddir) - checkout_args = ['artifact', 'checkout', '--force', 'target.bst'] + checkout_args = ["artifact", "checkout", "--force", "target.bst"] - checkout_args1 = checkout_args + ['--tar', tarball1] + checkout_args1 = checkout_args + ["--tar", tarball1] result = cli.run(project=project, args=checkout_args1) result.assert_success() - checkout_args2 = checkout_args + ['--tar', tarball2] + checkout_args2 = checkout_args + ["--tar", tarball2] result = cli.run(project=project, args=checkout_args2) result.assert_success() - with open(tarball1, 'rb') as f: + with open(tarball1, "rb") as f: contents = f.read() hash1 = hashlib.sha1(contents).hexdigest() - with open(tarball2, 'rb') as f: + with open(tarball2, "rb") as f: contents = f.read() hash2 = hashlib.sha1(contents).hexdigest() @@ -524,75 +519,75 @@ def test_build_checkout_tarball_is_deterministic(datafiles, cli): @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_tarball_links(datafiles, cli): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout.tar') - extract = os.path.join(cli.directory, 'extract') + checkout = os.path.join(cli.directory, "checkout.tar") + extract = os.path.join(cli.directory, "extract") # Create the link before running the tests. # This is needed for users working on Windows, git checks out symlinks as files which content is the name # of the symlink and the test therefore doesn't have the correct content os.symlink( os.path.join("..", "basicfile"), - os.path.join(project, "files", "files-and-links", "basicfolder", "basicsymlink") + os.path.join(project, "files", "files-and-links", "basicfolder", "basicsymlink"), ) - result = cli.run(project=project, args=['build', 'import-links.bst']) + result = cli.run(project=project, args=["build", "import-links.bst"]) result.assert_success() - builddir = os.path.join(cli.directory, 'build') + builddir = os.path.join(cli.directory, "build") assert os.path.isdir(builddir) assert not os.listdir(builddir) - checkout_args = ['artifact', 'checkout', '--tar', checkout, 'import-links.bst'] + checkout_args = ["artifact", "checkout", "--tar", checkout, "import-links.bst"] result = cli.run(project=project, args=checkout_args) result.assert_success() tar = tarfile.open(name=checkout, mode="r:") tar.extractall(extract) - assert open(os.path.join(extract, 'basicfolder', 'basicsymlink')).read() == "file contents\n" + assert open(os.path.join(extract, "basicfolder", "basicsymlink")).read() == "file contents\n" @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_links(datafiles, cli): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') + checkout = os.path.join(cli.directory, "checkout") # Create the link before running the tests. # This is needed for users working on Windows, git checks out symlinks as files which content is the name # of the symlink and the test therefore doesn't have the correct content os.symlink( os.path.join("..", "basicfile"), - os.path.join(project, "files", "files-and-links", "basicfolder", "basicsymlink") + os.path.join(project, "files", "files-and-links", "basicfolder", "basicsymlink"), ) - result = cli.run(project=project, args=['build', 'import-links.bst']) + result = cli.run(project=project, args=["build", "import-links.bst"]) result.assert_success() - builddir = os.path.join(cli.directory, 'build') + builddir = os.path.join(cli.directory, "build") assert os.path.isdir(builddir) assert not os.listdir(builddir) - checkout_args = ['artifact', 'checkout', '--directory', checkout, 'import-links.bst'] + checkout_args = ["artifact", "checkout", "--directory", checkout, "import-links.bst"] result = cli.run(project=project, args=checkout_args) result.assert_success() - assert open(os.path.join(checkout, 'basicfolder', 'basicsymlink')).read() == "file contents\n" + assert open(os.path.join(checkout, "basicfolder", "basicsymlink")).read() == "file contents\n" @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("hardlinks", [("copies"), ("hardlinks")]) def test_build_checkout_nonempty(datafiles, cli, hardlinks): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') + checkout = os.path.join(cli.directory, "checkout") filename = os.path.join(checkout, "file.txt") # First build it - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() # Assert that after a successful build, the builddir is empty - builddir = os.path.join(cli.directory, 'build') + builddir = os.path.join(cli.directory, "build") assert os.path.isdir(builddir) assert not os.listdir(builddir) @@ -602,10 +597,10 @@ def test_build_checkout_nonempty(datafiles, cli, hardlinks): f.write("Hello") # Prepare checkout args - checkout_args = ['artifact', 'checkout'] + checkout_args = ["artifact", "checkout"] if hardlinks == "hardlinks": - checkout_args += ['--hardlinks'] - checkout_args += ['target.bst', '--directory', checkout] + checkout_args += ["--hardlinks"] + checkout_args += ["target.bst", "--directory", checkout] # Now check it out result = cli.run(project=project, args=checkout_args) @@ -616,15 +611,15 @@ def test_build_checkout_nonempty(datafiles, cli, hardlinks): @pytest.mark.parametrize("hardlinks", [("copies"), ("hardlinks")]) def test_build_checkout_force(datafiles, cli, hardlinks): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') + checkout = os.path.join(cli.directory, "checkout") filename = os.path.join(checkout, "file.txt") # First build it - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() # Assert that after a successful build, the builddir is empty - builddir = os.path.join(cli.directory, 'build') + builddir = os.path.join(cli.directory, "build") assert os.path.isdir(builddir) assert not os.listdir(builddir) @@ -634,466 +629,388 @@ def test_build_checkout_force(datafiles, cli, hardlinks): f.write("Hello") # Prepare checkout args - checkout_args = ['artifact', 'checkout', '--force'] + checkout_args = ["artifact", "checkout", "--force"] if hardlinks == "hardlinks": - checkout_args += ['--hardlinks'] - checkout_args += ['target.bst', '--directory', checkout] + checkout_args += ["--hardlinks"] + checkout_args += ["target.bst", "--directory", checkout] # Now check it out result = cli.run(project=project, args=checkout_args) result.assert_success() # Check that the file we added is still there - filename = os.path.join(checkout, 'file.txt') + filename = os.path.join(checkout, "file.txt") assert os.path.exists(filename) # Check that the executable hello file is found in the checkout - filename = os.path.join(checkout, 'usr', 'bin', 'hello') + filename = os.path.join(checkout, "usr", "bin", "hello") assert os.path.exists(filename) # Check that the executable hello file is found in the checkout - filename = os.path.join(checkout, 'usr', 'include', 'pony.h') + filename = os.path.join(checkout, "usr", "include", "pony.h") assert os.path.exists(filename) @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_force_tarball(datafiles, cli): project = str(datafiles) - tarball = os.path.join(cli.directory, 'tarball.tar') + tarball = os.path.join(cli.directory, "tarball.tar") - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - builddir = os.path.join(cli.directory, 'build') + builddir = os.path.join(cli.directory, "build") assert os.path.isdir(builddir) assert not os.listdir(builddir) with open(tarball, "w") as f: f.write("Hello") - checkout_args = ['artifact', 'checkout', '--force', '--tar', tarball, 'target.bst'] + checkout_args = ["artifact", "checkout", "--force", "--tar", tarball, "target.bst"] result = cli.run(project=project, args=checkout_args) result.assert_success() tar = tarfile.TarFile(tarball) - assert os.path.join('.', 'usr', 'bin', 'hello') in tar.getnames() - assert os.path.join('.', 'usr', 'include', 'pony.h') in tar.getnames() + assert os.path.join(".", "usr", "bin", "hello") in tar.getnames() + assert os.path.join(".", "usr", "include", "pony.h") in tar.getnames() @pytest.mark.datafiles(DATA_DIR) def test_install_to_build(cli, datafiles): project = str(datafiles) - element = 'installed-to-build.bst' + element = "installed-to-build.bst" # Attempt building the element # We expect this to throw an ElementError, since the element will # attempt to stage into /buildstream/build, which is not allowed. - result = cli.run(project=project, args=strict_args(['build', element], True)) + result = cli.run(project=project, args=strict_args(["build", element], True)) result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.ELEMENT, None) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')]) +@pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")]) def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage): project = str(datafiles) - subproject_path = os.path.join(project, 'files', 'sub-project') - junction_path = os.path.join(project, 'elements', 'junction.bst') - element_path = os.path.join(project, 'elements', 'junction-dep.bst') + subproject_path = os.path.join(project, "files", "sub-project") + junction_path = os.path.join(project, "elements", "junction.bst") + element_path = os.path.join(project, "elements", "junction-dep.bst") - configure_project(project, { - 'ref-storage': ref_storage - }) + configure_project(project, {"ref-storage": ref_storage}) # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path, store_ref=False) # Create a stack element to depend on a cross junction element # - element = { - 'kind': 'stack', - 'depends': [ - { - 'junction': 'junction.bst', - 'filename': 'import-etc.bst' - } - ] - } + element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) # Now try to track it, this will bail with the appropriate error # informing the user to track the junction first - result = cli.run(project=project, args=['build', 'junction-dep.bst']) + result = cli.run(project=project, args=["build", "junction-dep.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT) # Assert that we have the expected provenance encoded into the error - element_node = _yaml.load(element_path, shortname='junction-dep.bst') - ref_node = element_node.get_sequence('depends').mapping_at(0) + element_node = _yaml.load(element_path, shortname="junction-dep.bst") + ref_node = element_node.get_sequence("depends").mapping_at(0) provenance = ref_node.get_provenance() assert str(provenance) in result.stderr @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')]) +@pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")]) def test_unfetched_junction(cli, tmpdir, datafiles, ref_storage): project = str(datafiles) - subproject_path = os.path.join(project, 'files', 'sub-project') - junction_path = os.path.join(project, 'elements', 'junction.bst') - element_path = os.path.join(project, 'elements', 'junction-dep.bst') + subproject_path = os.path.join(project, "files", "sub-project") + junction_path = os.path.join(project, "elements", "junction.bst") + element_path = os.path.join(project, "elements", "junction-dep.bst") - configure_project(project, { - 'ref-storage': ref_storage - }) + configure_project(project, {"ref-storage": ref_storage}) # Create a repo to hold the subproject and generate a junction element for it - ref = generate_junction(tmpdir, subproject_path, junction_path, store_ref=(ref_storage == 'inline')) + ref = generate_junction(tmpdir, subproject_path, junction_path, store_ref=(ref_storage == "inline")) # Create a stack element to depend on a cross junction element # - element = { - 'kind': 'stack', - 'depends': [ - { - 'junction': 'junction.bst', - 'filename': 'import-etc.bst' - } - ] - } + element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) # Dump a project.refs if we're using project.refs storage # - if ref_storage == 'project.refs': - project_refs = { - 'projects': { - 'test': { - 'junction.bst': [ - { - 'ref': ref - } - ] - } - } - } - _yaml.roundtrip_dump(project_refs, os.path.join(project, 'junction.refs')) + if ref_storage == "project.refs": + project_refs = {"projects": {"test": {"junction.bst": [{"ref": ref}]}}} + _yaml.roundtrip_dump(project_refs, os.path.join(project, "junction.refs")) # Now try to build it, this should automatically result in fetching # the junction itself at load time. - result = cli.run(project=project, args=['build', 'junction-dep.bst']) + result = cli.run(project=project, args=["build", "junction-dep.bst"]) result.assert_success() # Assert that it's cached now - assert cli.get_element_state(project, 'junction-dep.bst') == 'cached' + assert cli.get_element_state(project, "junction-dep.bst") == "cached" @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_junction(cli, tmpdir, datafiles): project = str(datafiles) - subproject_path = os.path.join(project, 'files', 'sub-project') - junction_path = os.path.join(project, 'elements', 'junction.bst') - element_path = os.path.join(project, 'elements', 'junction-dep.bst') - checkout = os.path.join(cli.directory, 'checkout') + subproject_path = os.path.join(project, "files", "sub-project") + junction_path = os.path.join(project, "elements", "junction.bst") + element_path = os.path.join(project, "elements", "junction-dep.bst") + checkout = os.path.join(cli.directory, "checkout") # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path) # Create a stack element to depend on a cross junction element # - element = { - 'kind': 'stack', - 'depends': [ - { - 'junction': 'junction.bst', - 'filename': 'import-etc.bst' - } - ] - } + element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) # Now try to build it, this should automatically result in fetching # the junction itself at load time. - result = cli.run(project=project, args=['build', 'junction-dep.bst']) + result = cli.run(project=project, args=["build", "junction-dep.bst"]) result.assert_success() # Assert that it's cached now - assert cli.get_element_state(project, 'junction-dep.bst') == 'cached' + assert cli.get_element_state(project, "junction-dep.bst") == "cached" # Now check it out - result = cli.run(project=project, args=[ - 'artifact', 'checkout', 'junction-dep.bst', '--directory', checkout - ]) + result = cli.run(project=project, args=["artifact", "checkout", "junction-dep.bst", "--directory", checkout]) result.assert_success() # Assert the content of /etc/animal.conf - filename = os.path.join(checkout, 'etc', 'animal.conf') + filename = os.path.join(checkout, "etc", "animal.conf") assert os.path.exists(filename) - with open(filename, 'r') as f: + with open(filename, "r") as f: contents = f.read() - assert contents == 'animal=Pony\n' + assert contents == "animal=Pony\n" # Test that default targets work with projects with junctions @pytest.mark.datafiles(DATA_DIR + "_world") def test_build_checkout_junction_default_targets(cli, tmpdir, datafiles): project = str(datafiles) - subproject_path = os.path.join(project, 'files', 'sub-project') - junction_path = os.path.join(project, 'elements', 'junction.bst') - element_path = os.path.join(project, 'elements', 'junction-dep.bst') - checkout = os.path.join(cli.directory, 'checkout') + subproject_path = os.path.join(project, "files", "sub-project") + junction_path = os.path.join(project, "elements", "junction.bst") + element_path = os.path.join(project, "elements", "junction-dep.bst") + checkout = os.path.join(cli.directory, "checkout") # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path) # Create a stack element to depend on a cross junction element # - element = { - 'kind': 'stack', - 'depends': [ - { - 'junction': 'junction.bst', - 'filename': 'import-etc.bst' - } - ] - } + element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) # Now try to build it, this should automatically result in fetching # the junction itself at load time. - result = cli.run(project=project, args=['build']) + result = cli.run(project=project, args=["build"]) result.assert_success() # Assert that it's cached now - assert cli.get_element_state(project, 'junction-dep.bst') == 'cached' + assert cli.get_element_state(project, "junction-dep.bst") == "cached" # Now check it out - result = cli.run(project=project, args=[ - 'artifact', 'checkout', 'junction-dep.bst', '--directory', checkout - ]) + result = cli.run(project=project, args=["artifact", "checkout", "junction-dep.bst", "--directory", checkout]) result.assert_success() # Assert the content of /etc/animal.conf - filename = os.path.join(checkout, 'etc', 'animal.conf') + filename = os.path.join(checkout, "etc", "animal.conf") assert os.path.exists(filename) - with open(filename, 'r') as f: + with open(filename, "r") as f: contents = f.read() - assert contents == 'animal=Pony\n' + assert contents == "animal=Pony\n" @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_workspaced_junction(cli, tmpdir, datafiles): project = str(datafiles) - subproject_path = os.path.join(project, 'files', 'sub-project') - junction_path = os.path.join(project, 'elements', 'junction.bst') - element_path = os.path.join(project, 'elements', 'junction-dep.bst') - workspace = os.path.join(cli.directory, 'workspace') - checkout = os.path.join(cli.directory, 'checkout') + subproject_path = os.path.join(project, "files", "sub-project") + junction_path = os.path.join(project, "elements", "junction.bst") + element_path = os.path.join(project, "elements", "junction-dep.bst") + workspace = os.path.join(cli.directory, "workspace") + checkout = os.path.join(cli.directory, "checkout") # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path) # Create a stack element to depend on a cross junction element # - element = { - 'kind': 'stack', - 'depends': [ - { - 'junction': 'junction.bst', - 'filename': 'import-etc.bst' - } - ] - } + element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) # Now open a workspace on the junction # - result = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, 'junction.bst']) + result = cli.run(project=project, args=["workspace", "open", "--directory", workspace, "junction.bst"]) result.assert_success() - filename = os.path.join(workspace, 'files', 'etc-files', 'etc', 'animal.conf') + filename = os.path.join(workspace, "files", "etc-files", "etc", "animal.conf") # Assert the content of /etc/animal.conf in the workspace assert os.path.exists(filename) - with open(filename, 'r') as f: + with open(filename, "r") as f: contents = f.read() - assert contents == 'animal=Pony\n' + assert contents == "animal=Pony\n" # Modify the content of the animal.conf in the workspace - with open(filename, 'w') as f: - f.write('animal=Horsy\n') + with open(filename, "w") as f: + f.write("animal=Horsy\n") # Now try to build it, this should automatically result in fetching # the junction itself at load time. - result = cli.run(project=project, args=['build', 'junction-dep.bst']) + result = cli.run(project=project, args=["build", "junction-dep.bst"]) result.assert_success() # Assert that it's cached now - assert cli.get_element_state(project, 'junction-dep.bst') == 'cached' + assert cli.get_element_state(project, "junction-dep.bst") == "cached" # Now check it out - result = cli.run(project=project, args=[ - 'artifact', 'checkout', 'junction-dep.bst', '--directory', checkout - ]) + result = cli.run(project=project, args=["artifact", "checkout", "junction-dep.bst", "--directory", checkout]) result.assert_success() # Assert the workspace modified content of /etc/animal.conf - filename = os.path.join(checkout, 'etc', 'animal.conf') + filename = os.path.join(checkout, "etc", "animal.conf") assert os.path.exists(filename) - with open(filename, 'r') as f: + with open(filename, "r") as f: contents = f.read() - assert contents == 'animal=Horsy\n' + assert contents == "animal=Horsy\n" @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_cross_junction(datafiles, cli, tmpdir): project = str(datafiles) - subproject_path = os.path.join(project, 'files', 'sub-project') - junction_path = os.path.join(project, 'elements', 'junction.bst') - checkout = os.path.join(cli.directory, 'checkout') + subproject_path = os.path.join(project, "files", "sub-project") + junction_path = os.path.join(project, "elements", "junction.bst") + checkout = os.path.join(cli.directory, "checkout") generate_junction(tmpdir, subproject_path, junction_path) - result = cli.run(project=project, args=['build', 'junction.bst:import-etc.bst']) + result = cli.run(project=project, args=["build", "junction.bst:import-etc.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'junction.bst:import-etc.bst', - '--directory', checkout]) + result = cli.run( + project=project, args=["artifact", "checkout", "junction.bst:import-etc.bst", "--directory", checkout] + ) result.assert_success() - filename = os.path.join(checkout, 'etc', 'animal.conf') + filename = os.path.join(checkout, "etc", "animal.conf") assert os.path.exists(filename) @pytest.mark.datafiles(DATA_DIR) def test_build_junction_short_notation(cli, tmpdir, datafiles): project = str(datafiles) - subproject_path = os.path.join(project, 'files', 'sub-project') - junction_path = os.path.join(project, 'elements', 'junction.bst') - element_path = os.path.join(project, 'elements', 'junction-dep.bst') - checkout = os.path.join(cli.directory, 'checkout') + subproject_path = os.path.join(project, "files", "sub-project") + junction_path = os.path.join(project, "elements", "junction.bst") + element_path = os.path.join(project, "elements", "junction-dep.bst") + checkout = os.path.join(cli.directory, "checkout") # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path) # Create a stack element to depend on a cross junction element, using # colon (:) as the separator - element = { - 'kind': 'stack', - 'depends': ['junction.bst:import-etc.bst'] - } + element = {"kind": "stack", "depends": ["junction.bst:import-etc.bst"]} _yaml.roundtrip_dump(element, element_path) # Now try to build it, this should automatically result in fetching # the junction itself at load time. - result = cli.run(project=project, args=['build', 'junction-dep.bst']) + result = cli.run(project=project, args=["build", "junction-dep.bst"]) result.assert_success() # Assert that it's cached now - assert cli.get_element_state(project, 'junction-dep.bst') == 'cached' + assert cli.get_element_state(project, "junction-dep.bst") == "cached" # Now check it out - result = cli.run(project=project, args=[ - 'artifact', 'checkout', 'junction-dep.bst', '--directory', checkout - ]) + result = cli.run(project=project, args=["artifact", "checkout", "junction-dep.bst", "--directory", checkout]) result.assert_success() # Assert the content of /etc/animal.conf - filename = os.path.join(checkout, 'etc', 'animal.conf') + filename = os.path.join(checkout, "etc", "animal.conf") assert os.path.exists(filename) - with open(filename, 'r') as f: + with open(filename, "r") as f: contents = f.read() - assert contents == 'animal=Pony\n' + assert contents == "animal=Pony\n" @pytest.mark.datafiles(DATA_DIR) def test_build_junction_short_notation_filename(cli, tmpdir, datafiles): project = str(datafiles) - subproject_path = os.path.join(project, 'files', 'sub-project') - junction_path = os.path.join(project, 'elements', 'junction.bst') - element_path = os.path.join(project, 'elements', 'junction-dep.bst') - checkout = os.path.join(cli.directory, 'checkout') + subproject_path = os.path.join(project, "files", "sub-project") + junction_path = os.path.join(project, "elements", "junction.bst") + element_path = os.path.join(project, "elements", "junction-dep.bst") + checkout = os.path.join(cli.directory, "checkout") # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path) # Create a stack element to depend on a cross junction element, using # colon (:) as the separator - element = { - 'kind': 'stack', - 'depends': [{'filename': 'junction.bst:import-etc.bst'}] - } + element = {"kind": "stack", "depends": [{"filename": "junction.bst:import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) # Now try to build it, this should automatically result in fetching # the junction itself at load time. - result = cli.run(project=project, args=['build', 'junction-dep.bst']) + result = cli.run(project=project, args=["build", "junction-dep.bst"]) result.assert_success() # Assert that it's cached now - assert cli.get_element_state(project, 'junction-dep.bst') == 'cached' + assert cli.get_element_state(project, "junction-dep.bst") == "cached" # Now check it out - result = cli.run(project=project, args=[ - 'artifact', 'checkout', 'junction-dep.bst', '--directory', checkout - ]) + result = cli.run(project=project, args=["artifact", "checkout", "junction-dep.bst", "--directory", checkout]) result.assert_success() # Assert the content of /etc/animal.conf - filename = os.path.join(checkout, 'etc', 'animal.conf') + filename = os.path.join(checkout, "etc", "animal.conf") assert os.path.exists(filename) - with open(filename, 'r') as f: + with open(filename, "r") as f: contents = f.read() - assert contents == 'animal=Pony\n' + assert contents == "animal=Pony\n" @pytest.mark.datafiles(DATA_DIR) def test_build_junction_short_notation_with_junction(cli, tmpdir, datafiles): project = str(datafiles) - subproject_path = os.path.join(project, 'files', 'sub-project') - junction_path = os.path.join(project, 'elements', 'junction.bst') - element_path = os.path.join(project, 'elements', 'junction-dep.bst') + subproject_path = os.path.join(project, "files", "sub-project") + junction_path = os.path.join(project, "elements", "junction.bst") + element_path = os.path.join(project, "elements", "junction-dep.bst") # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path) # Create a stack element to depend on a cross junction element, using # colon (:) as the separator - element = { - 'kind': 'stack', - 'depends': [{ - 'filename': 'junction.bst:import-etc.bst', - 'junction': 'junction.bst', - }] - } + element = {"kind": "stack", "depends": [{"filename": "junction.bst:import-etc.bst", "junction": "junction.bst",}]} _yaml.roundtrip_dump(element, element_path) # Now try to build it, this should fail as filenames should not contain # `:` when junction is explicity specified - result = cli.run(project=project, args=['build', 'junction-dep.bst']) + result = cli.run(project=project, args=["build", "junction-dep.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @pytest.mark.datafiles(DATA_DIR) def test_build_junction_transitive_short_notation_with_junction(cli, tmpdir, datafiles): project = str(datafiles) - subproject_path = os.path.join(project, 'files', 'sub-project') - junction_path = os.path.join(project, 'elements', 'junction.bst') - element_path = os.path.join(project, 'elements', 'junction-dep.bst') + subproject_path = os.path.join(project, "files", "sub-project") + junction_path = os.path.join(project, "elements", "junction.bst") + element_path = os.path.join(project, "elements", "junction-dep.bst") # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path) # Create a stack element to depend on a cross junction element, using # colon (:) as the separator - element = { - 'kind': 'stack', - 'depends': ['junction.bst:import-etc.bst:foo.bst'] - } + element = {"kind": "stack", "depends": ["junction.bst:import-etc.bst:foo.bst"]} _yaml.roundtrip_dump(element, element_path) # Now try to build it, this should fail as recursive lookups for # cross-junction elements is not allowed. - result = cli.run(project=project, args=['build', 'junction-dep.bst']) + result = cli.run(project=project, args=["build", "junction-dep.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @@ -1102,11 +1019,11 @@ def test_build_junction_transitive_short_notation_with_junction(cli, tmpdir, dat @pytest.mark.datafiles(DATA_DIR) def test_partial_artifact_checkout_fetch(cli, datafiles, tmpdir): project = str(datafiles) - checkout_dir = os.path.join(str(tmpdir), 'checkout') + checkout_dir = os.path.join(str(tmpdir), "checkout") - repo = create_repo('git', str(tmpdir)) + repo = create_repo("git", str(tmpdir)) repo.create(os.path.join(str(datafiles), "files")) - element_dir = os.path.join(str(tmpdir), 'elements') + element_dir = os.path.join(str(tmpdir), "elements") project = str(tmpdir) project_config = { "name": "partial-artifact-checkout-fetch", @@ -1118,38 +1035,33 @@ def test_partial_artifact_checkout_fetch(cli, datafiles, tmpdir): "kind": "import", "sources": [repo.source_config()], } - input_name = 'input.bst' + input_name = "input.bst" input_file = os.path.join(element_dir, input_name) _yaml.roundtrip_dump(input_config, input_file) - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: - cli.configure({'artifacts': { - 'url': share.repo, - 'push': True - }}) + cli.configure({"artifacts": {"url": share.repo, "push": True}}) - result = cli.run(project=project, args=['source', 'track', input_name]) + result = cli.run(project=project, args=["source", "track", input_name]) result.assert_success() - result = cli.run(project=project, args=['build', input_name]) + result = cli.run(project=project, args=["build", input_name]) result.assert_success() # A push artifact cache means we have to pull to push to them, so # delete some blobs from that CAS such that we have to fetch - digest = utils.sha256sum(os.path.join(project, 'files', 'bin-files', 'usr', 'bin', 'hello')) - objpath = os.path.join(cli.directory, 'cas', 'objects', digest[:2], digest[2:]) + digest = utils.sha256sum(os.path.join(project, "files", "bin-files", "usr", "bin", "hello")) + objpath = os.path.join(cli.directory, "cas", "objects", digest[:2], digest[2:]) os.unlink(objpath) # Verify that the build-only dependency is not (complete) in the local cache - result = cli.run(project=project, args=[ - 'artifact', 'checkout', input_name, - '--directory', checkout_dir]) - result.assert_main_error(ErrorDomain.STREAM, 'uncached-checkout-attempt') + result = cli.run(project=project, args=["artifact", "checkout", input_name, "--directory", checkout_dir]) + result.assert_main_error(ErrorDomain.STREAM, "uncached-checkout-attempt") # Verify that the pull method fetches relevant artifacts in order to stage - result = cli.run(project=project, args=[ - 'artifact', 'checkout', '--pull', input_name, - '--directory', checkout_dir]) + result = cli.run( + project=project, args=["artifact", "checkout", "--pull", input_name, "--directory", checkout_dir] + ) result.assert_success() # should have pulled whatever was deleted previous @@ -1159,18 +1071,13 @@ def test_partial_artifact_checkout_fetch(cli, datafiles, tmpdir): @pytest.mark.datafiles(DATA_DIR) def test_partial_checkout_fail(tmpdir, datafiles, cli): project = str(datafiles) - build_elt = 'import-bin.bst' - checkout_dir = os.path.join(str(tmpdir), 'checkout') + build_elt = "import-bin.bst" + checkout_dir = os.path.join(str(tmpdir), "checkout") - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: - cli.configure({'artifacts': { - 'url': share.repo, - 'push': True - }}) + cli.configure({"artifacts": {"url": share.repo, "push": True}}) - res = cli.run(project=project, args=[ - 'artifact', 'checkout', '--pull', build_elt, '--directory', - checkout_dir]) - res.assert_main_error(ErrorDomain.STREAM, 'uncached-checkout-attempt') - assert re.findall(r'Remote \((\S+)\) does not have artifact (\S+) cached', res.stderr) + res = cli.run(project=project, args=["artifact", "checkout", "--pull", build_elt, "--directory", checkout_dir]) + res.assert_main_error(ErrorDomain.STREAM, "uncached-checkout-attempt") + assert re.findall(r"Remote \((\S+)\) does not have artifact (\S+) cached", res.stderr) diff --git a/tests/frontend/completions.py b/tests/frontend/completions.py index 3352ddcc9..a5e3c8ed3 100644 --- a/tests/frontend/completions.py +++ b/tests/frontend/completions.py @@ -6,21 +6,9 @@ import pytest from buildstream.testing import cli # pylint: disable=unused-import # Project directory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - 'completions' -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "completions") -MAIN_COMMANDS = [ - 'artifact ', - 'build ', - 'help ', - 'init ', - 'shell ', - 'show ', - 'source ', - 'workspace ' -] +MAIN_COMMANDS = ["artifact ", "build ", "help ", "init ", "shell ", "show ", "source ", "workspace "] MAIN_OPTIONS = [ "--builders ", @@ -54,27 +42,22 @@ MAIN_OPTIONS = [ ] SOURCE_COMMANDS = [ - 'checkout ', - 'fetch ', - 'track ', + "checkout ", + "fetch ", + "track ", ] ARTIFACT_COMMANDS = [ - 'checkout ', - 'delete ', - 'push ', - 'pull ', - 'log ', - 'list-contents ', - 'show ', + "checkout ", + "delete ", + "push ", + "pull ", + "log ", + "list-contents ", + "show ", ] -WORKSPACE_COMMANDS = [ - 'close ', - 'list ', - 'open ', - 'reset ' -] +WORKSPACE_COMMANDS = ["close ", "list ", "open ", "reset "] PROJECT_ELEMENTS = [ "compose-all.bst", @@ -82,7 +65,7 @@ PROJECT_ELEMENTS = [ "compose-include-bin.bst", "import-bin.bst", "import-dev.bst", - "target.bst" + "target.bst", ] INVALID_ELEMENTS = [ @@ -94,11 +77,9 @@ MIXED_ELEMENTS = PROJECT_ELEMENTS + INVALID_ELEMENTS def assert_completion(cli, cmd, word_idx, expected, cwd=None): - result = cli.run(project='.', cwd=cwd, env={ - '_BST_COMPLETION': 'complete', - 'COMP_WORDS': cmd, - 'COMP_CWORD': str(word_idx) - }) + result = cli.run( + project=".", cwd=cwd, env={"_BST_COMPLETION": "complete", "COMP_WORDS": cmd, "COMP_CWORD": str(word_idx)} + ) words = [] if result.output: words = result.output.splitlines() @@ -112,11 +93,7 @@ def assert_completion(cli, cmd, word_idx, expected, cwd=None): def assert_completion_failed(cli, cmd, word_idx, expected, cwd=None): - result = cli.run(cwd=cwd, env={ - '_BST_COMPLETION': 'complete', - 'COMP_WORDS': cmd, - 'COMP_CWORD': str(word_idx) - }) + result = cli.run(cwd=cwd, env={"_BST_COMPLETION": "complete", "COMP_WORDS": cmd, "COMP_CWORD": str(word_idx)}) words = [] if result.output: words = result.output.splitlines() @@ -129,67 +106,76 @@ def assert_completion_failed(cli, cmd, word_idx, expected, cwd=None): assert words != expected -@pytest.mark.parametrize("cmd,word_idx,expected", [ - ('bst', 0, []), - ('bst ', 1, MAIN_COMMANDS), - ('bst artifact ', 2, ARTIFACT_COMMANDS), - ('bst source ', 2, SOURCE_COMMANDS), - ('bst w ', 1, ['workspace ']), - ('bst workspace ', 2, WORKSPACE_COMMANDS), -]) +@pytest.mark.parametrize( + "cmd,word_idx,expected", + [ + ("bst", 0, []), + ("bst ", 1, MAIN_COMMANDS), + ("bst artifact ", 2, ARTIFACT_COMMANDS), + ("bst source ", 2, SOURCE_COMMANDS), + ("bst w ", 1, ["workspace "]), + ("bst workspace ", 2, WORKSPACE_COMMANDS), + ], +) def test_commands(cli, cmd, word_idx, expected): assert_completion(cli, cmd, word_idx, expected) -@pytest.mark.parametrize("cmd,word_idx,expected", [ - ('bst -', 1, MAIN_OPTIONS), - ('bst --l', 1, ['--log-file ']), - - # Test that options of subcommands also complete - ('bst --no-colors build -', 3, ['--deps ', '-d ', - '--remote ', '-r ']), - - # Test the behavior of completing after an option that has a - # parameter that cannot be completed, vs an option that has - # no parameter - ('bst --fetchers ', 2, []), - ('bst --no-colors ', 2, MAIN_COMMANDS), -]) +@pytest.mark.parametrize( + "cmd,word_idx,expected", + [ + ("bst -", 1, MAIN_OPTIONS), + ("bst --l", 1, ["--log-file "]), + # Test that options of subcommands also complete + ("bst --no-colors build -", 3, ["--deps ", "-d ", "--remote ", "-r "]), + # Test the behavior of completing after an option that has a + # parameter that cannot be completed, vs an option that has + # no parameter + ("bst --fetchers ", 2, []), + ("bst --no-colors ", 2, MAIN_COMMANDS), + ], +) def test_options(cli, cmd, word_idx, expected): assert_completion(cli, cmd, word_idx, expected) -@pytest.mark.parametrize("cmd,word_idx,expected", [ - ('bst --on-error ', 2, ['continue ', 'quit ', 'terminate ']), - ('bst --cache-buildtrees ', 2, ['always ', 'auto ', 'never ']), - ('bst show --deps ', 3, ['all ', 'build ', 'none ', 'plan ', 'run ']), - ('bst show --deps=', 2, ['all ', 'build ', 'none ', 'plan ', 'run ']), - ('bst show --deps b', 3, ['build ']), - ('bst show --deps=b', 2, ['build ']), - ('bst show --deps r', 3, ['run ']), - ('bst source track --deps ', 4, ['all ', 'none ']), -]) +@pytest.mark.parametrize( + "cmd,word_idx,expected", + [ + ("bst --on-error ", 2, ["continue ", "quit ", "terminate "]), + ("bst --cache-buildtrees ", 2, ["always ", "auto ", "never "]), + ("bst show --deps ", 3, ["all ", "build ", "none ", "plan ", "run "]), + ("bst show --deps=", 2, ["all ", "build ", "none ", "plan ", "run "]), + ("bst show --deps b", 3, ["build "]), + ("bst show --deps=b", 2, ["build "]), + ("bst show --deps r", 3, ["run "]), + ("bst source track --deps ", 4, ["all ", "none "]), + ], +) def test_option_choice(cli, cmd, word_idx, expected): assert_completion(cli, cmd, word_idx, expected) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project')) -@pytest.mark.parametrize("cmd,word_idx,expected,subdir", [ - # Note that elements/ and files/ are partial completions and - # as such do not come with trailing whitespace - ('bst --config ', 2, ['cache/', 'elements/', 'files/', 'project.conf '], None), - ('bst --log-file ', 2, ['cache/', 'elements/', 'files/', 'project.conf '], None), - ('bst --config f', 2, ['files/'], None), - ('bst --log-file f', 2, ['files/'], None), - ('bst --config files', 2, ['files/bin-files/', 'files/dev-files/'], None), - ('bst --log-file files', 2, ['files/bin-files/', 'files/dev-files/'], None), - ('bst --config files/', 2, ['files/bin-files/', 'files/dev-files/'], None), - ('bst --log-file elements/', 2, [os.path.join('elements', e) + ' ' for e in PROJECT_ELEMENTS], None), - ('bst --config ../', 2, ['../cache/', '../elements/', '../files/', '../project.conf '], 'files'), - ('bst --config ../elements/', 2, [os.path.join('..', 'elements', e) + ' ' for e in PROJECT_ELEMENTS], 'files'), - ('bst --config ../nofile', 2, [], 'files'), - ('bst --config /pony/rainbow/nobodyhas/this/file', 2, [], 'files'), -]) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) +@pytest.mark.parametrize( + "cmd,word_idx,expected,subdir", + [ + # Note that elements/ and files/ are partial completions and + # as such do not come with trailing whitespace + ("bst --config ", 2, ["cache/", "elements/", "files/", "project.conf "], None), + ("bst --log-file ", 2, ["cache/", "elements/", "files/", "project.conf "], None), + ("bst --config f", 2, ["files/"], None), + ("bst --log-file f", 2, ["files/"], None), + ("bst --config files", 2, ["files/bin-files/", "files/dev-files/"], None), + ("bst --log-file files", 2, ["files/bin-files/", "files/dev-files/"], None), + ("bst --config files/", 2, ["files/bin-files/", "files/dev-files/"], None), + ("bst --log-file elements/", 2, [os.path.join("elements", e) + " " for e in PROJECT_ELEMENTS], None), + ("bst --config ../", 2, ["../cache/", "../elements/", "../files/", "../project.conf "], "files"), + ("bst --config ../elements/", 2, [os.path.join("..", "elements", e) + " " for e in PROJECT_ELEMENTS], "files"), + ("bst --config ../nofile", 2, [], "files"), + ("bst --config /pony/rainbow/nobodyhas/this/file", 2, [], "files"), + ], +) def test_option_file(datafiles, cli, cmd, word_idx, expected, subdir): cwd = str(datafiles) if subdir: @@ -197,15 +183,18 @@ def test_option_file(datafiles, cli, cmd, word_idx, expected, subdir): assert_completion(cli, cmd, word_idx, expected, cwd=cwd) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project')) -@pytest.mark.parametrize("cmd,word_idx,expected,subdir", [ - # Note that regular files like project.conf are not returned when - # completing for a directory - ('bst --directory ', 2, ['cache/', 'elements/', 'files/'], None), - ('bst --directory elements/', 2, [], None), - ('bst --directory ', 2, ['dev-files/', 'bin-files/'], 'files'), - ('bst --directory ../', 2, ['../cache/', '../elements/', '../files/'], 'files'), -]) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) +@pytest.mark.parametrize( + "cmd,word_idx,expected,subdir", + [ + # Note that regular files like project.conf are not returned when + # completing for a directory + ("bst --directory ", 2, ["cache/", "elements/", "files/"], None), + ("bst --directory elements/", 2, [], None), + ("bst --directory ", 2, ["dev-files/", "bin-files/"], "files"), + ("bst --directory ../", 2, ["../cache/", "../elements/", "../files/"], "files"), + ], +) def test_option_directory(datafiles, cli, cmd, word_idx, expected, subdir): cwd = str(datafiles) if subdir: @@ -214,56 +203,82 @@ def test_option_directory(datafiles, cli, cmd, word_idx, expected, subdir): @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("project,cmd,word_idx,expected,subdir", [ - # When running in the project directory - ('project', 'bst show ', 2, [e + ' ' for e in PROJECT_ELEMENTS], None), - ('project', 'bst build com', 2, - ['compose-all.bst ', 'compose-include-bin.bst ', 'compose-exclude-dev.bst '], None), - - # When running from the files subdir - ('project', 'bst show ', 2, [e + ' ' for e in PROJECT_ELEMENTS], 'files'), - ('project', 'bst build com', 2, - ['compose-all.bst ', 'compose-include-bin.bst ', 'compose-exclude-dev.bst '], 'files'), - - # When passing the project directory - ('project', 'bst --directory ../ show ', 4, [e + ' ' for e in PROJECT_ELEMENTS], 'files'), - ('project', 'bst --directory ../ build com', 4, - ['compose-all.bst ', 'compose-include-bin.bst ', 'compose-exclude-dev.bst '], 'files'), - - # Also try multi arguments together - ('project', 'bst --directory ../ artifact checkout t ', 5, ['target.bst '], 'files'), - ('project', 'bst --directory ../ artifact checkout --directory ', 6, - ['bin-files/', 'dev-files/'], 'files'), - - # When running in the project directory - ('no-element-path', 'bst show ', 2, - [e + ' ' for e in PROJECT_ELEMENTS] + ['files/'], None), - ('no-element-path', 'bst build com', 2, - ['compose-all.bst ', 'compose-include-bin.bst ', 'compose-exclude-dev.bst '], None), - - # When running from the files subdir - ('no-element-path', 'bst show ', 2, - [e + ' ' for e in PROJECT_ELEMENTS] + ['files/'], 'files'), - ('no-element-path', 'bst build com', 2, - ['compose-all.bst ', 'compose-include-bin.bst ', 'compose-exclude-dev.bst '], 'files'), - - # When passing the project directory - ('no-element-path', 'bst --directory ../ show ', 4, - [e + ' ' for e in PROJECT_ELEMENTS] + ['files/'], 'files'), - ('no-element-path', 'bst --directory ../ show f', 4, ['files/'], 'files'), - ('no-element-path', 'bst --directory ../ show files/', 4, ['files/bin-files/', 'files/dev-files/'], 'files'), - ('no-element-path', 'bst --directory ../ build com', 4, - ['compose-all.bst ', 'compose-include-bin.bst ', 'compose-exclude-dev.bst '], 'files'), - - # Also try multi arguments together - ('no-element-path', 'bst --directory ../ artifact checkout t ', 5, ['target.bst '], 'files'), - ('no-element-path', 'bst --directory ../ artifact checkout --directory ', 6, - ['bin-files/', 'dev-files/'], 'files'), - - # When element-path have sub-folders - ('sub-folders', 'bst show base', 2, ['base/wanted.bst '], None), - ('sub-folders', 'bst show base/', 2, ['base/wanted.bst '], None), -]) +@pytest.mark.parametrize( + "project,cmd,word_idx,expected,subdir", + [ + # When running in the project directory + ("project", "bst show ", 2, [e + " " for e in PROJECT_ELEMENTS], None), + ( + "project", + "bst build com", + 2, + ["compose-all.bst ", "compose-include-bin.bst ", "compose-exclude-dev.bst "], + None, + ), + # When running from the files subdir + ("project", "bst show ", 2, [e + " " for e in PROJECT_ELEMENTS], "files"), + ( + "project", + "bst build com", + 2, + ["compose-all.bst ", "compose-include-bin.bst ", "compose-exclude-dev.bst "], + "files", + ), + # When passing the project directory + ("project", "bst --directory ../ show ", 4, [e + " " for e in PROJECT_ELEMENTS], "files"), + ( + "project", + "bst --directory ../ build com", + 4, + ["compose-all.bst ", "compose-include-bin.bst ", "compose-exclude-dev.bst "], + "files", + ), + # Also try multi arguments together + ("project", "bst --directory ../ artifact checkout t ", 5, ["target.bst "], "files"), + ("project", "bst --directory ../ artifact checkout --directory ", 6, ["bin-files/", "dev-files/"], "files"), + # When running in the project directory + ("no-element-path", "bst show ", 2, [e + " " for e in PROJECT_ELEMENTS] + ["files/"], None), + ( + "no-element-path", + "bst build com", + 2, + ["compose-all.bst ", "compose-include-bin.bst ", "compose-exclude-dev.bst "], + None, + ), + # When running from the files subdir + ("no-element-path", "bst show ", 2, [e + " " for e in PROJECT_ELEMENTS] + ["files/"], "files"), + ( + "no-element-path", + "bst build com", + 2, + ["compose-all.bst ", "compose-include-bin.bst ", "compose-exclude-dev.bst "], + "files", + ), + # When passing the project directory + ("no-element-path", "bst --directory ../ show ", 4, [e + " " for e in PROJECT_ELEMENTS] + ["files/"], "files"), + ("no-element-path", "bst --directory ../ show f", 4, ["files/"], "files"), + ("no-element-path", "bst --directory ../ show files/", 4, ["files/bin-files/", "files/dev-files/"], "files"), + ( + "no-element-path", + "bst --directory ../ build com", + 4, + ["compose-all.bst ", "compose-include-bin.bst ", "compose-exclude-dev.bst "], + "files", + ), + # Also try multi arguments together + ("no-element-path", "bst --directory ../ artifact checkout t ", 5, ["target.bst "], "files"), + ( + "no-element-path", + "bst --directory ../ artifact checkout --directory ", + 6, + ["bin-files/", "dev-files/"], + "files", + ), + # When element-path have sub-folders + ("sub-folders", "bst show base", 2, ["base/wanted.bst "], None), + ("sub-folders", "bst show base/", 2, ["base/wanted.bst "], None), + ], +) def test_argument_element(datafiles, cli, project, cmd, word_idx, expected, subdir): cwd = os.path.join(str(datafiles), project) if subdir: @@ -272,11 +287,13 @@ def test_argument_element(datafiles, cli, project, cmd, word_idx, expected, subd @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("project,cmd,word_idx,expected,subdir", [ - - # When element has invalid suffix - ('project', 'bst --directory ../ show ', 4, [e + ' ' for e in MIXED_ELEMENTS], 'files') -]) +@pytest.mark.parametrize( + "project,cmd,word_idx,expected,subdir", + [ + # When element has invalid suffix + ("project", "bst --directory ../ show ", 4, [e + " " for e in MIXED_ELEMENTS], "files") + ], +) def test_argument_element_invalid(datafiles, cli, project, cmd, word_idx, expected, subdir): cwd = os.path.join(str(datafiles), project) if subdir: @@ -284,46 +301,45 @@ def test_argument_element_invalid(datafiles, cli, project, cmd, word_idx, expect assert_completion_failed(cli, cmd, word_idx, expected, cwd=cwd) -@pytest.mark.parametrize("cmd,word_idx,expected", [ - ('bst he', 1, ['help ']), - ('bst help ', 2, MAIN_COMMANDS), - ('bst help artifact ', 3, ARTIFACT_COMMANDS), - ('bst help in', 2, ['init ']), - ('bst help source ', 3, SOURCE_COMMANDS), - ('bst help artifact ', 3, ARTIFACT_COMMANDS), - ('bst help w', 2, ['workspace ']), - ('bst help workspace ', 3, WORKSPACE_COMMANDS), -]) +@pytest.mark.parametrize( + "cmd,word_idx,expected", + [ + ("bst he", 1, ["help "]), + ("bst help ", 2, MAIN_COMMANDS), + ("bst help artifact ", 3, ARTIFACT_COMMANDS), + ("bst help in", 2, ["init "]), + ("bst help source ", 3, SOURCE_COMMANDS), + ("bst help artifact ", 3, ARTIFACT_COMMANDS), + ("bst help w", 2, ["workspace "]), + ("bst help workspace ", 3, WORKSPACE_COMMANDS), + ], +) def test_help_commands(cli, cmd, word_idx, expected): assert_completion(cli, cmd, word_idx, expected) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) def test_argument_artifact(cli, datafiles): project = str(datafiles) # Build an import element with no dependencies (as there will only be ONE cache key) - result = cli.run(project=project, args=['build', 'import-bin.bst']) # Has no dependencies + result = cli.run(project=project, args=["build", "import-bin.bst"]) # Has no dependencies result.assert_success() # Get the key and the artifact ref ($project/$element_name/$key) - key = cli.get_element_key(project, 'import-bin.bst') - artifact = os.path.join('test', 'import-bin', key) + key = cli.get_element_key(project, "import-bin.bst") + artifact = os.path.join("test", "import-bin", key) # Test autocompletion of the artifact - cmds = [ - 'bst artifact log ', - 'bst artifact log t', - 'bst artifact log test/' - ] + cmds = ["bst artifact log ", "bst artifact log t", "bst artifact log test/"] for i, cmd in enumerate(cmds): word_idx = 3 - result = cli.run(project=project, cwd=project, env={ - '_BST_COMPLETION': 'complete', - 'COMP_WORDS': cmd, - 'COMP_CWORD': str(word_idx) - }) + result = cli.run( + project=project, + cwd=project, + env={"_BST_COMPLETION": "complete", "COMP_WORDS": cmd, "COMP_CWORD": str(word_idx)}, + ) if result.output: words = result.output.splitlines() # This leaves an extra space on each e.g. ['foo.bst '] @@ -332,7 +348,7 @@ def test_argument_artifact(cli, datafiles): if i == 0: expected = PROJECT_ELEMENTS + [artifact] # We should now be able to see the artifact elif i == 1: - expected = ['target.bst', artifact] + expected = ["target.bst", artifact] elif i == 2: expected = [artifact] diff --git a/tests/frontend/compose_splits.py b/tests/frontend/compose_splits.py index f1f9b73be..d333b031e 100644 --- a/tests/frontend/compose_splits.py +++ b/tests/frontend/compose_splits.py @@ -6,35 +6,27 @@ import pytest from buildstream.testing.runcli import cli # pylint: disable=unused-import # Project directory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project", -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",) -@pytest.mark.parametrize("target", [ - ('compose-include-bin.bst'), - ('compose-exclude-dev.bst') -]) +@pytest.mark.parametrize("target", [("compose-include-bin.bst"), ("compose-exclude-dev.bst")]) @pytest.mark.datafiles(DATA_DIR) def test_compose_splits(datafiles, cli, target): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') + checkout = os.path.join(cli.directory, "checkout") # First build it - result = cli.run(project=project, args=['build', target]) + result = cli.run(project=project, args=["build", target]) result.assert_success() # Now check it out - result = cli.run(project=project, args=[ - 'artifact', 'checkout', target, '--directory', checkout - ]) + result = cli.run(project=project, args=["artifact", "checkout", target, "--directory", checkout]) result.assert_success() # Check that the executable hello file is found in the checkout - filename = os.path.join(checkout, 'usr', 'bin', 'hello') + filename = os.path.join(checkout, "usr", "bin", "hello") assert os.path.exists(filename) # Check that the executable hello file is found in the checkout - filename = os.path.join(checkout, 'usr', 'include', 'pony.h') + filename = os.path.join(checkout, "usr", "include", "pony.h") assert not os.path.exists(filename) diff --git a/tests/frontend/configurable_warnings.py b/tests/frontend/configurable_warnings.py index caa91bb61..472df3eea 100644 --- a/tests/frontend/configurable_warnings.py +++ b/tests/frontend/configurable_warnings.py @@ -11,10 +11,7 @@ from buildstream import _yaml from buildstream.testing.runcli import cli # pylint: disable=unused-import from buildstream.testing._utils.site import HAVE_SANDBOX -TOP_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "configuredwarning" -) +TOP_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "configuredwarning") def get_project(fatal_warnings): @@ -22,17 +19,9 @@ def get_project(fatal_warnings): "name": "test", "element-path": "elements", "plugins": [ - { - "origin": "local", - "path": "plugins", - "elements": { - "warninga": 0, - "warningb": 0, - "corewarn": 0, - } - } + {"origin": "local", "path": "plugins", "elements": {"warninga": 0, "warningb": 0, "corewarn": 0,}} ], - "fatal-warnings": fatal_warnings + "fatal-warnings": fatal_warnings, } @@ -47,19 +36,21 @@ def build_project(datafiles, fatal_warnings): @pytest.mark.datafiles(TOP_DIR) -@pytest.mark.parametrize("element_name, fatal_warnings, expect_fatal, error_domain", [ - ("corewarn.bst", [CoreWarnings.OVERLAPS], True, ErrorDomain.STREAM), - ("warninga.bst", ["warninga:warning-a"], True, ErrorDomain.STREAM), - ("warningb.bst", ["warningb:warning-b"], True, ErrorDomain.STREAM), - ("corewarn.bst", [], False, None), - ("warninga.bst", [], False, None), - ("warningb.bst", [], False, None), - ("warninga.bst", [CoreWarnings.OVERLAPS], False, None), - ("warningb.bst", [CoreWarnings.OVERLAPS], False, None), -]) -def test_fatal_warnings(cli, datafiles, element_name, - fatal_warnings, expect_fatal, error_domain): - if HAVE_SANDBOX == 'buildbox' and error_domain != ErrorDomain.STREAM: +@pytest.mark.parametrize( + "element_name, fatal_warnings, expect_fatal, error_domain", + [ + ("corewarn.bst", [CoreWarnings.OVERLAPS], True, ErrorDomain.STREAM), + ("warninga.bst", ["warninga:warning-a"], True, ErrorDomain.STREAM), + ("warningb.bst", ["warningb:warning-b"], True, ErrorDomain.STREAM), + ("corewarn.bst", [], False, None), + ("warninga.bst", [], False, None), + ("warningb.bst", [], False, None), + ("warninga.bst", [CoreWarnings.OVERLAPS], False, None), + ("warningb.bst", [CoreWarnings.OVERLAPS], False, None), + ], +) +def test_fatal_warnings(cli, datafiles, element_name, fatal_warnings, expect_fatal, error_domain): + if HAVE_SANDBOX == "buildbox" and error_domain != ErrorDomain.STREAM: pytest.xfail() project_path = build_project(datafiles, fatal_warnings) diff --git a/tests/frontend/configuredwarning/plugins/corewarn.py b/tests/frontend/configuredwarning/plugins/corewarn.py index 1f263a0ce..bef0a4904 100644 --- a/tests/frontend/configuredwarning/plugins/corewarn.py +++ b/tests/frontend/configuredwarning/plugins/corewarn.py @@ -19,8 +19,7 @@ class CoreWarn(Element): pass def assemble(self, sandbox): - self.warn("Testing: CoreWarning produced during assemble", - warning_token=CoreWarnings.OVERLAPS) + self.warn("Testing: CoreWarning produced during assemble", warning_token=CoreWarnings.OVERLAPS) def setup(): diff --git a/tests/frontend/consistencyerror/plugins/consistencybug.py b/tests/frontend/consistencyerror/plugins/consistencybug.py index c60d81ea0..c442d883a 100644 --- a/tests/frontend/consistencyerror/plugins/consistencybug.py +++ b/tests/frontend/consistencyerror/plugins/consistencybug.py @@ -2,7 +2,6 @@ from buildstream import Source class ConsistencyBugSource(Source): - def configure(self, node): pass diff --git a/tests/frontend/consistencyerror/plugins/consistencyerror.py b/tests/frontend/consistencyerror/plugins/consistencyerror.py index bcbd1b5e9..125baf39c 100644 --- a/tests/frontend/consistencyerror/plugins/consistencyerror.py +++ b/tests/frontend/consistencyerror/plugins/consistencyerror.py @@ -2,7 +2,6 @@ from buildstream import Source, SourceError class ConsistencyErrorSource(Source): - def configure(self, node): pass @@ -15,8 +14,7 @@ class ConsistencyErrorSource(Source): def get_consistency(self): # Raise an error unconditionally - raise SourceError("Something went terribly wrong", - reason="the-consistency-error") + raise SourceError("Something went terribly wrong", reason="the-consistency-error") def get_ref(self): return None diff --git a/tests/frontend/cross_junction_workspace.py b/tests/frontend/cross_junction_workspace.py index ca21e7548..574de2bed 100644 --- a/tests/frontend/cross_junction_workspace.py +++ b/tests/frontend/cross_junction_workspace.py @@ -13,8 +13,8 @@ def prepare_junction_project(cli, tmpdir): os.makedirs(str(main_project)) os.makedirs(str(sub_project)) - _yaml.roundtrip_dump({'name': 'main'}, str(main_project.join("project.conf"))) - _yaml.roundtrip_dump({'name': 'sub'}, str(sub_project.join("project.conf"))) + _yaml.roundtrip_dump({"name": "main"}, str(main_project.join("project.conf"))) + _yaml.roundtrip_dump({"name": "sub"}, str(sub_project.join("project.conf"))) import_dir = tmpdir.join("import") os.makedirs(str(import_dir)) @@ -26,20 +26,20 @@ def prepare_junction_project(cli, tmpdir): import_repo = create_repo("git", str(import_repo_dir)) import_ref = import_repo.create(str(import_dir)) - _yaml.roundtrip_dump({'kind': 'import', - 'sources': [import_repo.source_config(ref=import_ref)]}, - str(sub_project.join("data.bst"))) + _yaml.roundtrip_dump( + {"kind": "import", "sources": [import_repo.source_config(ref=import_ref)]}, str(sub_project.join("data.bst")) + ) sub_repo_dir = tmpdir.join("sub_repo") os.makedirs(str(sub_repo_dir)) sub_repo = create_repo("git", str(sub_repo_dir)) sub_ref = sub_repo.create(str(sub_project)) - _yaml.roundtrip_dump({'kind': 'junction', - 'sources': [sub_repo.source_config(ref=sub_ref)]}, - str(main_project.join("sub.bst"))) + _yaml.roundtrip_dump( + {"kind": "junction", "sources": [sub_repo.source_config(ref=sub_ref)]}, str(main_project.join("sub.bst")) + ) - args = ['source', 'fetch', 'sub.bst'] + args = ["source", "fetch", "sub.bst"] result = cli.run(project=str(main_project), args=args) result.assert_success() @@ -50,13 +50,13 @@ def open_cross_junction(cli, tmpdir): project = prepare_junction_project(cli, tmpdir) workspace = tmpdir.join("workspace") - element = 'sub.bst:data.bst' - args = ['workspace', 'open', '--directory', str(workspace), element] + element = "sub.bst:data.bst" + args = ["workspace", "open", "--directory", str(workspace), element] result = cli.run(project=project, args=args) result.assert_success() - assert cli.get_element_state(project, element) == 'buildable' - assert os.path.exists(str(workspace.join('hello.txt'))) + assert cli.get_element_state(project, element) == "buildable" + assert os.path.exists(str(workspace.join("hello.txt"))) return project, workspace @@ -68,55 +68,55 @@ def test_open_cross_junction(cli, tmpdir): def test_list_cross_junction(cli, tmpdir): project, _ = open_cross_junction(cli, tmpdir) - element = 'sub.bst:data.bst' + element = "sub.bst:data.bst" - args = ['workspace', 'list'] + args = ["workspace", "list"] result = cli.run(project=project, args=args) result.assert_success() loaded = _yaml.load_data(result.output) - workspaces = loaded.get_sequence('workspaces') + workspaces = loaded.get_sequence("workspaces") assert len(workspaces) == 1 first_workspace = workspaces.mapping_at(0) - assert 'element' in first_workspace - assert first_workspace.get_str('element') == element + assert "element" in first_workspace + assert first_workspace.get_str("element") == element def test_close_cross_junction(cli, tmpdir): project, workspace = open_cross_junction(cli, tmpdir) - element = 'sub.bst:data.bst' - args = ['workspace', 'close', '--remove-dir', element] + element = "sub.bst:data.bst" + args = ["workspace", "close", "--remove-dir", element] result = cli.run(project=project, args=args) result.assert_success() assert not os.path.exists(str(workspace)) - args = ['workspace', 'list'] + args = ["workspace", "list"] result = cli.run(project=project, args=args) result.assert_success() loaded = _yaml.load_data(result.output) - workspaces = loaded.get_sequence('workspaces') + workspaces = loaded.get_sequence("workspaces") assert not workspaces def test_close_all_cross_junction(cli, tmpdir): project, workspace = open_cross_junction(cli, tmpdir) - args = ['workspace', 'close', '--remove-dir', '--all'] + args = ["workspace", "close", "--remove-dir", "--all"] result = cli.run(project=project, args=args) result.assert_success() assert not os.path.exists(str(workspace)) - args = ['workspace', 'list'] + args = ["workspace", "list"] result = cli.run(project=project, args=args) result.assert_success() loaded = _yaml.load_data(result.output) - workspaces = loaded.get_sequence('workspaces') + workspaces = loaded.get_sequence("workspaces") assert not workspaces @@ -124,17 +124,17 @@ def test_subdir_command_cross_junction(cli, tmpdir): # i.e. commands can be run successfully from a subdirectory of the # junction's workspace, in case project loading logic has gone wrong project = prepare_junction_project(cli, tmpdir) - workspace = os.path.join(str(tmpdir), 'workspace') - junction_element = 'sub.bst' + workspace = os.path.join(str(tmpdir), "workspace") + junction_element = "sub.bst" # Open the junction as a workspace - args = ['workspace', 'open', '--directory', workspace, junction_element] + args = ["workspace", "open", "--directory", workspace, junction_element] result = cli.run(project=project, args=args) result.assert_success() # Run commands from a subdirectory of the workspace newdir = os.path.join(str(workspace), "newdir") - element_name = 'data.bst' + element_name = "data.bst" os.makedirs(newdir) - result = cli.run(project=str(workspace), args=['-C', newdir, 'show', element_name]) + result = cli.run(project=str(workspace), args=["-C", newdir, "show", element_name]) result.assert_success() diff --git a/tests/frontend/fetch.py b/tests/frontend/fetch.py index 7ea357ac2..ff3667707 100644 --- a/tests/frontend/fetch.py +++ b/tests/frontend/fetch.py @@ -15,54 +15,48 @@ from . import configure_project # Project directory TOP_DIR = os.path.dirname(os.path.realpath(__file__)) -DATA_DIR = os.path.join(TOP_DIR, 'project') +DATA_DIR = os.path.join(TOP_DIR, "project") -@pytest.mark.datafiles(os.path.join(TOP_DIR, 'project_world')) +@pytest.mark.datafiles(os.path.join(TOP_DIR, "project_world")) def test_fetch_default_targets(cli, tmpdir, datafiles): project = str(datafiles) - element_path = os.path.join(project, 'elements') - element_name = 'fetch-test.bst' + element_path = os.path.join(project, "elements") + element_name = "fetch-test.bst" # Create our repo object of the given source type with # the bin files, and then collect the initial ref. # - repo = create_repo('git', str(tmpdir)) + repo = create_repo("git", str(tmpdir)) ref = repo.create(project) # Write out our test target - element = { - 'kind': 'import', - 'sources': [ - repo.source_config(ref=ref) - ] - } - _yaml.roundtrip_dump(element, - os.path.join(element_path, element_name)) + element = {"kind": "import", "sources": [repo.source_config(ref=ref)]} + _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) # Assert that a fetch is needed - assert cli.get_element_state(project, element_name) == 'fetch needed' + assert cli.get_element_state(project, element_name) == "fetch needed" # Now try to fetch it, using the default target feature - result = cli.run(project=project, args=['source', 'fetch']) + result = cli.run(project=project, args=["source", "fetch"]) result.assert_success() # Assert that we are now buildable because the source is # now cached. - assert cli.get_element_state(project, element_name) == 'buildable' + assert cli.get_element_state(project, element_name) == "buildable" -@pytest.mark.datafiles(os.path.join(TOP_DIR, 'consistencyerror')) +@pytest.mark.datafiles(os.path.join(TOP_DIR, "consistencyerror")) def test_fetch_consistency_error(cli, datafiles): project = str(datafiles) # When the error occurs outside of the scheduler at load time, # then the SourceError is reported directly as the main error. - result = cli.run(project=project, args=['source', 'fetch', 'error.bst']) - result.assert_main_error(ErrorDomain.SOURCE, 'the-consistency-error') + result = cli.run(project=project, args=["source", "fetch", "error.bst"]) + result.assert_main_error(ErrorDomain.SOURCE, "the-consistency-error") -@pytest.mark.datafiles(os.path.join(TOP_DIR, 'consistencyerror')) +@pytest.mark.datafiles(os.path.join(TOP_DIR, "consistencyerror")) def test_fetch_consistency_bug(cli, datafiles): project = str(datafiles) @@ -73,104 +67,68 @@ def test_fetch_consistency_bug(cli, datafiles): # for a fetch command, we could report this to the user # more gracefully as a BUG message. # - result = cli.run(project=project, args=['source', 'fetch', 'bug.bst']) + result = cli.run(project=project, args=["source", "fetch", "bug.bst"]) assert result.exc is not None assert str(result.exc) == "Something went terribly wrong" @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("strict", [True, False], ids=["strict", "no-strict"]) -@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')]) +@pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")]) def test_unfetched_junction(cli, tmpdir, datafiles, strict, ref_storage): project = str(datafiles) - subproject_path = os.path.join(project, 'files', 'sub-project') - junction_path = os.path.join(project, 'elements', 'junction.bst') - element_path = os.path.join(project, 'elements', 'junction-dep.bst') - - configure_project(project, { - 'ref-storage': ref_storage - }) - cli.configure({ - 'projects': { - 'test': { - 'strict': strict - } - } - }) + subproject_path = os.path.join(project, "files", "sub-project") + junction_path = os.path.join(project, "elements", "junction.bst") + element_path = os.path.join(project, "elements", "junction-dep.bst") + + configure_project(project, {"ref-storage": ref_storage}) + cli.configure({"projects": {"test": {"strict": strict}}}) # Create a repo to hold the subproject and generate a junction element for it - ref = generate_junction(tmpdir, subproject_path, junction_path, store_ref=(ref_storage == 'inline')) + ref = generate_junction(tmpdir, subproject_path, junction_path, store_ref=(ref_storage == "inline")) # Create a stack element to depend on a cross junction element # - element = { - 'kind': 'stack', - 'depends': [ - { - 'junction': 'junction.bst', - 'filename': 'import-etc.bst' - } - ] - } + element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) # Dump a project.refs if we're using project.refs storage # - if ref_storage == 'project.refs': - project_refs = { - 'projects': { - 'test': { - 'junction.bst': [ - { - 'ref': ref - } - ] - } - } - } - _yaml.roundtrip_dump(project_refs, os.path.join(project, 'junction.refs')) + if ref_storage == "project.refs": + project_refs = {"projects": {"test": {"junction.bst": [{"ref": ref}]}}} + _yaml.roundtrip_dump(project_refs, os.path.join(project, "junction.refs")) # Now try to fetch it, this should automatically result in fetching # the junction itself. - result = cli.run(project=project, args=['source', 'fetch', 'junction-dep.bst']) + result = cli.run(project=project, args=["source", "fetch", "junction-dep.bst"]) result.assert_success() @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')]) +@pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")]) def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage): project = str(datafiles) - subproject_path = os.path.join(project, 'files', 'sub-project') - junction_path = os.path.join(project, 'elements', 'junction.bst') - element_path = os.path.join(project, 'elements', 'junction-dep.bst') + subproject_path = os.path.join(project, "files", "sub-project") + junction_path = os.path.join(project, "elements", "junction.bst") + element_path = os.path.join(project, "elements", "junction-dep.bst") - configure_project(project, { - 'ref-storage': ref_storage - }) + configure_project(project, {"ref-storage": ref_storage}) # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path, store_ref=False) # Create a stack element to depend on a cross junction element # - element = { - 'kind': 'stack', - 'depends': [ - { - 'junction': 'junction.bst', - 'filename': 'import-etc.bst' - } - ] - } + element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) # Now try to fetch it, this will bail with the appropriate error # informing the user to track the junction first - result = cli.run(project=project, args=['source', 'fetch', 'junction-dep.bst']) + result = cli.run(project=project, args=["source", "fetch", "junction-dep.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT) # Assert that we have the expected provenance encoded into the error - element_node = _yaml.load(element_path, shortname='junction-dep.bst') - ref_node = element_node.get_sequence('depends').mapping_at(0) + element_node = _yaml.load(element_path, shortname="junction-dep.bst") + ref_node = element_node.get_sequence("depends").mapping_at(0) provenance = ref_node.get_provenance() assert str(provenance) in result.stderr diff --git a/tests/frontend/help.py b/tests/frontend/help.py index 3bc18499b..20a93160f 100644 --- a/tests/frontend/help.py +++ b/tests/frontend/help.py @@ -8,28 +8,21 @@ from buildstream.testing.runcli import cli # pylint: disable=unused-import def assert_help(cli_output): expected_start = "Usage: " if not cli_output.startswith(expected_start): - raise AssertionError("Help output expected to begin with '{}'," - .format(expected_start) + - " output was: {}" - .format(cli_output)) + raise AssertionError( + "Help output expected to begin with '{}',".format(expected_start) + " output was: {}".format(cli_output) + ) def test_help_main(cli): - result = cli.run(args=['--help']) + result = cli.run(args=["--help"]) result.assert_success() assert_help(result.output) -@pytest.mark.parametrize("command", [ - ('artifact'), - ('build'), - ('checkout'), - ('shell'), - ('show'), - ('source'), - ('workspace') -]) +@pytest.mark.parametrize( + "command", [("artifact"), ("build"), ("checkout"), ("shell"), ("show"), ("source"), ("workspace")] +) def test_help(cli, command): - result = cli.run(args=[command, '--help']) + result = cli.run(args=[command, "--help"]) result.assert_success() assert_help(result.output) diff --git a/tests/frontend/init.py b/tests/frontend/init.py index 0fdc0eda5..3b9a95c34 100644 --- a/tests/frontend/init.py +++ b/tests/frontend/init.py @@ -13,114 +13,106 @@ from buildstream._versions import BST_FORMAT_VERSION def test_defaults(cli, tmpdir): project = str(tmpdir) - project_path = os.path.join(project, 'project.conf') + project_path = os.path.join(project, "project.conf") - result = cli.run(args=['init', '--project-name', 'foo', project]) + result = cli.run(args=["init", "--project-name", "foo", project]) result.assert_success() project_conf = _yaml.load(project_path) - assert project_conf.get_str('name') == 'foo' - assert project_conf.get_str('format-version') == str(BST_FORMAT_VERSION) - assert project_conf.get_str('element-path') == 'elements' + assert project_conf.get_str("name") == "foo" + assert project_conf.get_str("format-version") == str(BST_FORMAT_VERSION) + assert project_conf.get_str("element-path") == "elements" def test_all_options(cli, tmpdir): project = str(tmpdir) - project_path = os.path.join(project, 'project.conf') - - result = cli.run(args=[ - 'init', - '--project-name', 'foo', - '--format-version', '2', - '--element-path', 'ponies', - project - ]) + project_path = os.path.join(project, "project.conf") + + result = cli.run( + args=["init", "--project-name", "foo", "--format-version", "2", "--element-path", "ponies", project] + ) result.assert_success() project_conf = _yaml.load(project_path) - assert project_conf.get_str('name') == 'foo' - assert project_conf.get_str('format-version') == str(2) - assert project_conf.get_str('element-path') == 'ponies' + assert project_conf.get_str("name") == "foo" + assert project_conf.get_str("format-version") == str(2) + assert project_conf.get_str("element-path") == "ponies" - elements_dir = os.path.join(project, 'ponies') + elements_dir = os.path.join(project, "ponies") assert os.path.isdir(elements_dir) def test_no_project_name(cli, tmpdir): - result = cli.run(args=['init', str(tmpdir)]) - result.assert_main_error(ErrorDomain.APP, 'unspecified-project-name') + result = cli.run(args=["init", str(tmpdir)]) + result.assert_main_error(ErrorDomain.APP, "unspecified-project-name") def test_project_exists(cli, tmpdir): project = str(tmpdir) - project_path = os.path.join(project, 'project.conf') - with open(project_path, 'w') as f: - f.write('name: pony\n') + project_path = os.path.join(project, "project.conf") + with open(project_path, "w") as f: + f.write("name: pony\n") - result = cli.run(args=['init', '--project-name', 'foo', project]) - result.assert_main_error(ErrorDomain.APP, 'project-exists') + result = cli.run(args=["init", "--project-name", "foo", project]) + result.assert_main_error(ErrorDomain.APP, "project-exists") def test_force_overwrite_project(cli, tmpdir): project = str(tmpdir) - project_path = os.path.join(project, 'project.conf') - with open(project_path, 'w') as f: - f.write('name: pony\n') + project_path = os.path.join(project, "project.conf") + with open(project_path, "w") as f: + f.write("name: pony\n") - result = cli.run(args=['init', '--project-name', 'foo', '--force', project]) + result = cli.run(args=["init", "--project-name", "foo", "--force", project]) result.assert_success() project_conf = _yaml.load(project_path) - assert project_conf.get_str('name') == 'foo' - assert project_conf.get_str('format-version') == str(BST_FORMAT_VERSION) + assert project_conf.get_str("name") == "foo" + assert project_conf.get_str("format-version") == str(BST_FORMAT_VERSION) def test_relative_path_directory_as_argument(cli, tmpdir): - project = os.path.join(str(tmpdir), 'child-directory') + project = os.path.join(str(tmpdir), "child-directory") os.makedirs(project, exist_ok=True) - project_path = os.path.join(project, 'project.conf') + project_path = os.path.join(project, "project.conf") rel_path = os.path.relpath(project) - result = cli.run(args=['init', '--project-name', 'foo', rel_path]) + result = cli.run(args=["init", "--project-name", "foo", rel_path]) result.assert_success() project_conf = _yaml.load(project_path) - assert project_conf.get_str('name') == 'foo' - assert project_conf.get_int('format-version') == BST_FORMAT_VERSION - assert project_conf.get_str('element-path') == 'elements' + assert project_conf.get_str("name") == "foo" + assert project_conf.get_int("format-version") == BST_FORMAT_VERSION + assert project_conf.get_str("element-path") == "elements" def test_set_directory_and_directory_as_argument(cli, tmpdir): - result = cli.run(args=['-C', '/foo/bar', 'init', '--project-name', 'foo', '/boo/far']) - result.assert_main_error(ErrorDomain.APP, 'init-with-set-directory') + result = cli.run(args=["-C", "/foo/bar", "init", "--project-name", "foo", "/boo/far"]) + result.assert_main_error(ErrorDomain.APP, "init-with-set-directory") -@pytest.mark.parametrize("project_name", [('Micheal Jackson'), ('one+one')]) +@pytest.mark.parametrize("project_name", [("Micheal Jackson"), ("one+one")]) def test_bad_project_name(cli, tmpdir, project_name): - result = cli.run(args=['init', '--project-name', str(tmpdir)]) + result = cli.run(args=["init", "--project-name", str(tmpdir)]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_SYMBOL_NAME) @pytest.mark.parametrize("format_version", [(str(-1)), (str(BST_FORMAT_VERSION + 1))]) def test_bad_format_version(cli, tmpdir, format_version): - result = cli.run(args=[ - 'init', '--project-name', 'foo', '--format-version', format_version, str(tmpdir) - ]) - result.assert_main_error(ErrorDomain.APP, 'invalid-format-version') + result = cli.run(args=["init", "--project-name", "foo", "--format-version", format_version, str(tmpdir)]) + result.assert_main_error(ErrorDomain.APP, "invalid-format-version") -@pytest.mark.parametrize("element_path", [('/absolute/path'), ('../outside/of/project')]) +@pytest.mark.parametrize("element_path", [("/absolute/path"), ("../outside/of/project")]) def test_bad_element_path(cli, tmpdir, element_path): - result = cli.run(args=[ - 'init', '--project-name', 'foo', '--element-path', element_path, str(tmpdir) - ]) - result.assert_main_error(ErrorDomain.APP, 'invalid-element-path') + result = cli.run(args=["init", "--project-name", "foo", "--element-path", element_path, str(tmpdir)]) + result.assert_main_error(ErrorDomain.APP, "invalid-element-path") -@pytest.mark.parametrize("element_path", [('foo'), ('foo/bar')]) +@pytest.mark.parametrize("element_path", [("foo"), ("foo/bar")]) def test_element_path_interactive(cli, tmp_path, monkeypatch, element_path): project = tmp_path - project_conf_path = project.joinpath('project.conf') + project_conf_path = project.joinpath("project.conf") class DummyInteractiveApp(App): def __init__(self, *args, **kwargs): @@ -132,17 +124,17 @@ def test_element_path_interactive(cli, tmp_path, monkeypatch, element_path): return DummyInteractiveApp(*args, **kwargs) def _init_project_interactive(self, *args, **kwargs): # pylint: disable=arguments-differ - return ('project_name', '0', element_path) + return ("project_name", "0", element_path) - monkeypatch.setattr(App, 'create', DummyInteractiveApp.create) + monkeypatch.setattr(App, "create", DummyInteractiveApp.create) - result = cli.run(args=['init', str(project)]) + result = cli.run(args=["init", str(project)]) result.assert_success() full_element_path = project.joinpath(element_path) assert full_element_path.exists() project_conf = _yaml.load(str(project_conf_path)) - assert project_conf.get_str('name') == 'project_name' - assert project_conf.get_str('format-version') == '0' - assert project_conf.get_str('element-path') == element_path + assert project_conf.get_str("name") == "project_name" + assert project_conf.get_str("format-version") == "0" + assert project_conf.get_str("element-path") == element_path diff --git a/tests/frontend/large_directory.py b/tests/frontend/large_directory.py index 921e2ddbe..ea29fd1ca 100644 --- a/tests/frontend/large_directory.py +++ b/tests/frontend/large_directory.py @@ -29,10 +29,7 @@ from tests.testutils import create_artifact_share, assert_shared # Project directory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project", -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",) @contextmanager @@ -40,7 +37,7 @@ def limit_grpc_message_length(limit): orig_insecure_channel = grpc.insecure_channel def new_insecure_channel(target): - return orig_insecure_channel(target, options=(('grpc.max_send_message_length', limit),)) + return orig_insecure_channel(target, options=(("grpc.max_send_message_length", limit),)) grpc.insecure_channel = new_insecure_channel try: @@ -58,29 +55,25 @@ def test_large_directory(cli, tmpdir, datafiles): MAX_MESSAGE_LENGTH = 1024 * 1024 NUM_FILES = MAX_MESSAGE_LENGTH // 64 + 1 - large_directory_dir = os.path.join(project, 'files', 'large-directory') + large_directory_dir = os.path.join(project, "files", "large-directory") os.mkdir(large_directory_dir) for i in range(NUM_FILES): - with open(os.path.join(large_directory_dir, str(i)), 'w') as f: + with open(os.path.join(large_directory_dir, str(i)), "w") as f: # The files need to have different content as we want different digests. f.write(str(i)) - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # Configure bst to push to the artifact share - cli.configure({ - 'artifacts': [ - {'url': share.repo, 'push': True}, - ] - }) + cli.configure({"artifacts": [{"url": share.repo, "push": True},]}) # Enforce 1 MB gRPC message limit with limit_grpc_message_length(MAX_MESSAGE_LENGTH): # Build and push - result = cli.run(project=project, args=['build', 'import-large-directory.bst']) + result = cli.run(project=project, args=["build", "import-large-directory.bst"]) result.assert_success() # Assert that we are now cached locally - assert cli.get_element_state(project, 'import-large-directory.bst') == 'cached' + assert cli.get_element_state(project, "import-large-directory.bst") == "cached" # Assert that the push was successful - assert_shared(cli, share, project, 'import-large-directory.bst') + assert_shared(cli, share, project, "import-large-directory.bst") diff --git a/tests/frontend/logging.py b/tests/frontend/logging.py index 462af821f..27ff88352 100644 --- a/tests/frontend/logging.py +++ b/tests/frontend/logging.py @@ -13,37 +13,28 @@ from buildstream._exceptions import ErrorDomain from buildstream.testing import cli # pylint: disable=unused-import # Project directory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project", -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",) @pytest.mark.datafiles(DATA_DIR) def test_default_logging(cli, tmpdir, datafiles): project = str(datafiles) - bin_files_path = os.path.join(project, 'files', 'bin-files') - element_path = os.path.join(project, 'elements') - element_name = 'fetch-test-git.bst' + bin_files_path = os.path.join(project, "files", "bin-files") + element_path = os.path.join(project, "elements") + element_name = "fetch-test-git.bst" # Create our repo object of the given source type with # the bin files, and then collect the initial ref. # - repo = create_repo('git', str(tmpdir)) + repo = create_repo("git", str(tmpdir)) ref = repo.create(bin_files_path) # Write out our test target - element = { - 'kind': 'import', - 'sources': [ - repo.source_config(ref=ref) - ] - } - _yaml.roundtrip_dump(element, - os.path.join(element_path, element_name)) + element = {"kind": "import", "sources": [repo.source_config(ref=ref)]} + _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) # Now try to fetch it - result = cli.run(project=project, args=['source', 'fetch', element_name]) + result = cli.run(project=project, args=["source", "fetch", element_name]) result.assert_success() m = re.search(r"\[\d\d:\d\d:\d\d\]\[\s*\]\[.*\] SUCCESS Checking sources", result.stderr) @@ -53,37 +44,34 @@ def test_default_logging(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_custom_logging(cli, tmpdir, datafiles): project = str(datafiles) - bin_files_path = os.path.join(project, 'files', 'bin-files') - element_path = os.path.join(project, 'elements') - element_name = 'fetch-test-git.bst' - - custom_log_format = ('%{elapsed},%{elapsed-us},%{wallclock},%{wallclock-us},' - '%{key},%{element},%{action},%{message}') - user_config = {'logging': {'message-format': custom_log_format}} + bin_files_path = os.path.join(project, "files", "bin-files") + element_path = os.path.join(project, "elements") + element_name = "fetch-test-git.bst" + + custom_log_format = ( + "%{elapsed},%{elapsed-us},%{wallclock},%{wallclock-us}," "%{key},%{element},%{action},%{message}" + ) + user_config = {"logging": {"message-format": custom_log_format}} cli.configure(user_config) # Create our repo object of the given source type with # the bin files, and then collect the initial ref. # - repo = create_repo('git', str(tmpdir)) + repo = create_repo("git", str(tmpdir)) ref = repo.create(bin_files_path) # Write out our test target - element = { - 'kind': 'import', - 'sources': [ - repo.source_config(ref=ref) - ] - } - _yaml.roundtrip_dump(element, - os.path.join(element_path, element_name)) + element = {"kind": "import", "sources": [repo.source_config(ref=ref)]} + _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) # Now try to fetch it - result = cli.run(project=project, args=['source', 'fetch', element_name]) + result = cli.run(project=project, args=["source", "fetch", element_name]) result.assert_success() - m = re.search(r"\d\d:\d\d:\d\d,\d\d:\d\d:\d\d.\d{6},\d\d:\d\d:\d\d,\d\d:\d\d:\d\d.\d{6}\s*,.*" - r",SUCCESS,Checking sources", result.stderr) + m = re.search( + r"\d\d:\d\d:\d\d,\d\d:\d\d:\d\d.\d{6},\d\d:\d\d:\d\d,\d\d:\d\d:\d\d.\d{6}\s*,.*" r",SUCCESS,Checking sources", + result.stderr, + ) assert m is not None @@ -92,19 +80,12 @@ def test_failed_build_listing(cli, datafiles): project = str(datafiles) element_names = [] for i in range(3): - element_name = 'testfail-{}.bst'.format(i) - element_path = os.path.join('elements', element_name) - element = { - 'kind': 'script', - 'config': { - 'commands': [ - 'false' - ] - } - } + element_name = "testfail-{}.bst".format(i) + element_path = os.path.join("elements", element_name) + element = {"kind": "script", "config": {"commands": ["false"]}} _yaml.roundtrip_dump(element, os.path.join(project, element_path)) element_names.append(element_name) - result = cli.run(project=project, args=['--on-error=continue', 'build', *element_names]) + result = cli.run(project=project, args=["--on-error=continue", "build", *element_names]) result.assert_main_error(ErrorDomain.STREAM, None) # Check that we re-print the failure summaries only in the "Failure Summary" @@ -115,10 +96,10 @@ def test_failed_build_listing(cli, datafiles): # testfail-0.bst: # [00:00:00][44f1b8c3][ build:testfail-0.bst ] FAILURE Running 'commands' # - failure_heading_pos = re.search(r'^Failure Summary$', result.stderr, re.MULTILINE).start() - pipeline_heading_pos = re.search(r'^Pipeline Summary$', result.stderr, re.MULTILINE).start() + failure_heading_pos = re.search(r"^Failure Summary$", result.stderr, re.MULTILINE).start() + pipeline_heading_pos = re.search(r"^Pipeline Summary$", result.stderr, re.MULTILINE).start() failure_summary_range = range(failure_heading_pos, pipeline_heading_pos) - matches = tuple(re.finditer(r'^\s+testfail-.\.bst:$', result.stderr, re.MULTILINE)) + matches = tuple(re.finditer(r"^\s+testfail-.\.bst:$", result.stderr, re.MULTILINE)) for m in matches: assert m.start() in failure_summary_range assert m.end() in failure_summary_range diff --git a/tests/frontend/main.py b/tests/frontend/main.py index 0df52e2c9..d864a0b1f 100644 --- a/tests/frontend/main.py +++ b/tests/frontend/main.py @@ -5,7 +5,7 @@ from buildstream._frontend.app import _prefix_choice_value_proc def test_prefix_choice_value_proc_full_match(): - value_proc = _prefix_choice_value_proc(['foo', 'bar', 'baz']) + value_proc = _prefix_choice_value_proc(["foo", "bar", "baz"]) assert value_proc("foo") == "foo" assert value_proc("bar") == "bar" @@ -13,13 +13,13 @@ def test_prefix_choice_value_proc_full_match(): def test_prefix_choice_value_proc_prefix_match(): - value_proc = _prefix_choice_value_proc(['foo']) + value_proc = _prefix_choice_value_proc(["foo"]) assert value_proc("f") == "foo" def test_prefix_choice_value_proc_ambigous_match(): - value_proc = _prefix_choice_value_proc(['bar', 'baz']) + value_proc = _prefix_choice_value_proc(["bar", "baz"]) assert value_proc("bar") == "bar" assert value_proc("baz") == "baz" @@ -28,7 +28,7 @@ def test_prefix_choice_value_proc_ambigous_match(): def test_prefix_choice_value_proc_value_not_in_choices(): - value_proc = _prefix_choice_value_proc(['bar', 'baz']) + value_proc = _prefix_choice_value_proc(["bar", "baz"]) with pytest.raises(click.UsageError): value_proc("foo") diff --git a/tests/frontend/mirror.py b/tests/frontend/mirror.py index 855155785..f71cb554d 100644 --- a/tests/frontend/mirror.py +++ b/tests/frontend/mirror.py @@ -11,15 +11,15 @@ from buildstream.testing import cli # pylint: disable=unused-import # Project directory TOP_DIR = os.path.dirname(os.path.realpath(__file__)) -DATA_DIR = os.path.join(TOP_DIR, 'project') +DATA_DIR = os.path.join(TOP_DIR, "project") def generate_element(output_file): element = { - 'kind': 'import', - 'sources': [ + "kind": "import", + "sources": [ { - 'kind': 'fetch_source', + "kind": "fetch_source", "output-text": output_file, "urls": ["foo:repo1", "bar:repo2"], "fetch-succeeds": { @@ -31,53 +31,24 @@ def generate_element(output_file): "RBA/repo2": False, "ooF/repo1": False, "raB/repo2": False, - } + }, } - ] + ], } return element def generate_project(): project = { - 'name': 'test', - 'element-path': 'elements', - 'aliases': { - 'foo': 'FOO/', - 'bar': 'BAR/', - }, - 'mirrors': [ - { - 'name': 'middle-earth', - 'aliases': { - 'foo': ['OOF/'], - 'bar': ['RAB/'], - }, - }, - { - 'name': 'arrakis', - 'aliases': { - 'foo': ['OFO/'], - 'bar': ['RBA/'], - }, - }, - { - 'name': 'oz', - 'aliases': { - 'foo': ['ooF/'], - 'bar': ['raB/'], - } - }, + "name": "test", + "element-path": "elements", + "aliases": {"foo": "FOO/", "bar": "BAR/",}, + "mirrors": [ + {"name": "middle-earth", "aliases": {"foo": ["OOF/"], "bar": ["RAB/"],},}, + {"name": "arrakis", "aliases": {"foo": ["OFO/"], "bar": ["RBA/"],},}, + {"name": "oz", "aliases": {"foo": ["ooF/"], "bar": ["raB/"],}}, ], - 'plugins': [ - { - 'origin': 'local', - 'path': 'sources', - 'sources': { - 'fetch_source': 0 - } - } - ] + "plugins": [{"origin": "local", "path": "sources", "sources": {"fetch_source": 0}}], } return project @@ -86,74 +57,58 @@ def generate_project(): @pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")]) @pytest.mark.parametrize("mirror", [("no-mirror"), ("mirror"), ("unrelated-mirror")]) def test_mirror_fetch_ref_storage(cli, tmpdir, datafiles, ref_storage, mirror): - bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr') - dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr') - upstream_repodir = os.path.join(str(tmpdir), 'upstream') - mirror_repodir = os.path.join(str(tmpdir), 'mirror') - project_dir = os.path.join(str(tmpdir), 'project') + bin_files_path = os.path.join(str(datafiles), "files", "bin-files", "usr") + dev_files_path = os.path.join(str(datafiles), "files", "dev-files", "usr") + upstream_repodir = os.path.join(str(tmpdir), "upstream") + mirror_repodir = os.path.join(str(tmpdir), "mirror") + project_dir = os.path.join(str(tmpdir), "project") os.makedirs(project_dir) - element_dir = os.path.join(project_dir, 'elements') + element_dir = os.path.join(project_dir, "elements") # Create repo objects of the upstream and mirror - upstream_repo = create_repo('tar', upstream_repodir) + upstream_repo = create_repo("tar", upstream_repodir) upstream_repo.create(bin_files_path) mirror_repo = upstream_repo.copy(mirror_repodir) upstream_ref = upstream_repo.create(dev_files_path) element = { - 'kind': 'import', - 'sources': [ - upstream_repo.source_config(ref=upstream_ref if ref_storage == 'inline' else None) - ] + "kind": "import", + "sources": [upstream_repo.source_config(ref=upstream_ref if ref_storage == "inline" else None)], } - element_name = 'test.bst' + element_name = "test.bst" element_path = os.path.join(element_dir, element_name) - full_repo = element['sources'][0]['url'] + full_repo = element["sources"][0]["url"] upstream_map, repo_name = os.path.split(full_repo) - alias = 'foo' - aliased_repo = alias + ':' + repo_name - element['sources'][0]['url'] = aliased_repo - full_mirror = mirror_repo.source_config()['url'] + alias = "foo" + aliased_repo = alias + ":" + repo_name + element["sources"][0]["url"] = aliased_repo + full_mirror = mirror_repo.source_config()["url"] mirror_map, _ = os.path.split(full_mirror) os.makedirs(element_dir) _yaml.roundtrip_dump(element, element_path) - if ref_storage == 'project.refs': + if ref_storage == "project.refs": # Manually set project.refs to avoid caching the repo prematurely - project_refs = {'projects': { - 'test': { - element_name: [ - {'ref': upstream_ref} - ] - } - }} - project_refs_path = os.path.join(project_dir, 'project.refs') + project_refs = {"projects": {"test": {element_name: [{"ref": upstream_ref}]}}} + project_refs_path = os.path.join(project_dir, "project.refs") _yaml.roundtrip_dump(project_refs, project_refs_path) project = { - 'name': 'test', - 'element-path': 'elements', - 'aliases': { - alias: upstream_map + "/" - }, - 'ref-storage': ref_storage + "name": "test", + "element-path": "elements", + "aliases": {alias: upstream_map + "/"}, + "ref-storage": ref_storage, } - if mirror != 'no-mirror': - mirror_data = [{ - 'name': 'middle-earth', - 'aliases': {alias: [mirror_map + '/']} - }] - if mirror == 'unrelated-mirror': - mirror_data.insert(0, { - 'name': 'narnia', - 'aliases': {'frob': ['http://www.example.com/repo']} - }) - project['mirrors'] = mirror_data - - project_file = os.path.join(project_dir, 'project.conf') + if mirror != "no-mirror": + mirror_data = [{"name": "middle-earth", "aliases": {alias: [mirror_map + "/"]}}] + if mirror == "unrelated-mirror": + mirror_data.insert(0, {"name": "narnia", "aliases": {"frob": ["http://www.example.com/repo"]}}) + project["mirrors"] = mirror_data + + project_file = os.path.join(project_dir, "project.conf") _yaml.roundtrip_dump(project, project_file) - result = cli.run(project=project_dir, args=['source', 'fetch', element_name]) + result = cli.run(project=project_dir, args=["source", "fetch", element_name]) result.assert_success() @@ -162,18 +117,18 @@ def test_mirror_fetch_ref_storage(cli, tmpdir, datafiles, ref_storage, mirror): def test_mirror_fetch_multi(cli, tmpdir): output_file = os.path.join(str(tmpdir), "output.txt") project_dir = str(tmpdir) - element_dir = os.path.join(project_dir, 'elements') + element_dir = os.path.join(project_dir, "elements") os.makedirs(element_dir, exist_ok=True) element_name = "test.bst" element_path = os.path.join(element_dir, element_name) element = generate_element(output_file) _yaml.roundtrip_dump(element, element_path) - project_file = os.path.join(project_dir, 'project.conf') + project_file = os.path.join(project_dir, "project.conf") project = generate_project() _yaml.roundtrip_dump(project, project_file) - result = cli.run(project=project_dir, args=['source', 'fetch', element_name]) + result = cli.run(project=project_dir, args=["source", "fetch", element_name]) result.assert_success() with open(output_file) as f: contents = f.read() @@ -186,18 +141,18 @@ def test_mirror_fetch_multi(cli, tmpdir): def test_mirror_fetch_default_cmdline(cli, tmpdir): output_file = os.path.join(str(tmpdir), "output.txt") project_dir = str(tmpdir) - element_dir = os.path.join(project_dir, 'elements') + element_dir = os.path.join(project_dir, "elements") os.makedirs(element_dir, exist_ok=True) element_name = "test.bst" element_path = os.path.join(element_dir, element_name) element = generate_element(output_file) _yaml.roundtrip_dump(element, element_path) - project_file = os.path.join(project_dir, 'project.conf') + project_file = os.path.join(project_dir, "project.conf") project = generate_project() _yaml.roundtrip_dump(project, project_file) - result = cli.run(project=project_dir, args=['--default-mirror', 'arrakis', 'source', 'fetch', element_name]) + result = cli.run(project=project_dir, args=["--default-mirror", "arrakis", "source", "fetch", element_name]) result.assert_success() with open(output_file) as f: contents = f.read() @@ -217,27 +172,21 @@ def test_mirror_fetch_default_cmdline(cli, tmpdir): def test_mirror_fetch_default_userconfig(cli, tmpdir): output_file = os.path.join(str(tmpdir), "output.txt") project_dir = str(tmpdir) - element_dir = os.path.join(project_dir, 'elements') + element_dir = os.path.join(project_dir, "elements") os.makedirs(element_dir, exist_ok=True) element_name = "test.bst" element_path = os.path.join(element_dir, element_name) element = generate_element(output_file) _yaml.roundtrip_dump(element, element_path) - project_file = os.path.join(project_dir, 'project.conf') + project_file = os.path.join(project_dir, "project.conf") project = generate_project() _yaml.roundtrip_dump(project, project_file) - userconfig = { - 'projects': { - 'test': { - 'default-mirror': 'oz' - } - } - } + userconfig = {"projects": {"test": {"default-mirror": "oz"}}} cli.configure(userconfig) - result = cli.run(project=project_dir, args=['source', 'fetch', element_name]) + result = cli.run(project=project_dir, args=["source", "fetch", element_name]) result.assert_success() with open(output_file) as f: contents = f.read() @@ -257,27 +206,21 @@ def test_mirror_fetch_default_userconfig(cli, tmpdir): def test_mirror_fetch_default_cmdline_overrides_config(cli, tmpdir): output_file = os.path.join(str(tmpdir), "output.txt") project_dir = str(tmpdir) - element_dir = os.path.join(project_dir, 'elements') + element_dir = os.path.join(project_dir, "elements") os.makedirs(element_dir, exist_ok=True) element_name = "test.bst" element_path = os.path.join(element_dir, element_name) element = generate_element(output_file) _yaml.roundtrip_dump(element, element_path) - project_file = os.path.join(project_dir, 'project.conf') + project_file = os.path.join(project_dir, "project.conf") project = generate_project() _yaml.roundtrip_dump(project, project_file) - userconfig = { - 'projects': { - 'test': { - 'default-mirror': 'oz' - } - } - } + userconfig = {"projects": {"test": {"default-mirror": "oz"}}} cli.configure(userconfig) - result = cli.run(project=project_dir, args=['--default-mirror', 'arrakis', 'source', 'fetch', element_name]) + result = cli.run(project=project_dir, args=["--default-mirror", "arrakis", "source", "fetch", element_name]) result.assert_success() with open(output_file) as f: contents = f.read() @@ -296,79 +239,65 @@ def test_mirror_fetch_default_cmdline_overrides_config(cli, tmpdir): def test_mirror_git_submodule_fetch(cli, tmpdir, datafiles): # Test that it behaves as expected with submodules, both defined in config # and discovered when fetching. - foo_file = os.path.join(str(datafiles), 'files', 'foo') - bar_file = os.path.join(str(datafiles), 'files', 'bar') - bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr') - dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr') - mirror_dir = os.path.join(str(datafiles), 'mirror') + foo_file = os.path.join(str(datafiles), "files", "foo") + bar_file = os.path.join(str(datafiles), "files", "bar") + bin_files_path = os.path.join(str(datafiles), "files", "bin-files", "usr") + dev_files_path = os.path.join(str(datafiles), "files", "dev-files", "usr") + mirror_dir = os.path.join(str(datafiles), "mirror") - defined_subrepo = create_repo('git', str(tmpdir), 'defined_subrepo') + defined_subrepo = create_repo("git", str(tmpdir), "defined_subrepo") defined_subrepo.create(bin_files_path) defined_subrepo.copy(mirror_dir) defined_subrepo.add_file(foo_file) - found_subrepo = create_repo('git', str(tmpdir), 'found_subrepo') + found_subrepo = create_repo("git", str(tmpdir), "found_subrepo") found_subrepo.create(dev_files_path) - main_repo = create_repo('git', str(tmpdir)) + main_repo = create_repo("git", str(tmpdir)) main_mirror_ref = main_repo.create(bin_files_path) - main_repo.add_submodule('defined', 'file://' + defined_subrepo.repo) - main_repo.add_submodule('found', 'file://' + found_subrepo.repo) + main_repo.add_submodule("defined", "file://" + defined_subrepo.repo) + main_repo.add_submodule("found", "file://" + found_subrepo.repo) main_mirror = main_repo.copy(mirror_dir) main_repo.add_file(bar_file) - project_dir = os.path.join(str(tmpdir), 'project') + project_dir = os.path.join(str(tmpdir), "project") os.makedirs(project_dir) - element_dir = os.path.join(project_dir, 'elements') + element_dir = os.path.join(project_dir, "elements") os.makedirs(element_dir) - element = { - 'kind': 'import', - 'sources': [ - main_repo.source_config(ref=main_mirror_ref) - ] - } - element_name = 'test.bst' + element = {"kind": "import", "sources": [main_repo.source_config(ref=main_mirror_ref)]} + element_name = "test.bst" element_path = os.path.join(element_dir, element_name) # Alias the main repo - full_repo = element['sources'][0]['url'] + full_repo = element["sources"][0]["url"] _, repo_name = os.path.split(full_repo) - alias = 'foo' - aliased_repo = alias + ':' + repo_name - element['sources'][0]['url'] = aliased_repo + alias = "foo" + aliased_repo = alias + ":" + repo_name + element["sources"][0]["url"] = aliased_repo # Hide the found subrepo - del element['sources'][0]['submodules']['found'] + del element["sources"][0]["submodules"]["found"] # Alias the defined subrepo - subrepo = element['sources'][0]['submodules']['defined']['url'] + subrepo = element["sources"][0]["submodules"]["defined"]["url"] _, repo_name = os.path.split(subrepo) - aliased_repo = alias + ':' + repo_name - element['sources'][0]['submodules']['defined']['url'] = aliased_repo + aliased_repo = alias + ":" + repo_name + element["sources"][0]["submodules"]["defined"]["url"] = aliased_repo _yaml.roundtrip_dump(element, element_path) - full_mirror = main_mirror.source_config()['url'] + full_mirror = main_mirror.source_config()["url"] mirror_map, _ = os.path.split(full_mirror) project = { - 'name': 'test', - 'element-path': 'elements', - 'aliases': { - alias: 'http://www.example.com/' - }, - 'mirrors': [ - { - 'name': 'middle-earth', - 'aliases': { - alias: [mirror_map + "/"], - }, - }, - ] + "name": "test", + "element-path": "elements", + "aliases": {alias: "http://www.example.com/"}, + "mirrors": [{"name": "middle-earth", "aliases": {alias: [mirror_map + "/"],},},], } - project_file = os.path.join(project_dir, 'project.conf') + project_file = os.path.join(project_dir, "project.conf") _yaml.roundtrip_dump(project, project_file) - result = cli.run(project=project_dir, args=['source', 'fetch', element_name]) + result = cli.run(project=project_dir, args=["source", "fetch", element_name]) result.assert_success() @@ -383,90 +312,76 @@ def test_mirror_fallback_git_only_submodules(cli, tmpdir, datafiles): # - overriden submodule is fetched from mirror. # - other submodule is fetched. - bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr') - dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr') + bin_files_path = os.path.join(str(datafiles), "files", "bin-files", "usr") + dev_files_path = os.path.join(str(datafiles), "files", "dev-files", "usr") - upstream_bin_repodir = os.path.join(str(tmpdir), 'bin-upstream') - mirror_bin_repodir = os.path.join(str(tmpdir), 'bin-mirror') - upstream_bin_repo = create_repo('git', upstream_bin_repodir) + upstream_bin_repodir = os.path.join(str(tmpdir), "bin-upstream") + mirror_bin_repodir = os.path.join(str(tmpdir), "bin-mirror") + upstream_bin_repo = create_repo("git", upstream_bin_repodir) upstream_bin_repo.create(bin_files_path) mirror_bin_repo = upstream_bin_repo.copy(mirror_bin_repodir) - dev_repodir = os.path.join(str(tmpdir), 'dev-upstream') - dev_repo = create_repo('git', dev_repodir) + dev_repodir = os.path.join(str(tmpdir), "dev-upstream") + dev_repo = create_repo("git", dev_repodir) dev_repo.create(dev_files_path) - main_files = os.path.join(str(tmpdir), 'main-files') + main_files = os.path.join(str(tmpdir), "main-files") os.makedirs(main_files) - with open(os.path.join(main_files, 'README'), 'w') as f: + with open(os.path.join(main_files, "README"), "w") as f: f.write("TEST\n") - main_repodir = os.path.join(str(tmpdir), 'main-upstream') - main_repo = create_repo('git', main_repodir) + main_repodir = os.path.join(str(tmpdir), "main-upstream") + main_repo = create_repo("git", main_repodir) main_repo.create(main_files) - upstream_url = 'file://{}'.format(upstream_bin_repo.repo) - main_repo.add_submodule('bin', url=upstream_url) - main_repo.add_submodule('dev', url='file://{}'.format(dev_repo.repo)) + upstream_url = "file://{}".format(upstream_bin_repo.repo) + main_repo.add_submodule("bin", url=upstream_url) + main_repo.add_submodule("dev", url="file://{}".format(dev_repo.repo)) # Unlist 'dev'. - del main_repo.submodules['dev'] + del main_repo.submodules["dev"] main_ref = main_repo.latest_commit() upstream_map, repo_name = os.path.split(upstream_url) - alias = 'foo' - aliased_repo = '{}:{}'.format(alias, repo_name) - main_repo.submodules['bin']['url'] = aliased_repo + alias = "foo" + aliased_repo = "{}:{}".format(alias, repo_name) + main_repo.submodules["bin"]["url"] = aliased_repo - full_mirror = mirror_bin_repo.source_config()['url'] + full_mirror = mirror_bin_repo.source_config()["url"] mirror_map, _ = os.path.split(full_mirror) - project_dir = os.path.join(str(tmpdir), 'project') + project_dir = os.path.join(str(tmpdir), "project") os.makedirs(project_dir) - element_dir = os.path.join(project_dir, 'elements') + element_dir = os.path.join(project_dir, "elements") - element = { - 'kind': 'import', - 'sources': [ - main_repo.source_config_extra(ref=main_ref, checkout_submodules=True) - ] - } - element_name = 'test.bst' + element = {"kind": "import", "sources": [main_repo.source_config_extra(ref=main_ref, checkout_submodules=True)]} + element_name = "test.bst" element_path = os.path.join(element_dir, element_name) os.makedirs(element_dir) _yaml.roundtrip_dump(element, element_path) project = { - 'name': 'test', - 'element-path': 'elements', - 'aliases': { - alias: upstream_map + "/" - }, - 'mirrors': [ - { - 'name': 'middle-earth', - 'aliases': { - alias: [mirror_map + "/"], - } - } - ] + "name": "test", + "element-path": "elements", + "aliases": {alias: upstream_map + "/"}, + "mirrors": [{"name": "middle-earth", "aliases": {alias: [mirror_map + "/"],}}], } - project_file = os.path.join(project_dir, 'project.conf') + project_file = os.path.join(project_dir, "project.conf") _yaml.roundtrip_dump(project, project_file) # Now make the upstream unavailable. - os.rename(upstream_bin_repo.repo, '{}.bak'.format(upstream_bin_repo.repo)) - result = cli.run(project=project_dir, args=['source', 'fetch', element_name]) + os.rename(upstream_bin_repo.repo, "{}.bak".format(upstream_bin_repo.repo)) + result = cli.run(project=project_dir, args=["source", "fetch", element_name]) result.assert_success() - result = cli.run(project=project_dir, args=['build', element_name]) + result = cli.run(project=project_dir, args=["build", element_name]) result.assert_success() - checkout = os.path.join(str(tmpdir), 'checkout') - result = cli.run(project=project_dir, args=['artifact', 'checkout', element_name, '--directory', checkout]) + checkout = os.path.join(str(tmpdir), "checkout") + result = cli.run(project=project_dir, args=["artifact", "checkout", element_name, "--directory", checkout]) result.assert_success() - assert os.path.exists(os.path.join(checkout, 'bin', 'bin', 'hello')) - assert os.path.exists(os.path.join(checkout, 'dev', 'include', 'pony.h')) + assert os.path.exists(os.path.join(checkout, "bin", "bin", "hello")) + assert os.path.exists(os.path.join(checkout, "dev", "include", "pony.h")) @pytest.mark.datafiles(DATA_DIR) @@ -476,90 +391,79 @@ def test_mirror_fallback_git_with_submodules(cli, tmpdir, datafiles): # We expect: # - we will fetch submodules anyway - bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr') - dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr') + bin_files_path = os.path.join(str(datafiles), "files", "bin-files", "usr") + dev_files_path = os.path.join(str(datafiles), "files", "dev-files", "usr") - bin_repodir = os.path.join(str(tmpdir), 'bin-repo') - bin_repo = create_repo('git', bin_repodir) + bin_repodir = os.path.join(str(tmpdir), "bin-repo") + bin_repo = create_repo("git", bin_repodir) bin_repo.create(bin_files_path) - dev_repodir = os.path.join(str(tmpdir), 'dev-repo') - dev_repo = create_repo('git', dev_repodir) + dev_repodir = os.path.join(str(tmpdir), "dev-repo") + dev_repo = create_repo("git", dev_repodir) dev_repo.create(dev_files_path) - main_files = os.path.join(str(tmpdir), 'main-files') + main_files = os.path.join(str(tmpdir), "main-files") os.makedirs(main_files) - with open(os.path.join(main_files, 'README'), 'w') as f: + with open(os.path.join(main_files, "README"), "w") as f: f.write("TEST\n") - upstream_main_repodir = os.path.join(str(tmpdir), 'main-upstream') - upstream_main_repo = create_repo('git', upstream_main_repodir) + upstream_main_repodir = os.path.join(str(tmpdir), "main-upstream") + upstream_main_repo = create_repo("git", upstream_main_repodir) upstream_main_repo.create(main_files) - upstream_main_repo.add_submodule('bin', url='file://{}'.format(bin_repo.repo)) - upstream_main_repo.add_submodule('dev', url='file://{}'.format(dev_repo.repo)) + upstream_main_repo.add_submodule("bin", url="file://{}".format(bin_repo.repo)) + upstream_main_repo.add_submodule("dev", url="file://{}".format(dev_repo.repo)) # Unlist submodules. - del upstream_main_repo.submodules['bin'] - del upstream_main_repo.submodules['dev'] + del upstream_main_repo.submodules["bin"] + del upstream_main_repo.submodules["dev"] upstream_main_ref = upstream_main_repo.latest_commit() - mirror_main_repodir = os.path.join(str(tmpdir), 'main-mirror') + mirror_main_repodir = os.path.join(str(tmpdir), "main-mirror") mirror_main_repo = upstream_main_repo.copy(mirror_main_repodir) - upstream_url = mirror_main_repo.source_config()['url'] + upstream_url = mirror_main_repo.source_config()["url"] upstream_map, repo_name = os.path.split(upstream_url) - alias = 'foo' - aliased_repo = '{}:{}'.format(alias, repo_name) + alias = "foo" + aliased_repo = "{}:{}".format(alias, repo_name) - full_mirror = mirror_main_repo.source_config()['url'] + full_mirror = mirror_main_repo.source_config()["url"] mirror_map, _ = os.path.split(full_mirror) - project_dir = os.path.join(str(tmpdir), 'project') + project_dir = os.path.join(str(tmpdir), "project") os.makedirs(project_dir) - element_dir = os.path.join(project_dir, 'elements') + element_dir = os.path.join(project_dir, "elements") element = { - 'kind': 'import', - 'sources': [ - upstream_main_repo.source_config_extra(ref=upstream_main_ref, checkout_submodules=True) - ] + "kind": "import", + "sources": [upstream_main_repo.source_config_extra(ref=upstream_main_ref, checkout_submodules=True)], } - element['sources'][0]['url'] = aliased_repo - element_name = 'test.bst' + element["sources"][0]["url"] = aliased_repo + element_name = "test.bst" element_path = os.path.join(element_dir, element_name) os.makedirs(element_dir) _yaml.roundtrip_dump(element, element_path) project = { - 'name': 'test', - 'element-path': 'elements', - 'aliases': { - alias: upstream_map + "/" - }, - 'mirrors': [ - { - 'name': 'middle-earth', - 'aliases': { - alias: [mirror_map + "/"], - } - } - ] + "name": "test", + "element-path": "elements", + "aliases": {alias: upstream_map + "/"}, + "mirrors": [{"name": "middle-earth", "aliases": {alias: [mirror_map + "/"],}}], } - project_file = os.path.join(project_dir, 'project.conf') + project_file = os.path.join(project_dir, "project.conf") _yaml.roundtrip_dump(project, project_file) # Now make the upstream unavailable. - os.rename(upstream_main_repo.repo, '{}.bak'.format(upstream_main_repo.repo)) - result = cli.run(project=project_dir, args=['source', 'fetch', element_name]) + os.rename(upstream_main_repo.repo, "{}.bak".format(upstream_main_repo.repo)) + result = cli.run(project=project_dir, args=["source", "fetch", element_name]) result.assert_success() - result = cli.run(project=project_dir, args=['build', element_name]) + result = cli.run(project=project_dir, args=["build", element_name]) result.assert_success() - checkout = os.path.join(str(tmpdir), 'checkout') - result = cli.run(project=project_dir, args=['artifact', 'checkout', element_name, '--directory', checkout]) + checkout = os.path.join(str(tmpdir), "checkout") + result = cli.run(project=project_dir, args=["artifact", "checkout", element_name, "--directory", checkout]) result.assert_success() - assert os.path.exists(os.path.join(checkout, 'bin', 'bin', 'hello')) - assert os.path.exists(os.path.join(checkout, 'dev', 'include', 'pony.h')) + assert os.path.exists(os.path.join(checkout, "bin", "bin", "hello")) + assert os.path.exists(os.path.join(checkout, "dev", "include", "pony.h")) diff --git a/tests/frontend/order.py b/tests/frontend/order.py index c62377419..fbeb7c398 100644 --- a/tests/frontend/order.py +++ b/tests/frontend/order.py @@ -9,10 +9,7 @@ from buildstream.testing import cli # pylint: disable=unused-import from buildstream import _yaml # Project directory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project", -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",) # create_element() @@ -25,18 +22,12 @@ DATA_DIR = os.path.join( # Returns: # (Repo): The corresponding git repository created for the element def create_element(project, name, dependencies): - dev_files_path = os.path.join(project, 'files', 'dev-files') - element_path = os.path.join(project, 'elements') - repo = create_repo('git', project, "{}-repo".format(name)) + dev_files_path = os.path.join(project, "files", "dev-files") + element_path = os.path.join(project, "elements") + repo = create_repo("git", project, "{}-repo".format(name)) ref = repo.create(dev_files_path) - element = { - 'kind': 'import', - 'sources': [ - repo.source_config(ref=ref) - ], - 'depends': dependencies - } + element = {"kind": "import", "sources": [repo.source_config(ref=ref)], "depends": dependencies} _yaml.roundtrip_dump(element, os.path.join(element_path, name)) return repo @@ -56,37 +47,37 @@ def create_element(project, name, dependencies): # expected (list): A list of element names in the expected order # @pytest.mark.datafiles(os.path.join(DATA_DIR)) -@pytest.mark.parametrize("target,template,expected", [ - # First simple test - ('3.bst', { - '0.bst': ['1.bst'], - '1.bst': [], - '2.bst': ['0.bst'], - '3.bst': ['0.bst', '1.bst', '2.bst'] - }, ['1.bst', '0.bst', '2.bst', '3.bst']), - - # A more complicated test with build of build dependencies - ('target.bst', { - 'a.bst': [], - 'base.bst': [], - 'timezones.bst': [], - 'middleware.bst': [{'filename': 'base.bst', 'type': 'build'}], - 'app.bst': [{'filename': 'middleware.bst', 'type': 'build'}], - 'target.bst': ['a.bst', 'base.bst', 'middleware.bst', 'app.bst', 'timezones.bst'] - }, ['base.bst', 'middleware.bst', 'a.bst', 'app.bst', 'timezones.bst', 'target.bst']), -]) -@pytest.mark.parametrize("operation", [('show'), ('fetch'), ('build')]) +@pytest.mark.parametrize( + "target,template,expected", + [ + # First simple test + ( + "3.bst", + {"0.bst": ["1.bst"], "1.bst": [], "2.bst": ["0.bst"], "3.bst": ["0.bst", "1.bst", "2.bst"]}, + ["1.bst", "0.bst", "2.bst", "3.bst"], + ), + # A more complicated test with build of build dependencies + ( + "target.bst", + { + "a.bst": [], + "base.bst": [], + "timezones.bst": [], + "middleware.bst": [{"filename": "base.bst", "type": "build"}], + "app.bst": [{"filename": "middleware.bst", "type": "build"}], + "target.bst": ["a.bst", "base.bst", "middleware.bst", "app.bst", "timezones.bst"], + }, + ["base.bst", "middleware.bst", "a.bst", "app.bst", "timezones.bst", "target.bst"], + ), + ], +) +@pytest.mark.parametrize("operation", [("show"), ("fetch"), ("build")]) def test_order(cli, datafiles, operation, target, template, expected): project = str(datafiles) # Configure to only allow one fetcher at a time, make it easy to # determine what is being planned in what order. - cli.configure({ - 'scheduler': { - 'fetchers': 1, - 'builders': 1 - } - }) + cli.configure({"scheduler": {"fetchers": 1, "builders": 1}}) # Build the project from the template, make import elements # all with the same repo @@ -95,13 +86,13 @@ def test_order(cli, datafiles, operation, target, template, expected): create_element(project, element, dependencies) # Run test and collect results - if operation == 'show': - result = cli.run(args=['show', '--deps', 'plan', '--format', '%{name}', target], project=project, silent=True) + if operation == "show": + result = cli.run(args=["show", "--deps", "plan", "--format", "%{name}", target], project=project, silent=True) result.assert_success() results = result.output.splitlines() else: - if operation == 'fetch': - result = cli.run(args=['source', 'fetch', target], project=project, silent=True) + if operation == "fetch": + result = cli.run(args=["source", "fetch", target], project=project, silent=True) else: result = cli.run(args=[operation, target], project=project, silent=True) result.assert_success() diff --git a/tests/frontend/overlaps.py b/tests/frontend/overlaps.py index eb2cd4a86..e1c5a7c94 100644 --- a/tests/frontend/overlaps.py +++ b/tests/frontend/overlaps.py @@ -10,16 +10,11 @@ from buildstream.plugin import CoreWarnings from tests.testutils import generate_junction # Project directory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "overlaps" -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "overlaps") def gen_project(project_dir, fail_on_overlap, use_fatal_warnings=True, project_name="test"): - template = { - "name": project_name - } + template = {"name": project_name} if use_fatal_warnings: template["fatal-warnings"] = [CoreWarnings.OVERLAPS] if fail_on_overlap else [] else: @@ -33,8 +28,7 @@ def gen_project(project_dir, fail_on_overlap, use_fatal_warnings=True, project_n def test_overlaps(cli, datafiles, use_fatal_warnings): project_dir = str(datafiles) gen_project(project_dir, False, use_fatal_warnings) - result = cli.run(project=project_dir, silent=True, args=[ - 'build', 'collect.bst']) + result = cli.run(project=project_dir, silent=True, args=["build", "collect.bst"]) result.assert_success() @@ -43,8 +37,7 @@ def test_overlaps(cli, datafiles, use_fatal_warnings): def test_overlaps_error(cli, datafiles, use_fatal_warnings): project_dir = str(datafiles) gen_project(project_dir, True, use_fatal_warnings) - result = cli.run(project=project_dir, silent=True, args=[ - 'build', 'collect.bst']) + result = cli.run(project=project_dir, silent=True, args=["build", "collect.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.OVERLAPS) @@ -53,8 +46,7 @@ def test_overlaps_error(cli, datafiles, use_fatal_warnings): def test_overlaps_whitelist(cli, datafiles): project_dir = str(datafiles) gen_project(project_dir, True) - result = cli.run(project=project_dir, silent=True, args=[ - 'build', 'collect-whitelisted.bst']) + result = cli.run(project=project_dir, silent=True, args=["build", "collect-whitelisted.bst"]) result.assert_success() @@ -62,8 +54,7 @@ def test_overlaps_whitelist(cli, datafiles): def test_overlaps_whitelist_ignored(cli, datafiles): project_dir = str(datafiles) gen_project(project_dir, False) - result = cli.run(project=project_dir, silent=True, args=[ - 'build', 'collect-whitelisted.bst']) + result = cli.run(project=project_dir, silent=True, args=["build", "collect-whitelisted.bst"]) result.assert_success() @@ -74,8 +65,7 @@ def test_overlaps_whitelist_on_overlapper(cli, datafiles): # it'll still fail because A doesn't permit overlaps. project_dir = str(datafiles) gen_project(project_dir, True) - result = cli.run(project=project_dir, silent=True, args=[ - 'build', 'collect-partially-whitelisted.bst']) + result = cli.run(project=project_dir, silent=True, args=["build", "collect-partially-whitelisted.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.OVERLAPS) @@ -87,21 +77,20 @@ def test_overlaps_script(cli, datafiles, use_fatal_warnings): # Element.stage_dependency_artifacts() with Scope.RUN project_dir = str(datafiles) gen_project(project_dir, False, use_fatal_warnings) - result = cli.run(project=project_dir, silent=True, args=[ - 'build', 'script.bst']) + result = cli.run(project=project_dir, silent=True, args=["build", "script.bst"]) result.assert_success() @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("project_policy", [('fail'), ('warn')]) -@pytest.mark.parametrize("subproject_policy", [('fail'), ('warn')]) +@pytest.mark.parametrize("project_policy", [("fail"), ("warn")]) +@pytest.mark.parametrize("subproject_policy", [("fail"), ("warn")]) def test_overlap_subproject(cli, tmpdir, datafiles, project_policy, subproject_policy): project_dir = str(datafiles) - subproject_dir = os.path.join(project_dir, 'sub-project') - junction_path = os.path.join(project_dir, 'sub-project.bst') + subproject_dir = os.path.join(project_dir, "sub-project") + junction_path = os.path.join(project_dir, "sub-project.bst") - gen_project(project_dir, bool(project_policy == 'fail'), project_name='test') - gen_project(subproject_dir, bool(subproject_policy == 'fail'), project_name='subtest') + gen_project(project_dir, bool(project_policy == "fail"), project_name="test") + gen_project(subproject_dir, bool(subproject_policy == "fail"), project_name="subtest") generate_junction(tmpdir, subproject_dir, junction_path) # Here we have a dependency chain where the project element @@ -110,8 +99,8 @@ def test_overlap_subproject(cli, tmpdir, datafiles, project_policy, subproject_p # Test that overlap error vs warning policy for this overlap # is always controlled by the project and not the subproject. # - result = cli.run(project=project_dir, silent=True, args=['build', 'sub-collect.bst']) - if project_policy == 'fail': + result = cli.run(project=project_dir, silent=True, args=["build", "sub-collect.bst"]) + if project_policy == "fail": result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.OVERLAPS) else: diff --git a/tests/frontend/progress.py b/tests/frontend/progress.py index e3b127f3b..45cba0b50 100644 --- a/tests/frontend/progress.py +++ b/tests/frontend/progress.py @@ -11,125 +11,97 @@ from buildstream._exceptions import ErrorDomain, LoadErrorReason from tests.testutils import generate_junction # Project directory -DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), ) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)),) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) def test_show_progress_tally(cli, datafiles): # Check that the progress reporting messages give correct tallies project = str(datafiles) - result = cli.run(project=project, args=['show', 'compose-all.bst']) + result = cli.run(project=project, args=["show", "compose-all.bst"]) result.assert_success() assert " 3 subtasks processed" in result.stderr assert "3 of 3 subtasks processed" in result.stderr -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) def test_junction_tally(cli, tmpdir, datafiles): # Check that the progress reporting messages count elements in junctions project = str(datafiles) - subproject_path = os.path.join(project, 'files', 'sub-project') - junction_path = os.path.join(project, 'elements', 'junction.bst') - element_path = os.path.join(project, 'elements', 'junction-dep.bst') + subproject_path = os.path.join(project, "files", "sub-project") + junction_path = os.path.join(project, "elements", "junction.bst") + element_path = os.path.join(project, "elements", "junction-dep.bst") # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path, store_ref=True) # Create a stack element to depend on a cross junction element # - element = { - 'kind': 'stack', - 'depends': [{ - 'junction': 'junction.bst', - 'filename': 'import-etc.bst' - }] - } + element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) - result = cli.run(project=project, - silent=True, - args=['source', 'fetch', 'junction.bst']) + result = cli.run(project=project, silent=True, args=["source", "fetch", "junction.bst"]) result.assert_success() # Assert the correct progress tallies are in the logging - result = cli.run(project=project, args=['show', 'junction-dep.bst']) + result = cli.run(project=project, args=["show", "junction-dep.bst"]) assert " 2 subtasks processed" in result.stderr assert "2 of 2 subtasks processed" in result.stderr -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) def test_nested_junction_tally(cli, tmpdir, datafiles): # Check that the progress reporting messages count elements in # junctions of junctions project = str(datafiles) - sub1_path = os.path.join(project, 'files', 'sub-project') - sub2_path = os.path.join(project, 'files', 'sub2-project') + sub1_path = os.path.join(project, "files", "sub-project") + sub2_path = os.path.join(project, "files", "sub2-project") # A junction element which pulls sub1 into sub2 - sub1_element = os.path.join(project, 'files', 'sub2-project', 'elements', 'sub-junction.bst') + sub1_element = os.path.join(project, "files", "sub2-project", "elements", "sub-junction.bst") # A junction element which pulls sub2 into the main project - sub2_element = os.path.join(project, 'elements', 'junction.bst') - element_path = os.path.join(project, 'elements', 'junction-dep.bst') + sub2_element = os.path.join(project, "elements", "junction.bst") + element_path = os.path.join(project, "elements", "junction-dep.bst") generate_junction(tmpdir / "sub-project", sub1_path, sub1_element, store_ref=True) generate_junction(tmpdir / "sub2-project", sub2_path, sub2_element, store_ref=True) # Create a stack element to depend on a cross junction element # - element = { - 'kind': 'stack', - 'depends': [{ - 'junction': 'junction.bst', - 'filename': 'import-sub.bst' - }] - } + element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-sub.bst"}]} _yaml.roundtrip_dump(element, element_path) - result = cli.run(project=project, - silent=True, - args=['source', 'fetch', 'junction.bst']) + result = cli.run(project=project, silent=True, args=["source", "fetch", "junction.bst"]) result.assert_success() # Assert the correct progress tallies are in the logging - result = cli.run(project=project, args=['show', 'junction-dep.bst']) + result = cli.run(project=project, args=["show", "junction-dep.bst"]) assert " 3 subtasks processed" in result.stderr assert "3 of 3 subtasks processed" in result.stderr -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) def test_junction_dep_tally(cli, tmpdir, datafiles): # Check that the progress reporting messages count elements in junctions project = str(datafiles) - subproject_path = os.path.join(project, 'files', 'sub-project') - junction_path = os.path.join(project, 'elements', 'junction.bst') - element_path = os.path.join(project, 'elements', 'junction-dep.bst') + subproject_path = os.path.join(project, "files", "sub-project") + junction_path = os.path.join(project, "elements", "junction.bst") + element_path = os.path.join(project, "elements", "junction-dep.bst") # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path, store_ref=True) # Add dependencies to the junction (not allowed, but let's do it # anyway) - with open(junction_path, 'a') as f: - deps = { - 'depends': [ - 'manual.bst' - ] - } + with open(junction_path, "a") as f: + deps = {"depends": ["manual.bst"]} _yaml.roundtrip_dump(deps, f) # Create a stack element to depend on a cross junction element # - element = { - 'kind': 'stack', - 'depends': [{ - 'junction': 'junction.bst', - 'filename': 'import-etc.bst' - }] - } + element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) - result = cli.run(project=project, - silent=True, - args=['source', 'fetch', 'junction-dep.bst']) + result = cli.run(project=project, silent=True, args=["source", "fetch", "junction-dep.bst"]) # Since we aren't allowed to specify any dependencies on a # junction, we should fail diff --git a/tests/frontend/project/sources/fetch_source.py b/tests/frontend/project/sources/fetch_source.py index ac3020ec2..51bfe1049 100644 --- a/tests/frontend/project/sources/fetch_source.py +++ b/tests/frontend/project/sources/fetch_source.py @@ -22,14 +22,10 @@ class FetchFetcher(SourceFetcher): self.mark_download_url(url) def fetch(self, alias_override=None): - url = self.source.translate_url(self.original_url, - alias_override=alias_override, - primary=self.primary) + url = self.source.translate_url(self.original_url, alias_override=alias_override, primary=self.primary) with open(self.source.output_file, "a") as f: success = url in self.source.fetch_succeeds and self.source.fetch_succeeds[url] - message = "Fetch {} {} from {}\n".format(self.original_url, - "succeeded" if success else "failed", - url) + message = "Fetch {} {} from {}\n".format(self.original_url, "succeeded" if success else "failed", url) f.write(message) if not success: raise SourceError("Failed to fetch {}".format(url)) @@ -38,12 +34,9 @@ class FetchFetcher(SourceFetcher): class FetchSource(Source): # Read config to know which URLs to fetch def configure(self, node): - self.original_urls = node.get_str_list('urls') - self.output_file = node.get_str('output-text') - self.fetch_succeeds = { - key: value.as_bool() - for key, value in node.get_mapping('fetch-succeeds', {}).items() - } + self.original_urls = node.get_str_list("urls") + self.output_file = node.get_str("output-text") + self.fetch_succeeds = {key: value.as_bool() for key, value in node.get_mapping("fetch-succeeds", {}).items()} # First URL is the primary one for this test # diff --git a/tests/frontend/pull.py b/tests/frontend/pull.py index 234f1133d..3e726ffcb 100644 --- a/tests/frontend/pull.py +++ b/tests/frontend/pull.py @@ -12,10 +12,7 @@ from tests.testutils import create_artifact_share, generate_junction, assert_sha # Project directory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project", -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",) # Tests that: @@ -27,40 +24,38 @@ DATA_DIR = os.path.join( def test_push_pull_all(cli, tmpdir, datafiles): project = str(datafiles) - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # First build the target element and push to the remote. - cli.configure({ - 'artifacts': {'url': share.repo, 'push': True} - }) - result = cli.run(project=project, args=['build', 'target.bst']) + cli.configure({"artifacts": {"url": share.repo, "push": True}}) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - assert cli.get_element_state(project, 'target.bst') == 'cached' + assert cli.get_element_state(project, "target.bst") == "cached" # Assert that everything is now cached in the remote. - all_elements = ['target.bst', 'import-bin.bst', 'import-dev.bst', 'compose-all.bst'] + all_elements = ["target.bst", "import-bin.bst", "import-dev.bst", "compose-all.bst"] for element_name in all_elements: assert_shared(cli, share, project, element_name) # Now we've pushed, delete the user's local artifact cache # directory and try to redownload it from the share # - casdir = os.path.join(cli.directory, 'cas') + casdir = os.path.join(cli.directory, "cas") shutil.rmtree(casdir) - artifactdir = os.path.join(cli.directory, 'artifacts') + artifactdir = os.path.join(cli.directory, "artifacts") shutil.rmtree(artifactdir) # Assert that nothing is cached locally anymore states = cli.get_element_states(project, all_elements) - assert not any(states[e] == 'cached' for e in all_elements) + assert not any(states[e] == "cached" for e in all_elements) # Now try bst artifact pull - result = cli.run(project=project, args=['artifact', 'pull', '--deps', 'all', 'target.bst']) + result = cli.run(project=project, args=["artifact", "pull", "--deps", "all", "target.bst"]) result.assert_success() # And assert that it's again in the local cache, without having built states = cli.get_element_states(project, all_elements) - assert not any(states[e] != 'cached' for e in all_elements) + assert not any(states[e] != "cached" for e in all_elements) # Tests that: @@ -68,51 +63,47 @@ def test_push_pull_all(cli, tmpdir, datafiles): # * `bst artifact push` (default targets) pushes all built elements to configured 'push' cache # * `bst artifact pull` (default targets) downloads everything from cache after local deletion # -@pytest.mark.datafiles(DATA_DIR + '_world') +@pytest.mark.datafiles(DATA_DIR + "_world") def test_push_pull_default_targets(cli, tmpdir, datafiles): project = str(datafiles) - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # First build the target elements - cli.configure({ - 'artifacts': {'url': share.repo} - }) - result = cli.run(project=project, args=['build']) + cli.configure({"artifacts": {"url": share.repo}}) + result = cli.run(project=project, args=["build"]) result.assert_success() - assert cli.get_element_state(project, 'target.bst') == 'cached' + assert cli.get_element_state(project, "target.bst") == "cached" # Push all elements - cli.configure({ - 'artifacts': {'url': share.repo, 'push': True} - }) - result = cli.run(project=project, args=['artifact', 'push']) + cli.configure({"artifacts": {"url": share.repo, "push": True}}) + result = cli.run(project=project, args=["artifact", "push"]) result.assert_success() # Assert that everything is now cached in the remote. - all_elements = ['target.bst', 'import-bin.bst', 'import-dev.bst', 'compose-all.bst'] + all_elements = ["target.bst", "import-bin.bst", "import-dev.bst", "compose-all.bst"] for element_name in all_elements: assert_shared(cli, share, project, element_name) # Now we've pushed, delete the user's local artifact cache # directory and try to redownload it from the share # - casdir = os.path.join(cli.directory, 'cas') + casdir = os.path.join(cli.directory, "cas") shutil.rmtree(casdir) - artifactdir = os.path.join(cli.directory, 'artifacts') + artifactdir = os.path.join(cli.directory, "artifacts") shutil.rmtree(artifactdir) # Assert that nothing is cached locally anymore states = cli.get_element_states(project, all_elements) - assert not any(states[e] == 'cached' for e in all_elements) + assert not any(states[e] == "cached" for e in all_elements) # Now try bst artifact pull - result = cli.run(project=project, args=['artifact', 'pull']) + result = cli.run(project=project, args=["artifact", "pull"]) result.assert_success() # And assert that it's again in the local cache, without having built states = cli.get_element_states(project, all_elements) - assert not any(states[e] != 'cached' for e in all_elements) + assert not any(states[e] != "cached" for e in all_elements) # Tests that: @@ -124,38 +115,34 @@ def test_push_pull_default_targets(cli, tmpdir, datafiles): def test_pull_secondary_cache(cli, tmpdir, datafiles): project = str(datafiles) - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare1')) as share1,\ - create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) as share2: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare1")) as share1, create_artifact_share( + os.path.join(str(tmpdir), "artifactshare2") + ) as share2: # Build the target and push it to share2 only. - cli.configure({ - 'artifacts': [ - {'url': share1.repo, 'push': False}, - {'url': share2.repo, 'push': True}, - ] - }) - result = cli.run(project=project, args=['build', 'target.bst']) + cli.configure({"artifacts": [{"url": share1.repo, "push": False}, {"url": share2.repo, "push": True},]}) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - assert_not_shared(cli, share1, project, 'target.bst') - assert_shared(cli, share2, project, 'target.bst') + assert_not_shared(cli, share1, project, "target.bst") + assert_shared(cli, share2, project, "target.bst") # Delete the user's local artifact cache. - casdir = os.path.join(cli.directory, 'cas') + casdir = os.path.join(cli.directory, "cas") shutil.rmtree(casdir) - artifactdir = os.path.join(cli.directory, 'artifacts') + artifactdir = os.path.join(cli.directory, "artifacts") shutil.rmtree(artifactdir) # Assert that the element is not cached anymore. - assert cli.get_element_state(project, 'target.bst') != 'cached' + assert cli.get_element_state(project, "target.bst") != "cached" # Now try bst artifact pull - result = cli.run(project=project, args=['artifact', 'pull', 'target.bst']) + result = cli.run(project=project, args=["artifact", "pull", "target.bst"]) result.assert_success() # And assert that it's again in the local cache, without having built, # i.e. we found it in share2. - assert cli.get_element_state(project, 'target.bst') == 'cached' + assert cli.get_element_state(project, "target.bst") == "cached" # Tests that: @@ -167,47 +154,45 @@ def test_pull_secondary_cache(cli, tmpdir, datafiles): def test_push_pull_specific_remote(cli, tmpdir, datafiles): project = str(datafiles) - with create_artifact_share(os.path.join(str(tmpdir), 'goodartifactshare')) as good_share,\ - create_artifact_share(os.path.join(str(tmpdir), 'badartifactshare')) as bad_share: + with create_artifact_share(os.path.join(str(tmpdir), "goodartifactshare")) as good_share, create_artifact_share( + os.path.join(str(tmpdir), "badartifactshare") + ) as bad_share: # Build the target so we have it cached locally only. - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - state = cli.get_element_state(project, 'target.bst') - assert state == 'cached' + state = cli.get_element_state(project, "target.bst") + assert state == "cached" # Configure the default push location to be bad_share; we will assert that # nothing actually gets pushed there. - cli.configure({ - 'artifacts': {'url': bad_share.repo, 'push': True}, - }) + cli.configure( + {"artifacts": {"url": bad_share.repo, "push": True},} + ) # Now try `bst artifact push` to the good_share. - result = cli.run(project=project, args=[ - 'artifact', 'push', 'target.bst', '--remote', good_share.repo - ]) + result = cli.run(project=project, args=["artifact", "push", "target.bst", "--remote", good_share.repo]) result.assert_success() # Assert that all the artifacts are in the share we pushed # to, and not the other. - assert_shared(cli, good_share, project, 'target.bst') - assert_not_shared(cli, bad_share, project, 'target.bst') + assert_shared(cli, good_share, project, "target.bst") + assert_not_shared(cli, bad_share, project, "target.bst") # Now we've pushed, delete the user's local artifact cache # directory and try to redownload it from the good_share. # - casdir = os.path.join(cli.directory, 'cas') + casdir = os.path.join(cli.directory, "cas") shutil.rmtree(casdir) - artifactdir = os.path.join(cli.directory, 'artifacts') + artifactdir = os.path.join(cli.directory, "artifacts") shutil.rmtree(artifactdir) - result = cli.run(project=project, args=['artifact', 'pull', 'target.bst', '--remote', - good_share.repo]) + result = cli.run(project=project, args=["artifact", "pull", "target.bst", "--remote", good_share.repo]) result.assert_success() # And assert that it's again in the local cache, without having built - assert cli.get_element_state(project, 'target.bst') == 'cached' + assert cli.get_element_state(project, "target.bst") == "cached" # Tests that: @@ -218,123 +203,114 @@ def test_push_pull_specific_remote(cli, tmpdir, datafiles): def test_push_pull_non_strict(cli, tmpdir, datafiles): project = str(datafiles) - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # First build the target element and push to the remote. - cli.configure({ - 'artifacts': {'url': share.repo, 'push': True}, - 'projects': { - 'test': {'strict': False} - } - }) - result = cli.run(project=project, args=['build', 'target.bst']) + cli.configure({"artifacts": {"url": share.repo, "push": True}, "projects": {"test": {"strict": False}}}) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - assert cli.get_element_state(project, 'target.bst') == 'cached' + assert cli.get_element_state(project, "target.bst") == "cached" # Assert that everything is now cached in the remote. - all_elements = ['target.bst', 'import-bin.bst', 'import-dev.bst', 'compose-all.bst'] + all_elements = ["target.bst", "import-bin.bst", "import-dev.bst", "compose-all.bst"] for element_name in all_elements: assert_shared(cli, share, project, element_name) # Now we've pushed, delete the user's local artifact cache # directory and try to redownload it from the share # - casdir = os.path.join(cli.directory, 'cas') + casdir = os.path.join(cli.directory, "cas") shutil.rmtree(casdir) - artifactdir = os.path.join(cli.directory, 'artifacts') + artifactdir = os.path.join(cli.directory, "artifacts") shutil.rmtree(artifactdir) # Assert that nothing is cached locally anymore for element_name in all_elements: - assert cli.get_element_state(project, element_name) != 'cached' + assert cli.get_element_state(project, element_name) != "cached" # Add a file to force change in strict cache key of import-bin.bst - with open(os.path.join(str(project), 'files', 'bin-files', 'usr', 'bin', 'world'), 'w') as f: - f.write('world') + with open(os.path.join(str(project), "files", "bin-files", "usr", "bin", "world"), "w") as f: + f.write("world") # Assert that the workspaced element requires a rebuild - assert cli.get_element_state(project, 'import-bin.bst') == 'buildable' + assert cli.get_element_state(project, "import-bin.bst") == "buildable" # Assert that the target is still waiting due to --no-strict - assert cli.get_element_state(project, 'target.bst') == 'waiting' + assert cli.get_element_state(project, "target.bst") == "waiting" # Now try bst artifact pull - result = cli.run(project=project, args=['artifact', 'pull', '--deps', 'all', 'target.bst']) + result = cli.run(project=project, args=["artifact", "pull", "--deps", "all", "target.bst"]) result.assert_success() # And assert that the target is again in the local cache, without having built - assert cli.get_element_state(project, 'target.bst') == 'cached' + assert cli.get_element_state(project, "target.bst") == "cached" @pytest.mark.datafiles(DATA_DIR) def test_push_pull_cross_junction(cli, tmpdir, datafiles): project = str(datafiles) - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: - subproject_path = os.path.join(project, 'files', 'sub-project') - junction_path = os.path.join(project, 'elements', 'junction.bst') + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: + subproject_path = os.path.join(project, "files", "sub-project") + junction_path = os.path.join(project, "elements", "junction.bst") generate_junction(tmpdir, subproject_path, junction_path, store_ref=True) # First build the target element and push to the remote. - cli.configure({ - 'artifacts': {'url': share.repo, 'push': True} - }) - result = cli.run(project=project, args=['build', 'junction.bst:import-etc.bst']) + cli.configure({"artifacts": {"url": share.repo, "push": True}}) + result = cli.run(project=project, args=["build", "junction.bst:import-etc.bst"]) result.assert_success() - assert cli.get_element_state(project, 'junction.bst:import-etc.bst') == 'cached' + assert cli.get_element_state(project, "junction.bst:import-etc.bst") == "cached" - cache_dir = os.path.join(project, 'cache', 'cas') + cache_dir = os.path.join(project, "cache", "cas") shutil.rmtree(cache_dir) - artifact_dir = os.path.join(project, 'cache', 'artifacts') + artifact_dir = os.path.join(project, "cache", "artifacts") shutil.rmtree(artifact_dir) - assert cli.get_element_state(project, 'junction.bst:import-etc.bst') == 'buildable' + assert cli.get_element_state(project, "junction.bst:import-etc.bst") == "buildable" # Now try bst artifact pull - result = cli.run(project=project, args=['artifact', 'pull', 'junction.bst:import-etc.bst']) + result = cli.run(project=project, args=["artifact", "pull", "junction.bst:import-etc.bst"]) result.assert_success() # And assert that it's again in the local cache, without having built - assert cli.get_element_state(project, 'junction.bst:import-etc.bst') == 'cached' + assert cli.get_element_state(project, "junction.bst:import-etc.bst") == "cached" @pytest.mark.datafiles(DATA_DIR) def test_pull_missing_blob(cli, tmpdir, datafiles): project = str(datafiles) - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # First build the target element and push to the remote. - cli.configure({ - 'artifacts': {'url': share.repo, 'push': True} - }) - result = cli.run(project=project, args=['build', 'target.bst']) + cli.configure({"artifacts": {"url": share.repo, "push": True}}) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - assert cli.get_element_state(project, 'target.bst') == 'cached' + assert cli.get_element_state(project, "target.bst") == "cached" # Assert that everything is now cached in the remote. - all_elements = ['target.bst', 'import-bin.bst', 'import-dev.bst', 'compose-all.bst'] + all_elements = ["target.bst", "import-bin.bst", "import-dev.bst", "compose-all.bst"] for element_name in all_elements: assert_shared(cli, share, project, element_name) # Now we've pushed, delete the user's local artifact cache # directory and try to redownload it from the share # - casdir = os.path.join(cli.directory, 'cas') + casdir = os.path.join(cli.directory, "cas") shutil.rmtree(casdir) - artifactdir = os.path.join(cli.directory, 'artifacts') + artifactdir = os.path.join(cli.directory, "artifacts") shutil.rmtree(artifactdir) # Assert that nothing is cached locally anymore for element_name in all_elements: - assert cli.get_element_state(project, element_name) != 'cached' + assert cli.get_element_state(project, element_name) != "cached" # Now delete blobs in the remote without deleting the artifact ref. # This simulates scenarios with concurrent artifact expiry. - remote_objdir = os.path.join(share.repodir, 'cas', 'objects') + remote_objdir = os.path.join(share.repodir, "cas", "objects") shutil.rmtree(remote_objdir) # Now try bst build - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() # Assert that no artifacts were pulled @@ -344,9 +320,9 @@ def test_pull_missing_blob(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_pull_missing_local_blob(cli, tmpdir, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename) - repo = create_repo('git', str(tmpdir)) + repo = create_repo("git", str(tmpdir)) repo.create(os.path.join(str(datafiles), "files")) - element_dir = os.path.join(str(tmpdir), 'elements') + element_dir = os.path.join(str(tmpdir), "elements") project = str(tmpdir) project_config = { "name": "pull-missing-local-blob", @@ -358,43 +334,36 @@ def test_pull_missing_local_blob(cli, tmpdir, datafiles): "kind": "import", "sources": [repo.source_config()], } - input_name = 'input.bst' + input_name = "input.bst" input_file = os.path.join(element_dir, input_name) _yaml.roundtrip_dump(input_config, input_file) - depends_name = 'depends.bst' - depends_config = { - "kind": "stack", - "depends": [ - {"filename": input_name, "type": "build"} - ] - } + depends_name = "depends.bst" + depends_config = {"kind": "stack", "depends": [{"filename": input_name, "type": "build"}]} depends_file = os.path.join(element_dir, depends_name) _yaml.roundtrip_dump(depends_config, depends_file) - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # First build the import-bin element and push to the remote. - cli.configure({ - 'artifacts': {'url': share.repo, 'push': True} - }) + cli.configure({"artifacts": {"url": share.repo, "push": True}}) - result = cli.run(project=project, args=['source', 'track', input_name]) + result = cli.run(project=project, args=["source", "track", input_name]) result.assert_success() - result = cli.run(project=project, args=['build', input_name]) + result = cli.run(project=project, args=["build", input_name]) result.assert_success() - assert cli.get_element_state(project, input_name) == 'cached' + assert cli.get_element_state(project, input_name) == "cached" # Delete a file blob from the local cache. # This is a placeholder to test partial CAS handling until we support # partial artifact pulling (or blob-based CAS expiry). # - digest = utils.sha256sum(os.path.join(project, 'files', 'bin-files', 'usr', 'bin', 'hello')) - objpath = os.path.join(cli.directory, 'cas', 'objects', digest[:2], digest[2:]) + digest = utils.sha256sum(os.path.join(project, "files", "bin-files", "usr", "bin", "hello")) + objpath = os.path.join(cli.directory, "cas", "objects", digest[:2], digest[2:]) os.unlink(objpath) # Now try bst build - result = cli.run(project=project, args=['build', depends_name]) + result = cli.run(project=project, args=["build", depends_name]) result.assert_success() # Assert that the import-bin artifact was pulled (completing the partial artifact) @@ -406,16 +375,13 @@ def test_pull_missing_notifies_user(caplog, cli, tmpdir, datafiles): project = str(datafiles) caplog.set_level(1) - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: - cli.configure({ - 'artifacts': {'url': share.repo} - }) - result = cli.run(project=project, args=['build', 'target.bst']) + cli.configure({"artifacts": {"url": share.repo}}) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - assert not result.get_pulled_elements(), \ - "No elements should have been pulled since the cache was empty" + assert not result.get_pulled_elements(), "No elements should have been pulled since the cache was empty" assert "INFO Remote ({}) does not have".format(share.repo) in result.stderr assert "SKIPPED Pull" in result.stderr @@ -426,25 +392,23 @@ def test_build_remote_option(caplog, cli, tmpdir, datafiles): project = str(datafiles) caplog.set_level(1) - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare1')) as shareuser,\ - create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) as shareproject,\ - create_artifact_share(os.path.join(str(tmpdir), 'artifactshare3')) as sharecli: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare1")) as shareuser, create_artifact_share( + os.path.join(str(tmpdir), "artifactshare2") + ) as shareproject, create_artifact_share(os.path.join(str(tmpdir), "artifactshare3")) as sharecli: # Add shareproject repo url to project.conf with open(os.path.join(project, "project.conf"), "a") as projconf: projconf.write("artifacts:\n url: {}\n push: True".format(shareproject.repo)) # Configure shareuser remote in user conf - cli.configure({ - 'artifacts': {'url': shareuser.repo, 'push': True} - }) + cli.configure({"artifacts": {"url": shareuser.repo, "push": True}}) # Push the artifacts to the shareuser and shareproject remotes. # Assert that shareuser and shareproject have the artfifacts cached, # but sharecli doesn't, then delete locally cached elements - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - all_elements = ['target.bst', 'import-bin.bst', 'compose-all.bst'] + all_elements = ["target.bst", "import-bin.bst", "compose-all.bst"] for element_name in all_elements: assert element_name in result.get_pushed_elements() assert_not_shared(cli, sharecli, project, element_name) @@ -455,7 +419,7 @@ def test_build_remote_option(caplog, cli, tmpdir, datafiles): # Now check that a build with cli set as sharecli results in nothing being pulled, # as it doesn't have them cached and shareuser/shareproject should be ignored. This # will however result in the artifacts being built and pushed to it - result = cli.run(project=project, args=['build', '--remote', sharecli.repo, 'target.bst']) + result = cli.run(project=project, args=["build", "--remote", sharecli.repo, "target.bst"]) result.assert_success() for element_name in all_elements: assert element_name not in result.get_pulled_elements() @@ -464,10 +428,10 @@ def test_build_remote_option(caplog, cli, tmpdir, datafiles): # Now check that a clean build with cli set as sharecli should result in artifacts only # being pulled from it, as that was provided via the cli and is populated - result = cli.run(project=project, args=['build', '--remote', sharecli.repo, 'target.bst']) + result = cli.run(project=project, args=["build", "--remote", sharecli.repo, "target.bst"]) result.assert_success() for element_name in all_elements: - assert cli.get_element_state(project, element_name) == 'cached' + assert cli.get_element_state(project, element_name) == "cached" assert element_name in result.get_pulled_elements() assert shareproject.repo not in result.stderr assert shareuser.repo not in result.stderr @@ -477,70 +441,66 @@ def test_build_remote_option(caplog, cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_pull_access_rights(cli, tmpdir, datafiles): project = str(datafiles) - checkout = os.path.join(str(tmpdir), 'checkout') + checkout = os.path.join(str(tmpdir), "checkout") # Work-around datafiles not preserving mode - os.chmod(os.path.join(project, 'files/bin-files/usr/bin/hello'), 0o0755) + os.chmod(os.path.join(project, "files/bin-files/usr/bin/hello"), 0o0755) # We need a big file that does not go into a batch to test a different # code path - os.makedirs(os.path.join(project, 'files/dev-files/usr/share'), exist_ok=True) - with open(os.path.join(project, 'files/dev-files/usr/share/big-file'), 'w') as f: - buf = ' ' * 4096 + os.makedirs(os.path.join(project, "files/dev-files/usr/share"), exist_ok=True) + with open(os.path.join(project, "files/dev-files/usr/share/big-file"), "w") as f: + buf = " " * 4096 for _ in range(1024): f.write(buf) - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: - cli.configure({ - 'artifacts': {'url': share.repo, 'push': True} - }) - result = cli.run(project=project, args=['build', 'compose-all.bst']) + cli.configure({"artifacts": {"url": share.repo, "push": True}}) + result = cli.run(project=project, args=["build", "compose-all.bst"]) result.assert_success() - result = cli.run(project=project, - args=['artifact', 'checkout', - '--hardlinks', '--no-integrate', - 'compose-all.bst', - '--directory', checkout]) + result = cli.run( + project=project, + args=["artifact", "checkout", "--hardlinks", "--no-integrate", "compose-all.bst", "--directory", checkout], + ) result.assert_success() - st = os.lstat(os.path.join(checkout, 'usr/include/pony.h')) + st = os.lstat(os.path.join(checkout, "usr/include/pony.h")) assert stat.S_ISREG(st.st_mode) assert stat.S_IMODE(st.st_mode) == 0o0644 - st = os.lstat(os.path.join(checkout, 'usr/bin/hello')) + st = os.lstat(os.path.join(checkout, "usr/bin/hello")) assert stat.S_ISREG(st.st_mode) assert stat.S_IMODE(st.st_mode) == 0o0755 - st = os.lstat(os.path.join(checkout, 'usr/share/big-file')) + st = os.lstat(os.path.join(checkout, "usr/share/big-file")) assert stat.S_ISREG(st.st_mode) assert stat.S_IMODE(st.st_mode) == 0o0644 shutil.rmtree(checkout) - casdir = os.path.join(cli.directory, 'cas') + casdir = os.path.join(cli.directory, "cas") shutil.rmtree(casdir) - result = cli.run(project=project, args=['artifact', 'pull', 'compose-all.bst']) + result = cli.run(project=project, args=["artifact", "pull", "compose-all.bst"]) result.assert_success() - result = cli.run(project=project, - args=['artifact', 'checkout', - '--hardlinks', '--no-integrate', - 'compose-all.bst', - '--directory', checkout]) + result = cli.run( + project=project, + args=["artifact", "checkout", "--hardlinks", "--no-integrate", "compose-all.bst", "--directory", checkout], + ) result.assert_success() - st = os.lstat(os.path.join(checkout, 'usr/include/pony.h')) + st = os.lstat(os.path.join(checkout, "usr/include/pony.h")) assert stat.S_ISREG(st.st_mode) assert stat.S_IMODE(st.st_mode) == 0o0644 - st = os.lstat(os.path.join(checkout, 'usr/bin/hello')) + st = os.lstat(os.path.join(checkout, "usr/bin/hello")) assert stat.S_ISREG(st.st_mode) assert stat.S_IMODE(st.st_mode) == 0o0755 - st = os.lstat(os.path.join(checkout, 'usr/share/big-file')) + st = os.lstat(os.path.join(checkout, "usr/share/big-file")) assert stat.S_ISREG(st.st_mode) assert stat.S_IMODE(st.st_mode) == 0o0644 @@ -549,39 +509,37 @@ def test_pull_access_rights(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_pull_artifact(cli, tmpdir, datafiles): project = str(datafiles) - element = 'target.bst' + element = "target.bst" # Configure a local cache - local_cache = os.path.join(str(tmpdir), 'cache') - cli.configure({'cachedir': local_cache}) + local_cache = os.path.join(str(tmpdir), "cache") + cli.configure({"cachedir": local_cache}) - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # First build the target element and push to the remote. - cli.configure({ - 'artifacts': {'url': share.repo, 'push': True} - }) + cli.configure({"artifacts": {"url": share.repo, "push": True}}) - result = cli.run(project=project, args=['build', element]) + result = cli.run(project=project, args=["build", element]) result.assert_success() # Assert that the *artifact* is cached locally cache_key = cli.get_element_key(project, element) - artifact_ref = os.path.join('test', os.path.splitext(element)[0], cache_key) - assert os.path.exists(os.path.join(local_cache, 'artifacts', 'refs', artifact_ref)) + artifact_ref = os.path.join("test", os.path.splitext(element)[0], cache_key) + assert os.path.exists(os.path.join(local_cache, "artifacts", "refs", artifact_ref)) # Assert that the target is shared (note that assert shared will use the artifact name) assert_shared(cli, share, project, element) # Now we've pushed, remove the local cache - shutil.rmtree(os.path.join(local_cache, 'artifacts')) + shutil.rmtree(os.path.join(local_cache, "artifacts")) # Assert that nothing is cached locally anymore - assert not os.path.exists(os.path.join(local_cache, 'artifacts', 'refs', artifact_ref)) + assert not os.path.exists(os.path.join(local_cache, "artifacts", "refs", artifact_ref)) # Now try bst artifact pull - result = cli.run(project=project, args=['artifact', 'pull', artifact_ref]) + result = cli.run(project=project, args=["artifact", "pull", artifact_ref]) result.assert_success() # And assert that it's again in the local cache, without having built - assert os.path.exists(os.path.join(local_cache, 'artifacts', 'refs', artifact_ref)) + assert os.path.exists(os.path.join(local_cache, "artifacts", "refs", artifact_ref)) diff --git a/tests/frontend/push.py b/tests/frontend/push.py index 31f96cbdf..583b57399 100644 --- a/tests/frontend/push.py +++ b/tests/frontend/push.py @@ -28,15 +28,18 @@ import pytest from buildstream._exceptions import ErrorDomain from buildstream.testing import cli # pylint: disable=unused-import -from tests.testutils import create_artifact_share, create_element_size, generate_junction, \ - wait_for_cache_granularity, assert_shared, assert_not_shared +from tests.testutils import ( + create_artifact_share, + create_element_size, + generate_junction, + wait_for_cache_granularity, + assert_shared, + assert_not_shared, +) # Project directory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project", -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",) # Tests that: @@ -49,100 +52,85 @@ def test_push(cli, tmpdir, datafiles): project = str(datafiles) # First build the project without the artifact cache configured - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() # Assert that we are now cached locally - assert cli.get_element_state(project, 'target.bst') == 'cached' + assert cli.get_element_state(project, "target.bst") == "cached" # Set up two artifact shares. - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare1')) as share1: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare1")) as share1: - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) as share2: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare2")) as share2: # Try pushing with no remotes configured. This should fail. - result = cli.run(project=project, args=['artifact', 'push', 'target.bst']) + result = cli.run(project=project, args=["artifact", "push", "target.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) # Configure bst to pull but not push from a cache and run `bst artifact push`. # This should also fail. - cli.configure({ - 'artifacts': {'url': share1.repo, 'push': False}, - }) - result = cli.run(project=project, args=['artifact', 'push', 'target.bst']) + cli.configure( + {"artifacts": {"url": share1.repo, "push": False},} + ) + result = cli.run(project=project, args=["artifact", "push", "target.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) # Configure bst to push to one of the caches and run `bst artifact push`. This works. - cli.configure({ - 'artifacts': [ - {'url': share1.repo, 'push': False}, - {'url': share2.repo, 'push': True}, - ] - }) - cli.run(project=project, args=['artifact', 'push', 'target.bst']) + cli.configure({"artifacts": [{"url": share1.repo, "push": False}, {"url": share2.repo, "push": True},]}) + cli.run(project=project, args=["artifact", "push", "target.bst"]) - assert_not_shared(cli, share1, project, 'target.bst') - assert_shared(cli, share2, project, 'target.bst') + assert_not_shared(cli, share1, project, "target.bst") + assert_shared(cli, share2, project, "target.bst") # Now try pushing to both - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) as share2: - cli.configure({ - 'artifacts': [ - {'url': share1.repo, 'push': True}, - {'url': share2.repo, 'push': True}, - ] - }) - cli.run(project=project, args=['artifact', 'push', 'target.bst']) + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare2")) as share2: + cli.configure({"artifacts": [{"url": share1.repo, "push": True}, {"url": share2.repo, "push": True},]}) + cli.run(project=project, args=["artifact", "push", "target.bst"]) - assert_shared(cli, share1, project, 'target.bst') - assert_shared(cli, share2, project, 'target.bst') + assert_shared(cli, share1, project, "target.bst") + assert_shared(cli, share2, project, "target.bst") # Tests `bst artifact push $artifact_ref` @pytest.mark.datafiles(DATA_DIR) def test_push_artifact(cli, tmpdir, datafiles): project = str(datafiles) - element = 'target.bst' + element = "target.bst" # Configure a local cache - local_cache = os.path.join(str(tmpdir), 'cache') - cli.configure({'cachedir': local_cache}) + local_cache = os.path.join(str(tmpdir), "cache") + cli.configure({"cachedir": local_cache}) - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # First build it without the artifact cache configured - result = cli.run(project=project, args=['build', element]) + result = cli.run(project=project, args=["build", element]) result.assert_success() # Assert that the *artifact* is cached locally cache_key = cli.get_element_key(project, element) - artifact_ref = os.path.join('test', os.path.splitext(element)[0], cache_key) - assert os.path.exists(os.path.join(local_cache, 'artifacts', 'refs', artifact_ref)) + artifact_ref = os.path.join("test", os.path.splitext(element)[0], cache_key) + assert os.path.exists(os.path.join(local_cache, "artifacts", "refs", artifact_ref)) # Configure artifact share - cli.configure({ - # - # FIXME: This test hangs "sometimes" if we allow - # concurrent push. - # - # It's not too bad to ignore since we're - # using the local artifact cache functionality - # only, but it should probably be fixed. - # - 'scheduler': { - 'pushers': 1 - }, - 'artifacts': { - 'url': share.repo, - 'push': True, + cli.configure( + { + # + # FIXME: This test hangs "sometimes" if we allow + # concurrent push. + # + # It's not too bad to ignore since we're + # using the local artifact cache functionality + # only, but it should probably be fixed. + # + "scheduler": {"pushers": 1}, + "artifacts": {"url": share.repo, "push": True,}, } - }) + ) # Now try bst artifact push all the deps - result = cli.run(project=project, args=[ - 'artifact', 'push', artifact_ref - ]) + result = cli.run(project=project, args=["artifact", "push", artifact_ref]) result.assert_success() # And finally assert that all the artifacts are in the share @@ -162,27 +150,23 @@ def test_push_fails(cli, tmpdir, datafiles): project = str(datafiles) # Set up the share - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # Configure bst to be able to push to the share - cli.configure({ - 'artifacts': [ - {'url': share.repo, 'push': True}, - ] - }) + cli.configure({"artifacts": [{"url": share.repo, "push": True},]}) # First ensure that the target is *NOT* cache - assert cli.get_element_state(project, 'target.bst') != 'cached' + assert cli.get_element_state(project, "target.bst") != "cached" # Now try and push the target - result = cli.run(project=project, args=['artifact', 'push', 'target.bst']) + result = cli.run(project=project, args=["artifact", "push", "target.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) assert "Push failed: target.bst is not cached" in result.stderr # Now ensure that deps are also not cached - assert cli.get_element_state(project, 'import-bin.bst') != 'cached' - assert cli.get_element_state(project, 'import-dev.bst') != 'cached' - assert cli.get_element_state(project, 'compose-all.bst') != 'cached' + assert cli.get_element_state(project, "import-bin.bst") != "cached" + assert cli.get_element_state(project, "import-dev.bst") != "cached" + assert cli.get_element_state(project, "compose-all.bst") != "cached" # Tests that: @@ -194,45 +178,42 @@ def test_push_fails_with_on_error_continue(cli, tmpdir, datafiles): project = str(datafiles) # Set up the share - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # First build the target (and its deps) - result = cli.run(project=project, args=['build', 'target.bst']) - assert cli.get_element_state(project, 'target.bst') == 'cached' - assert cli.get_element_state(project, 'import-dev.bst') == 'cached' + result = cli.run(project=project, args=["build", "target.bst"]) + assert cli.get_element_state(project, "target.bst") == "cached" + assert cli.get_element_state(project, "import-dev.bst") == "cached" # Now delete the artifact of a dependency and ensure it is not in the cache - result = cli.run(project=project, args=['artifact', 'delete', 'import-dev.bst']) - assert cli.get_element_state(project, 'import-dev.bst') != 'cached' + result = cli.run(project=project, args=["artifact", "delete", "import-dev.bst"]) + assert cli.get_element_state(project, "import-dev.bst") != "cached" # Configure bst to be able to push to the share - cli.configure({ - 'artifacts': [ - {'url': share.repo, 'push': True}, - ] - }) + cli.configure({"artifacts": [{"url": share.repo, "push": True},]}) # Now try and push the target with its deps using --on-error continue # and assert that push failed, but what could be pushed was pushed - result = cli.run(project=project, - args=['--on-error=continue', 'artifact', 'push', '--deps', 'all', 'target.bst']) + result = cli.run( + project=project, args=["--on-error=continue", "artifact", "push", "--deps", "all", "target.bst"] + ) # The overall process should return as failed result.assert_main_error(ErrorDomain.STREAM, None) # We should still have pushed what we could - assert_shared(cli, share, project, 'import-bin.bst') - assert_shared(cli, share, project, 'compose-all.bst') - assert_shared(cli, share, project, 'target.bst') + assert_shared(cli, share, project, "import-bin.bst") + assert_shared(cli, share, project, "compose-all.bst") + assert_shared(cli, share, project, "target.bst") - assert_not_shared(cli, share, project, 'import-dev.bst') + assert_not_shared(cli, share, project, "import-dev.bst") errors = [ "import-dev.bst is not cached", ( "Error while pushing. The following elements were not pushed as they are not yet cached:\n" "\n" "\timport-dev.bst\n" - ) + ), ] for error in errors: assert error in result.stderr @@ -244,91 +225,81 @@ def test_push_fails_with_on_error_continue(cli, tmpdir, datafiles): def test_push_all(cli, tmpdir, datafiles): project = str(datafiles) - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # First build it without the artifact cache configured - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() # Assert that we are now cached locally - assert cli.get_element_state(project, 'target.bst') == 'cached' + assert cli.get_element_state(project, "target.bst") == "cached" # Configure artifact share - cli.configure({ - # - # FIXME: This test hangs "sometimes" if we allow - # concurrent push. - # - # It's not too bad to ignore since we're - # using the local artifact cache functionality - # only, but it should probably be fixed. - # - 'scheduler': { - 'pushers': 1 - }, - 'artifacts': { - 'url': share.repo, - 'push': True, + cli.configure( + { + # + # FIXME: This test hangs "sometimes" if we allow + # concurrent push. + # + # It's not too bad to ignore since we're + # using the local artifact cache functionality + # only, but it should probably be fixed. + # + "scheduler": {"pushers": 1}, + "artifacts": {"url": share.repo, "push": True,}, } - }) + ) # Now try bst artifact push all the deps - result = cli.run(project=project, args=[ - 'artifact', 'push', 'target.bst', - '--deps', 'all' - ]) + result = cli.run(project=project, args=["artifact", "push", "target.bst", "--deps", "all"]) result.assert_success() # And finally assert that all the artifacts are in the share - assert_shared(cli, share, project, 'target.bst') - assert_shared(cli, share, project, 'import-bin.bst') - assert_shared(cli, share, project, 'import-dev.bst') - assert_shared(cli, share, project, 'compose-all.bst') + assert_shared(cli, share, project, "target.bst") + assert_shared(cli, share, project, "import-bin.bst") + assert_shared(cli, share, project, "import-dev.bst") + assert_shared(cli, share, project, "compose-all.bst") + # Tests that `bst artifact push --deps run $artifact_ref` fails @pytest.mark.datafiles(DATA_DIR) def test_push_artifacts_all_deps_fails(cli, tmpdir, datafiles): project = str(datafiles) - element = 'checkout-deps.bst' + element = "checkout-deps.bst" # Configure a local cache - local_cache = os.path.join(str(tmpdir), 'cache') - cli.configure({'cachedir': local_cache}) + local_cache = os.path.join(str(tmpdir), "cache") + cli.configure({"cachedir": local_cache}) - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # First build it without the artifact cache configured - result = cli.run(project=project, args=['build', element]) + result = cli.run(project=project, args=["build", element]) result.assert_success() # Assert that the *artifact* is cached locally cache_key = cli.get_element_key(project, element) - artifact_ref = os.path.join('test', os.path.splitext(element)[0], cache_key) - assert os.path.exists(os.path.join(local_cache, 'artifacts', 'refs', artifact_ref)) + artifact_ref = os.path.join("test", os.path.splitext(element)[0], cache_key) + assert os.path.exists(os.path.join(local_cache, "artifacts", "refs", artifact_ref)) # Configure artifact share - cli.configure({ - # - # FIXME: This test hangs "sometimes" if we allow - # concurrent push. - # - # It's not too bad to ignore since we're - # using the local artifact cache functionality - # only, but it should probably be fixed. - # - 'scheduler': { - 'pushers': 1 - }, - 'artifacts': { - 'url': share.repo, - 'push': True, + cli.configure( + { + # + # FIXME: This test hangs "sometimes" if we allow + # concurrent push. + # + # It's not too bad to ignore since we're + # using the local artifact cache functionality + # only, but it should probably be fixed. + # + "scheduler": {"pushers": 1}, + "artifacts": {"url": share.repo, "push": True,}, } - }) + ) # Now try bst artifact push all the deps - result = cli.run(project=project, args=[ - 'artifact', 'push', '--deps', 'all', artifact_ref - ]) + result = cli.run(project=project, args=["artifact", "push", "--deps", "all", artifact_ref]) result.assert_main_error(ErrorDomain.STREAM, None) assert "Error: '--deps all' is not supported for artifact refs" in result.stderr @@ -342,47 +313,43 @@ def test_push_after_pull(cli, tmpdir, datafiles): project = str(datafiles) # Set up two artifact shares. - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare1')) as share1,\ - create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) as share2: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare1")) as share1, create_artifact_share( + os.path.join(str(tmpdir), "artifactshare2") + ) as share2: # Set the scene: share1 has the artifact, share2 does not. # - cli.configure({ - 'artifacts': {'url': share1.repo, 'push': True}, - }) + cli.configure( + {"artifacts": {"url": share1.repo, "push": True},} + ) - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - cli.remove_artifact_from_cache(project, 'target.bst') + cli.remove_artifact_from_cache(project, "target.bst") - assert_shared(cli, share1, project, 'target.bst') - assert_not_shared(cli, share2, project, 'target.bst') - assert cli.get_element_state(project, 'target.bst') != 'cached' + assert_shared(cli, share1, project, "target.bst") + assert_not_shared(cli, share2, project, "target.bst") + assert cli.get_element_state(project, "target.bst") != "cached" # Now run the build again. Correct `bst build` behaviour is to download the # artifact from share1 but not push it back again. # - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - assert result.get_pulled_elements() == ['target.bst'] + assert result.get_pulled_elements() == ["target.bst"] assert result.get_pushed_elements() == [] # Delete the artifact locally again. - cli.remove_artifact_from_cache(project, 'target.bst') + cli.remove_artifact_from_cache(project, "target.bst") # Now we add share2 into the mix as a second push remote. This time, # `bst build` should push to share2 after pulling from share1. - cli.configure({ - 'artifacts': [ - {'url': share1.repo, 'push': True}, - {'url': share2.repo, 'push': True}, - ] - }) - result = cli.run(project=project, args=['build', 'target.bst']) + cli.configure({"artifacts": [{"url": share1.repo, "push": True}, {"url": share2.repo, "push": True},]}) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - assert result.get_pulled_elements() == ['target.bst'] - assert result.get_pushed_elements() == ['target.bst'] + assert result.get_pulled_elements() == ["target.bst"] + assert result.get_pushed_elements() == ["target.bst"] # Ensure that when an artifact's size exceeds available disk space @@ -391,52 +358,51 @@ def test_push_after_pull(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_artifact_expires(cli, datafiles, tmpdir): project = str(datafiles) - element_path = 'elements' + element_path = "elements" # Create an artifact share (remote artifact cache) in the tmpdir/artifactshare # Set a 22 MB quota - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare'), - quota=int(22e6)) as share: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare"), quota=int(22e6)) as share: # Configure bst to push to the cache - cli.configure({ - 'artifacts': {'url': share.repo, 'push': True}, - }) + cli.configure( + {"artifacts": {"url": share.repo, "push": True},} + ) # Create and build an element of 15 MB - create_element_size('element1.bst', project, element_path, [], int(15e6)) - result = cli.run(project=project, args=['build', 'element1.bst']) + create_element_size("element1.bst", project, element_path, [], int(15e6)) + result = cli.run(project=project, args=["build", "element1.bst"]) result.assert_success() # Create and build an element of 5 MB - create_element_size('element2.bst', project, element_path, [], int(5e6)) - result = cli.run(project=project, args=['build', 'element2.bst']) + create_element_size("element2.bst", project, element_path, [], int(5e6)) + result = cli.run(project=project, args=["build", "element2.bst"]) result.assert_success() # check that element's 1 and 2 are cached both locally and remotely - states = cli.get_element_states(project, ['element1.bst', 'element2.bst']) + states = cli.get_element_states(project, ["element1.bst", "element2.bst"]) assert states == { "element1.bst": "cached", "element2.bst": "cached", } - assert_shared(cli, share, project, 'element1.bst') - assert_shared(cli, share, project, 'element2.bst') + assert_shared(cli, share, project, "element1.bst") + assert_shared(cli, share, project, "element2.bst") # Create and build another element of 5 MB (This will exceed the free disk space available) - create_element_size('element3.bst', project, element_path, [], int(5e6)) - result = cli.run(project=project, args=['build', 'element3.bst']) + create_element_size("element3.bst", project, element_path, [], int(5e6)) + result = cli.run(project=project, args=["build", "element3.bst"]) result.assert_success() # Ensure it is cached both locally and remotely - assert cli.get_element_state(project, 'element3.bst') == 'cached' - assert_shared(cli, share, project, 'element3.bst') + assert cli.get_element_state(project, "element3.bst") == "cached" + assert_shared(cli, share, project, "element3.bst") # Ensure element1 has been removed from the share - assert_not_shared(cli, share, project, 'element1.bst') + assert_not_shared(cli, share, project, "element1.bst") # Ensure that elemen2 remains - assert_shared(cli, share, project, 'element2.bst') + assert_shared(cli, share, project, "element2.bst") # Test that a large artifact, whose size exceeds the quota, is not pushed @@ -444,26 +410,25 @@ def test_artifact_expires(cli, datafiles, tmpdir): @pytest.mark.datafiles(DATA_DIR) def test_artifact_too_large(cli, datafiles, tmpdir): project = str(datafiles) - element_path = 'elements' + element_path = "elements" # Create an artifact share (remote cache) in tmpdir/artifactshare # Mock a file system with 5 MB total space - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare'), - quota=int(5e6)) as share: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare"), quota=int(5e6)) as share: # Configure bst to push to the remote cache - cli.configure({ - 'artifacts': {'url': share.repo, 'push': True}, - }) + cli.configure( + {"artifacts": {"url": share.repo, "push": True},} + ) # Create and push a 3MB element - create_element_size('small_element.bst', project, element_path, [], int(3e6)) - result = cli.run(project=project, args=['build', 'small_element.bst']) + create_element_size("small_element.bst", project, element_path, [], int(3e6)) + result = cli.run(project=project, args=["build", "small_element.bst"]) result.assert_success() # Create and try to push a 6MB element. - create_element_size('large_element.bst', project, element_path, [], int(6e6)) - result = cli.run(project=project, args=['build', 'large_element.bst']) + create_element_size("large_element.bst", project, element_path, [], int(6e6)) + result = cli.run(project=project, args=["build", "large_element.bst"]) # This should fail; the server will refuse to store the CAS # blobs for the artifact, and then fail to find the files for # the uploaded artifact proto. @@ -476,100 +441,98 @@ def test_artifact_too_large(cli, datafiles, tmpdir): result.assert_main_error(ErrorDomain.STREAM, None) # Ensure that the small artifact is still in the share - states = cli.get_element_states(project, ['small_element.bst', 'large_element.bst']) - assert states['small_element.bst'] == 'cached' - assert_shared(cli, share, project, 'small_element.bst') + states = cli.get_element_states(project, ["small_element.bst", "large_element.bst"]) + assert states["small_element.bst"] == "cached" + assert_shared(cli, share, project, "small_element.bst") # Ensure that the artifact is cached locally but NOT remotely - assert states['large_element.bst'] == 'cached' - assert_not_shared(cli, share, project, 'large_element.bst') + assert states["large_element.bst"] == "cached" + assert_not_shared(cli, share, project, "large_element.bst") # Test that when an element is pulled recently, it is not considered the LRU element. @pytest.mark.datafiles(DATA_DIR) def test_recently_pulled_artifact_does_not_expire(cli, datafiles, tmpdir): project = str(datafiles) - element_path = 'elements' + element_path = "elements" # Create an artifact share (remote cache) in tmpdir/artifactshare # Set a 22 MB quota - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare'), - quota=int(22e6)) as share: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare"), quota=int(22e6)) as share: # Configure bst to push to the cache - cli.configure({ - 'artifacts': {'url': share.repo, 'push': True}, - }) + cli.configure( + {"artifacts": {"url": share.repo, "push": True},} + ) # Create and build 2 elements, one 5 MB and one 15 MB. - create_element_size('element1.bst', project, element_path, [], int(5e6)) - result = cli.run(project=project, args=['build', 'element1.bst']) + create_element_size("element1.bst", project, element_path, [], int(5e6)) + result = cli.run(project=project, args=["build", "element1.bst"]) result.assert_success() - create_element_size('element2.bst', project, element_path, [], int(15e6)) - result = cli.run(project=project, args=['build', 'element2.bst']) + create_element_size("element2.bst", project, element_path, [], int(15e6)) + result = cli.run(project=project, args=["build", "element2.bst"]) result.assert_success() # Ensure they are cached locally - states = cli.get_element_states(project, ['element1.bst', 'element2.bst']) + states = cli.get_element_states(project, ["element1.bst", "element2.bst"]) assert states == { "element1.bst": "cached", "element2.bst": "cached", } # Ensure that they have been pushed to the cache - assert_shared(cli, share, project, 'element1.bst') - assert_shared(cli, share, project, 'element2.bst') + assert_shared(cli, share, project, "element1.bst") + assert_shared(cli, share, project, "element2.bst") # Remove element1 from the local cache - cli.remove_artifact_from_cache(project, 'element1.bst') - assert cli.get_element_state(project, 'element1.bst') != 'cached' + cli.remove_artifact_from_cache(project, "element1.bst") + assert cli.get_element_state(project, "element1.bst") != "cached" # Pull the element1 from the remote cache (this should update its mtime) - result = cli.run(project=project, args=['artifact', 'pull', 'element1.bst', '--remote', - share.repo]) + result = cli.run(project=project, args=["artifact", "pull", "element1.bst", "--remote", share.repo]) result.assert_success() # Ensure element1 is cached locally - assert cli.get_element_state(project, 'element1.bst') == 'cached' + assert cli.get_element_state(project, "element1.bst") == "cached" wait_for_cache_granularity() # Create and build the element3 (of 5 MB) - create_element_size('element3.bst', project, element_path, [], int(5e6)) - result = cli.run(project=project, args=['build', 'element3.bst']) + create_element_size("element3.bst", project, element_path, [], int(5e6)) + result = cli.run(project=project, args=["build", "element3.bst"]) result.assert_success() # Make sure it's cached locally and remotely - assert cli.get_element_state(project, 'element3.bst') == 'cached' - assert_shared(cli, share, project, 'element3.bst') + assert cli.get_element_state(project, "element3.bst") == "cached" + assert_shared(cli, share, project, "element3.bst") # Ensure that element2 was deleted from the share and element1 remains - assert_not_shared(cli, share, project, 'element2.bst') - assert_shared(cli, share, project, 'element1.bst') + assert_not_shared(cli, share, project, "element2.bst") + assert_shared(cli, share, project, "element1.bst") @pytest.mark.datafiles(DATA_DIR) def test_push_cross_junction(cli, tmpdir, datafiles): project = str(datafiles) - subproject_path = os.path.join(project, 'files', 'sub-project') - junction_path = os.path.join(project, 'elements', 'junction.bst') + subproject_path = os.path.join(project, "files", "sub-project") + junction_path = os.path.join(project, "elements", "junction.bst") generate_junction(tmpdir, subproject_path, junction_path, store_ref=True) - result = cli.run(project=project, args=['build', 'junction.bst:import-etc.bst']) + result = cli.run(project=project, args=["build", "junction.bst:import-etc.bst"]) result.assert_success() - assert cli.get_element_state(project, 'junction.bst:import-etc.bst') == 'cached' + assert cli.get_element_state(project, "junction.bst:import-etc.bst") == "cached" - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: - cli.configure({ - 'artifacts': {'url': share.repo, 'push': True}, - }) - cli.run(project=project, args=['artifact', 'push', 'junction.bst:import-etc.bst']) + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: + cli.configure( + {"artifacts": {"url": share.repo, "push": True},} + ) + cli.run(project=project, args=["artifact", "push", "junction.bst:import-etc.bst"]) - cache_key = cli.get_element_key(project, 'junction.bst:import-etc.bst') - assert share.get_artifact(cli.get_artifact_name(project, 'subtest', 'import-etc.bst', cache_key=cache_key)) + cache_key = cli.get_element_key(project, "junction.bst:import-etc.bst") + assert share.get_artifact(cli.get_artifact_name(project, "subtest", "import-etc.bst", cache_key=cache_key)) @pytest.mark.datafiles(DATA_DIR) @@ -577,17 +540,15 @@ def test_push_already_cached(caplog, cli, tmpdir, datafiles): project = str(datafiles) caplog.set_level(1) - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: - cli.configure({ - 'artifacts': {'url': share.repo, 'push': True} - }) - result = cli.run(project=project, args=['build', 'target.bst']) + cli.configure({"artifacts": {"url": share.repo, "push": True}}) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() assert "SKIPPED Push" not in result.stderr - result = cli.run(project=project, args=['artifact', 'push', 'target.bst']) + result = cli.run(project=project, args=["artifact", "push", "target.bst"]) result.assert_success() assert not result.get_pushed_elements(), "No elements should have been pushed since the cache was populated" @@ -600,24 +561,22 @@ def test_build_remote_option(caplog, cli, tmpdir, datafiles): project = str(datafiles) caplog.set_level(1) - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare1')) as shareuser,\ - create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) as shareproject,\ - create_artifact_share(os.path.join(str(tmpdir), 'artifactshare3')) as sharecli: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare1")) as shareuser, create_artifact_share( + os.path.join(str(tmpdir), "artifactshare2") + ) as shareproject, create_artifact_share(os.path.join(str(tmpdir), "artifactshare3")) as sharecli: # Add shareproject repo url to project.conf with open(os.path.join(project, "project.conf"), "a") as projconf: projconf.write("artifacts:\n url: {}\n push: True".format(shareproject.repo)) # Configure shareuser remote in user conf - cli.configure({ - 'artifacts': {'url': shareuser.repo, 'push': True} - }) + cli.configure({"artifacts": {"url": shareuser.repo, "push": True}}) - result = cli.run(project=project, args=['build', '--remote', sharecli.repo, 'target.bst']) + result = cli.run(project=project, args=["build", "--remote", sharecli.repo, "target.bst"]) # Artifacts should have only been pushed to sharecli, as that was provided via the cli result.assert_success() - all_elements = ['target.bst', 'import-bin.bst', 'compose-all.bst'] + all_elements = ["target.bst", "import-bin.bst", "compose-all.bst"] for element_name in all_elements: assert element_name in result.get_pushed_elements() assert_shared(cli, sharecli, project, element_name) @@ -632,26 +591,16 @@ def test_build_remote_option(caplog, cli, tmpdir, datafiles): # This is a regression test for issue #990 # @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("buildtrees", [('buildtrees'), ('normal')]) +@pytest.mark.parametrize("buildtrees", [("buildtrees"), ("normal")]) def test_push_no_strict(caplog, cli, tmpdir, datafiles, buildtrees): project = os.path.join(datafiles.dirname, datafiles.basename) caplog.set_level(1) - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: - cli.configure({ - 'artifacts': { - 'url': share.repo, - 'push': True - }, - 'projects': { - 'test': { - 'strict': False - } - } - }) + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: + cli.configure({"artifacts": {"url": share.repo, "push": True}, "projects": {"test": {"strict": False}}}) # First get us a build - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() # Now cause one of the dependenies to change their cache key @@ -659,12 +608,12 @@ def test_push_no_strict(caplog, cli, tmpdir, datafiles, buildtrees): # Here we just add a file, causing the strong cache key of the # import-bin.bst element to change due to the local files it # imports changing. - path = os.path.join(project, 'files', 'bin-files', 'newfile') - with open(path, 'w') as f: + path = os.path.join(project, "files", "bin-files", "newfile") + with open(path, "w") as f: f.write("PONY !") # Now build again after having changed the dependencies - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() # Now run `bst artifact push`. @@ -673,8 +622,8 @@ def test_push_no_strict(caplog, cli, tmpdir, datafiles, buildtrees): # a pull queue to be added to the `push` command, the behavior # around this is different. args = [] - if buildtrees == 'buildtrees': - args += ['--pull-buildtrees'] - args += ['artifact', 'push', '--deps', 'all', 'target.bst'] + if buildtrees == "buildtrees": + args += ["--pull-buildtrees"] + args += ["artifact", "push", "--deps", "all", "target.bst"] result = cli.run(project=project, args=args) result.assert_success() diff --git a/tests/frontend/rebuild.py b/tests/frontend/rebuild.py index 1cdb45d11..1aef8e423 100644 --- a/tests/frontend/rebuild.py +++ b/tests/frontend/rebuild.py @@ -6,15 +6,12 @@ import pytest from buildstream.testing import cli # pylint: disable=unused-import # Project directory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project", -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",) def strict_args(args, strict): if strict != "strict": - return ['--no-strict', *args] + return ["--no-strict", *args] return args @@ -24,15 +21,15 @@ def test_rebuild(datafiles, cli, strict): project = str(datafiles) # First build intermediate target.bst - result = cli.run(project=project, args=strict_args(['build', 'target.bst'], strict)) + result = cli.run(project=project, args=strict_args(["build", "target.bst"], strict)) result.assert_success() # Modify base import - with open(os.path.join(project, 'files', 'dev-files', 'usr', 'include', 'new.h'), "w") as f: + with open(os.path.join(project, "files", "dev-files", "usr", "include", "new.h"), "w") as f: f.write("#define NEW") # Rebuild base import and build top-level rebuild-target.bst # In non-strict mode, this does not rebuild intermediate target.bst, # which means that a weakly cached target.bst will be staged as dependency. - result = cli.run(project=project, args=strict_args(['build', 'rebuild-target.bst'], strict)) + result = cli.run(project=project, args=strict_args(["build", "rebuild-target.bst"], strict)) result.assert_success() diff --git a/tests/frontend/remote-caches.py b/tests/frontend/remote-caches.py index 6ee57df23..b112e0882 100644 --- a/tests/frontend/remote-caches.py +++ b/tests/frontend/remote-caches.py @@ -28,7 +28,7 @@ from buildstream import _yaml from tests.testutils import create_artifact_share, create_element_size -DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'project') +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") def message_handler(message, context): @@ -37,54 +37,46 @@ def message_handler(message, context): @pytest.mark.datafiles(DATA_DIR) def test_source_artifact_caches(cli, tmpdir, datafiles): - cachedir = os.path.join(str(tmpdir), 'cache') + cachedir = os.path.join(str(tmpdir), "cache") project_dir = str(datafiles) - element_path = os.path.join(project_dir, 'elements') + element_path = os.path.join(project_dir, "elements") - with create_artifact_share(os.path.join(str(tmpdir), 'share')) as share: - user_config_file = str(tmpdir.join('buildstream.conf')) + with create_artifact_share(os.path.join(str(tmpdir), "share")) as share: + user_config_file = str(tmpdir.join("buildstream.conf")) user_config = { - 'scheduler': { - 'pushers': 1 - }, - 'source-caches': { - 'url': share.repo, - 'push': True, - }, - 'artifacts': { - 'url': share.repo, - 'push': True, - }, - 'cachedir': cachedir + "scheduler": {"pushers": 1}, + "source-caches": {"url": share.repo, "push": True,}, + "artifacts": {"url": share.repo, "push": True,}, + "cachedir": cachedir, } _yaml.roundtrip_dump(user_config, file=user_config_file) cli.configure(user_config) - create_element_size('repo.bst', project_dir, element_path, [], 10000) + create_element_size("repo.bst", project_dir, element_path, [], 10000) - res = cli.run(project=project_dir, args=['build', 'repo.bst']) + res = cli.run(project=project_dir, args=["build", "repo.bst"]) res.assert_success() assert "Pushed source " in res.stderr assert "Pushed artifact " in res.stderr # delete local sources and artifacts and check it pulls them - shutil.rmtree(os.path.join(cachedir, 'cas')) - shutil.rmtree(os.path.join(cachedir, 'sources')) + shutil.rmtree(os.path.join(cachedir, "cas")) + shutil.rmtree(os.path.join(cachedir, "sources")) # this should just fetch the artifacts - res = cli.run(project=project_dir, args=['build', 'repo.bst']) + res = cli.run(project=project_dir, args=["build", "repo.bst"]) res.assert_success() assert "Pulled artifact " in res.stderr assert "Pulled source " not in res.stderr # remove the artifact from the repo and check it pulls sources, builds # and then pushes the artifacts - shutil.rmtree(os.path.join(cachedir, 'cas')) - shutil.rmtree(os.path.join(cachedir, 'artifacts')) - print(os.listdir(os.path.join(share.repodir, 'artifacts', 'refs'))) - shutil.rmtree(os.path.join(share.repodir, 'artifacts', 'refs', 'test')) + shutil.rmtree(os.path.join(cachedir, "cas")) + shutil.rmtree(os.path.join(cachedir, "artifacts")) + print(os.listdir(os.path.join(share.repodir, "artifacts", "refs"))) + shutil.rmtree(os.path.join(share.repodir, "artifacts", "refs", "test")) - res = cli.run(project=project_dir, args=['build', 'repo.bst']) + res = cli.run(project=project_dir, args=["build", "repo.bst"]) res.assert_success() assert "Remote ({}) does not have artifact ".format(share.repo) in res.stderr assert "Pulled source" in res.stderr diff --git a/tests/frontend/show.py b/tests/frontend/show.py index bc51d2967..b08670a30 100644 --- a/tests/frontend/show.py +++ b/tests/frontend/show.py @@ -15,85 +15,77 @@ from tests.testutils import generate_junction from . import configure_project # Project directory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)),) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project')) -@pytest.mark.parametrize("target,fmt,expected", [ - ('import-bin.bst', '%{name}', 'import-bin.bst'), - ('import-bin.bst', '%{state}', 'buildable'), - ('compose-all.bst', '%{state}', 'waiting') -]) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) +@pytest.mark.parametrize( + "target,fmt,expected", + [ + ("import-bin.bst", "%{name}", "import-bin.bst"), + ("import-bin.bst", "%{state}", "buildable"), + ("compose-all.bst", "%{state}", "waiting"), + ], +) def test_show(cli, datafiles, target, fmt, expected): project = str(datafiles) - result = cli.run(project=project, silent=True, args=[ - 'show', - '--deps', 'none', - '--format', fmt, - target]) + result = cli.run(project=project, silent=True, args=["show", "--deps", "none", "--format", fmt, target]) result.assert_success() if result.output.strip() != expected: - raise AssertionError("Expected output:\n{}\nInstead received output:\n{}" - .format(expected, result.output)) + raise AssertionError("Expected output:\n{}\nInstead received output:\n{}".format(expected, result.output)) -@pytest.mark.datafiles(os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "invalid_element_path", -)) +@pytest.mark.datafiles(os.path.join(os.path.dirname(os.path.realpath(__file__)), "invalid_element_path",)) def test_show_invalid_element_path(cli, datafiles): project = str(datafiles) - cli.run(project=project, silent=True, args=['show', "foo.bst"]) + cli.run(project=project, silent=True, args=["show", "foo.bst"]) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project_default')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "project_default")) def test_show_default(cli, datafiles): project = str(datafiles) - result = cli.run(project=project, silent=True, args=[ - 'show']) + result = cli.run(project=project, silent=True, args=["show"]) result.assert_success() # Get the result output of "[state sha element]" and turn into a list results = result.output.strip().split(" ") - expected = 'target2.bst' + expected = "target2.bst" assert results[2] == expected -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project_fail')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "project_fail")) def test_show_fail(cli, datafiles): project = str(datafiles) - result = cli.run(project=project, silent=True, args=[ - 'show']) + result = cli.run(project=project, silent=True, args=["show"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project')) -@pytest.mark.parametrize("target,except_,expected", [ - ('target.bst', 'import-bin.bst', ['import-dev.bst', 'compose-all.bst', 'target.bst']), - ('target.bst', 'import-dev.bst', ['import-bin.bst', 'compose-all.bst', 'target.bst']), - ('target.bst', 'compose-all.bst', ['import-bin.bst', 'target.bst']), - ('compose-all.bst', 'import-bin.bst', ['import-dev.bst', 'compose-all.bst']) -]) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) +@pytest.mark.parametrize( + "target,except_,expected", + [ + ("target.bst", "import-bin.bst", ["import-dev.bst", "compose-all.bst", "target.bst"]), + ("target.bst", "import-dev.bst", ["import-bin.bst", "compose-all.bst", "target.bst"]), + ("target.bst", "compose-all.bst", ["import-bin.bst", "target.bst"]), + ("compose-all.bst", "import-bin.bst", ["import-dev.bst", "compose-all.bst"]), + ], +) def test_show_except_simple(cli, datafiles, target, except_, expected): project = str(datafiles) - result = cli.run(project=project, silent=True, args=[ - 'show', - '--deps', 'all', - '--format', '%{name}', - '--except', except_, - target]) + result = cli.run( + project=project, + silent=True, + args=["show", "--deps", "all", "--format", "%{name}", "--except", except_, target], + ) result.assert_success() results = result.output.strip().splitlines() if results != expected: - raise AssertionError("Expected elements:\n{}\nInstead received elements:\n{}" - .format(expected, results)) + raise AssertionError("Expected elements:\n{}\nInstead received elements:\n{}".format(expected, results)) # This test checks various constructions of a pipeline @@ -101,104 +93,116 @@ def test_show_except_simple(cli, datafiles, target, except_, expected): # each data set provides the targets, exceptions and expected # result list. # -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'exceptions')) -@pytest.mark.parametrize("targets,exceptions,expected", [ - - # Test without exceptions, lets just see the whole list here - (['build.bst'], None, [ - 'fourth-level-1.bst', - 'third-level-1.bst', - 'fourth-level-2.bst', - 'third-level-2.bst', - 'fourth-level-3.bst', - 'third-level-3.bst', - 'second-level-1.bst', - 'first-level-1.bst', - 'first-level-2.bst', - 'build.bst', - ]), - - # Test one target and excepting a part of the pipeline, this - # removes forth-level-1 and third-level-1 - (['build.bst'], ['third-level-1.bst'], [ - 'fourth-level-2.bst', - 'third-level-2.bst', - 'fourth-level-3.bst', - 'third-level-3.bst', - 'second-level-1.bst', - 'first-level-1.bst', - 'first-level-2.bst', - 'build.bst', - ]), - - # Test one target and excepting a part of the pipeline, check that - # excepted dependencies remain in the pipeline if depended on from - # outside of the except element - (['build.bst'], ['second-level-1.bst'], [ - 'fourth-level-2.bst', - 'third-level-2.bst', # first-level-2 depends on this, so not excepted - 'first-level-1.bst', - 'first-level-2.bst', - 'build.bst', - ]), - - # The same as the above test, but excluding the toplevel build.bst, - # instead only select the two toplevel dependencies as targets - (['first-level-1.bst', 'first-level-2.bst'], ['second-level-1.bst'], [ - 'fourth-level-2.bst', - 'third-level-2.bst', # first-level-2 depends on this, so not excepted - 'first-level-1.bst', - 'first-level-2.bst', - ]), - - # Test one target and excepting an element outisde the pipeline - (['build.bst'], ['unrelated-1.bst'], [ - 'fourth-level-2.bst', - 'third-level-2.bst', # first-level-2 depends on this, so not excepted - 'first-level-1.bst', - 'first-level-2.bst', - 'build.bst', - ]), - - # Test one target and excepting two elements - (['build.bst'], ['unrelated-1.bst', 'unrelated-2.bst'], [ - 'first-level-1.bst', - 'build.bst', - ]), -]) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "exceptions")) +@pytest.mark.parametrize( + "targets,exceptions,expected", + [ + # Test without exceptions, lets just see the whole list here + ( + ["build.bst"], + None, + [ + "fourth-level-1.bst", + "third-level-1.bst", + "fourth-level-2.bst", + "third-level-2.bst", + "fourth-level-3.bst", + "third-level-3.bst", + "second-level-1.bst", + "first-level-1.bst", + "first-level-2.bst", + "build.bst", + ], + ), + # Test one target and excepting a part of the pipeline, this + # removes forth-level-1 and third-level-1 + ( + ["build.bst"], + ["third-level-1.bst"], + [ + "fourth-level-2.bst", + "third-level-2.bst", + "fourth-level-3.bst", + "third-level-3.bst", + "second-level-1.bst", + "first-level-1.bst", + "first-level-2.bst", + "build.bst", + ], + ), + # Test one target and excepting a part of the pipeline, check that + # excepted dependencies remain in the pipeline if depended on from + # outside of the except element + ( + ["build.bst"], + ["second-level-1.bst"], + [ + "fourth-level-2.bst", + "third-level-2.bst", # first-level-2 depends on this, so not excepted + "first-level-1.bst", + "first-level-2.bst", + "build.bst", + ], + ), + # The same as the above test, but excluding the toplevel build.bst, + # instead only select the two toplevel dependencies as targets + ( + ["first-level-1.bst", "first-level-2.bst"], + ["second-level-1.bst"], + [ + "fourth-level-2.bst", + "third-level-2.bst", # first-level-2 depends on this, so not excepted + "first-level-1.bst", + "first-level-2.bst", + ], + ), + # Test one target and excepting an element outisde the pipeline + ( + ["build.bst"], + ["unrelated-1.bst"], + [ + "fourth-level-2.bst", + "third-level-2.bst", # first-level-2 depends on this, so not excepted + "first-level-1.bst", + "first-level-2.bst", + "build.bst", + ], + ), + # Test one target and excepting two elements + (["build.bst"], ["unrelated-1.bst", "unrelated-2.bst"], ["first-level-1.bst", "build.bst",]), + ], +) def test_show_except(cli, datafiles, targets, exceptions, expected): basedir = str(datafiles) - results = cli.get_pipeline(basedir, targets, except_=exceptions, scope='all') + results = cli.get_pipeline(basedir, targets, except_=exceptions, scope="all") if results != expected: - raise AssertionError("Expected elements:\n{}\nInstead received elements:\n{}" - .format(expected, results)) + raise AssertionError("Expected elements:\n{}\nInstead received elements:\n{}".format(expected, results)) ############################################################### # Testing multiple targets # ############################################################### -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) def test_parallel_order(cli, datafiles): project = str(datafiles) - elements = ['multiple_targets/order/0.bst', - 'multiple_targets/order/1.bst'] + elements = ["multiple_targets/order/0.bst", "multiple_targets/order/1.bst"] - args = ['show', '-d', 'plan', '-f', '%{name}', *elements] + args = ["show", "-d", "plan", "-f", "%{name}", *elements] result = cli.run(project=project, args=args) result.assert_success() # Get the planned order names = result.output.splitlines() - names = [name[len('multiple_targets/order/'):] for name in names] + names = [name[len("multiple_targets/order/") :] for name in names] # Create all possible 'correct' topological orderings orderings = itertools.product( - [('5.bst', '6.bst')], - itertools.permutations(['4.bst', '7.bst']), - itertools.permutations(['3.bst', '8.bst']), - itertools.permutations(['2.bst', '9.bst']), - itertools.permutations(['0.bst', '1.bst', 'run.bst']) + [("5.bst", "6.bst")], + itertools.permutations(["4.bst", "7.bst"]), + itertools.permutations(["3.bst", "8.bst"]), + itertools.permutations(["2.bst", "9.bst"]), + itertools.permutations(["0.bst", "1.bst", "run.bst"]), ) orderings = [list(itertools.chain.from_iterable(perm)) for perm in orderings] @@ -206,126 +210,96 @@ def test_parallel_order(cli, datafiles): assert names in orderings, "We got: {}".format(", ".join(names)) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) def test_target_is_dependency(cli, datafiles): project = str(datafiles) - elements = ['multiple_targets/dependency/zebry.bst', - 'multiple_targets/dependency/horsey.bst'] + elements = ["multiple_targets/dependency/zebry.bst", "multiple_targets/dependency/horsey.bst"] - args = ['show', '-d', 'plan', '-f', '%{name}', *elements] + args = ["show", "-d", "plan", "-f", "%{name}", *elements] result = cli.run(project=project, args=args) result.assert_success() # Get the planned order names = result.output.splitlines() - names = [name[len('multiple_targets/dependency/'):] for name in names] + names = [name[len("multiple_targets/dependency/") :] for name in names] - assert names == ['pony.bst', 'horsey.bst', 'zebry.bst'] + assert names == ["pony.bst", "horsey.bst", "zebry.bst"] -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project')) -@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')]) -@pytest.mark.parametrize("element_name", ['junction-dep.bst', 'junction.bst:import-etc.bst']) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) +@pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")]) +@pytest.mark.parametrize("element_name", ["junction-dep.bst", "junction.bst:import-etc.bst"]) @pytest.mark.parametrize("workspaced", [True, False], ids=["workspace", "no-workspace"]) def test_unfetched_junction(cli, tmpdir, datafiles, ref_storage, element_name, workspaced): project = str(datafiles) - subproject_path = os.path.join(project, 'files', 'sub-project') - junction_path = os.path.join(project, 'elements', 'junction.bst') - element_path = os.path.join(project, 'elements', 'junction-dep.bst') + subproject_path = os.path.join(project, "files", "sub-project") + junction_path = os.path.join(project, "elements", "junction.bst") + element_path = os.path.join(project, "elements", "junction-dep.bst") - configure_project(project, { - 'ref-storage': ref_storage - }) + configure_project(project, {"ref-storage": ref_storage}) # Create a repo to hold the subproject and generate a junction element for it - ref = generate_junction(tmpdir, subproject_path, junction_path, store_ref=(ref_storage == 'inline')) + ref = generate_junction(tmpdir, subproject_path, junction_path, store_ref=(ref_storage == "inline")) # Create a stack element to depend on a cross junction element # - element = { - 'kind': 'stack', - 'depends': [ - { - 'junction': 'junction.bst', - 'filename': 'import-etc.bst' - } - ] - } + element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) # Dump a project.refs if we're using project.refs storage # - if ref_storage == 'project.refs': - project_refs = { - 'projects': { - 'test': { - 'junction.bst': [ - { - 'ref': ref - } - ] - } - } - } - _yaml.roundtrip_dump(project_refs, os.path.join(project, 'junction.refs')) + if ref_storage == "project.refs": + project_refs = {"projects": {"test": {"junction.bst": [{"ref": ref}]}}} + _yaml.roundtrip_dump(project_refs, os.path.join(project, "junction.refs")) # Open a workspace if we're testing workspaced behavior if workspaced: - result = cli.run(project=project, silent=True, args=[ - 'workspace', 'open', '--no-checkout', '--directory', subproject_path, 'junction.bst' - ]) + result = cli.run( + project=project, + silent=True, + args=["workspace", "open", "--no-checkout", "--directory", subproject_path, "junction.bst"], + ) result.assert_success() # Assert successful bst show (requires implicit subproject fetching) - result = cli.run(project=project, silent=True, args=[ - 'show', element_name]) + result = cli.run(project=project, silent=True, args=["show", element_name]) result.assert_success() -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project')) -@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')]) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) +@pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")]) @pytest.mark.parametrize("workspaced", [True, False], ids=["workspace", "no-workspace"]) def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage, workspaced): project = str(datafiles) - subproject_path = os.path.join(project, 'files', 'sub-project') - junction_path = os.path.join(project, 'elements', 'junction.bst') - element_path = os.path.join(project, 'elements', 'junction-dep.bst') + subproject_path = os.path.join(project, "files", "sub-project") + junction_path = os.path.join(project, "elements", "junction.bst") + element_path = os.path.join(project, "elements", "junction-dep.bst") - configure_project(project, { - 'ref-storage': ref_storage - }) + configure_project(project, {"ref-storage": ref_storage}) # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path, store_ref=False) # Create a stack element to depend on a cross junction element # - element = { - 'kind': 'stack', - 'depends': [ - { - 'junction': 'junction.bst', - 'filename': 'import-etc.bst' - } - ] - } + element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) # Open a workspace if we're testing workspaced behavior if workspaced: - result = cli.run(project=project, silent=True, args=[ - 'workspace', 'open', '--no-checkout', '--directory', subproject_path, 'junction.bst' - ]) + result = cli.run( + project=project, + silent=True, + args=["workspace", "open", "--no-checkout", "--directory", subproject_path, "junction.bst"], + ) result.assert_success() # Assert the correct error when trying to show the pipeline - dep_result = cli.run(project=project, silent=True, args=[ - 'show', 'junction-dep.bst']) + dep_result = cli.run(project=project, silent=True, args=["show", "junction-dep.bst"]) # Assert the correct error when trying to show the pipeline - etc_result = cli.run(project=project, silent=True, args=[ - 'show', 'junction.bst:import-etc.bst']) + etc_result = cli.run(project=project, silent=True, args=["show", "junction.bst:import-etc.bst"]) # If a workspace is open, no ref is needed if workspaced: @@ -333,8 +307,8 @@ def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage, workspaced): etc_result.assert_success() else: # Assert that we have the expected provenance encoded into the error - element_node = _yaml.load(element_path, shortname='junction-dep.bst') - ref_node = element_node.get_sequence('depends').mapping_at(0) + element_node = _yaml.load(element_path, shortname="junction-dep.bst") + ref_node = element_node.get_sequence("depends").mapping_at(0) provenance = ref_node.get_provenance() assert str(provenance) in dep_result.stderr @@ -342,49 +316,41 @@ def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage, workspaced): etc_result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project')) -@pytest.mark.parametrize("element_name", ['junction-dep.bst', 'junction.bst:import-etc.bst']) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) +@pytest.mark.parametrize("element_name", ["junction-dep.bst", "junction.bst:import-etc.bst"]) @pytest.mark.parametrize("workspaced", [True, False], ids=["workspace", "no-workspace"]) def test_fetched_junction(cli, tmpdir, datafiles, element_name, workspaced): project = str(datafiles) project = os.path.join(datafiles.dirname, datafiles.basename) - subproject_path = os.path.join(project, 'files', 'sub-project') - junction_path = os.path.join(project, 'elements', 'junction.bst') - element_path = os.path.join(project, 'elements', 'junction-dep.bst') + subproject_path = os.path.join(project, "files", "sub-project") + junction_path = os.path.join(project, "elements", "junction.bst") + element_path = os.path.join(project, "elements", "junction-dep.bst") # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path, store_ref=True) # Create a stack element to depend on a cross junction element # - element = { - 'kind': 'stack', - 'depends': [ - { - 'junction': 'junction.bst', - 'filename': 'import-etc.bst' - } - ] - } + element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) - result = cli.run(project=project, silent=True, args=[ - 'source', 'fetch', 'junction.bst']) + result = cli.run(project=project, silent=True, args=["source", "fetch", "junction.bst"]) result.assert_success() # Open a workspace if we're testing workspaced behavior if workspaced: - result = cli.run(project=project, silent=True, args=[ - 'workspace', 'open', '--no-checkout', '--directory', subproject_path, 'junction.bst' - ]) + result = cli.run( + project=project, + silent=True, + args=["workspace", "open", "--no-checkout", "--directory", subproject_path, "junction.bst"], + ) result.assert_success() # Assert the correct error when trying to show the pipeline - result = cli.run(project=project, silent=True, args=[ - 'show', '--format', '%{name}-%{state}', element_name]) + result = cli.run(project=project, silent=True, args=["show", "--format", "%{name}-%{state}", element_name]) results = result.output.strip().splitlines() - assert 'junction.bst:import-etc.bst-buildable' in results + assert "junction.bst:import-etc.bst-buildable" in results ############################################################### @@ -404,8 +370,7 @@ def test_exceed_max_recursion_depth(cli, tmpdir, dependency_depth): """ os.mkdir(project_path) - result = cli.run(silent=True, - args=['init', '--project-name', project_name, project_path]) + result = cli.run(silent=True, args=["init", "--project-name", project_name, project_path]) result.assert_success() sourcefiles_path = os.path.join(project_path, "files") @@ -414,22 +379,20 @@ def test_exceed_max_recursion_depth(cli, tmpdir, dependency_depth): element_path = os.path.join(project_path, "elements") for i in range(0, dependency_depth + 1): element = { - 'kind': 'import', - 'sources': [{'kind': 'local', - 'path': 'files/source{}'.format(str(i))}], - 'depends': ['element{}.bst'.format(str(i - 1))] + "kind": "import", + "sources": [{"kind": "local", "path": "files/source{}".format(str(i))}], + "depends": ["element{}.bst".format(str(i - 1))], } if i == 0: - del element['depends'] + del element["depends"] _yaml.roundtrip_dump(element, os.path.join(element_path, "element{}.bst".format(str(i)))) source = os.path.join(sourcefiles_path, "source{}".format(str(i))) - open(source, 'x').close() + open(source, "x").close() assert os.path.exists(source) setup_test() - result = cli.run(project=project_path, silent=True, - args=['show', "element{}.bst".format(str(dependency_depth))]) + result = cli.run(project=project_path, silent=True, args=["show", "element{}.bst".format(str(dependency_depth))]) recursion_limit = sys.getrecursionlimit() if dependency_depth <= recursion_limit: @@ -445,71 +408,62 @@ def test_exceed_max_recursion_depth(cli, tmpdir, dependency_depth): ############################################################### # Testing format symbols # ############################################################### -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project')) -@pytest.mark.parametrize("dep_kind, expected_deps", [ - ('%{deps}', '[import-dev.bst, import-bin.bst]'), - ('%{build-deps}', '[import-dev.bst]'), - ('%{runtime-deps}', '[import-bin.bst]') -]) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) +@pytest.mark.parametrize( + "dep_kind, expected_deps", + [ + ("%{deps}", "[import-dev.bst, import-bin.bst]"), + ("%{build-deps}", "[import-dev.bst]"), + ("%{runtime-deps}", "[import-bin.bst]"), + ], +) def test_format_deps(cli, datafiles, dep_kind, expected_deps): project = str(datafiles) - target = 'checkout-deps.bst' - result = cli.run(project=project, silent=True, args=[ - 'show', - '--deps', 'none', - '--format', '%{name}: ' + dep_kind, - target]) + target = "checkout-deps.bst" + result = cli.run( + project=project, silent=True, args=["show", "--deps", "none", "--format", "%{name}: " + dep_kind, target] + ) result.assert_success() - expected = '{name}: {deps}'.format(name=target, deps=expected_deps) + expected = "{name}: {deps}".format(name=target, deps=expected_deps) if result.output.strip() != expected: - raise AssertionError("Expected output:\n{}\nInstead received output:\n{}" - .format(expected, result.output)) + raise AssertionError("Expected output:\n{}\nInstead received output:\n{}".format(expected, result.output)) # This tests the resolved value of the 'max-jobs' variable, # ensuring at least that the variables are resolved according # to how the user has configured max-jobs # -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project')) -@pytest.mark.parametrize("cli_value, config_value", [ - (None, None), - (None, '16'), - ('16', None), - ('5', '16'), - ('0', '16'), - ('16', '0'), -]) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) +@pytest.mark.parametrize( + "cli_value, config_value", [(None, None), (None, "16"), ("16", None), ("5", "16"), ("0", "16"), ("16", "0"),] +) def test_max_jobs(cli, datafiles, cli_value, config_value): project = str(datafiles) - target = 'target.bst' + target = "target.bst" # Specify `--max-jobs` if this test sets it args = [] if cli_value is not None: - args += ['--max-jobs', cli_value] - args += ['show', '--deps', 'none', '--format', '%{vars}', target] + args += ["--max-jobs", cli_value] + args += ["show", "--deps", "none", "--format", "%{vars}", target] # Specify `max-jobs` in user configuration if this test sets it if config_value is not None: - cli.configure({ - 'build': { - 'max-jobs': config_value - } - }) + cli.configure({"build": {"max-jobs": config_value}}) result = cli.run(project=project, silent=True, args=args) result.assert_success() loaded = _yaml.load_data(result.output) - loaded_value = loaded.get_int('max-jobs') + loaded_value = loaded.get_int("max-jobs") # We expect the value provided on the command line to take # precedence over the configuration file value, if specified. # # If neither are specified then we expect the default - expected_value = cli_value or config_value or '0' + expected_value = cli_value or config_value or "0" - if expected_value == '0': + if expected_value == "0": # If we are expecting the automatic behavior of using the maximum # number of cores available, just check that it is a value > 0 assert loaded_value > 0, "Automatic setting of max-jobs didnt work" @@ -534,39 +488,32 @@ def test_max_jobs(cli, datafiles, cli_value, config_value): # depends on the changing import element, and one which # depends on it regularly. # -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'strict-depends')) -@pytest.mark.parametrize("target, expected_state", [ - ("non-strict-depends.bst", "cached"), - ("strict-depends.bst", "waiting"), -]) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "strict-depends")) +@pytest.mark.parametrize( + "target, expected_state", [("non-strict-depends.bst", "cached"), ("strict-depends.bst", "waiting"),] +) def test_strict_dependencies(cli, datafiles, target, expected_state): project = str(datafiles) # Configure non strict mode, this will have # an effect on the build and the `bst show` # commands run via cli.get_element_states() - cli.configure({ - 'projects': { - 'test': { - 'strict': False - } - } - }) + cli.configure({"projects": {"test": {"strict": False}}}) - result = cli.run(project=project, silent=True, args=['build', target]) + result = cli.run(project=project, silent=True, args=["build", target]) result.assert_success() - states = cli.get_element_states(project, ['base.bst', target]) - assert states['base.bst'] == 'cached' - assert states[target] == 'cached' + states = cli.get_element_states(project, ["base.bst", target]) + assert states["base.bst"] == "cached" + assert states[target] == "cached" # Now modify the file, effectively causing the common base.bst # dependency to change it's cache key - hello_path = os.path.join(project, 'files', 'hello.txt') - with open(hello_path, 'w') as f: + hello_path = os.path.join(project, "files", "hello.txt") + with open(hello_path, "w") as f: f.write("Goodbye") # Now assert that we have the states we expect as a result - states = cli.get_element_states(project, ['base.bst', target]) - assert states['base.bst'] == 'buildable' + states = cli.get_element_states(project, ["base.bst", target]) + assert states["base.bst"] == "buildable" assert states[target] == expected_state diff --git a/tests/frontend/source_checkout.py b/tests/frontend/source_checkout.py index 8d6bae83b..624e997cd 100644 --- a/tests/frontend/source_checkout.py +++ b/tests/frontend/source_checkout.py @@ -11,23 +11,20 @@ from buildstream.testing import cli # pylint: disable=unused-import from buildstream import utils, _yaml # Project directory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - 'project', -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",) def generate_remote_import_element(input_path, output_path): return { - 'kind': 'import', - 'sources': [ + "kind": "import", + "sources": [ { - 'kind': 'remote', - 'url': 'file://{}'.format(input_path), - 'filename': output_path, - 'ref': utils.sha256sum(input_path), + "kind": "remote", + "url": "file://{}".format(input_path), + "filename": output_path, + "ref": utils.sha256sum(input_path), } - ] + ], } @@ -35,65 +32,60 @@ def generate_remote_import_element(input_path, output_path): @pytest.mark.parametrize( "with_workspace,guess_element", [(True, True), (True, False), (False, False)], - ids=["workspace-guess", "workspace-no-guess", "no-workspace-no-guess"] + ids=["workspace-guess", "workspace-no-guess", "no-workspace-no-guess"], ) def test_source_checkout(datafiles, cli, tmpdir_factory, with_workspace, guess_element): tmpdir = tmpdir_factory.mktemp("") project = str(datafiles) - checkout = os.path.join(cli.directory, 'source-checkout') - target = 'checkout-deps.bst' - workspace = os.path.join(str(tmpdir), 'workspace') + checkout = os.path.join(cli.directory, "source-checkout") + target = "checkout-deps.bst" + workspace = os.path.join(str(tmpdir), "workspace") elm_cmd = [target] if not guess_element else [] if with_workspace: - ws_cmd = ['-C', workspace] + ws_cmd = ["-C", workspace] result = cli.run(project=project, args=["workspace", "open", "--directory", workspace, target]) result.assert_success() else: ws_cmd = [] - args = ws_cmd + ['source', 'checkout', '--deps', 'none', '--directory', checkout, *elm_cmd] + args = ws_cmd + ["source", "checkout", "--deps", "none", "--directory", checkout, *elm_cmd] result = cli.run(project=project, args=args) result.assert_success() - assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config')) + assert os.path.exists(os.path.join(checkout, "checkout-deps", "etc", "buildstream", "config")) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize('force_flag', ['--force', '-f']) +@pytest.mark.parametrize("force_flag", ["--force", "-f"]) def test_source_checkout_force(datafiles, cli, force_flag): project = str(datafiles) - checkout = os.path.join(cli.directory, 'source-checkout') - target = 'checkout-deps.bst' + checkout = os.path.join(cli.directory, "source-checkout") + target = "checkout-deps.bst" # Make the checkout directory with 'some-thing' inside it - os.makedirs(os.path.join(checkout, 'some-thing')) + os.makedirs(os.path.join(checkout, "some-thing")) - result = cli.run(project=project, args=['source', 'checkout', - force_flag, - '--deps', 'none', - '--directory', checkout, - target]) + result = cli.run( + project=project, args=["source", "checkout", force_flag, "--deps", "none", "--directory", checkout, target] + ) result.assert_success() - assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config')) + assert os.path.exists(os.path.join(checkout, "checkout-deps", "etc", "buildstream", "config")) @pytest.mark.datafiles(DATA_DIR) def test_source_checkout_tar(datafiles, cli): project = str(datafiles) - tar = os.path.join(cli.directory, 'source-checkout.tar') - target = 'checkout-deps.bst' + tar = os.path.join(cli.directory, "source-checkout.tar") + target = "checkout-deps.bst" - result = cli.run(project=project, args=['source', 'checkout', - '--tar', tar, - '--deps', 'none', - target]) + result = cli.run(project=project, args=["source", "checkout", "--tar", tar, "--deps", "none", target]) result.assert_success() assert os.path.exists(tar) with tarfile.open(tar) as tf: - expected_content = os.path.join(tar, 'checkout-deps', 'etc', 'buildstream', 'config') + expected_content = os.path.join(tar, "checkout-deps", "etc", "buildstream", "config") tar_members = [f.name for f in tf] for member in tar_members: assert member in expected_content @@ -105,111 +97,106 @@ def test_source_checkout_compressed_tar(datafiles, cli, compression): project = str(datafiles) tarfile_name = "source-checkout.tar" + compression tar = os.path.join(cli.directory, tarfile_name) - target = 'checkout-deps.bst' + target = "checkout-deps.bst" - result = cli.run(project=project, args=['source', 'checkout', - '--tar', tar, - '--compression', compression, - '--deps', 'none', - target]) + result = cli.run( + project=project, + args=["source", "checkout", "--tar", tar, "--compression", compression, "--deps", "none", target], + ) result.assert_success() - tar = tarfile.open(name=tar, mode='r:' + compression) - assert os.path.join('checkout-deps', 'etc', 'buildstream', 'config') in tar.getnames() + tar = tarfile.open(name=tar, mode="r:" + compression) + assert os.path.join("checkout-deps", "etc", "buildstream", "config") in tar.getnames() @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize('deps', [('build'), ('none'), ('run'), ('all')]) +@pytest.mark.parametrize("deps", [("build"), ("none"), ("run"), ("all")]) def test_source_checkout_deps(datafiles, cli, deps): project = str(datafiles) - checkout = os.path.join(cli.directory, 'source-checkout') - target = 'checkout-deps.bst' + checkout = os.path.join(cli.directory, "source-checkout") + target = "checkout-deps.bst" - result = cli.run(project=project, args=['source', 'checkout', - '--directory', checkout, - '--deps', deps, - target]) + result = cli.run(project=project, args=["source", "checkout", "--directory", checkout, "--deps", deps, target]) result.assert_success() # Sources of the target - if deps == 'build': - assert not os.path.exists(os.path.join(checkout, 'checkout-deps')) + if deps == "build": + assert not os.path.exists(os.path.join(checkout, "checkout-deps")) else: - assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config')) + assert os.path.exists(os.path.join(checkout, "checkout-deps", "etc", "buildstream", "config")) # Sources of the target's build dependencies - if deps in ('build', 'all'): - assert os.path.exists(os.path.join(checkout, 'import-dev', 'usr', 'include', 'pony.h')) + if deps in ("build", "all"): + assert os.path.exists(os.path.join(checkout, "import-dev", "usr", "include", "pony.h")) else: - assert not os.path.exists(os.path.join(checkout, 'import-dev')) + assert not os.path.exists(os.path.join(checkout, "import-dev")) # Sources of the target's runtime dependencies - if deps in ('run', 'all'): - assert os.path.exists(os.path.join(checkout, 'import-bin', 'usr', 'bin', 'hello')) + if deps in ("run", "all"): + assert os.path.exists(os.path.join(checkout, "import-bin", "usr", "bin", "hello")) else: - assert not os.path.exists(os.path.join(checkout, 'import-bin')) + assert not os.path.exists(os.path.join(checkout, "import-bin")) @pytest.mark.datafiles(DATA_DIR) def test_source_checkout_except(datafiles, cli): project = str(datafiles) - checkout = os.path.join(cli.directory, 'source-checkout') - target = 'checkout-deps.bst' - - result = cli.run(project=project, args=['source', 'checkout', - '--directory', checkout, - '--deps', 'all', - '--except', 'import-bin.bst', - target]) + checkout = os.path.join(cli.directory, "source-checkout") + target = "checkout-deps.bst" + + result = cli.run( + project=project, + args=["source", "checkout", "--directory", checkout, "--deps", "all", "--except", "import-bin.bst", target], + ) result.assert_success() # Sources for the target should be present - assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config')) + assert os.path.exists(os.path.join(checkout, "checkout-deps", "etc", "buildstream", "config")) # Sources for import-bin.bst should not be present - assert not os.path.exists(os.path.join(checkout, 'import-bin')) + assert not os.path.exists(os.path.join(checkout, "import-bin")) # Sources for other dependencies should be present - assert os.path.exists(os.path.join(checkout, 'import-dev', 'usr', 'include', 'pony.h')) + assert os.path.exists(os.path.join(checkout, "import-dev", "usr", "include", "pony.h")) @pytest.mark.datafiles(DATA_DIR) def test_source_checkout_fetch(datafiles, cli): project = str(datafiles) - checkout = os.path.join(cli.directory, 'source-checkout') - target = 'remote-import-dev.bst' - target_path = os.path.join(project, 'elements', target) + checkout = os.path.join(cli.directory, "source-checkout") + target = "remote-import-dev.bst" + target_path = os.path.join(project, "elements", target) # Create an element with remote source element = generate_remote_import_element( - os.path.join(project, 'files', 'dev-files', 'usr', 'include', 'pony.h'), - 'pony.h') + os.path.join(project, "files", "dev-files", "usr", "include", "pony.h"), "pony.h" + ) _yaml.roundtrip_dump(element, target_path) # Testing implicit fetching requires that we do not have the sources # cached already - assert cli.get_element_state(project, target) == 'fetch needed' + assert cli.get_element_state(project, target) == "fetch needed" - args = ['source', 'checkout'] + args = ["source", "checkout"] args += [target, checkout] - result = cli.run(project=project, args=['source', 'checkout', '--directory', checkout, target]) + result = cli.run(project=project, args=["source", "checkout", "--directory", checkout, target]) result.assert_success() - assert os.path.exists(os.path.join(checkout, 'remote-import-dev', 'pony.h')) + assert os.path.exists(os.path.join(checkout, "remote-import-dev", "pony.h")) @pytest.mark.datafiles(DATA_DIR) def test_source_checkout_build_scripts(cli, tmpdir, datafiles): project_path = str(datafiles) - element_name = 'source-bundle/source-bundle-hello.bst' - normal_name = 'source-bundle-source-bundle-hello' - checkout = os.path.join(str(tmpdir), 'source-checkout') + element_name = "source-bundle/source-bundle-hello.bst" + normal_name = "source-bundle-source-bundle-hello" + checkout = os.path.join(str(tmpdir), "source-checkout") - args = ['source', 'checkout', '--include-build-scripts', '--directory', checkout, element_name] + args = ["source", "checkout", "--include-build-scripts", "--directory", checkout, element_name] result = cli.run(project=project_path, args=args) result.assert_success() # There sould be a script for each element (just one in this case) and a top level build script - expected_scripts = ['build.sh', 'build-' + normal_name] + expected_scripts = ["build.sh", "build-" + normal_name] for script in expected_scripts: assert script in os.listdir(checkout) @@ -217,17 +204,17 @@ def test_source_checkout_build_scripts(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_source_checkout_tar_buildscripts(cli, tmpdir, datafiles): project_path = str(datafiles) - element_name = 'source-bundle/source-bundle-hello.bst' - normal_name = 'source-bundle-source-bundle-hello' - tar_file = os.path.join(str(tmpdir), 'source-checkout.tar') + element_name = "source-bundle/source-bundle-hello.bst" + normal_name = "source-bundle-source-bundle-hello" + tar_file = os.path.join(str(tmpdir), "source-checkout.tar") - args = ['source', 'checkout', '--include-build-scripts', '--tar', tar_file, element_name] + args = ["source", "checkout", "--include-build-scripts", "--tar", tar_file, element_name] result = cli.run(project=project_path, args=args) result.assert_success() - expected_scripts = ['build.sh', 'build-' + normal_name] + expected_scripts = ["build.sh", "build-" + normal_name] - with tarfile.open(tar_file, 'r') as tf: + with tarfile.open(tar_file, "r") as tf: for script in expected_scripts: assert script in tf.getnames() @@ -236,14 +223,11 @@ def test_source_checkout_tar_buildscripts(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_source_checkout_options_tar_and_dir_conflict(cli, tmpdir, datafiles): project = str(datafiles) - checkout = os.path.join(cli.directory, 'source-checkout') - tar_file = os.path.join(str(tmpdir), 'source-checkout.tar') - target = 'checkout-deps.bst' + checkout = os.path.join(cli.directory, "source-checkout") + tar_file = os.path.join(str(tmpdir), "source-checkout.tar") + target = "checkout-deps.bst" - result = cli.run(project=project, args=['source', 'checkout', - '--directory', checkout, - '--tar', tar_file, - target]) + result = cli.run(project=project, args=["source", "checkout", "--directory", checkout, "--tar", tar_file, target]) assert result.exit_code != 0 assert "ERROR: options --directory and --tar conflict" in result.stderr @@ -253,13 +237,12 @@ def test_source_checkout_options_tar_and_dir_conflict(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_source_checkout_compression_without_tar(cli, tmpdir, datafiles): project = str(datafiles) - checkout = os.path.join(cli.directory, 'source-checkout') - target = 'checkout-deps.bst' + checkout = os.path.join(cli.directory, "source-checkout") + target = "checkout-deps.bst" - result = cli.run(project=project, args=['source', 'checkout', - '--directory', checkout, - '--compression', 'xz', - target]) + result = cli.run( + project=project, args=["source", "checkout", "--directory", checkout, "--compression", "xz", target] + ) assert result.exit_code != 0 assert "ERROR: --compression specified without --tar" in result.stderr diff --git a/tests/frontend/track.py b/tests/frontend/track.py index a628043d8..477c81556 100644 --- a/tests/frontend/track.py +++ b/tests/frontend/track.py @@ -14,18 +14,13 @@ from . import configure_project # Project directory TOP_DIR = os.path.dirname(os.path.realpath(__file__)) -DATA_DIR = os.path.join(TOP_DIR, 'project') +DATA_DIR = os.path.join(TOP_DIR, "project") def generate_element(repo, element_path, dep_name=None): - element = { - 'kind': 'import', - 'sources': [ - repo.source_config() - ] - } + element = {"kind": "import", "sources": [repo.source_config()]} if dep_name: - element['depends'] = [dep_name] + element["depends"] = [dep_name] _yaml.roundtrip_dump(element, element_path) @@ -33,21 +28,20 @@ def generate_element(repo, element_path, dep_name=None): @pytest.mark.datafiles(DATA_DIR) def test_track_single(cli, tmpdir, datafiles): project = str(datafiles) - dev_files_path = os.path.join(project, 'files', 'dev-files') - element_path = os.path.join(project, 'elements') - element_dep_name = 'track-test-dep.bst' - element_target_name = 'track-test-target.bst' + dev_files_path = os.path.join(project, "files", "dev-files") + element_path = os.path.join(project, "elements") + element_dep_name = "track-test-dep.bst" + element_target_name = "track-test-target.bst" # Create our repo object of the given source type with # the dev files, and then collect the initial ref. # - repo = create_repo('git', str(tmpdir)) + repo = create_repo("git", str(tmpdir)) repo.create(dev_files_path) # Write out our test targets generate_element(repo, os.path.join(element_path, element_dep_name)) - generate_element(repo, os.path.join(element_path, element_target_name), - dep_name=element_dep_name) + generate_element(repo, os.path.join(element_path, element_target_name), dep_name=element_dep_name) # Assert that tracking is needed for both elements states = cli.get_element_states(project, [element_target_name]) @@ -57,71 +51,69 @@ def test_track_single(cli, tmpdir, datafiles): } # Now first try to track only one element - result = cli.run(project=project, args=[ - 'source', 'track', '--deps', 'none', - element_target_name]) + result = cli.run(project=project, args=["source", "track", "--deps", "none", element_target_name]) result.assert_success() # And now fetch it - result = cli.run(project=project, args=[ - 'source', 'fetch', '--deps', 'none', - element_target_name]) + result = cli.run(project=project, args=["source", "fetch", "--deps", "none", element_target_name]) result.assert_success() # Assert that the dependency is waiting and the target has still never been tracked states = cli.get_element_states(project, [element_target_name]) assert states == { - element_dep_name: 'no reference', - element_target_name: 'waiting', + element_dep_name: "no reference", + element_target_name: "waiting", } @pytest.mark.datafiles(os.path.join(TOP_DIR)) -@pytest.mark.parametrize("ref_storage", [('inline'), ('project-refs')]) +@pytest.mark.parametrize("ref_storage", [("inline"), ("project-refs")]) def test_track_optional(cli, tmpdir, datafiles, ref_storage): - project = os.path.join(datafiles.dirname, datafiles.basename, 'track-optional-' + ref_storage) - dev_files_path = os.path.join(project, 'files') - element_path = os.path.join(project, 'target.bst') + project = os.path.join(datafiles.dirname, datafiles.basename, "track-optional-" + ref_storage) + dev_files_path = os.path.join(project, "files") + element_path = os.path.join(project, "target.bst") # Create our repo object of the given source type with # the dev files, and then collect the initial ref. # - repo = create_repo('git', str(tmpdir)) + repo = create_repo("git", str(tmpdir)) repo.create(dev_files_path) # Now create an optional test branch and add a commit to that, # so two branches with different heads now exist. # - repo.branch('test') + repo.branch("test") repo.add_commit() # Substitute the {repo} for the git repo we created with open(element_path) as f: target_bst = f.read() target_bst = target_bst.format(repo=repo.repo) - with open(element_path, 'w') as f: + with open(element_path, "w") as f: f.write(target_bst) # First track for both options # # We want to track and persist the ref separately in this test # - result = cli.run(project=project, args=['--option', 'test', 'False', 'source', 'track', 'target.bst']) + result = cli.run(project=project, args=["--option", "test", "False", "source", "track", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['--option', 'test', 'True', 'source', 'track', 'target.bst']) + result = cli.run(project=project, args=["--option", "test", "True", "source", "track", "target.bst"]) result.assert_success() # Now fetch the key for both options # - result = cli.run(project=project, args=[ - '--option', 'test', 'False', 'show', '--deps', 'none', '--format', '%{key}', 'target.bst' - ]) + result = cli.run( + project=project, + args=["--option", "test", "False", "show", "--deps", "none", "--format", "%{key}", "target.bst"], + ) result.assert_success() master_key = result.output - result = cli.run(project=project, args=[ - '--option', 'test', 'True', 'show', '--deps', 'none', '--format', '%{key}', 'target.bst' - ]) + result = cli.run( + project=project, + args=["--option", "test", "True", "show", "--deps", "none", "--format", "%{key}", "target.bst"], + ) result.assert_success() test_key = result.output @@ -130,81 +122,72 @@ def test_track_optional(cli, tmpdir, datafiles, ref_storage): assert test_key != master_key -@pytest.mark.datafiles(os.path.join(TOP_DIR, 'track-cross-junction')) -@pytest.mark.parametrize("cross_junction", [('cross'), ('nocross')]) -@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')]) +@pytest.mark.datafiles(os.path.join(TOP_DIR, "track-cross-junction")) +@pytest.mark.parametrize("cross_junction", [("cross"), ("nocross")]) +@pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")]) def test_track_cross_junction(cli, tmpdir, datafiles, cross_junction, ref_storage): project = str(datafiles) - dev_files_path = os.path.join(project, 'files') - target_path = os.path.join(project, 'target.bst') - subtarget_path = os.path.join(project, 'subproject', 'subtarget.bst') + dev_files_path = os.path.join(project, "files") + target_path = os.path.join(project, "target.bst") + subtarget_path = os.path.join(project, "subproject", "subtarget.bst") # Create our repo object of the given source type with # the dev files, and then collect the initial ref. # - repo = create_repo('git', str(tmpdir)) + repo = create_repo("git", str(tmpdir)) repo.create(dev_files_path) # Generate two elements using the git source, one in # the main project and one in the subproject. - generate_element(repo, target_path, dep_name='subproject.bst') + generate_element(repo, target_path, dep_name="subproject.bst") generate_element(repo, subtarget_path) # Generate project.conf # - project_conf = { - 'name': 'test', - 'ref-storage': ref_storage - } - _yaml.roundtrip_dump(project_conf, os.path.join(project, 'project.conf')) + project_conf = {"name": "test", "ref-storage": ref_storage} + _yaml.roundtrip_dump(project_conf, os.path.join(project, "project.conf")) # # FIXME: This can be simplified when we have support # for addressing of junctioned elements. # def get_subproject_element_state(): - result = cli.run(project=project, args=[ - 'show', '--deps', 'all', - '--format', '%{name}|%{state}', 'target.bst' - ]) + result = cli.run(project=project, args=["show", "--deps", "all", "--format", "%{name}|%{state}", "target.bst"]) result.assert_success() # Create two dimentional list of the result, # first line should be the junctioned element - lines = [ - line.split('|') - for line in result.output.splitlines() - ] - assert lines[0][0] == 'subproject-junction.bst:subtarget.bst' + lines = [line.split("|") for line in result.output.splitlines()] + assert lines[0][0] == "subproject-junction.bst:subtarget.bst" return lines[0][1] # # Assert that we have no reference yet for the cross junction element # - assert get_subproject_element_state() == 'no reference' + assert get_subproject_element_state() == "no reference" # Track recursively across the junction - args = ['source', 'track', '--deps', 'all'] - if cross_junction == 'cross': - args += ['--cross-junctions'] - args += ['target.bst'] + args = ["source", "track", "--deps", "all"] + if cross_junction == "cross": + args += ["--cross-junctions"] + args += ["target.bst"] result = cli.run(project=project, args=args) - if ref_storage == 'inline': + if ref_storage == "inline": - if cross_junction == 'cross': + if cross_junction == "cross": # # Cross junction tracking is not allowed when the toplevel project # is using inline ref storage. # - result.assert_main_error(ErrorDomain.PIPELINE, 'untrackable-sources') + result.assert_main_error(ErrorDomain.PIPELINE, "untrackable-sources") else: # # No cross juction tracking was requested # result.assert_success() - assert get_subproject_element_state() == 'no reference' + assert get_subproject_element_state() == "no reference" else: # # Tracking is allowed with project.refs ref storage @@ -214,117 +197,97 @@ def test_track_cross_junction(cli, tmpdir, datafiles, cross_junction, ref_storag # # If cross junction tracking was enabled, we should now be buildable # - if cross_junction == 'cross': - assert get_subproject_element_state() == 'buildable' + if cross_junction == "cross": + assert get_subproject_element_state() == "buildable" else: - assert get_subproject_element_state() == 'no reference' + assert get_subproject_element_state() == "no reference" -@pytest.mark.datafiles(os.path.join(TOP_DIR, 'consistencyerror')) +@pytest.mark.datafiles(os.path.join(TOP_DIR, "consistencyerror")) def test_track_consistency_error(cli, datafiles): project = str(datafiles) # Track the element causing a consistency error - result = cli.run(project=project, args=['source', 'track', 'error.bst']) + result = cli.run(project=project, args=["source", "track", "error.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) - result.assert_task_error(ErrorDomain.SOURCE, 'the-consistency-error') + result.assert_task_error(ErrorDomain.SOURCE, "the-consistency-error") -@pytest.mark.datafiles(os.path.join(TOP_DIR, 'consistencyerror')) +@pytest.mark.datafiles(os.path.join(TOP_DIR, "consistencyerror")) def test_track_consistency_bug(cli, datafiles): project = str(datafiles) # Track the element causing an unhandled exception - result = cli.run(project=project, args=['source', 'track', 'bug.bst']) + result = cli.run(project=project, args=["source", "track", "bug.bst"]) # We expect BuildStream to fail gracefully, with no recorded exception. result.assert_main_error(ErrorDomain.STREAM, None) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')]) +@pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")]) def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage): project = str(datafiles) - subproject_path = os.path.join(project, 'files', 'sub-project') - junction_path = os.path.join(project, 'elements', 'junction.bst') - element_path = os.path.join(project, 'elements', 'junction-dep.bst') + subproject_path = os.path.join(project, "files", "sub-project") + junction_path = os.path.join(project, "elements", "junction.bst") + element_path = os.path.join(project, "elements", "junction-dep.bst") - configure_project(project, { - 'ref-storage': ref_storage - }) + configure_project(project, {"ref-storage": ref_storage}) # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path, store_ref=False) # Create a stack element to depend on a cross junction element # - element = { - 'kind': 'stack', - 'depends': [ - { - 'junction': 'junction.bst', - 'filename': 'import-etc.bst' - } - ] - } + element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) # Now try to track it, this will bail with the appropriate error # informing the user to track the junction first - result = cli.run(project=project, args=['source', 'track', 'junction-dep.bst']) + result = cli.run(project=project, args=["source", "track", "junction-dep.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT) # Assert that we have the expected provenance encoded into the error - element_node = _yaml.load(element_path, shortname='junction-dep.bst') - ref_node = element_node.get_sequence('depends').mapping_at(0) + element_node = _yaml.load(element_path, shortname="junction-dep.bst") + ref_node = element_node.get_sequence("depends").mapping_at(0) provenance = ref_node.get_provenance() assert str(provenance) in result.stderr @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')]) +@pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")]) def test_junction_element(cli, tmpdir, datafiles, ref_storage): project = str(datafiles) - subproject_path = os.path.join(project, 'files', 'sub-project') - junction_path = os.path.join(project, 'elements', 'junction.bst') - element_path = os.path.join(project, 'elements', 'junction-dep.bst') + subproject_path = os.path.join(project, "files", "sub-project") + junction_path = os.path.join(project, "elements", "junction.bst") + element_path = os.path.join(project, "elements", "junction-dep.bst") - configure_project(project, { - 'ref-storage': ref_storage - }) + configure_project(project, {"ref-storage": ref_storage}) # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path, store_ref=False) # Create a stack element to depend on a cross junction element # - element = { - 'kind': 'stack', - 'depends': [ - { - 'junction': 'junction.bst', - 'filename': 'import-etc.bst' - } - ] - } + element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) # First demonstrate that showing the pipeline yields an error - result = cli.run(project=project, args=['show', 'junction-dep.bst']) + result = cli.run(project=project, args=["show", "junction-dep.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT) # Assert that we have the expected provenance encoded into the error - element_node = _yaml.load(element_path, shortname='junction-dep.bst') - ref_node = element_node.get_sequence('depends').mapping_at(0) + element_node = _yaml.load(element_path, shortname="junction-dep.bst") + ref_node = element_node.get_sequence("depends").mapping_at(0) provenance = ref_node.get_provenance() assert str(provenance) in result.stderr # Now track the junction itself - result = cli.run(project=project, args=['source', 'track', 'junction.bst']) + result = cli.run(project=project, args=["source", "track", "junction.bst"]) result.assert_success() # Now assert element state (via bst show under the hood) of the dep again - assert cli.get_element_state(project, 'junction-dep.bst') == 'waiting' + assert cli.get_element_state(project, "junction-dep.bst") == "waiting" @pytest.mark.datafiles(DATA_DIR) @@ -333,15 +296,13 @@ def test_track_error_cannot_write_file(cli, tmpdir, datafiles): pytest.skip("This is not testable with root permissions") project = str(datafiles) - dev_files_path = os.path.join(project, 'files', 'dev-files') - element_path = os.path.join(project, 'elements') - element_name = 'track-test.bst' + dev_files_path = os.path.join(project, "files", "dev-files") + element_path = os.path.join(project, "elements") + element_name = "track-test.bst" - configure_project(project, { - 'ref-storage': 'inline' - }) + configure_project(project, {"ref-storage": "inline"}) - repo = create_repo('git', str(tmpdir)) + repo = create_repo("git", str(tmpdir)) repo.create(dev_files_path) element_full_path = os.path.join(element_path, element_name) @@ -352,9 +313,9 @@ def test_track_error_cannot_write_file(cli, tmpdir, datafiles): read_mask = stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH os.chmod(element_path, stat.S_IMODE(st.st_mode) & ~read_mask) - result = cli.run(project=project, args=['source', 'track', element_name]) + result = cli.run(project=project, args=["source", "track", element_name]) result.assert_main_error(ErrorDomain.STREAM, None) - result.assert_task_error(ErrorDomain.SOURCE, 'save-ref-error') + result.assert_task_error(ErrorDomain.SOURCE, "save-ref-error") finally: os.chmod(element_path, stat.S_IMODE(st.st_mode)) @@ -362,14 +323,14 @@ def test_track_error_cannot_write_file(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_no_needless_overwrite(cli, tmpdir, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename) - dev_files_path = os.path.join(project, 'files', 'dev-files') - element_path = os.path.join(project, 'elements') - target = 'track-test-target.bst' + dev_files_path = os.path.join(project, "files", "dev-files") + element_path = os.path.join(project, "elements") + target = "track-test-target.bst" # Create our repo object of the given source type with # the dev files, and then collect the initial ref. # - repo = create_repo('git', str(tmpdir)) + repo = create_repo("git", str(tmpdir)) repo.create(dev_files_path) # Write out our test target and assert it exists @@ -380,10 +341,10 @@ def test_no_needless_overwrite(cli, tmpdir, datafiles): # Assert tracking is needed states = cli.get_element_states(project, [target]) - assert states[target] == 'no reference' + assert states[target] == "no reference" # Perform the track - result = cli.run(project=project, args=['source', 'track', target]) + result = cli.run(project=project, args=["source", "track", target]) result.assert_success() track1_mtime = os.path.getmtime(path_to_target) @@ -391,7 +352,7 @@ def test_no_needless_overwrite(cli, tmpdir, datafiles): assert creation_mtime != track1_mtime # Now (needlessly) track again - result = cli.run(project=project, args=['source', 'track', target]) + result = cli.run(project=project, args=["source", "track", target]) result.assert_success() track2_mtime = os.path.getmtime(path_to_target) diff --git a/tests/frontend/version.py b/tests/frontend/version.py index e7db19915..279a51747 100644 --- a/tests/frontend/version.py +++ b/tests/frontend/version.py @@ -12,13 +12,13 @@ def assert_version(cli_version_output): major, minor = utils.get_bst_version() expected_start = "{}.{}".format(major, minor) if not cli_version_output.startswith(expected_start): - raise AssertionError("Version output expected to begin with '{}'," - .format(expected_start) + - " output was: {}" - .format(cli_version_output)) + raise AssertionError( + "Version output expected to begin with '{}',".format(expected_start) + + " output was: {}".format(cli_version_output) + ) def test_version(cli): - result = cli.run(args=['--version']) + result = cli.run(args=["--version"]) result.assert_success() assert_version(result.output) diff --git a/tests/frontend/workspace.py b/tests/frontend/workspace.py index 0c7e63525..80db8cc98 100644 --- a/tests/frontend/workspace.py +++ b/tests/frontend/workspace.py @@ -45,13 +45,10 @@ from tests.testutils import create_artifact_share, create_element_size, wait_for repo_kinds = [(kind) for kind in ALL_REPO_KINDS] # Project directory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project", -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",) -class WorkspaceCreator(): +class WorkspaceCreator: def __init__(self, cli, tmpdir, datafiles, project_path=None): self.cli = cli self.tmpdir = tmpdir @@ -63,17 +60,16 @@ class WorkspaceCreator(): shutil.copytree(str(datafiles), project_path) self.project_path = project_path - self.bin_files_path = os.path.join(project_path, 'files', 'bin-files') + self.bin_files_path = os.path.join(project_path, "files", "bin-files") - self.workspace_cmd = os.path.join(self.project_path, 'workspace_cmd') + self.workspace_cmd = os.path.join(self.project_path, "workspace_cmd") - def create_workspace_element(self, kind, suffix='', workspace_dir=None, - element_attrs=None): - element_name = 'workspace-test-{}{}.bst'.format(kind, suffix) - element_path = os.path.join(self.project_path, 'elements') + def create_workspace_element(self, kind, suffix="", workspace_dir=None, element_attrs=None): + element_name = "workspace-test-{}{}.bst".format(kind, suffix) + element_path = os.path.join(self.project_path, "elements") if not workspace_dir: workspace_dir = os.path.join(self.workspace_cmd, element_name) - if workspace_dir[-4:] == '.bst': + if workspace_dir[-4:] == ".bst": workspace_dir = workspace_dir[:-4] # Create our repo object of the given source type with @@ -82,59 +78,48 @@ class WorkspaceCreator(): ref = repo.create(self.bin_files_path) # Write out our test target - element = { - 'kind': 'import', - 'sources': [ - repo.source_config(ref=ref) - ] - } + element = {"kind": "import", "sources": [repo.source_config(ref=ref)]} if element_attrs: element = {**element, **element_attrs} - _yaml.roundtrip_dump(element, - os.path.join(element_path, element_name)) + _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) return element_name, element_path, workspace_dir - def create_workspace_elements(self, kinds, suffixs=None, workspace_dir_usr=None, - element_attrs=None): + def create_workspace_elements(self, kinds, suffixs=None, workspace_dir_usr=None, element_attrs=None): element_tuples = [] if suffixs is None: - suffixs = ['', ] * len(kinds) + suffixs = ["",] * len(kinds) else: if len(suffixs) != len(kinds): raise "terable error" for suffix, kind in zip(suffixs, kinds): - element_name, _, workspace_dir = \ - self.create_workspace_element(kind, suffix, workspace_dir_usr, - element_attrs) + element_name, _, workspace_dir = self.create_workspace_element( + kind, suffix, workspace_dir_usr, element_attrs + ) element_tuples.append((element_name, workspace_dir)) # Assert that there is a fetch is needed - states = self.cli.get_element_states(self.project_path, [ - e for e, _ in element_tuples - ]) - assert not any(states[e] != 'fetch needed' for e, _ in element_tuples) + states = self.cli.get_element_states(self.project_path, [e for e, _ in element_tuples]) + assert not any(states[e] != "fetch needed" for e, _ in element_tuples) return element_tuples - def open_workspaces(self, kinds, suffixs=None, workspace_dir=None, - element_attrs=None, no_checkout=False): + def open_workspaces(self, kinds, suffixs=None, workspace_dir=None, element_attrs=None, no_checkout=False): - element_tuples = self.create_workspace_elements(kinds, suffixs, workspace_dir, - element_attrs) + element_tuples = self.create_workspace_elements(kinds, suffixs, workspace_dir, element_attrs) os.makedirs(self.workspace_cmd, exist_ok=True) # Now open the workspace, this should have the effect of automatically # tracking & fetching the source from the repo. - args = ['workspace', 'open'] + args = ["workspace", "open"] if no_checkout: - args.append('--no-checkout') + args.append("--no-checkout") if workspace_dir is not None: assert len(element_tuples) == 1, "test logic error" _, workspace_dir = element_tuples[0] - args.extend(['--directory', workspace_dir]) + args.extend(["--directory", workspace_dir]) args.extend([element_name for element_name, workspace_dir_suffix in element_tuples]) result = self.cli.run(cwd=self.workspace_cmd, project=self.project_path, args=args) @@ -143,24 +128,30 @@ class WorkspaceCreator(): if not no_checkout: # Assert that we are now buildable because the source is now cached. - states = self.cli.get_element_states(self.project_path, [ - e for e, _ in element_tuples - ]) - assert not any(states[e] != 'buildable' for e, _ in element_tuples) + states = self.cli.get_element_states(self.project_path, [e for e, _ in element_tuples]) + assert not any(states[e] != "buildable" for e, _ in element_tuples) # Check that the executable hello file is found in each workspace for _, workspace in element_tuples: - filename = os.path.join(workspace, 'usr', 'bin', 'hello') + filename = os.path.join(workspace, "usr", "bin", "hello") assert os.path.exists(filename) return element_tuples -def open_workspace(cli, tmpdir, datafiles, kind, suffix='', workspace_dir=None, - project_path=None, element_attrs=None, no_checkout=False): +def open_workspace( + cli, + tmpdir, + datafiles, + kind, + suffix="", + workspace_dir=None, + project_path=None, + element_attrs=None, + no_checkout=False, +): workspace_object = WorkspaceCreator(cli, tmpdir, datafiles, project_path) - workspaces = workspace_object.open_workspaces((kind, ), (suffix, ), workspace_dir, - element_attrs, no_checkout) + workspaces = workspace_object.open_workspaces((kind,), (suffix,), workspace_dir, element_attrs, no_checkout) assert len(workspaces) == 1 element_name, workspace = workspaces[0] return element_name, workspace_object.project_path, workspace @@ -176,11 +167,10 @@ def test_open_bzr_customize(cli, tmpdir, datafiles): # Check that the correct origin branch is set element_config = _yaml.load(os.path.join(project, "elements", element_name)) - source_config = element_config.get_sequence('sources').mapping_at(0) + source_config = element_config.get_sequence("sources").mapping_at(0) output = subprocess.check_output(["bzr", "info"], cwd=workspace) - stripped_url = source_config.get_str('url').lstrip("file:///") - expected_output_str = ("checkout of branch: /{}/{}" - .format(stripped_url, source_config.get_str('track'))) + stripped_url = source_config.get_str("url").lstrip("file:///") + expected_output_str = "checkout of branch: /{}/{}".format(stripped_url, source_config.get_str("track")) assert expected_output_str in str(output) @@ -193,13 +183,13 @@ def test_open_multi(cli, tmpdir, datafiles): for (elname, workspace), kind in zip(workspaces, repo_kinds): assert kind in elname workspace_lsdir = os.listdir(workspace) - if kind == 'git': - assert '.git' in workspace_lsdir - elif kind == 'bzr': - assert '.bzr' in workspace_lsdir + if kind == "git": + assert ".git" in workspace_lsdir + elif kind == "bzr": + assert ".bzr" in workspace_lsdir else: - assert '.git' not in workspace_lsdir - assert '.bzr' not in workspace_lsdir + assert ".git" not in workspace_lsdir + assert ".bzr" not in workspace_lsdir @pytest.mark.skipif(os.geteuid() == 0, reason="root may have CAP_DAC_OVERRIDE and ignore permissions") @@ -212,9 +202,9 @@ def test_open_multi_unwritable(cli, tmpdir, datafiles): # Now open the workspace, this should have the effect of automatically # tracking & fetching the source from the repo. - args = ['workspace', 'open'] + args = ["workspace", "open"] args.extend([element_name for element_name, workspace_dir_suffix in element_tuples]) - cli.configure({'workspacedir': workspace_object.workspace_cmd}) + cli.configure({"workspacedir": workspace_object.workspace_cmd}) cwdstat = os.stat(workspace_object.workspace_cmd) try: @@ -239,14 +229,15 @@ def test_open_multi_with_directory(cli, tmpdir, datafiles): # Now open the workspace, this should have the effect of automatically # tracking & fetching the source from the repo. - args = ['workspace', 'open'] - args.extend(['--directory', 'any/dir/should/fail']) + args = ["workspace", "open"] + args.extend(["--directory", "any/dir/should/fail"]) args.extend([element_name for element_name, workspace_dir_suffix in element_tuples]) - result = workspace_object.cli.run(cwd=workspace_object.workspace_cmd, project=workspace_object.project_path, - args=args) + result = workspace_object.cli.run( + cwd=workspace_object.workspace_cmd, project=workspace_object.project_path, args=args + ) - result.assert_main_error(ErrorDomain.STREAM, 'directory-with-multiple-elements') + result.assert_main_error(ErrorDomain.STREAM, "directory-with-multiple-elements") @pytest.mark.datafiles(DATA_DIR) @@ -254,31 +245,30 @@ def test_open_defaultlocation(cli, tmpdir, datafiles): workspace_object = WorkspaceCreator(cli, tmpdir, datafiles) # pylint: disable=unbalanced-tuple-unpacking - ((element_name, workspace_dir), ) = workspace_object.create_workspace_elements(['git'], ['git']) + ((element_name, workspace_dir),) = workspace_object.create_workspace_elements(["git"], ["git"]) os.makedirs(workspace_object.workspace_cmd, exist_ok=True) # Now open the workspace, this should have the effect of automatically # tracking & fetching the source from the repo. - args = ['workspace', 'open'] + args = ["workspace", "open"] args.append(element_name) # In the other tests we set the cmd to workspace_object.workspace_cmd with the optional # argument, cwd for the workspace_object.cli.run function. But hear we set the default # workspace location to workspace_object.workspace_cmd and run the cli.run function with # no cwd option so that it runs in the project directory. - cli.configure({'workspacedir': workspace_object.workspace_cmd}) - result = workspace_object.cli.run(project=workspace_object.project_path, - args=args) + cli.configure({"workspacedir": workspace_object.workspace_cmd}) + result = workspace_object.cli.run(project=workspace_object.project_path, args=args) result.assert_success() - assert cli.get_element_state(workspace_object.project_path, element_name) == 'buildable' + assert cli.get_element_state(workspace_object.project_path, element_name) == "buildable" # Check that the executable hello file is found in the workspace # even though the cli.run function was not run with cwd = workspace_object.workspace_cmd # the workspace should be created in there as we used the 'workspacedir' configuration # option. - filename = os.path.join(workspace_dir, 'usr', 'bin', 'hello') + filename = os.path.join(workspace_dir, "usr", "bin", "hello") assert os.path.exists(filename) @@ -287,117 +277,110 @@ def test_open_defaultlocation_exists(cli, tmpdir, datafiles): workspace_object = WorkspaceCreator(cli, tmpdir, datafiles) # pylint: disable=unbalanced-tuple-unpacking - ((element_name, workspace_dir), ) = workspace_object.create_workspace_elements(['git'], ['git']) + ((element_name, workspace_dir),) = workspace_object.create_workspace_elements(["git"], ["git"]) os.makedirs(workspace_object.workspace_cmd, exist_ok=True) - with open(workspace_dir, 'w') as fl: - fl.write('foo') + with open(workspace_dir, "w") as fl: + fl.write("foo") # Now open the workspace, this should have the effect of automatically # tracking & fetching the source from the repo. - args = ['workspace', 'open'] + args = ["workspace", "open"] args.append(element_name) # In the other tests we set the cmd to workspace_object.workspace_cmd with the optional # argument, cwd for the workspace_object.cli.run function. But hear we set the default # workspace location to workspace_object.workspace_cmd and run the cli.run function with # no cwd option so that it runs in the project directory. - cli.configure({'workspacedir': workspace_object.workspace_cmd}) - result = workspace_object.cli.run(project=workspace_object.project_path, - args=args) + cli.configure({"workspacedir": workspace_object.workspace_cmd}) + result = workspace_object.cli.run(project=workspace_object.project_path, args=args) - result.assert_main_error(ErrorDomain.STREAM, 'bad-directory') + result.assert_main_error(ErrorDomain.STREAM, "bad-directory") @pytest.mark.datafiles(DATA_DIR) def test_open_track(cli, tmpdir, datafiles): - open_workspace(cli, tmpdir, datafiles, 'git') + open_workspace(cli, tmpdir, datafiles, "git") @pytest.mark.datafiles(DATA_DIR) def test_open_noclose_open(cli, tmpdir, datafiles): # opening the same workspace twice without closing it should fail - element_name, project, _ = open_workspace(cli, tmpdir, datafiles, 'git') + element_name, project, _ = open_workspace(cli, tmpdir, datafiles, "git") - result = cli.run(project=project, args=['workspace', 'open', element_name]) + result = cli.run(project=project, args=["workspace", "open", element_name]) result.assert_main_error(ErrorDomain.STREAM, None) @pytest.mark.datafiles(DATA_DIR) def test_open_force(cli, tmpdir, datafiles): - element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git') + element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git") # Close the workspace - result = cli.run(project=project, args=[ - 'workspace', 'close', element_name - ]) + result = cli.run(project=project, args=["workspace", "close", element_name]) result.assert_success() # Assert the workspace dir still exists assert os.path.exists(workspace) # Now open the workspace again with --force, this should happily succeed - result = cli.run(project=project, args=[ - 'workspace', 'open', '--force', '--directory', workspace, element_name - ]) + result = cli.run(project=project, args=["workspace", "open", "--force", "--directory", workspace, element_name]) result.assert_success() @pytest.mark.datafiles(DATA_DIR) def test_open_force_open(cli, tmpdir, datafiles): - element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git') + element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git") - result = cli.run(project=project, args=['workspace', 'close', element_name]) + result = cli.run(project=project, args=["workspace", "close", element_name]) result.assert_success() # Assert the workspace dir exists assert os.path.exists(workspace) # Now open the workspace again with --force, this should happily succeed - result = cli.run(project=project, args=[ - 'workspace', 'open', '--force', '--directory', workspace, element_name - ]) + result = cli.run(project=project, args=["workspace", "open", "--force", "--directory", workspace, element_name]) result.assert_success() # Regression test for #1086. @pytest.mark.datafiles(DATA_DIR) def test_open_force_open_no_checkout(cli, tmpdir, datafiles): - element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git') - hello_path = os.path.join(workspace, 'hello.txt') + element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git") + hello_path = os.path.join(workspace, "hello.txt") # Assert the workspace dir exists assert os.path.exists(workspace) # Create a new file in the workspace - with open(hello_path, 'w') as f: - f.write('hello') + with open(hello_path, "w") as f: + f.write("hello") # Now open the workspace again with --force and --no-checkout - result = cli.run(project=project, args=[ - 'workspace', 'open', '--force', '--no-checkout', '--directory', workspace, element_name - ]) + result = cli.run( + project=project, args=["workspace", "open", "--force", "--no-checkout", "--directory", workspace, element_name] + ) result.assert_success() # Ensure that our files were not overwritten assert os.path.exists(hello_path) with open(hello_path) as f: - assert f.read() == 'hello' + assert f.read() == "hello" @pytest.mark.datafiles(DATA_DIR) def test_open_force_different_workspace(cli, tmpdir, datafiles): - _, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git', "-alpha") + _, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", "-alpha") # Assert the workspace dir exists assert os.path.exists(workspace) - hello_path = os.path.join(workspace, 'usr', 'bin', 'hello') - hello1_path = os.path.join(workspace, 'usr', 'bin', 'hello1') + hello_path = os.path.join(workspace, "usr", "bin", "hello") + hello1_path = os.path.join(workspace, "usr", "bin", "hello1") tmpdir = os.path.join(str(tmpdir), "-beta") shutil.move(hello_path, hello1_path) - element_name2, _, workspace2 = open_workspace(cli, tmpdir, datafiles, 'git', "-beta") + element_name2, _, workspace2 = open_workspace(cli, tmpdir, datafiles, "git", "-beta") # Assert the workspace dir exists assert os.path.exists(workspace2) @@ -406,15 +389,13 @@ def test_open_force_different_workspace(cli, tmpdir, datafiles): assert os.path.exists(hello1_path) # Assert that workspace 2 contains the unmodified file - assert os.path.exists(os.path.join(workspace2, 'usr', 'bin', 'hello')) + assert os.path.exists(os.path.join(workspace2, "usr", "bin", "hello")) - result = cli.run(project=project, args=['workspace', 'close', element_name2]) + result = cli.run(project=project, args=["workspace", "close", element_name2]) result.assert_success() # Now open the workspace again with --force, this should happily succeed - result = cli.run(project=project, args=[ - 'workspace', 'open', '--force', '--directory', workspace, element_name2 - ]) + result = cli.run(project=project, args=["workspace", "open", "--force", "--directory", workspace, element_name2]) result.assert_success() # Assert that the file in workspace 1 has been replaced @@ -425,12 +406,10 @@ def test_open_force_different_workspace(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_close(cli, tmpdir, datafiles): - element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git') + element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git") # Close the workspace - result = cli.run(project=project, args=[ - 'workspace', 'close', '--remove-dir', element_name - ]) + result = cli.run(project=project, args=["workspace", "close", "--remove-dir", element_name]) result.assert_success() # Assert the workspace dir has been deleted @@ -440,17 +419,15 @@ def test_close(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_close_external_after_move_project(cli, tmpdir, datafiles): workspace_dir = os.path.join(str(tmpdir), "workspace") - project_path = os.path.join(str(tmpdir), 'initial_project') - element_name, _, _ = open_workspace(cli, tmpdir, datafiles, 'git', "", workspace_dir, project_path) + project_path = os.path.join(str(tmpdir), "initial_project") + element_name, _, _ = open_workspace(cli, tmpdir, datafiles, "git", "", workspace_dir, project_path) assert os.path.exists(workspace_dir) - moved_dir = os.path.join(str(tmpdir), 'external_project') + moved_dir = os.path.join(str(tmpdir), "external_project") shutil.move(project_path, moved_dir) assert os.path.exists(moved_dir) # Close the workspace - result = cli.run(project=moved_dir, args=[ - 'workspace', 'close', '--remove-dir', element_name - ]) + result = cli.run(project=moved_dir, args=["workspace", "close", "--remove-dir", element_name]) result.assert_success() # Assert the workspace dir has been deleted @@ -459,36 +436,33 @@ def test_close_external_after_move_project(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_close_internal_after_move_project(cli, tmpdir, datafiles): - initial_dir = os.path.join(str(tmpdir), 'initial_project') - initial_workspace = os.path.join(initial_dir, 'workspace') - element_name, _, _ = open_workspace(cli, tmpdir, datafiles, 'git', - workspace_dir=initial_workspace, project_path=initial_dir) - moved_dir = os.path.join(str(tmpdir), 'internal_project') + initial_dir = os.path.join(str(tmpdir), "initial_project") + initial_workspace = os.path.join(initial_dir, "workspace") + element_name, _, _ = open_workspace( + cli, tmpdir, datafiles, "git", workspace_dir=initial_workspace, project_path=initial_dir + ) + moved_dir = os.path.join(str(tmpdir), "internal_project") shutil.move(initial_dir, moved_dir) assert os.path.exists(moved_dir) # Close the workspace - result = cli.run(project=moved_dir, args=[ - 'workspace', 'close', '--remove-dir', element_name - ]) + result = cli.run(project=moved_dir, args=["workspace", "close", "--remove-dir", element_name]) result.assert_success() # Assert the workspace dir has been deleted - workspace = os.path.join(moved_dir, 'workspace') + workspace = os.path.join(moved_dir, "workspace") assert not os.path.exists(workspace) @pytest.mark.datafiles(DATA_DIR) def test_close_removed(cli, tmpdir, datafiles): - element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git') + element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git") # Remove it first, closing the workspace should work shutil.rmtree(workspace) # Close the workspace - result = cli.run(project=project, args=[ - 'workspace', 'close', element_name - ]) + result = cli.run(project=project, args=["workspace", "close", element_name]) result.assert_success() # Assert the workspace dir has been deleted @@ -497,8 +471,8 @@ def test_close_removed(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_close_nonexistant_element(cli, tmpdir, datafiles): - element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git') - element_path = os.path.join(datafiles.dirname, datafiles.basename, 'elements', element_name) + element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git") + element_path = os.path.join(datafiles.dirname, datafiles.basename, "elements", element_name) # First brutally remove the element.bst file, ensuring that # the element does not exist anymore in the project where @@ -506,9 +480,7 @@ def test_close_nonexistant_element(cli, tmpdir, datafiles): os.remove(element_path) # Close the workspace - result = cli.run(project=project, args=[ - 'workspace', 'close', '--remove-dir', element_name - ]) + result = cli.run(project=project, args=["workspace", "close", "--remove-dir", element_name]) result.assert_success() # Assert the workspace dir has been deleted @@ -517,17 +489,13 @@ def test_close_nonexistant_element(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_close_multiple(cli, tmpdir, datafiles): - tmpdir_alpha = os.path.join(str(tmpdir), 'alpha') - tmpdir_beta = os.path.join(str(tmpdir), 'beta') - alpha, project, workspace_alpha = open_workspace( - cli, tmpdir_alpha, datafiles, 'git', suffix='-alpha') - beta, project, workspace_beta = open_workspace( - cli, tmpdir_beta, datafiles, 'git', suffix='-beta') + tmpdir_alpha = os.path.join(str(tmpdir), "alpha") + tmpdir_beta = os.path.join(str(tmpdir), "beta") + alpha, project, workspace_alpha = open_workspace(cli, tmpdir_alpha, datafiles, "git", suffix="-alpha") + beta, project, workspace_beta = open_workspace(cli, tmpdir_beta, datafiles, "git", suffix="-beta") # Close the workspaces - result = cli.run(project=project, args=[ - 'workspace', 'close', '--remove-dir', alpha, beta - ]) + result = cli.run(project=project, args=["workspace", "close", "--remove-dir", alpha, beta]) result.assert_success() # Assert the workspace dirs have been deleted @@ -537,17 +505,13 @@ def test_close_multiple(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_close_all(cli, tmpdir, datafiles): - tmpdir_alpha = os.path.join(str(tmpdir), 'alpha') - tmpdir_beta = os.path.join(str(tmpdir), 'beta') - _, project, workspace_alpha = open_workspace( - cli, tmpdir_alpha, datafiles, 'git', suffix='-alpha') - _, project, workspace_beta = open_workspace( - cli, tmpdir_beta, datafiles, 'git', suffix='-beta') + tmpdir_alpha = os.path.join(str(tmpdir), "alpha") + tmpdir_beta = os.path.join(str(tmpdir), "beta") + _, project, workspace_alpha = open_workspace(cli, tmpdir_alpha, datafiles, "git", suffix="-alpha") + _, project, workspace_beta = open_workspace(cli, tmpdir_beta, datafiles, "git", suffix="-beta") # Close the workspaces - result = cli.run(project=project, args=[ - 'workspace', 'close', '--remove-dir', '--all' - ]) + result = cli.run(project=project, args=["workspace", "close", "--remove-dir", "--all"]) result.assert_success() # Assert the workspace dirs have been deleted @@ -558,45 +522,43 @@ def test_close_all(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_reset(cli, tmpdir, datafiles): # Open the workspace - element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git') + element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git") # Modify workspace - shutil.rmtree(os.path.join(workspace, 'usr', 'bin')) - os.makedirs(os.path.join(workspace, 'etc')) - with open(os.path.join(workspace, 'etc', 'pony.conf'), 'w') as f: + shutil.rmtree(os.path.join(workspace, "usr", "bin")) + os.makedirs(os.path.join(workspace, "etc")) + with open(os.path.join(workspace, "etc", "pony.conf"), "w") as f: f.write("PONY='pink'") # Now reset the open workspace, this should have the # effect of reverting our changes. - result = cli.run(project=project, args=[ - 'workspace', 'reset', element_name - ]) + result = cli.run(project=project, args=["workspace", "reset", element_name]) result.assert_success() - assert os.path.exists(os.path.join(workspace, 'usr', 'bin', 'hello')) - assert not os.path.exists(os.path.join(workspace, 'etc', 'pony.conf')) + assert os.path.exists(os.path.join(workspace, "usr", "bin", "hello")) + assert not os.path.exists(os.path.join(workspace, "etc", "pony.conf")) @pytest.mark.datafiles(DATA_DIR) def test_reset_soft(cli, tmpdir, datafiles): # Open the workspace - element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git') + element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git") - assert cli.get_element_state(project, element_name) == 'buildable' + assert cli.get_element_state(project, element_name) == "buildable" - hello_path = os.path.join(workspace, 'usr', 'bin', 'hello') - pony_path = os.path.join(workspace, 'etc', 'pony.conf') + hello_path = os.path.join(workspace, "usr", "bin", "hello") + pony_path = os.path.join(workspace, "etc", "pony.conf") - assert os.path.exists(os.path.join(workspace, 'usr', 'bin')) + assert os.path.exists(os.path.join(workspace, "usr", "bin")) assert os.path.exists(hello_path) assert not os.path.exists(pony_path) key_1 = cli.get_element_key(project, element_name) - assert key_1 != "{:?<64}".format('') - result = cli.run(project=project, args=['build', element_name]) + assert key_1 != "{:?<64}".format("") + result = cli.run(project=project, args=["build", element_name]) result.assert_success() - assert cli.get_element_state(project, element_name) == 'cached' + assert cli.get_element_state(project, element_name) == "cached" key_2 = cli.get_element_key(project, element_name) - assert key_2 != "{:?<64}".format('') + assert key_2 != "{:?<64}".format("") # workspace keys are not recalculated assert key_1 == key_2 @@ -604,99 +566,87 @@ def test_reset_soft(cli, tmpdir, datafiles): wait_for_cache_granularity() # Modify workspace - shutil.rmtree(os.path.join(workspace, 'usr', 'bin')) - os.makedirs(os.path.join(workspace, 'etc')) - with open(os.path.join(workspace, 'etc', 'pony.conf'), 'w') as f: + shutil.rmtree(os.path.join(workspace, "usr", "bin")) + os.makedirs(os.path.join(workspace, "etc")) + with open(os.path.join(workspace, "etc", "pony.conf"), "w") as f: f.write("PONY='pink'") - assert not os.path.exists(os.path.join(workspace, 'usr', 'bin')) + assert not os.path.exists(os.path.join(workspace, "usr", "bin")) assert os.path.exists(pony_path) # Now soft-reset the open workspace, this should not revert the changes - result = cli.run(project=project, args=[ - 'workspace', 'reset', '--soft', element_name - ]) + result = cli.run(project=project, args=["workspace", "reset", "--soft", element_name]) result.assert_success() # we removed this dir - assert not os.path.exists(os.path.join(workspace, 'usr', 'bin')) + assert not os.path.exists(os.path.join(workspace, "usr", "bin")) # and added this one - assert os.path.exists(os.path.join(workspace, 'etc', 'pony.conf')) + assert os.path.exists(os.path.join(workspace, "etc", "pony.conf")) - assert cli.get_element_state(project, element_name) == 'buildable' + assert cli.get_element_state(project, element_name) == "buildable" key_3 = cli.get_element_key(project, element_name) - assert key_3 != "{:?<64}".format('') + assert key_3 != "{:?<64}".format("") assert key_1 != key_3 @pytest.mark.datafiles(DATA_DIR) def test_reset_multiple(cli, tmpdir, datafiles): # Open the workspaces - tmpdir_alpha = os.path.join(str(tmpdir), 'alpha') - tmpdir_beta = os.path.join(str(tmpdir), 'beta') - alpha, project, workspace_alpha = open_workspace( - cli, tmpdir_alpha, datafiles, 'git', suffix='-alpha') - beta, project, workspace_beta = open_workspace( - cli, tmpdir_beta, datafiles, 'git', suffix='-beta') + tmpdir_alpha = os.path.join(str(tmpdir), "alpha") + tmpdir_beta = os.path.join(str(tmpdir), "beta") + alpha, project, workspace_alpha = open_workspace(cli, tmpdir_alpha, datafiles, "git", suffix="-alpha") + beta, project, workspace_beta = open_workspace(cli, tmpdir_beta, datafiles, "git", suffix="-beta") # Modify workspaces - shutil.rmtree(os.path.join(workspace_alpha, 'usr', 'bin')) - os.makedirs(os.path.join(workspace_beta, 'etc')) - with open(os.path.join(workspace_beta, 'etc', 'pony.conf'), 'w') as f: + shutil.rmtree(os.path.join(workspace_alpha, "usr", "bin")) + os.makedirs(os.path.join(workspace_beta, "etc")) + with open(os.path.join(workspace_beta, "etc", "pony.conf"), "w") as f: f.write("PONY='pink'") # Now reset the open workspaces, this should have the # effect of reverting our changes. - result = cli.run(project=project, args=[ - 'workspace', 'reset', alpha, beta, - ]) + result = cli.run(project=project, args=["workspace", "reset", alpha, beta,]) result.assert_success() - assert os.path.exists(os.path.join(workspace_alpha, 'usr', 'bin', 'hello')) - assert not os.path.exists(os.path.join(workspace_beta, 'etc', 'pony.conf')) + assert os.path.exists(os.path.join(workspace_alpha, "usr", "bin", "hello")) + assert not os.path.exists(os.path.join(workspace_beta, "etc", "pony.conf")) @pytest.mark.datafiles(DATA_DIR) def test_reset_all(cli, tmpdir, datafiles): # Open the workspaces - tmpdir_alpha = os.path.join(str(tmpdir), 'alpha') - tmpdir_beta = os.path.join(str(tmpdir), 'beta') - _, project, workspace_alpha = open_workspace( - cli, tmpdir_alpha, datafiles, 'git', suffix='-alpha') - _, project, workspace_beta = open_workspace( - cli, tmpdir_beta, datafiles, 'git', suffix='-beta') + tmpdir_alpha = os.path.join(str(tmpdir), "alpha") + tmpdir_beta = os.path.join(str(tmpdir), "beta") + _, project, workspace_alpha = open_workspace(cli, tmpdir_alpha, datafiles, "git", suffix="-alpha") + _, project, workspace_beta = open_workspace(cli, tmpdir_beta, datafiles, "git", suffix="-beta") # Modify workspaces - shutil.rmtree(os.path.join(workspace_alpha, 'usr', 'bin')) - os.makedirs(os.path.join(workspace_beta, 'etc')) - with open(os.path.join(workspace_beta, 'etc', 'pony.conf'), 'w') as f: + shutil.rmtree(os.path.join(workspace_alpha, "usr", "bin")) + os.makedirs(os.path.join(workspace_beta, "etc")) + with open(os.path.join(workspace_beta, "etc", "pony.conf"), "w") as f: f.write("PONY='pink'") # Now reset the open workspace, this should have the # effect of reverting our changes. - result = cli.run(project=project, args=[ - 'workspace', 'reset', '--all' - ]) + result = cli.run(project=project, args=["workspace", "reset", "--all"]) result.assert_success() - assert os.path.exists(os.path.join(workspace_alpha, 'usr', 'bin', 'hello')) - assert not os.path.exists(os.path.join(workspace_beta, 'etc', 'pony.conf')) + assert os.path.exists(os.path.join(workspace_alpha, "usr", "bin", "hello")) + assert not os.path.exists(os.path.join(workspace_beta, "etc", "pony.conf")) @pytest.mark.datafiles(DATA_DIR) def test_list(cli, tmpdir, datafiles): - element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git') + element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git") # Now list the workspaces - result = cli.run(project=project, args=[ - 'workspace', 'list' - ]) + result = cli.run(project=project, args=["workspace", "list"]) result.assert_success() loaded = _yaml.load_data(result.output) - workspaces = loaded.get_sequence('workspaces') + workspaces = loaded.get_sequence("workspaces") assert len(workspaces) == 1 space = workspaces.mapping_at(0) - assert space.get_str('element') == element_name - assert space.get_str('directory') == workspace + assert space.get_str("element") == element_name + assert space.get_str("directory") == workspace @pytest.mark.datafiles(DATA_DIR) @@ -705,117 +655,99 @@ def test_list(cli, tmpdir, datafiles): @pytest.mark.parametrize( "from_workspace,guess_element", [(False, False), (True, True), (True, False)], - ids=["project-no-guess", "workspace-guess", "workspace-no-guess"]) + ids=["project-no-guess", "workspace-guess", "workspace-no-guess"], +) def test_build(cli, tmpdir_factory, datafiles, kind, strict, from_workspace, guess_element): - tmpdir = tmpdir_factory.mktemp('') + tmpdir = tmpdir_factory.mktemp("") element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, kind, False) - checkout = os.path.join(str(tmpdir), 'checkout') - args_dir = ['-C', workspace] if from_workspace else [] + checkout = os.path.join(str(tmpdir), "checkout") + args_dir = ["-C", workspace] if from_workspace else [] args_elm = [element_name] if not guess_element else [] # Modify workspace - shutil.rmtree(os.path.join(workspace, 'usr', 'bin')) - os.makedirs(os.path.join(workspace, 'etc')) - with open(os.path.join(workspace, 'etc', 'pony.conf'), 'w') as f: + shutil.rmtree(os.path.join(workspace, "usr", "bin")) + os.makedirs(os.path.join(workspace, "etc")) + with open(os.path.join(workspace, "etc", "pony.conf"), "w") as f: f.write("PONY='pink'") # Configure strict mode strict_mode = True - if strict != 'strict': + if strict != "strict": strict_mode = False - cli.configure({ - 'projects': { - 'test': { - 'strict': strict_mode - } - } - }) + cli.configure({"projects": {"test": {"strict": strict_mode}}}) # Build modified workspace - assert cli.get_element_state(project, element_name) == 'buildable' + assert cli.get_element_state(project, element_name) == "buildable" key_1 = cli.get_element_key(project, element_name) - assert key_1 != "{:?<64}".format('') - result = cli.run(project=project, args=args_dir + ['build', *args_elm]) + assert key_1 != "{:?<64}".format("") + result = cli.run(project=project, args=args_dir + ["build", *args_elm]) result.assert_success() - assert cli.get_element_state(project, element_name) == 'cached' + assert cli.get_element_state(project, element_name) == "cached" key_2 = cli.get_element_key(project, element_name) - assert key_2 != "{:?<64}".format('') + assert key_2 != "{:?<64}".format("") # workspace keys are not recalculated assert key_1 == key_2 # Checkout the result - result = cli.run(project=project, - args=args_dir + ['artifact', 'checkout', '--directory', checkout, *args_elm]) + result = cli.run(project=project, args=args_dir + ["artifact", "checkout", "--directory", checkout, *args_elm]) result.assert_success() # Check that the pony.conf from the modified workspace exists - filename = os.path.join(checkout, 'etc', 'pony.conf') + filename = os.path.join(checkout, "etc", "pony.conf") assert os.path.exists(filename) # Check that the original /usr/bin/hello is not in the checkout - assert not os.path.exists(os.path.join(checkout, 'usr', 'bin', 'hello')) + assert not os.path.exists(os.path.join(checkout, "usr", "bin", "hello")) @pytest.mark.datafiles(DATA_DIR) def test_buildable_no_ref(cli, tmpdir, datafiles): project = str(datafiles) - element_name = 'workspace-test-no-ref.bst' - element_path = os.path.join(project, 'elements') + element_name = "workspace-test-no-ref.bst" + element_path = os.path.join(project, "elements") # Write out our test target without any source ref - repo = create_repo('git', str(tmpdir)) - element = { - 'kind': 'import', - 'sources': [ - repo.source_config() - ] - } - _yaml.roundtrip_dump(element, - os.path.join(element_path, element_name)) + repo = create_repo("git", str(tmpdir)) + element = {"kind": "import", "sources": [repo.source_config()]} + _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) # Assert that this target is not buildable when no workspace is associated. - assert cli.get_element_state(project, element_name) == 'no reference' + assert cli.get_element_state(project, element_name) == "no reference" # Now open the workspace. We don't need to checkout the source though. - workspace = os.path.join(str(tmpdir), 'workspace-no-ref') + workspace = os.path.join(str(tmpdir), "workspace-no-ref") os.makedirs(workspace) - args = ['workspace', 'open', '--no-checkout', '--directory', workspace, element_name] + args = ["workspace", "open", "--no-checkout", "--directory", workspace, element_name] result = cli.run(project=project, args=args) result.assert_success() # Assert that the target is now buildable. - assert cli.get_element_state(project, element_name) == 'buildable' + assert cli.get_element_state(project, element_name) == "buildable" @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("modification", [("addfile"), ("removefile"), ("modifyfile")]) @pytest.mark.parametrize("strict", [("strict"), ("non-strict")]) def test_detect_modifications(cli, tmpdir, datafiles, modification, strict): - element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git') - checkout = os.path.join(str(tmpdir), 'checkout') + element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git") + checkout = os.path.join(str(tmpdir), "checkout") # Configure strict mode strict_mode = True - if strict != 'strict': + if strict != "strict": strict_mode = False - cli.configure({ - 'projects': { - 'test': { - 'strict': strict_mode - } - } - }) + cli.configure({"projects": {"test": {"strict": strict_mode}}}) # Build clean workspace - assert cli.get_element_state(project, element_name) == 'buildable' + assert cli.get_element_state(project, element_name) == "buildable" key_1 = cli.get_element_key(project, element_name) - assert key_1 != "{:?<64}".format('') - result = cli.run(project=project, args=['build', element_name]) + assert key_1 != "{:?<64}".format("") + result = cli.run(project=project, args=["build", element_name]) result.assert_success() - assert cli.get_element_state(project, element_name) == 'cached' + assert cli.get_element_state(project, element_name) == "cached" key_2 = cli.get_element_key(project, element_name) - assert key_2 != "{:?<64}".format('') + assert key_2 != "{:?<64}".format("") # workspace keys are not recalculated assert key_1 == key_2 @@ -825,32 +757,32 @@ def test_detect_modifications(cli, tmpdir, datafiles, modification, strict): # Modify the workspace in various different ways, ensuring we # properly detect the changes. # - if modification == 'addfile': - os.makedirs(os.path.join(workspace, 'etc')) - with open(os.path.join(workspace, 'etc', 'pony.conf'), 'w') as f: + if modification == "addfile": + os.makedirs(os.path.join(workspace, "etc")) + with open(os.path.join(workspace, "etc", "pony.conf"), "w") as f: f.write("PONY='pink'") - elif modification == 'removefile': - os.remove(os.path.join(workspace, 'usr', 'bin', 'hello')) - elif modification == 'modifyfile': - with open(os.path.join(workspace, 'usr', 'bin', 'hello'), 'w') as f: - f.write('cookie') + elif modification == "removefile": + os.remove(os.path.join(workspace, "usr", "bin", "hello")) + elif modification == "modifyfile": + with open(os.path.join(workspace, "usr", "bin", "hello"), "w") as f: + f.write("cookie") else: # This cannot be reached assert 0 # First assert that the state is properly detected - assert cli.get_element_state(project, element_name) == 'buildable' + assert cli.get_element_state(project, element_name) == "buildable" key_3 = cli.get_element_key(project, element_name) - assert key_3 != "{:?<64}".format('') + assert key_3 != "{:?<64}".format("") # Since there are different things going on at `bst build` time # than `bst show` time, we also want to build / checkout again, # and ensure that the result contains what we expect. - result = cli.run(project=project, args=['build', element_name]) + result = cli.run(project=project, args=["build", element_name]) result.assert_success() - assert cli.get_element_state(project, element_name) == 'cached' + assert cli.get_element_state(project, element_name) == "cached" key_4 = cli.get_element_key(project, element_name) - assert key_4 != "{:?<64}".format('') + assert key_4 != "{:?<64}".format("") # workspace keys are not recalculated assert key_3 == key_4 @@ -858,22 +790,20 @@ def test_detect_modifications(cli, tmpdir, datafiles, modification, strict): assert key_1 != key_3 # Checkout the result - result = cli.run(project=project, args=[ - 'artifact', 'checkout', element_name, '--directory', checkout - ]) + result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) result.assert_success() # Check the result for the changes we made # - if modification == 'addfile': - filename = os.path.join(checkout, 'etc', 'pony.conf') + if modification == "addfile": + filename = os.path.join(checkout, "etc", "pony.conf") assert os.path.exists(filename) - elif modification == 'removefile': - assert not os.path.exists(os.path.join(checkout, 'usr', 'bin', 'hello')) - elif modification == 'modifyfile': - with open(os.path.join(workspace, 'usr', 'bin', 'hello'), 'r') as f: + elif modification == "removefile": + assert not os.path.exists(os.path.join(checkout, "usr", "bin", "hello")) + elif modification == "modifyfile": + with open(os.path.join(workspace, "usr", "bin", "hello"), "r") as f: data = f.read() - assert data == 'cookie' + assert data == "cookie" else: # This cannot be reached assert 0 @@ -882,143 +812,111 @@ def test_detect_modifications(cli, tmpdir, datafiles, modification, strict): # Ensure that various versions that should not be accepted raise a # LoadError.INVALID_DATA @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("workspace_cfg", [ - # Test loading a negative workspace version - {"format-version": -1}, - # Test loading version 0 with two sources - { - "format-version": 0, - "alpha.bst": { - 0: "/workspaces/bravo", - 1: "/workspaces/charlie", - } - }, - # Test loading a version with decimals - {"format-version": 0.5}, - # Test loading a future version - {"format-version": BST_WORKSPACE_FORMAT_VERSION + 1} -]) +@pytest.mark.parametrize( + "workspace_cfg", + [ + # Test loading a negative workspace version + {"format-version": -1}, + # Test loading version 0 with two sources + {"format-version": 0, "alpha.bst": {0: "/workspaces/bravo", 1: "/workspaces/charlie",}}, + # Test loading a version with decimals + {"format-version": 0.5}, + # Test loading a future version + {"format-version": BST_WORKSPACE_FORMAT_VERSION + 1}, + ], +) def test_list_unsupported_workspace(cli, datafiles, workspace_cfg): project = str(datafiles) - os.makedirs(os.path.join(project, '.bst')) - workspace_config_path = os.path.join(project, '.bst', 'workspaces.yml') + os.makedirs(os.path.join(project, ".bst")) + workspace_config_path = os.path.join(project, ".bst", "workspaces.yml") _yaml.roundtrip_dump(workspace_cfg, workspace_config_path) - result = cli.run(project=project, args=['workspace', 'list']) + result = cli.run(project=project, args=["workspace", "list"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) # Ensure that various versions that should be accepted are parsed # correctly. @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("workspace_cfg,expected", [ - # Test loading version 0 without a dict - ({ - "alpha.bst": "/workspaces/bravo" - }, { - "format-version": BST_WORKSPACE_FORMAT_VERSION, - "workspaces": { - "alpha.bst": { - "prepared": False, - "path": "/workspaces/bravo", - "running_files": {} - } - } - }), - # Test loading version 0 with only one source - ({ - "alpha.bst": { - 0: "/workspaces/bravo" - } - }, { - "format-version": BST_WORKSPACE_FORMAT_VERSION, - "workspaces": { - "alpha.bst": { - "prepared": False, - "path": "/workspaces/bravo", - "running_files": {} - } - } - }), - # Test loading version 1 - ({ - "format-version": 1, - "workspaces": { - "alpha.bst": { - "path": "/workspaces/bravo" - } - } - }, { - "format-version": BST_WORKSPACE_FORMAT_VERSION, - "workspaces": { - "alpha.bst": { - "prepared": False, - "path": "/workspaces/bravo", - "running_files": {} - } - } - }), - # Test loading version 2 - ({ - "format-version": 2, - "workspaces": { - "alpha.bst": { - "path": "/workspaces/bravo", - "last_successful": "some_key", - "running_files": { - "beta.bst": ["some_file"] - } - } - } - }, { - "format-version": BST_WORKSPACE_FORMAT_VERSION, - "workspaces": { - "alpha.bst": { - "prepared": False, - "path": "/workspaces/bravo", - "last_successful": "some_key", - "running_files": { - "beta.bst": ["some_file"] - } - } - } - }), - # Test loading version 3 - ({ - "format-version": 3, - "workspaces": { - "alpha.bst": { - "prepared": True, - "path": "/workspaces/bravo", - "running_files": {} - } - } - }, { - "format-version": BST_WORKSPACE_FORMAT_VERSION, - "workspaces": { - "alpha.bst": { - "prepared": True, - "path": "/workspaces/bravo", - "running_files": {} - } - } - }) -]) +@pytest.mark.parametrize( + "workspace_cfg,expected", + [ + # Test loading version 0 without a dict + ( + {"alpha.bst": "/workspaces/bravo"}, + { + "format-version": BST_WORKSPACE_FORMAT_VERSION, + "workspaces": {"alpha.bst": {"prepared": False, "path": "/workspaces/bravo", "running_files": {}}}, + }, + ), + # Test loading version 0 with only one source + ( + {"alpha.bst": {0: "/workspaces/bravo"}}, + { + "format-version": BST_WORKSPACE_FORMAT_VERSION, + "workspaces": {"alpha.bst": {"prepared": False, "path": "/workspaces/bravo", "running_files": {}}}, + }, + ), + # Test loading version 1 + ( + {"format-version": 1, "workspaces": {"alpha.bst": {"path": "/workspaces/bravo"}}}, + { + "format-version": BST_WORKSPACE_FORMAT_VERSION, + "workspaces": {"alpha.bst": {"prepared": False, "path": "/workspaces/bravo", "running_files": {}}}, + }, + ), + # Test loading version 2 + ( + { + "format-version": 2, + "workspaces": { + "alpha.bst": { + "path": "/workspaces/bravo", + "last_successful": "some_key", + "running_files": {"beta.bst": ["some_file"]}, + } + }, + }, + { + "format-version": BST_WORKSPACE_FORMAT_VERSION, + "workspaces": { + "alpha.bst": { + "prepared": False, + "path": "/workspaces/bravo", + "last_successful": "some_key", + "running_files": {"beta.bst": ["some_file"]}, + } + }, + }, + ), + # Test loading version 3 + ( + { + "format-version": 3, + "workspaces": {"alpha.bst": {"prepared": True, "path": "/workspaces/bravo", "running_files": {}}}, + }, + { + "format-version": BST_WORKSPACE_FORMAT_VERSION, + "workspaces": {"alpha.bst": {"prepared": True, "path": "/workspaces/bravo", "running_files": {}}}, + }, + ), + ], +) def test_list_supported_workspace(cli, tmpdir, datafiles, workspace_cfg, expected): def parse_dict_as_yaml(node): - tempfile = os.path.join(str(tmpdir), 'yaml_dump') + tempfile = os.path.join(str(tmpdir), "yaml_dump") _yaml.roundtrip_dump(node, tempfile) return _yaml.load(tempfile).strip_node_info() project = str(datafiles) - os.makedirs(os.path.join(project, '.bst')) - workspace_config_path = os.path.join(project, '.bst', 'workspaces.yml') + os.makedirs(os.path.join(project, ".bst")) + workspace_config_path = os.path.join(project, ".bst", "workspaces.yml") _yaml.roundtrip_dump(workspace_cfg, workspace_config_path) # Check that we can still read workspace config that is in old format - result = cli.run(project=project, args=['workspace', 'list']) + result = cli.run(project=project, args=["workspace", "list"]) result.assert_success() loaded_config = _yaml.load(workspace_config_path).strip_node_info() @@ -1028,31 +926,25 @@ def test_list_supported_workspace(cli, tmpdir, datafiles, workspace_cfg, expecte assert loaded_config == parse_dict_as_yaml(workspace_cfg) # Create a test bst file - bin_files_path = os.path.join(project, 'files', 'bin-files') - element_path = os.path.join(project, 'elements') - element_name = 'workspace-test.bst' - workspace = os.path.join(str(tmpdir), 'workspace') + bin_files_path = os.path.join(project, "files", "bin-files") + element_path = os.path.join(project, "elements") + element_name = "workspace-test.bst" + workspace = os.path.join(str(tmpdir), "workspace") # Create our repo object of the given source type with # the bin files, and then collect the initial ref. # - repo = create_repo('git', str(tmpdir)) + repo = create_repo("git", str(tmpdir)) ref = repo.create(bin_files_path) # Write out our test target - element = { - 'kind': 'import', - 'sources': [ - repo.source_config(ref=ref) - ] - } - _yaml.roundtrip_dump(element, - os.path.join(element_path, element_name)) + element = {"kind": "import", "sources": [repo.source_config(ref=ref)]} + _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) # Make a change to the workspaces file - result = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name]) + result = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name]) result.assert_success() - result = cli.run(project=project, args=['workspace', 'close', '--remove-dir', element_name]) + result = cli.run(project=project, args=["workspace", "close", "--remove-dir", element_name]) result.assert_success() # Check that workspace config is converted correctly if necessary @@ -1062,73 +954,55 @@ def test_list_supported_workspace(cli, tmpdir, datafiles, workspace_cfg, expecte @pytest.mark.datafiles(DATA_DIR) def test_inconsitent_pipeline_message(cli, tmpdir, datafiles): - element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git') + element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git") shutil.rmtree(workspace) - result = cli.run(project=project, args=[ - 'build', element_name - ]) + result = cli.run(project=project, args=["build", element_name]) result.assert_main_error(ErrorDomain.PIPELINE, "inconsistent-pipeline-workspaced") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("strict", [("strict"), ("non-strict")]) def test_cache_key_workspace_in_dependencies(cli, tmpdir, datafiles, strict): - checkout = os.path.join(str(tmpdir), 'checkout') - element_name, project, workspace = open_workspace(cli, os.path.join(str(tmpdir), 'repo-a'), - datafiles, 'git') + checkout = os.path.join(str(tmpdir), "checkout") + element_name, project, workspace = open_workspace(cli, os.path.join(str(tmpdir), "repo-a"), datafiles, "git") - element_path = os.path.join(project, 'elements') - back_dep_element_name = 'workspace-test-back-dep.bst' + element_path = os.path.join(project, "elements") + back_dep_element_name = "workspace-test-back-dep.bst" # Write out our test target - element = { - 'kind': 'compose', - 'depends': [ - { - 'filename': element_name, - 'type': 'build' - } - ] - } - _yaml.roundtrip_dump(element, - os.path.join(element_path, back_dep_element_name)) + element = {"kind": "compose", "depends": [{"filename": element_name, "type": "build"}]} + _yaml.roundtrip_dump(element, os.path.join(element_path, back_dep_element_name)) # Modify workspace - shutil.rmtree(os.path.join(workspace, 'usr', 'bin')) - os.makedirs(os.path.join(workspace, 'etc')) - with open(os.path.join(workspace, 'etc', 'pony.conf'), 'w') as f: + shutil.rmtree(os.path.join(workspace, "usr", "bin")) + os.makedirs(os.path.join(workspace, "etc")) + with open(os.path.join(workspace, "etc", "pony.conf"), "w") as f: f.write("PONY='pink'") # Configure strict mode strict_mode = True - if strict != 'strict': + if strict != "strict": strict_mode = False - cli.configure({ - 'projects': { - 'test': { - 'strict': strict_mode - } - } - }) + cli.configure({"projects": {"test": {"strict": strict_mode}}}) # Build artifact with dependency's modified workspace - assert cli.get_element_state(project, element_name) == 'buildable' + assert cli.get_element_state(project, element_name) == "buildable" key_a1 = cli.get_element_key(project, element_name) - assert key_a1 != "{:?<64}".format('') - assert cli.get_element_state(project, back_dep_element_name) == 'waiting' + assert key_a1 != "{:?<64}".format("") + assert cli.get_element_state(project, back_dep_element_name) == "waiting" key_b1 = cli.get_element_key(project, back_dep_element_name) - assert key_b1 != "{:?<64}".format('') - result = cli.run(project=project, args=['build', back_dep_element_name]) + assert key_b1 != "{:?<64}".format("") + result = cli.run(project=project, args=["build", back_dep_element_name]) result.assert_success() - assert cli.get_element_state(project, element_name) == 'cached' + assert cli.get_element_state(project, element_name) == "cached" key_a2 = cli.get_element_key(project, element_name) - assert key_a2 != "{:?<64}".format('') - assert cli.get_element_state(project, back_dep_element_name) == 'cached' + assert key_a2 != "{:?<64}".format("") + assert cli.get_element_state(project, back_dep_element_name) == "cached" key_b2 = cli.get_element_key(project, back_dep_element_name) - assert key_b2 != "{:?<64}".format('') - result = cli.run(project=project, args=['build', back_dep_element_name]) + assert key_b2 != "{:?<64}".format("") + result = cli.run(project=project, args=["build", back_dep_element_name]) result.assert_success() # workspace keys are not recalculated @@ -1136,31 +1010,21 @@ def test_cache_key_workspace_in_dependencies(cli, tmpdir, datafiles, strict): assert key_b1 == key_b2 # Checkout the result - result = cli.run(project=project, args=[ - 'artifact', 'checkout', back_dep_element_name, '--directory', checkout - ]) + result = cli.run(project=project, args=["artifact", "checkout", back_dep_element_name, "--directory", checkout]) result.assert_success() # Check that the pony.conf from the modified workspace exists - filename = os.path.join(checkout, 'etc', 'pony.conf') + filename = os.path.join(checkout, "etc", "pony.conf") assert os.path.exists(filename) # Check that the original /usr/bin/hello is not in the checkout - assert not os.path.exists(os.path.join(checkout, 'usr', 'bin', 'hello')) + assert not os.path.exists(os.path.join(checkout, "usr", "bin", "hello")) @pytest.mark.datafiles(DATA_DIR) def test_multiple_failed_builds(cli, tmpdir, datafiles): - element_config = { - "kind": "manual", - "config": { - "configure-commands": [ - "unknown_command_that_will_fail" - ] - } - } - element_name, project, _ = open_workspace(cli, tmpdir, datafiles, - "git", element_attrs=element_config) + element_config = {"kind": "manual", "config": {"configure-commands": ["unknown_command_that_will_fail"]}} + element_name, project, _ = open_workspace(cli, tmpdir, datafiles, "git", element_attrs=element_config) for _ in range(2): result = cli.run(project=project, args=["build", element_name]) @@ -1169,60 +1033,57 @@ def test_multiple_failed_builds(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize('subdir', [True, False], ids=["subdir", "no-subdir"]) +@pytest.mark.parametrize("subdir", [True, False], ids=["subdir", "no-subdir"]) @pytest.mark.parametrize("guess_element", [True, False], ids=["guess", "no-guess"]) def test_external_fetch(cli, datafiles, tmpdir_factory, subdir, guess_element): # An element with an open workspace can't be fetched, but we still expect fetches # to fetch any dependencies - tmpdir = tmpdir_factory.mktemp('') - depend_element = 'fetchable.bst' + tmpdir = tmpdir_factory.mktemp("") + depend_element = "fetchable.bst" # Create an element to fetch (local sources do not need to fetch) - create_element_size(depend_element, str(datafiles), 'elements', [], 1024) + create_element_size(depend_element, str(datafiles), "elements", [], 1024) element_name, project, workspace = open_workspace( - cli, tmpdir, datafiles, "git", no_checkout=True, - element_attrs={'depends': [depend_element]} + cli, tmpdir, datafiles, "git", no_checkout=True, element_attrs={"depends": [depend_element]} ) arg_elm = [element_name] if not guess_element else [] if subdir: - call_dir = os.path.join(workspace, 'usr') + call_dir = os.path.join(workspace, "usr") os.makedirs(call_dir, exist_ok=True) else: call_dir = workspace # Assert that the depended element is not fetched yet - assert cli.get_element_state(str(datafiles), depend_element) == 'fetch needed' + assert cli.get_element_state(str(datafiles), depend_element) == "fetch needed" # Fetch the workspaced element - result = cli.run(project=project, args=['-C', call_dir, 'source', 'fetch', *arg_elm]) + result = cli.run(project=project, args=["-C", call_dir, "source", "fetch", *arg_elm]) result.assert_success() # Assert that the depended element has now been fetched - assert cli.get_element_state(str(datafiles), depend_element) == 'buildable' + assert cli.get_element_state(str(datafiles), depend_element) == "buildable" @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("guess_element", [True, False], ids=["guess", "no-guess"]) def test_external_push_pull(cli, datafiles, tmpdir_factory, guess_element): # Pushing and pulling to/from an artifact cache works from an external workspace - tmpdir = tmpdir_factory.mktemp('') + tmpdir = tmpdir_factory.mktemp("") element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git") arg_elm = [element_name] if not guess_element else [] - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: - result = cli.run(project=project, args=['-C', workspace, 'build', element_name]) + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: + result = cli.run(project=project, args=["-C", workspace, "build", element_name]) result.assert_success() - cli.configure({ - 'artifacts': {'url': share.repo, 'push': True} - }) + cli.configure({"artifacts": {"url": share.repo, "push": True}}) - result = cli.run(project=project, args=['-C', workspace, 'artifact', 'push', *arg_elm]) + result = cli.run(project=project, args=["-C", workspace, "artifact", "push", *arg_elm]) result.assert_success() - result = cli.run(project=project, args=['-C', workspace, 'artifact', 'pull', '--deps', 'all', *arg_elm]) + result = cli.run(project=project, args=["-C", workspace, "artifact", "pull", "--deps", "all", *arg_elm]) result.assert_success() @@ -1232,35 +1093,35 @@ def test_external_push_pull(cli, datafiles, tmpdir_factory, guess_element): @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("guess_element", [True, False], ids=["guess", "no-guess"]) def test_external_track(cli, datafiles, tmpdir_factory, guess_element): - tmpdir = tmpdir_factory.mktemp('') + tmpdir = tmpdir_factory.mktemp("") element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git") - element_file = os.path.join(str(datafiles), 'elements', element_name) + element_file = os.path.join(str(datafiles), "elements", element_name) arg_elm = [element_name] if not guess_element else [] # Delete the ref from the source so that we can detect if the # element has been tracked after closing the workspace element_contents = _yaml.load(element_file) - ref1 = element_contents.get_sequence('sources').mapping_at(0).get_str('ref') - del element_contents.get_sequence('sources').mapping_at(0)['ref'] + ref1 = element_contents.get_sequence("sources").mapping_at(0).get_str("ref") + del element_contents.get_sequence("sources").mapping_at(0)["ref"] _yaml.roundtrip_dump(element_contents, element_file) - result = cli.run(project=project, args=['-C', workspace, 'source', 'track', *arg_elm]) + result = cli.run(project=project, args=["-C", workspace, "source", "track", *arg_elm]) result.assert_success() # Element is not tracked now element_contents = _yaml.load(element_file) - assert 'ref' not in element_contents.get_sequence('sources').mapping_at(0) + assert "ref" not in element_contents.get_sequence("sources").mapping_at(0) # close the workspace - result = cli.run(project=project, args=['-C', workspace, 'workspace', 'close', *arg_elm]) + result = cli.run(project=project, args=["-C", workspace, "workspace", "close", *arg_elm]) result.assert_success() # and retrack the element - result = cli.run(project=project, args=['source', 'track', element_name]) + result = cli.run(project=project, args=["source", "track", element_name]) result.assert_success() element_contents = _yaml.load(element_file) - ref2 = element_contents.get_sequence('sources').mapping_at(0).get_str('ref') + ref2 = element_contents.get_sequence("sources").mapping_at(0).get_str("ref") # these values should be equivalent assert ref1 == ref2 @@ -1268,62 +1129,63 @@ def test_external_track(cli, datafiles, tmpdir_factory, guess_element): @pytest.mark.datafiles(DATA_DIR) def test_external_open_other(cli, datafiles, tmpdir_factory): # From inside an external workspace, open another workspace - tmpdir1 = tmpdir_factory.mktemp('') - tmpdir2 = tmpdir_factory.mktemp('') + tmpdir1 = tmpdir_factory.mktemp("") + tmpdir2 = tmpdir_factory.mktemp("") # Making use of the assumption that it's the same project in both invocations of open_workspace _, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", suffix="-alpha") beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", suffix="-beta") # Closing the other element first, because I'm too lazy to create an # element without opening it - result = cli.run(project=project, args=['workspace', 'close', beta_element]) + result = cli.run(project=project, args=["workspace", "close", beta_element]) result.assert_success() - result = cli.run(project=project, args=[ - '-C', alpha_workspace, 'workspace', 'open', '--force', '--directory', beta_workspace, beta_element - ]) + result = cli.run( + project=project, + args=["-C", alpha_workspace, "workspace", "open", "--force", "--directory", beta_workspace, beta_element], + ) result.assert_success() @pytest.mark.datafiles(DATA_DIR) def test_external_close_other(cli, datafiles, tmpdir_factory): # From inside an external workspace, close the other workspace - tmpdir1 = tmpdir_factory.mktemp('') - tmpdir2 = tmpdir_factory.mktemp('') + tmpdir1 = tmpdir_factory.mktemp("") + tmpdir2 = tmpdir_factory.mktemp("") # Making use of the assumption that it's the same project in both invocations of open_workspace _, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", suffix="-alpha") beta_element, _, _ = open_workspace(cli, tmpdir2, datafiles, "git", suffix="-beta") - result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'close', beta_element]) + result = cli.run(project=project, args=["-C", alpha_workspace, "workspace", "close", beta_element]) result.assert_success() - assert 'you can no longer run BuildStream' not in result.stderr + assert "you can no longer run BuildStream" not in result.stderr @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("guess_element", [True, False], ids=["guess", "no-guess"]) def test_external_close_self(cli, datafiles, tmpdir_factory, guess_element): # From inside an external workspace, close it - tmpdir1 = tmpdir_factory.mktemp('') - tmpdir2 = tmpdir_factory.mktemp('') + tmpdir1 = tmpdir_factory.mktemp("") + tmpdir2 = tmpdir_factory.mktemp("") # Making use of the assumption that it's the same project in both invocations of open_workspace alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", suffix="-alpha") _, _, _ = open_workspace(cli, tmpdir2, datafiles, "git", suffix="-beta") arg_elm = [alpha_element] if not guess_element else [] - result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'close', *arg_elm]) + result = cli.run(project=project, args=["-C", alpha_workspace, "workspace", "close", *arg_elm]) result.assert_success() - assert 'you can no longer run BuildStream' in result.stderr + assert "you can no longer run BuildStream" in result.stderr @pytest.mark.datafiles(DATA_DIR) def test_external_reset_other(cli, datafiles, tmpdir_factory): - tmpdir1 = tmpdir_factory.mktemp('') - tmpdir2 = tmpdir_factory.mktemp('') + tmpdir1 = tmpdir_factory.mktemp("") + tmpdir2 = tmpdir_factory.mktemp("") # Making use of the assumption that it's the same project in both invocations of open_workspace _, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", suffix="-alpha") beta_element, _, _ = open_workspace(cli, tmpdir2, datafiles, "git", suffix="-beta") - result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'reset', beta_element]) + result = cli.run(project=project, args=["-C", alpha_workspace, "workspace", "reset", beta_element]) result.assert_success() @@ -1334,21 +1196,21 @@ def test_external_reset_self(cli, datafiles, tmpdir, guess_element): arg_elm = [element] if not guess_element else [] # Command succeeds - result = cli.run(project=project, args=['-C', workspace, 'workspace', 'reset', *arg_elm]) + result = cli.run(project=project, args=["-C", workspace, "workspace", "reset", *arg_elm]) result.assert_success() # Successive commands still work (i.e. .bstproject.yaml hasn't been deleted) - result = cli.run(project=project, args=['-C', workspace, 'workspace', 'list']) + result = cli.run(project=project, args=["-C", workspace, "workspace", "list"]) result.assert_success() @pytest.mark.datafiles(DATA_DIR) def test_external_list(cli, datafiles, tmpdir_factory): - tmpdir = tmpdir_factory.mktemp('') + tmpdir = tmpdir_factory.mktemp("") # Making use of the assumption that it's the same project in both invocations of open_workspace _, project, workspace = open_workspace(cli, tmpdir, datafiles, "git") - result = cli.run(project=project, args=['-C', workspace, 'workspace', 'list']) + result = cli.run(project=project, args=["-C", workspace, "workspace", "list"]) result.assert_success() @@ -1359,26 +1221,18 @@ def test_multisource_workspace(cli, datafiles, tmpdir): project = str(datafiles) element_name = "multisource.bst" element = { - 'kind': 'import', - 'sources': [{ - 'kind': 'local', - 'path': 'files/bin-files' - }, { - 'kind': 'local', - 'path': 'files/dev-files' - }] + "kind": "import", + "sources": [{"kind": "local", "path": "files/bin-files"}, {"kind": "local", "path": "files/dev-files"}], } - element_path = os.path.join(project, 'elements', element_name) + element_path = os.path.join(project, "elements", element_name) _yaml.roundtrip_dump(element, element_path) - workspace_dir = os.path.join(str(tmpdir), 'multisource') - res = cli.run(project=project, - args=['workspace', 'open', 'multisource.bst', - '--directory', workspace_dir]) + workspace_dir = os.path.join(str(tmpdir), "multisource") + res = cli.run(project=project, args=["workspace", "open", "multisource.bst", "--directory", workspace_dir]) res.assert_success() - directories = os.listdir(os.path.join(workspace_dir, 'usr')) - assert 'bin' in directories and 'include' in directories + directories = os.listdir(os.path.join(workspace_dir, "usr")) + assert "bin" in directories and "include" in directories # This strange test tests against a regression raised in issue #919, @@ -1387,9 +1241,7 @@ def test_multisource_workspace(cli, datafiles, tmpdir): # but just successfully builds the workspaced element and happily # exits without completing the build. # -TEST_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)) -) +TEST_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__))) @pytest.mark.datafiles(TEST_DIR) @@ -1397,73 +1249,61 @@ TEST_DIR = os.path.join( ["case", "non_workspaced_elements_state"], [ ("workspaced-build-dep", ["waiting", "waiting", "waiting", "waiting", "waiting"]), - ("workspaced-runtime-dep", ["buildable", "buildable", "waiting", "waiting", "waiting"]) + ("workspaced-runtime-dep", ["buildable", "buildable", "waiting", "waiting", "waiting"]), ], ) @pytest.mark.parametrize("strict", [("strict"), ("non-strict")]) def test_build_all(cli, tmpdir, datafiles, case, strict, non_workspaced_elements_state): project = os.path.join(str(datafiles), case) - workspace = os.path.join(str(tmpdir), 'workspace') + workspace = os.path.join(str(tmpdir), "workspace") non_leaf_elements = ["elem2.bst", "elem3.bst", "stack.bst", "elem4.bst", "elem5.bst"] all_elements = ["elem1.bst", *non_leaf_elements] # Configure strict mode strict_mode = True - if strict != 'strict': + if strict != "strict": strict_mode = False - cli.configure({ - 'projects': { - 'test': { - 'strict': strict_mode - } - } - }) + cli.configure({"projects": {"test": {"strict": strict_mode}}}) # First open the workspace - result = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, 'elem1.bst']) + result = cli.run(project=project, args=["workspace", "open", "--directory", workspace, "elem1.bst"]) result.assert_success() # Ensure all elements are waiting build the first - assert cli.get_element_states(project, all_elements) == \ - dict(zip(all_elements, ['buildable', *non_workspaced_elements_state])) + assert cli.get_element_states(project, all_elements) == dict( + zip(all_elements, ["buildable", *non_workspaced_elements_state]) + ) # Now build the targets elem4.bst and elem5.bst - result = cli.run(project=project, args=['build', 'elem4.bst', 'elem5.bst']) + result = cli.run(project=project, args=["build", "elem4.bst", "elem5.bst"]) result.assert_success() # Assert that the target is built - assert cli.get_element_states(project, all_elements) == \ - {elem: "cached" for elem in all_elements} + assert cli.get_element_states(project, all_elements) == {elem: "cached" for elem in all_elements} @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize('strict', ['strict', 'non-strict']) +@pytest.mark.parametrize("strict", ["strict", "non-strict"]) def test_show_workspace_logs(cli, tmpdir, datafiles, strict): project = str(datafiles) - workspace = os.path.join(str(tmpdir), 'workspace') - target = 'manual.bst' + workspace = os.path.join(str(tmpdir), "workspace") + target = "manual.bst" # Configure strict mode strict_mode = True - if strict != 'strict': + if strict != "strict": strict_mode = False - cli.configure({ - 'projects': { - 'test': { - 'strict': strict_mode - } - } - }) + cli.configure({"projects": {"test": {"strict": strict_mode}}}) # First open the workspace - result = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, target]) + result = cli.run(project=project, args=["workspace", "open", "--directory", workspace, target]) result.assert_success() # Build the element - result = cli.run(project=project, args=['build', target]) - result.assert_task_error(ErrorDomain.SANDBOX, 'missing-command') + result = cli.run(project=project, args=["build", target]) + result.assert_task_error(ErrorDomain.SANDBOX, "missing-command") - result = cli.run(project=project, args=['artifact', 'log', target]) + result = cli.run(project=project, args=["artifact", "log", target]) result.assert_success() # Assert that the log is not empty diff --git a/tests/integration/artifact.py b/tests/integration/artifact.py index 59b7bfaad..e21dd4296 100644 --- a/tests/integration/artifact.py +++ b/tests/integration/artifact.py @@ -36,10 +36,7 @@ pytestmark = pytest.mark.integration # Project directory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project", -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",) # A test to capture the integration of the cachebuildtrees @@ -48,70 +45,64 @@ DATA_DIR = os.path.join( # Dse this really need a sandbox? @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_cache_buildtrees(cli, tmpdir, datafiles): project = str(datafiles) - element_name = 'autotools/amhello.bst' + element_name = "autotools/amhello.bst" cwd = str(tmpdir) # Create artifact shares for pull & push testing - with create_artifact_share(os.path.join(str(tmpdir), 'share1')) as share1,\ - create_artifact_share(os.path.join(str(tmpdir), 'share2')) as share2,\ - create_artifact_share(os.path.join(str(tmpdir), 'share3')) as share3: - cli.configure({ - 'artifacts': {'url': share1.repo, 'push': True}, - 'cachedir': str(tmpdir) - }) + with create_artifact_share(os.path.join(str(tmpdir), "share1")) as share1, create_artifact_share( + os.path.join(str(tmpdir), "share2") + ) as share2, create_artifact_share(os.path.join(str(tmpdir), "share3")) as share3: + cli.configure({"artifacts": {"url": share1.repo, "push": True}, "cachedir": str(tmpdir)}) # Build autotools element with the default behavior of caching buildtrees # only when necessary. The artifact should be successfully pushed to the share1 remote # and cached locally with an 'empty' buildtree digest, as it's not a # dangling ref - result = cli.run(project=project, args=['build', element_name]) + result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 - assert cli.get_element_state(project, element_name) == 'cached' - assert share1.get_artifact(cli.get_artifact_name(project, 'test', element_name)) + assert cli.get_element_state(project, element_name) == "cached" + assert share1.get_artifact(cli.get_artifact_name(project, "test", element_name)) # The buildtree dir should not exist, as we set the config to not cache buildtrees. - artifact_name = cli.get_artifact_name(project, 'test', element_name) + artifact_name = cli.get_artifact_name(project, "test", element_name) assert share1.get_artifact(artifact_name) with cli.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert not buildtreedir # Delete the local cached artifacts, and assert the when pulled with --pull-buildtrees # that is was cached in share1 as expected without a buildtree dir - shutil.rmtree(os.path.join(str(tmpdir), 'cas')) - shutil.rmtree(os.path.join(str(tmpdir), 'artifacts')) - assert cli.get_element_state(project, element_name) != 'cached' - result = cli.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name]) + shutil.rmtree(os.path.join(str(tmpdir), "cas")) + shutil.rmtree(os.path.join(str(tmpdir), "artifacts")) + assert cli.get_element_state(project, element_name) != "cached" + result = cli.run(project=project, args=["--pull-buildtrees", "artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() with cli.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert not buildtreedir - shutil.rmtree(os.path.join(str(tmpdir), 'cas')) - shutil.rmtree(os.path.join(str(tmpdir), 'artifacts')) + shutil.rmtree(os.path.join(str(tmpdir), "cas")) + shutil.rmtree(os.path.join(str(tmpdir), "artifacts")) # Assert that the default behaviour of pull to not include buildtrees on the artifact # in share1 which was purposely cached with an empty one behaves as expected. As such the # pulled artifact will have a dangling ref for the buildtree dir, regardless of content, # leading to no buildtreedir being extracted - result = cli.run(project=project, args=['artifact', 'pull', element_name]) + result = cli.run(project=project, args=["artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() with cli.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert not buildtreedir - shutil.rmtree(os.path.join(str(tmpdir), 'cas')) - shutil.rmtree(os.path.join(str(tmpdir), 'artifacts')) + shutil.rmtree(os.path.join(str(tmpdir), "cas")) + shutil.rmtree(os.path.join(str(tmpdir), "artifacts")) # Repeat building the artifacts, this time with cache-buildtrees set to # 'always' via the cli, as such the buildtree dir should not be empty - cli.configure({ - 'artifacts': {'url': share2.repo, 'push': True}, - 'cachedir': str(tmpdir) - }) - result = cli.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name]) + cli.configure({"artifacts": {"url": share2.repo, "push": True}, "cachedir": str(tmpdir)}) + result = cli.run(project=project, args=["--cache-buildtrees", "always", "build", element_name]) assert result.exit_code == 0 - assert cli.get_element_state(project, element_name) == 'cached' - assert share2.get_artifact(cli.get_artifact_name(project, 'test', element_name)) + assert cli.get_element_state(project, element_name) == "cached" + assert share2.get_artifact(cli.get_artifact_name(project, "test", element_name)) # Cache key will be the same however the digest hash will have changed as expected, so reconstruct paths with cli.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: @@ -120,28 +111,30 @@ def test_cache_buildtrees(cli, tmpdir, datafiles): # Delete the local cached artifacts, and assert that when pulled with --pull-buildtrees # that it was cached in share2 as expected with a populated buildtree dir - shutil.rmtree(os.path.join(str(tmpdir), 'cas')) - shutil.rmtree(os.path.join(str(tmpdir), 'artifacts')) - assert cli.get_element_state(project, element_name) != 'cached' - result = cli.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name]) + shutil.rmtree(os.path.join(str(tmpdir), "cas")) + shutil.rmtree(os.path.join(str(tmpdir), "artifacts")) + assert cli.get_element_state(project, element_name) != "cached" + result = cli.run(project=project, args=["--pull-buildtrees", "artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() with cli.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert os.path.isdir(buildtreedir) assert os.listdir(buildtreedir) - shutil.rmtree(os.path.join(str(tmpdir), 'cas')) - shutil.rmtree(os.path.join(str(tmpdir), 'artifacts')) + shutil.rmtree(os.path.join(str(tmpdir), "cas")) + shutil.rmtree(os.path.join(str(tmpdir), "artifacts")) # Clarify that the user config option for cache-buildtrees works as the cli # main option does. Point to share3 which does not have the artifacts cached to force # a build - cli.configure({ - 'artifacts': {'url': share3.repo, 'push': True}, - 'cachedir': str(tmpdir), - 'cache': {'cache-buildtrees': 'always'} - }) - result = cli.run(project=project, args=['build', element_name]) + cli.configure( + { + "artifacts": {"url": share3.repo, "push": True}, + "cachedir": str(tmpdir), + "cache": {"cache-buildtrees": "always"}, + } + ) + result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 - assert cli.get_element_state(project, element_name) == 'cached' + assert cli.get_element_state(project, element_name) == "cached" with cli.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert os.path.isdir(buildtreedir) assert os.listdir(buildtreedir) diff --git a/tests/integration/autotools.py b/tests/integration/autotools.py index 16cb38af7..7a3b946b2 100644 --- a/tests/integration/autotools.py +++ b/tests/integration/autotools.py @@ -12,67 +12,80 @@ from buildstream.testing._utils.site import HAVE_SANDBOX pytestmark = pytest.mark.integration -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project" -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") # Test that an autotools build 'works' - we use the autotools sample # amhello project for this. @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_autotools_build(cli, datafiles): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') - element_name = 'autotools/amhello.bst' + checkout = os.path.join(cli.directory, "checkout") + element_name = "autotools/amhello.bst" - result = cli.run(project=project, args=['build', element_name]) + result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 - result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout]) + result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert result.exit_code == 0 - assert_contains(checkout, ['/usr', '/usr/lib', '/usr/bin', - '/usr/share', - '/usr/bin/hello', '/usr/share/doc', - '/usr/share/doc/amhello', - '/usr/share/doc/amhello/README']) + assert_contains( + checkout, + [ + "/usr", + "/usr/lib", + "/usr/bin", + "/usr/share", + "/usr/bin/hello", + "/usr/share/doc", + "/usr/share/doc/amhello", + "/usr/share/doc/amhello/README", + ], + ) # Test that an autotools build 'works' - we use the autotools sample # amhello project for this. @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') -@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") +@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox") def test_autotools_confroot_build(cli, datafiles): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') - element_name = 'autotools/amhelloconfroot.bst' + checkout = os.path.join(cli.directory, "checkout") + element_name = "autotools/amhelloconfroot.bst" - result = cli.run(project=project, args=['build', element_name]) + result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 - result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout]) + result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert result.exit_code == 0 - assert_contains(checkout, ['/usr', '/usr/lib', '/usr/bin', - '/usr/share', - '/usr/bin/hello', '/usr/share/doc', - '/usr/share/doc/amhello', - '/usr/share/doc/amhello/README']) + assert_contains( + checkout, + [ + "/usr", + "/usr/lib", + "/usr/bin", + "/usr/share", + "/usr/bin/hello", + "/usr/share/doc", + "/usr/share/doc/amhello", + "/usr/share/doc/amhello/README", + ], + ) # Test running an executable built with autotools @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_autotools_run(cli, datafiles): project = str(datafiles) - element_name = 'autotools/amhello.bst' + element_name = "autotools/amhello.bst" - result = cli.run(project=project, args=['build', element_name]) + result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 - result = cli.run(project=project, args=['shell', element_name, '/usr/bin/hello']) + result = cli.run(project=project, args=["shell", element_name, "/usr/bin/hello"]) assert result.exit_code == 0 - assert result.output == 'Hello World!\nThis is amhello 1.0.\n' + assert result.output == "Hello World!\nThis is amhello 1.0.\n" diff --git a/tests/integration/build-uid.py b/tests/integration/build-uid.py index 66f9b3fbc..2fc78d263 100644 --- a/tests/integration/build-uid.py +++ b/tests/integration/build-uid.py @@ -10,55 +10,38 @@ from buildstream.testing._utils.site import HAVE_SANDBOX, IS_LINUX pytestmark = pytest.mark.integration -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project" -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") -@pytest.mark.skipif(not IS_LINUX or HAVE_SANDBOX != "bwrap", reason='Only available on linux with bubblewrap') +@pytest.mark.skipif(not IS_LINUX or HAVE_SANDBOX != "bwrap", reason="Only available on linux with bubblewrap") @pytest.mark.datafiles(DATA_DIR) def test_build_uid_overridden(cli, datafiles): project = str(datafiles) - element_name = 'build-uid/build-uid.bst' - - project_config = { - 'name': 'build-uid-test', - 'sandbox': { - 'build-uid': 800, - 'build-gid': 900 - } - } - - result = cli.run_project_config( - project=project, project_config=project_config, args=['build', element_name]) + element_name = "build-uid/build-uid.bst" + + project_config = {"name": "build-uid-test", "sandbox": {"build-uid": 800, "build-gid": 900}} + + result = cli.run_project_config(project=project, project_config=project_config, args=["build", element_name]) assert result.exit_code == 0 -@pytest.mark.skipif(not IS_LINUX or HAVE_SANDBOX != "bwrap", reason='Only available on linux with bubbelwrap') +@pytest.mark.skipif(not IS_LINUX or HAVE_SANDBOX != "bwrap", reason="Only available on linux with bubbelwrap") @pytest.mark.datafiles(DATA_DIR) def test_build_uid_in_project(cli, datafiles): project = str(datafiles) - element_name = 'build-uid/build-uid-1023.bst' - - project_config = { - 'name': 'build-uid-test', - 'sandbox': { - 'build-uid': 1023, - 'build-gid': 3490 - } - } - - result = cli.run_project_config( - project=project, project_config=project_config, args=['build', element_name]) + element_name = "build-uid/build-uid-1023.bst" + + project_config = {"name": "build-uid-test", "sandbox": {"build-uid": 1023, "build-gid": 3490}} + + result = cli.run_project_config(project=project, project_config=project_config, args=["build", element_name]) assert result.exit_code == 0 @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(HAVE_SANDBOX != "bwrap", reason='Only available with a functioning sandbox') +@pytest.mark.skipif(HAVE_SANDBOX != "bwrap", reason="Only available with a functioning sandbox") def test_build_uid_default(cli, datafiles): project = str(datafiles) - element_name = 'build-uid/build-uid-default.bst' + element_name = "build-uid/build-uid-default.bst" - result = cli.run(project=project, args=['build', element_name]) + result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 diff --git a/tests/integration/cachedfail.py b/tests/integration/cachedfail.py index 63ad2d4d3..9d68635e8 100644 --- a/tests/integration/cachedfail.py +++ b/tests/integration/cachedfail.py @@ -31,158 +31,115 @@ from tests.testutils import create_artifact_share pytestmark = pytest.mark.integration -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project" -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_build_checkout_cached_fail(cli, datafiles): project = str(datafiles) - element_path = os.path.join(project, 'elements', 'element.bst') - checkout = os.path.join(cli.directory, 'checkout') + element_path = os.path.join(project, "elements", "element.bst") + checkout = os.path.join(cli.directory, "checkout") # Write out our test target element = { - 'kind': 'script', - 'depends': [ - { - 'filename': 'base.bst', - 'type': 'build', - }, - ], - 'config': { - 'commands': [ - 'touch %{install-root}/foo', - 'false', - ], - }, + "kind": "script", + "depends": [{"filename": "base.bst", "type": "build",},], + "config": {"commands": ["touch %{install-root}/foo", "false",],}, } _yaml.roundtrip_dump(element, element_path) # Try to build it, this should result in a failure that contains the content - result = cli.run(project=project, args=['build', 'element.bst']) + result = cli.run(project=project, args=["build", "element.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) # Assert that it's cached in a failed artifact - assert cli.get_element_state(project, 'element.bst') == 'failed' + assert cli.get_element_state(project, "element.bst") == "failed" # Now check it out - result = cli.run(project=project, args=[ - 'artifact', 'checkout', 'element.bst', '--directory', checkout - ]) + result = cli.run(project=project, args=["artifact", "checkout", "element.bst", "--directory", checkout]) result.assert_success() # Check that the checkout contains the file created before failure - filename = os.path.join(checkout, 'foo') + filename = os.path.join(checkout, "foo") assert os.path.exists(filename) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_build_depend_on_cached_fail(cli, datafiles): project = str(datafiles) - dep_path = os.path.join(project, 'elements', 'dep.bst') - target_path = os.path.join(project, 'elements', 'target.bst') + dep_path = os.path.join(project, "elements", "dep.bst") + target_path = os.path.join(project, "elements", "target.bst") dep = { - 'kind': 'script', - 'depends': [ - { - 'filename': 'base.bst', - 'type': 'build', - }, - ], - 'config': { - 'commands': [ - 'touch %{install-root}/foo', - 'false', - ], - }, + "kind": "script", + "depends": [{"filename": "base.bst", "type": "build",},], + "config": {"commands": ["touch %{install-root}/foo", "false",],}, } _yaml.roundtrip_dump(dep, dep_path) target = { - 'kind': 'script', - 'depends': [ - { - 'filename': 'base.bst', - 'type': 'build', - }, - { - 'filename': 'dep.bst', - 'type': 'build', - }, - ], - 'config': { - 'commands': [ - 'test -e /foo', - ], - }, + "kind": "script", + "depends": [{"filename": "base.bst", "type": "build",}, {"filename": "dep.bst", "type": "build",},], + "config": {"commands": ["test -e /foo",],}, } _yaml.roundtrip_dump(target, target_path) # Try to build it, this should result in caching a failure to build dep - result = cli.run(project=project, args=['build', 'dep.bst']) + result = cli.run(project=project, args=["build", "dep.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) # Assert that it's cached in a failed artifact - assert cli.get_element_state(project, 'dep.bst') == 'failed' + assert cli.get_element_state(project, "dep.bst") == "failed" # Now we should fail because we've a cached fail of dep - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) # Assert that it's not yet built, since one of its dependencies isn't ready. - assert cli.get_element_state(project, 'target.bst') == 'waiting' + assert cli.get_element_state(project, "target.bst") == "waiting" -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("on_error", ("continue", "quit")) def test_push_cached_fail(cli, tmpdir, datafiles, on_error): - if on_error == 'quit': - pytest.xfail('https://gitlab.com/BuildStream/buildstream/issues/534') + if on_error == "quit": + pytest.xfail("https://gitlab.com/BuildStream/buildstream/issues/534") project = str(datafiles) - element_path = os.path.join(project, 'elements', 'element.bst') + element_path = os.path.join(project, "elements", "element.bst") # Write out our test target element = { - 'kind': 'script', - 'depends': [ - { - 'filename': 'base.bst', - 'type': 'build', - }, - ], - 'config': { - 'commands': [ - 'false', + "kind": "script", + "depends": [{"filename": "base.bst", "type": "build",},], + "config": { + "commands": [ + "false", # Ensure unique cache key for different test variants - 'TEST="{}"'.format(os.environ.get('PYTEST_CURRENT_TEST')), + 'TEST="{}"'.format(os.environ.get("PYTEST_CURRENT_TEST")), ], }, } _yaml.roundtrip_dump(element, element_path) - with create_artifact_share(os.path.join(str(tmpdir), 'remote')) as share: - cli.configure({ - 'artifacts': {'url': share.repo, 'push': True}, - }) + with create_artifact_share(os.path.join(str(tmpdir), "remote")) as share: + cli.configure( + {"artifacts": {"url": share.repo, "push": True},} + ) # Build the element, continuing to finish active jobs on error. - result = cli.run(project=project, args=['--on-error={}'.format(on_error), 'build', 'element.bst']) + result = cli.run(project=project, args=["--on-error={}".format(on_error), "build", "element.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) # This element should have failed - assert cli.get_element_state(project, 'element.bst') == 'failed' + assert cli.get_element_state(project, "element.bst") == "failed" # This element should have been pushed to the remote - assert share.get_artifact(cli.get_artifact_name(project, 'test', 'element.bst')) + assert share.get_artifact(cli.get_artifact_name(project, "test", "element.bst")) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("on_error", ("continue", "quit")) def test_push_failed_missing_shell(cli, tmpdir, datafiles, on_error): @@ -191,78 +148,69 @@ def test_push_failed_missing_shell(cli, tmpdir, datafiles, on_error): When we don't have a valid shell, the artifact will be empty, not even the root directory. This ensures we handle the case of an entirely empty artifact correctly. """ - if on_error == 'quit': - pytest.xfail('https://gitlab.com/BuildStream/buildstream/issues/534') + if on_error == "quit": + pytest.xfail("https://gitlab.com/BuildStream/buildstream/issues/534") project = str(datafiles) - element_path = os.path.join(project, 'elements', 'element.bst') + element_path = os.path.join(project, "elements", "element.bst") # Write out our test target element = { - 'kind': 'script', - 'config': { - 'commands': [ - 'false', + "kind": "script", + "config": { + "commands": [ + "false", # Ensure unique cache key for different test variants - 'TEST="{}"'.format(os.environ.get('PYTEST_CURRENT_TEST')), + 'TEST="{}"'.format(os.environ.get("PYTEST_CURRENT_TEST")), ], }, } _yaml.roundtrip_dump(element, element_path) - with create_artifact_share(os.path.join(str(tmpdir), 'remote')) as share: - cli.configure({ - 'artifacts': {'url': share.repo, 'push': True}, - }) + with create_artifact_share(os.path.join(str(tmpdir), "remote")) as share: + cli.configure( + {"artifacts": {"url": share.repo, "push": True},} + ) # Build the element, continuing to finish active jobs on error. - result = cli.run(project=project, args=['--on-error={}'.format(on_error), 'build', 'element.bst']) + result = cli.run(project=project, args=["--on-error={}".format(on_error), "build", "element.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) # This element should have failed - assert cli.get_element_state(project, 'element.bst') == 'failed' + assert cli.get_element_state(project, "element.bst") == "failed" # This element should have been pushed to the remote - assert share.get_artifact(cli.get_artifact_name(project, 'test', 'element.bst')) + assert share.get_artifact(cli.get_artifact_name(project, "test", "element.bst")) -@pytest.mark.skipif(HAVE_SANDBOX != 'bwrap', reason='Only available with bubblewrap on Linux') +@pytest.mark.skipif(HAVE_SANDBOX != "bwrap", reason="Only available with bubblewrap on Linux") @pytest.mark.datafiles(DATA_DIR) def test_host_tools_errors_are_not_cached(cli, datafiles, tmp_path): # Create symlink to buildbox-casd to work with custom PATH - buildbox_casd = tmp_path.joinpath('bin/buildbox-casd') + buildbox_casd = tmp_path.joinpath("bin/buildbox-casd") buildbox_casd.parent.mkdir() - os.symlink(utils.get_host_tool('buildbox-casd'), str(buildbox_casd)) + os.symlink(utils.get_host_tool("buildbox-casd"), str(buildbox_casd)) project = str(datafiles) - element_path = os.path.join(project, 'elements', 'element.bst') + element_path = os.path.join(project, "elements", "element.bst") # Write out our test target element = { - 'kind': 'script', - 'depends': [ - { - 'filename': 'base.bst', - 'type': 'build', - }, - ], - 'config': { - 'commands': [ - 'true', - ], - }, + "kind": "script", + "depends": [{"filename": "base.bst", "type": "build",},], + "config": {"commands": ["true",],}, } _yaml.roundtrip_dump(element, element_path) # Build without access to host tools, this will fail result1 = cli.run( project=project, - args=['build', 'element.bst'], - env={'PATH': str(tmp_path.joinpath('bin')), - 'BST_FORCE_SANDBOX': None}) - result1.assert_task_error(ErrorDomain.SANDBOX, 'unavailable-local-sandbox') - assert cli.get_element_state(project, 'element.bst') == 'buildable' + args=["build", "element.bst"], + env={"PATH": str(tmp_path.joinpath("bin")), "BST_FORCE_SANDBOX": None}, + ) + result1.assert_task_error(ErrorDomain.SANDBOX, "unavailable-local-sandbox") + assert cli.get_element_state(project, "element.bst") == "buildable" # When rebuilding, this should work - result2 = cli.run(project=project, args=['build', 'element.bst']) + result2 = cli.run(project=project, args=["build", "element.bst"]) result2.assert_success() - assert cli.get_element_state(project, 'element.bst') == 'cached' + assert cli.get_element_state(project, "element.bst") == "cached" diff --git a/tests/integration/cmake.py b/tests/integration/cmake.py index a0298c2c3..f5d2f6cc9 100644 --- a/tests/integration/cmake.py +++ b/tests/integration/cmake.py @@ -12,57 +12,57 @@ from buildstream.testing._utils.site import HAVE_SANDBOX pytestmark = pytest.mark.integration -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project" -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_cmake_build(cli, datafiles): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') - element_name = 'cmake/cmakehello.bst' + checkout = os.path.join(cli.directory, "checkout") + element_name = "cmake/cmakehello.bst" - result = cli.run(project=project, args=['build', element_name]) + result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 - result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout]) + result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert result.exit_code == 0 - assert_contains(checkout, ['/usr', '/usr/bin', '/usr/bin/hello']) + assert_contains(checkout, ["/usr", "/usr/bin", "/usr/bin/hello"]) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') -@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") +@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox") def test_cmake_confroot_build(cli, datafiles): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') - element_name = 'cmake/cmakeconfroothello.bst' + checkout = os.path.join(cli.directory, "checkout") + element_name = "cmake/cmakeconfroothello.bst" - result = cli.run(project=project, args=['build', element_name]) + result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 - result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout]) + result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert result.exit_code == 0 - assert_contains(checkout, ['/usr', '/usr/bin', '/usr/bin/hello']) + assert_contains(checkout, ["/usr", "/usr/bin", "/usr/bin/hello"]) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_cmake_run(cli, datafiles): project = str(datafiles) - element_name = 'cmake/cmakehello.bst' + element_name = "cmake/cmakehello.bst" - result = cli.run(project=project, args=['build', element_name]) + result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 - result = cli.run(project=project, args=['shell', element_name, '/usr/bin/hello']) + result = cli.run(project=project, args=["shell", element_name, "/usr/bin/hello"]) assert result.exit_code == 0 - assert result.output == """Hello World! + assert ( + result.output + == """Hello World! This is hello. """ + ) diff --git a/tests/integration/compose-symlinks.py b/tests/integration/compose-symlinks.py index 061d8f8e4..7b807d1d5 100644 --- a/tests/integration/compose-symlinks.py +++ b/tests/integration/compose-symlinks.py @@ -10,10 +10,7 @@ from buildstream.testing import cli_integration as cli # pylint: disable=unused pytestmark = pytest.mark.integration -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project" -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") # Test that staging a file inside a directory symlink fails. @@ -26,11 +23,11 @@ def test_compose_symlinks(cli, tmpdir, datafiles): # Symlinks do not survive being placed in a source distribution # ('setup.py sdist'), so we have to create the one we need here. - project_files = os.path.join(project, 'files', 'compose-symlinks', 'base') - symlink_file = os.path.join(project_files, 'sbin') - os.symlink(os.path.join('usr', 'sbin'), symlink_file, target_is_directory=True) + project_files = os.path.join(project, "files", "compose-symlinks", "base") + symlink_file = os.path.join(project_files, "sbin") + os.symlink(os.path.join("usr", "sbin"), symlink_file, target_is_directory=True) - result = cli.run(project=project, args=['build', 'compose-symlinks/compose.bst']) + result = cli.run(project=project, args=["build", "compose-symlinks/compose.bst"]) assert result.exit_code == -1 - assert 'Destination is a symlink, not a directory: /sbin' in result.stderr + assert "Destination is a symlink, not a directory: /sbin" in result.stderr diff --git a/tests/integration/compose.py b/tests/integration/compose.py index 3562ed94b..638dbc8dd 100644 --- a/tests/integration/compose.py +++ b/tests/integration/compose.py @@ -14,10 +14,7 @@ from buildstream.testing._utils.site import HAVE_SANDBOX pytestmark = pytest.mark.integration -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project" -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") def create_compose_element(name, path, config=None): @@ -25,83 +22,110 @@ def create_compose_element(name, path, config=None): config = {} element = { - 'kind': 'compose', - 'depends': [{ - 'filename': 'compose/amhello.bst', - 'type': 'build' - }, { - 'filename': 'compose/test.bst', - 'type': 'build' - }], - 'config': config + "kind": "compose", + "depends": [ + {"filename": "compose/amhello.bst", "type": "build"}, + {"filename": "compose/test.bst", "type": "build"}, + ], + "config": config, } os.makedirs(os.path.dirname(os.path.join(path, name)), exist_ok=True) _yaml.roundtrip_dump(element, os.path.join(path, name)) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("include_domains,exclude_domains,expected", [ - # Test flat inclusion - ([], [], ['/usr', '/usr/bin', - '/usr/share', - '/usr/bin/hello', - '/usr/share/doc', '/usr/share/doc/amhello', - '/usr/share/doc/amhello/README', - '/tests', '/tests/test']), - # Test only runtime - (['runtime'], [], ['/usr', '/usr/share', - '/usr/bin', '/usr/bin/hello']), - # Test with runtime and doc - (['runtime', 'doc'], [], ['/usr', '/usr/share', - '/usr/bin', '/usr/bin/hello', - '/usr/share/doc', '/usr/share/doc/amhello', - '/usr/share/doc/amhello/README']), - # Test with only runtime excluded - ([], ['runtime'], ['/usr', '/usr/share', - '/usr/share/doc', '/usr/share/doc/amhello', - '/usr/share/doc/amhello/README', - '/tests', '/tests/test']), - # Test with runtime and doc excluded - ([], ['runtime', 'doc'], ['/usr', '/usr/share', - '/tests', '/tests/test']), - # Test with runtime simultaneously in- and excluded - (['runtime'], ['runtime'], ['/usr', '/usr/share']), - # Test with runtime included and doc excluded - (['runtime'], ['doc'], ['/usr', '/usr/share', - '/usr/bin', '/usr/bin/hello']), - # Test including a custom 'test' domain - (['test'], [], ['/usr', '/usr/share', - '/tests', '/tests/test']), - # Test excluding a custom 'test' domain - ([], ['test'], ['/usr', '/usr/bin', - '/usr/share', - '/usr/bin/hello', - '/usr/share/doc', '/usr/share/doc/amhello', - '/usr/share/doc/amhello/README']) -]) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') -def test_compose_include(cli, datafiles, include_domains, - exclude_domains, expected): +@pytest.mark.parametrize( + "include_domains,exclude_domains,expected", + [ + # Test flat inclusion + ( + [], + [], + [ + "/usr", + "/usr/bin", + "/usr/share", + "/usr/bin/hello", + "/usr/share/doc", + "/usr/share/doc/amhello", + "/usr/share/doc/amhello/README", + "/tests", + "/tests/test", + ], + ), + # Test only runtime + (["runtime"], [], ["/usr", "/usr/share", "/usr/bin", "/usr/bin/hello"]), + # Test with runtime and doc + ( + ["runtime", "doc"], + [], + [ + "/usr", + "/usr/share", + "/usr/bin", + "/usr/bin/hello", + "/usr/share/doc", + "/usr/share/doc/amhello", + "/usr/share/doc/amhello/README", + ], + ), + # Test with only runtime excluded + ( + [], + ["runtime"], + [ + "/usr", + "/usr/share", + "/usr/share/doc", + "/usr/share/doc/amhello", + "/usr/share/doc/amhello/README", + "/tests", + "/tests/test", + ], + ), + # Test with runtime and doc excluded + ([], ["runtime", "doc"], ["/usr", "/usr/share", "/tests", "/tests/test"]), + # Test with runtime simultaneously in- and excluded + (["runtime"], ["runtime"], ["/usr", "/usr/share"]), + # Test with runtime included and doc excluded + (["runtime"], ["doc"], ["/usr", "/usr/share", "/usr/bin", "/usr/bin/hello"]), + # Test including a custom 'test' domain + (["test"], [], ["/usr", "/usr/share", "/tests", "/tests/test"]), + # Test excluding a custom 'test' domain + ( + [], + ["test"], + [ + "/usr", + "/usr/bin", + "/usr/share", + "/usr/bin/hello", + "/usr/share/doc", + "/usr/share/doc/amhello", + "/usr/share/doc/amhello/README", + ], + ), + ], +) +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") +def test_compose_include(cli, datafiles, include_domains, exclude_domains, expected): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') - element_path = os.path.join(project, 'elements') - element_name = 'compose/compose-amhello.bst' + checkout = os.path.join(cli.directory, "checkout") + element_path = os.path.join(project, "elements") + element_name = "compose/compose-amhello.bst" # Create a yaml configuration from the specified include and # exclude domains - config = { - 'include': include_domains, - 'exclude': exclude_domains - } + config = {"include": include_domains, "exclude": exclude_domains} create_compose_element(element_name, element_path, config=config) - result = cli.run(project=project, args=['source', 'track', 'compose/amhello.bst']) + result = cli.run(project=project, args=["source", "track", "compose/amhello.bst"]) assert result.exit_code == 0 - result = cli.run(project=project, args=['build', element_name]) + result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 - result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout]) + result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert result.exit_code == 0 assert set(walk_dir(checkout)) == set(expected) diff --git a/tests/integration/filter.py b/tests/integration/filter.py index 2a32d4010..f13521cc8 100644 --- a/tests/integration/filter.py +++ b/tests/integration/filter.py @@ -13,27 +13,26 @@ from buildstream.testing._utils.site import HAVE_SANDBOX pytestmark = pytest.mark.integration -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - 'project' -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") @pytest.mark.datafiles(os.path.join(DATA_DIR)) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_filter_pass_integration(datafiles, cli): project = str(datafiles) # Passing integration commands should build nicely - result = cli.run(project=project, args=['build', 'filter/filter.bst']) + result = cli.run(project=project, args=["build", "filter/filter.bst"]) result.assert_success() # Checking out the element should work - checkout_dir = os.path.join(project, 'filter') - result = cli.run(project=project, args=['artifact', 'checkout', '--integrate', '--hardlinks', - '--directory', checkout_dir, 'filter/filter.bst']) + checkout_dir = os.path.join(project, "filter") + result = cli.run( + project=project, + args=["artifact", "checkout", "--integrate", "--hardlinks", "--directory", checkout_dir, "filter/filter.bst"], + ) result.assert_success() # Check that the integration command was run - assert_contains(checkout_dir, ['/foo']) + assert_contains(checkout_dir, ["/foo"]) shutil.rmtree(checkout_dir) diff --git a/tests/integration/import.py b/tests/integration/import.py index bac92cadf..f7510e2e5 100644 --- a/tests/integration/import.py +++ b/tests/integration/import.py @@ -13,50 +13,46 @@ from buildstream.testing.integration import walk_dir pytestmark = pytest.mark.integration -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project" -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") def create_import_element(name, path, source, target, source_path): element = { - 'kind': 'import', - 'sources': [{ - 'kind': 'local', - 'path': source_path - }], - 'config': { - 'source': source, - 'target': target - } + "kind": "import", + "sources": [{"kind": "local", "path": source_path}], + "config": {"source": source, "target": target}, } os.makedirs(os.path.dirname(os.path.join(path, name)), exist_ok=True) _yaml.roundtrip_dump(element, os.path.join(path, name)) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("source,target,path,expected", [ - ('/', '/', 'files/import-source', ['/test.txt', '/subdir', - '/subdir/test.txt']), - ('/subdir', '/', 'files/import-source', ['/test.txt']), - ('/', '/', 'files/import-source/subdir', ['/test.txt']), - ('/', '/output', 'files/import-source', ['/output', '/output/test.txt', - '/output/subdir', - '/output/subdir/test.txt']), -]) +@pytest.mark.parametrize( + "source,target,path,expected", + [ + ("/", "/", "files/import-source", ["/test.txt", "/subdir", "/subdir/test.txt"]), + ("/subdir", "/", "files/import-source", ["/test.txt"]), + ("/", "/", "files/import-source/subdir", ["/test.txt"]), + ( + "/", + "/output", + "files/import-source", + ["/output", "/output/test.txt", "/output/subdir", "/output/subdir/test.txt"], + ), + ], +) def test_import(cli, datafiles, source, target, path, expected): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') - element_path = os.path.join(project, 'elements') - element_name = 'import/import.bst' + checkout = os.path.join(cli.directory, "checkout") + element_path = os.path.join(project, "elements") + element_name = "import/import.bst" create_import_element(element_name, element_path, source, target, path) - res = cli.run(project=project, args=['build', element_name]) + res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 - cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout]) + cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert res.exit_code == 0 assert set(walk_dir(checkout)) == set(expected) diff --git a/tests/integration/make.py b/tests/integration/make.py index 664e7ca7a..109777622 100644 --- a/tests/integration/make.py +++ b/tests/integration/make.py @@ -12,41 +12,37 @@ from buildstream.testing._utils.site import HAVE_SANDBOX pytestmark = pytest.mark.integration -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project" -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") # Test that a make build 'works' - we use the make sample # makehello project for this. @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_make_build(cli, datafiles): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') - element_name = 'make/makehello.bst' + checkout = os.path.join(cli.directory, "checkout") + element_name = "make/makehello.bst" - result = cli.run(project=project, args=['build', element_name]) + result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 - result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout]) + result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert result.exit_code == 0 - assert_contains(checkout, ['/usr', '/usr/bin', - '/usr/bin/hello']) + assert_contains(checkout, ["/usr", "/usr/bin", "/usr/bin/hello"]) # Test running an executable built with make @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_make_run(cli, datafiles): project = str(datafiles) - element_name = 'make/makehello.bst' + element_name = "make/makehello.bst" - result = cli.run(project=project, args=['build', element_name]) + result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 - result = cli.run(project=project, args=['shell', element_name, '/usr/bin/hello']) + result = cli.run(project=project, args=["shell", element_name, "/usr/bin/hello"]) assert result.exit_code == 0 - assert result.output == 'Hello, world\n' + assert result.output == "Hello, world\n" diff --git a/tests/integration/manual.py b/tests/integration/manual.py index 2ac7f74d0..c6a84b062 100644 --- a/tests/integration/manual.py +++ b/tests/integration/manual.py @@ -13,141 +13,139 @@ from buildstream.testing._utils.site import HAVE_SANDBOX pytestmark = pytest.mark.integration -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project" -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") def create_manual_element(name, path, config, variables, environment): element = { - 'kind': 'manual', - 'depends': [{ - 'filename': 'base.bst', - 'type': 'build' - }], - 'config': config, - 'variables': variables, - 'environment': environment + "kind": "manual", + "depends": [{"filename": "base.bst", "type": "build"}], + "config": config, + "variables": variables, + "environment": environment, } os.makedirs(os.path.dirname(os.path.join(path, name)), exist_ok=True) _yaml.roundtrip_dump(element, os.path.join(path, name)) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_manual_element(cli, datafiles): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') - element_path = os.path.join(project, 'elements') - element_name = 'import/import.bst' - - create_manual_element(element_name, element_path, { - 'configure-commands': ["echo './configure' >> test"], - 'build-commands': ["echo 'make' >> test"], - 'install-commands': [ - "echo 'make install' >> test", - "cp test %{install-root}" - ], - 'strip-commands': ["echo 'strip' >> %{install-root}/test"] - }, {}, {}) - - res = cli.run(project=project, args=['build', element_name]) + checkout = os.path.join(cli.directory, "checkout") + element_path = os.path.join(project, "elements") + element_name = "import/import.bst" + + create_manual_element( + element_name, + element_path, + { + "configure-commands": ["echo './configure' >> test"], + "build-commands": ["echo 'make' >> test"], + "install-commands": ["echo 'make install' >> test", "cp test %{install-root}"], + "strip-commands": ["echo 'strip' >> %{install-root}/test"], + }, + {}, + {}, + ) + + res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 - cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout]) + cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert res.exit_code == 0 - with open(os.path.join(checkout, 'test')) as f: + with open(os.path.join(checkout, "test")) as f: text = f.read() - assert text == """./configure + assert ( + text + == """./configure make make install strip """ + ) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_manual_element_environment(cli, datafiles): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') - element_path = os.path.join(project, 'elements') - element_name = 'import/import.bst' - - create_manual_element(element_name, element_path, { - 'install-commands': [ - "echo $V >> test", - "cp test %{install-root}" - ] - }, { - }, { - 'V': 2 - }) - - res = cli.run(project=project, args=['build', element_name]) + checkout = os.path.join(cli.directory, "checkout") + element_path = os.path.join(project, "elements") + element_name = "import/import.bst" + + create_manual_element( + element_name, element_path, {"install-commands": ["echo $V >> test", "cp test %{install-root}"]}, {}, {"V": 2} + ) + + res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 - cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout]) + cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert res.exit_code == 0 - with open(os.path.join(checkout, 'test')) as f: + with open(os.path.join(checkout, "test")) as f: text = f.read() assert text == "2\n" @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_manual_element_noparallel(cli, datafiles): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') - element_path = os.path.join(project, 'elements') - element_name = 'import/import.bst' - - create_manual_element(element_name, element_path, { - 'install-commands': [ - "echo $MAKEFLAGS >> test", - "echo $V >> test", - "cp test %{install-root}" - ] - }, { - 'notparallel': True - }, { - 'MAKEFLAGS': '-j%{max-jobs} -Wall', - 'V': 2 - }) - - res = cli.run(project=project, args=['build', element_name]) + checkout = os.path.join(cli.directory, "checkout") + element_path = os.path.join(project, "elements") + element_name = "import/import.bst" + + create_manual_element( + element_name, + element_path, + {"install-commands": ["echo $MAKEFLAGS >> test", "echo $V >> test", "cp test %{install-root}"]}, + {"notparallel": True}, + {"MAKEFLAGS": "-j%{max-jobs} -Wall", "V": 2}, + ) + + res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 - cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout]) + cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert res.exit_code == 0 - with open(os.path.join(checkout, 'test')) as f: + with open(os.path.join(checkout, "test")) as f: text = f.read() - assert text == """-j1 -Wall + assert ( + text + == """-j1 -Wall 2 """ + ) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_manual_element_logging(cli, datafiles): project = str(datafiles) - element_path = os.path.join(project, 'elements') - element_name = 'import/import.bst' - - create_manual_element(element_name, element_path, { - 'configure-commands': ["echo configure"], - 'build-commands': ["echo build"], - 'install-commands': ["echo install"], - 'strip-commands': ["echo strip"] - }, {}, {}) - - res = cli.run(project=project, args=['build', element_name]) + element_path = os.path.join(project, "elements") + element_name = "import/import.bst" + + create_manual_element( + element_name, + element_path, + { + "configure-commands": ["echo configure"], + "build-commands": ["echo build"], + "install-commands": ["echo install"], + "strip-commands": ["echo strip"], + }, + {}, + {}, + ) + + res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 # Verify that individual commands are logged diff --git a/tests/integration/messages.py b/tests/integration/messages.py index 42725fc5b..f35b778d6 100644 --- a/tests/integration/messages.py +++ b/tests/integration/messages.py @@ -33,29 +33,20 @@ pytestmark = pytest.mark.integration # Project directory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project", -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_disable_message_lines(cli, datafiles): project = str(datafiles) - element_path = os.path.join(project, 'elements') - element_name = 'message.bst' + element_path = os.path.join(project, "elements") + element_name = "message.bst" element = { - 'kind': 'manual', - 'depends': [{ - 'filename': 'base.bst' - }], - 'config': { - 'build-commands': - ['echo "Silly message"'], - 'strip-commands': [] - } + "kind": "manual", + "depends": [{"filename": "base.bst"}], + "config": {"build-commands": ['echo "Silly message"'], "strip-commands": []}, } os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True) @@ -68,43 +59,34 @@ def test_disable_message_lines(cli, datafiles): # Let's now build it again, but with --message-lines 0 cli.remove_artifact_from_cache(project, element_name) - result = cli.run(project=project, args=["--message-lines", "0", - "build", element_name]) + result = cli.run(project=project, args=["--message-lines", "0", "build", element_name]) result.assert_success() assert "Message contains " not in result.stderr @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_disable_error_lines(cli, datafiles): project = str(datafiles) - element_path = os.path.join(project, 'elements') - element_name = 'message.bst' + element_path = os.path.join(project, "elements") + element_name = "message.bst" element = { - 'kind': 'manual', - 'depends': [{ - 'filename': 'base.bst' - }], - 'config': { - 'build-commands': - ['This is a syntax error > >'], - 'strip-commands': [] - } + "kind": "manual", + "depends": [{"filename": "base.bst"}], + "config": {"build-commands": ["This is a syntax error > >"], "strip-commands": []}, } os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True) _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) # First we check that we get the syntax error - result = cli.run(project=project, args=["--error-lines", "0", - "build", element_name]) + result = cli.run(project=project, args=["--error-lines", "0", "build", element_name]) result.assert_main_error(ErrorDomain.STREAM, None) assert "This is a syntax error" in result.stderr # Let's now build it again, but with --error-lines 0 cli.remove_artifact_from_cache(project, element_name) - result = cli.run(project=project, args=["--error-lines", "0", - "build", element_name]) + result = cli.run(project=project, args=["--error-lines", "0", "build", element_name]) result.assert_main_error(ErrorDomain.STREAM, None) assert "Printing the last" not in result.stderr diff --git a/tests/integration/pip_element.py b/tests/integration/pip_element.py index da0badcb3..bfc4f8098 100644 --- a/tests/integration/pip_element.py +++ b/tests/integration/pip_element.py @@ -17,119 +17,101 @@ from tests.testutils import setup_pypi_repo # pylint: disable=unused-import pytestmark = pytest.mark.integration -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project" -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_pip_build(cli, datafiles): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') - element_path = os.path.join(project, 'elements') - element_name = 'pip/hello.bst' + checkout = os.path.join(cli.directory, "checkout") + element_path = os.path.join(project, "elements") + element_name = "pip/hello.bst" element = { - 'kind': 'pip', - 'variables': { - 'pip': 'pip3' - }, - 'depends': [{ - 'filename': 'base.bst' - }], - 'sources': [{ - 'kind': 'tar', - 'url': 'file://{}/files/hello.tar.xz'.format(project), - 'ref': 'ad96570b552498807abec33c06210bf68378d854ced6753b77916c5ed517610d' - - }] + "kind": "pip", + "variables": {"pip": "pip3"}, + "depends": [{"filename": "base.bst"}], + "sources": [ + { + "kind": "tar", + "url": "file://{}/files/hello.tar.xz".format(project), + "ref": "ad96570b552498807abec33c06210bf68378d854ced6753b77916c5ed517610d", + } + ], } os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True) _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) - result = cli.run(project=project, args=['build', element_name]) + result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 - result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout]) + result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert result.exit_code == 0 - assert_contains(checkout, ['/usr', '/usr/lib', '/usr/bin', - '/usr/bin/hello', '/usr/lib/python3.6']) + assert_contains(checkout, ["/usr", "/usr/lib", "/usr/bin", "/usr/bin/hello", "/usr/lib/python3.6"]) # Test running an executable built with pip @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_pip_run(cli, datafiles): # Create and build our test element test_pip_build(cli, datafiles) project = str(datafiles) - element_name = 'pip/hello.bst' + element_name = "pip/hello.bst" - result = cli.run(project=project, args=['shell', element_name, '/usr/bin/hello']) + result = cli.run(project=project, args=["shell", element_name, "/usr/bin/hello"]) assert result.exit_code == 0 - assert result.output == 'Hello, world!\n' + assert result.output == "Hello, world!\n" @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_pip_element_should_install_pip_deps(cli, datafiles, setup_pypi_repo): project = str(datafiles) - elements_path = os.path.join(project, 'elements') - element_name = 'pip/hello.bst' + elements_path = os.path.join(project, "elements") + element_name = "pip/hello.bst" # check that exotically named packages are imported correctly - myreqs_packages = 'alohalib' - dependencies = ['app2', 'app.3', 'app-4', 'app_5', 'app.no.6', 'app-no-7', 'app_no_8'] - mock_packages = { - myreqs_packages: { - package: {} for package in dependencies - } - } + myreqs_packages = "alohalib" + dependencies = ["app2", "app.3", "app-4", "app_5", "app.no.6", "app-no-7", "app_no_8"] + mock_packages = {myreqs_packages: {package: {} for package in dependencies}} # set up directories - pypi_repo = os.path.join(project, 'files', 'pypi-repo') + pypi_repo = os.path.join(project, "files", "pypi-repo") os.makedirs(pypi_repo, exist_ok=True) os.makedirs(os.path.dirname(os.path.join(elements_path, element_name)), exist_ok=True) setup_pypi_repo(mock_packages, pypi_repo) # create pip element element = { - 'kind': 'pip', - 'variables': { - 'pip': 'pip3' - }, - 'depends': [{ - 'filename': 'base.bst' - }], - 'sources': [ + "kind": "pip", + "variables": {"pip": "pip3"}, + "depends": [{"filename": "base.bst"}], + "sources": [ { - 'kind': 'tar', - 'url': 'file://{}/files/hello.tar.xz'.format(project), + "kind": "tar", + "url": "file://{}/files/hello.tar.xz".format(project), # FIXME: remove hardcoded ref once issue #1010 is closed - 'ref': 'ad96570b552498807abec33c06210bf68378d854ced6753b77916c5ed517610d' + "ref": "ad96570b552498807abec33c06210bf68378d854ced6753b77916c5ed517610d", }, - { - 'kind': 'pip', - 'url': 'file://{}'.format(os.path.realpath(pypi_repo)), - 'packages': [myreqs_packages], - } - ] + {"kind": "pip", "url": "file://{}".format(os.path.realpath(pypi_repo)), "packages": [myreqs_packages],}, + ], } _yaml.roundtrip_dump(element, os.path.join(elements_path, element_name)) - result = cli.run(project=project, args=['source', 'track', element_name]) + result = cli.run(project=project, args=["source", "track", element_name]) assert result.exit_code == 0 - result = cli.run(project=project, args=['build', element_name]) + result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 # get installed packages in sandbox installed_packages = set( - cli.run(project=project, args=['shell', element_name, 'pip3', 'freeze']).output.split('\n')) + cli.run(project=project, args=["shell", element_name, "pip3", "freeze"]).output.split("\n") + ) # compare with packages that are expected to be installed - pip_source_packages = {package.replace('_', "-") + '==0.1' for package in dependencies + [myreqs_packages]} + pip_source_packages = {package.replace("_", "-") + "==0.1" for package in dependencies + [myreqs_packages]} assert pip_source_packages.issubset(installed_packages) diff --git a/tests/integration/pip_source.py b/tests/integration/pip_source.py index c221910a6..5d314974d 100644 --- a/tests/integration/pip_source.py +++ b/tests/integration/pip_source.py @@ -16,180 +16,162 @@ from tests.testutils.python_repo import setup_pypi_repo # pylint: disable=unuse pytestmark = pytest.mark.integration -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project" -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") @pytest.mark.datafiles(DATA_DIR) def test_pip_source_import_packages(cli, datafiles, setup_pypi_repo): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') - element_path = os.path.join(project, 'elements') - element_name = 'pip/hello.bst' + checkout = os.path.join(cli.directory, "checkout") + element_path = os.path.join(project, "elements") + element_name = "pip/hello.bst" # check that exotically named packages are imported correctly - myreqs_packages = 'hellolib' - dependencies = ['app2', 'app.3', 'app-4', 'app_5', 'app.no.6', 'app-no-7', 'app_no_8'] - mock_packages = { - myreqs_packages: { - package: {} for package in dependencies - } - } + myreqs_packages = "hellolib" + dependencies = ["app2", "app.3", "app-4", "app_5", "app.no.6", "app-no-7", "app_no_8"] + mock_packages = {myreqs_packages: {package: {} for package in dependencies}} # create mock pypi repository - pypi_repo = os.path.join(project, 'files', 'pypi-repo') + pypi_repo = os.path.join(project, "files", "pypi-repo") os.makedirs(pypi_repo, exist_ok=True) setup_pypi_repo(mock_packages, pypi_repo) element = { - 'kind': 'import', - 'sources': [ - { - 'kind': 'local', - 'path': 'files/pip-source' - }, - { - 'kind': 'pip', - 'url': 'file://{}'.format(os.path.realpath(pypi_repo)), - 'packages': [myreqs_packages] - } - ] + "kind": "import", + "sources": [ + {"kind": "local", "path": "files/pip-source"}, + {"kind": "pip", "url": "file://{}".format(os.path.realpath(pypi_repo)), "packages": [myreqs_packages]}, + ], } os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True) _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) - result = cli.run(project=project, args=['source', 'track', element_name]) + result = cli.run(project=project, args=["source", "track", element_name]) assert result.exit_code == 0 - result = cli.run(project=project, args=['build', element_name]) + result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 - result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout]) + result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert result.exit_code == 0 - assert_contains(checkout, ['/.bst_pip_downloads', - '/.bst_pip_downloads/hellolib-0.1.tar.gz', - '/.bst_pip_downloads/app2-0.1.tar.gz', - '/.bst_pip_downloads/app.3-0.1.tar.gz', - '/.bst_pip_downloads/app-4-0.1.tar.gz', - '/.bst_pip_downloads/app_5-0.1.tar.gz', - '/.bst_pip_downloads/app.no.6-0.1.tar.gz', - '/.bst_pip_downloads/app-no-7-0.1.tar.gz', - '/.bst_pip_downloads/app_no_8-0.1.tar.gz']) + assert_contains( + checkout, + [ + "/.bst_pip_downloads", + "/.bst_pip_downloads/hellolib-0.1.tar.gz", + "/.bst_pip_downloads/app2-0.1.tar.gz", + "/.bst_pip_downloads/app.3-0.1.tar.gz", + "/.bst_pip_downloads/app-4-0.1.tar.gz", + "/.bst_pip_downloads/app_5-0.1.tar.gz", + "/.bst_pip_downloads/app.no.6-0.1.tar.gz", + "/.bst_pip_downloads/app-no-7-0.1.tar.gz", + "/.bst_pip_downloads/app_no_8-0.1.tar.gz", + ], + ) @pytest.mark.datafiles(DATA_DIR) def test_pip_source_import_requirements_files(cli, datafiles, setup_pypi_repo): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') - element_path = os.path.join(project, 'elements') - element_name = 'pip/hello.bst' + checkout = os.path.join(cli.directory, "checkout") + element_path = os.path.join(project, "elements") + element_name = "pip/hello.bst" # check that exotically named packages are imported correctly - myreqs_packages = 'hellolib' - dependencies = ['app2', 'app.3', 'app-4', 'app_5', 'app.no.6', 'app-no-7', 'app_no_8'] - mock_packages = { - myreqs_packages: { - package: {} for package in dependencies - } - } + myreqs_packages = "hellolib" + dependencies = ["app2", "app.3", "app-4", "app_5", "app.no.6", "app-no-7", "app_no_8"] + mock_packages = {myreqs_packages: {package: {} for package in dependencies}} # create mock pypi repository - pypi_repo = os.path.join(project, 'files', 'pypi-repo') + pypi_repo = os.path.join(project, "files", "pypi-repo") os.makedirs(pypi_repo, exist_ok=True) setup_pypi_repo(mock_packages, pypi_repo) element = { - 'kind': 'import', - 'sources': [ + "kind": "import", + "sources": [ + {"kind": "local", "path": "files/pip-source"}, { - 'kind': 'local', - 'path': 'files/pip-source' + "kind": "pip", + "url": "file://{}".format(os.path.realpath(pypi_repo)), + "requirements-files": ["myreqs.txt"], }, - { - 'kind': 'pip', - 'url': 'file://{}'.format(os.path.realpath(pypi_repo)), - 'requirements-files': ['myreqs.txt'], - } - ] + ], } os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True) _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) - result = cli.run(project=project, args=['source', 'track', element_name]) + result = cli.run(project=project, args=["source", "track", element_name]) assert result.exit_code == 0 - result = cli.run(project=project, args=['build', element_name]) + result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 - result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout]) + result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert result.exit_code == 0 - assert_contains(checkout, ['/.bst_pip_downloads', - '/.bst_pip_downloads/hellolib-0.1.tar.gz', - '/.bst_pip_downloads/app2-0.1.tar.gz', - '/.bst_pip_downloads/app.3-0.1.tar.gz', - '/.bst_pip_downloads/app-4-0.1.tar.gz', - '/.bst_pip_downloads/app_5-0.1.tar.gz', - '/.bst_pip_downloads/app.no.6-0.1.tar.gz', - '/.bst_pip_downloads/app-no-7-0.1.tar.gz', - '/.bst_pip_downloads/app_no_8-0.1.tar.gz']) + assert_contains( + checkout, + [ + "/.bst_pip_downloads", + "/.bst_pip_downloads/hellolib-0.1.tar.gz", + "/.bst_pip_downloads/app2-0.1.tar.gz", + "/.bst_pip_downloads/app.3-0.1.tar.gz", + "/.bst_pip_downloads/app-4-0.1.tar.gz", + "/.bst_pip_downloads/app_5-0.1.tar.gz", + "/.bst_pip_downloads/app.no.6-0.1.tar.gz", + "/.bst_pip_downloads/app-no-7-0.1.tar.gz", + "/.bst_pip_downloads/app_no_8-0.1.tar.gz", + ], + ) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_pip_source_build(cli, datafiles, setup_pypi_repo): project = str(datafiles) - element_path = os.path.join(project, 'elements') - element_name = 'pip/hello.bst' + element_path = os.path.join(project, "elements") + element_name = "pip/hello.bst" # check that exotically named packages are imported correctly - myreqs_packages = 'hellolib' - dependencies = ['app2', 'app.3', 'app-4', 'app_5', 'app.no.6', 'app-no-7', 'app_no_8'] - mock_packages = { - myreqs_packages: { - package: {} for package in dependencies - } - } + myreqs_packages = "hellolib" + dependencies = ["app2", "app.3", "app-4", "app_5", "app.no.6", "app-no-7", "app_no_8"] + mock_packages = {myreqs_packages: {package: {} for package in dependencies}} # create mock pypi repository - pypi_repo = os.path.join(project, 'files', 'pypi-repo') + pypi_repo = os.path.join(project, "files", "pypi-repo") os.makedirs(pypi_repo, exist_ok=True) setup_pypi_repo(mock_packages, pypi_repo) element = { - 'kind': 'manual', - 'depends': ['base.bst'], - 'sources': [ + "kind": "manual", + "depends": ["base.bst"], + "sources": [ + {"kind": "local", "path": "files/pip-source"}, { - 'kind': 'local', - 'path': 'files/pip-source' + "kind": "pip", + "url": "file://{}".format(os.path.realpath(pypi_repo)), + "requirements-files": ["myreqs.txt"], + "packages": dependencies, }, - { - 'kind': 'pip', - 'url': 'file://{}'.format(os.path.realpath(pypi_repo)), - 'requirements-files': ['myreqs.txt'], - 'packages': dependencies - } ], - 'config': { - 'install-commands': [ - 'pip3 install --no-index --prefix %{install-root}/usr .bst_pip_downloads/*.tar.gz', - 'install app1.py %{install-root}/usr/bin/' + "config": { + "install-commands": [ + "pip3 install --no-index --prefix %{install-root}/usr .bst_pip_downloads/*.tar.gz", + "install app1.py %{install-root}/usr/bin/", ] - } + }, } os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True) _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) - result = cli.run(project=project, args=['source', 'track', element_name]) + result = cli.run(project=project, args=["source", "track", element_name]) assert result.exit_code == 0 - result = cli.run(project=project, args=['build', element_name]) + result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 - result = cli.run(project=project, args=['shell', element_name, '/usr/bin/app1.py']) + result = cli.run(project=project, args=["shell", element_name, "/usr/bin/app1.py"]) assert result.exit_code == 0 assert result.output == "Hello App1! This is hellolib\n" diff --git a/tests/integration/project/files/pip-source/app1.py b/tests/integration/project/files/pip-source/app1.py index ab1005ba4..b96d14b00 100644 --- a/tests/integration/project/files/pip-source/app1.py +++ b/tests/integration/project/files/pip-source/app1.py @@ -4,8 +4,8 @@ from hellolib import hello def main(): - hello('App1') + hello("App1") -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/tests/integration/pullbuildtrees.py b/tests/integration/pullbuildtrees.py index a1b188e5a..b43a07121 100644 --- a/tests/integration/pullbuildtrees.py +++ b/tests/integration/pullbuildtrees.py @@ -16,22 +16,21 @@ from tests.testutils import create_artifact_share pytestmark = pytest.mark.integration -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project" -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") # Remove artifact cache & set cli.config value of pull-buildtrees # to false, which is the default user context. The cache has to be # cleared as just forcefully removing the refpath leaves dangling objects. def default_state(cli, tmpdir, share): - shutil.rmtree(os.path.join(str(tmpdir), 'cas')) - cli.configure({ - 'artifacts': {'url': share.repo, 'push': False}, - 'cachedir': str(tmpdir), - 'cache': {'pull-buildtrees': False}, - }) + shutil.rmtree(os.path.join(str(tmpdir), "cas")) + cli.configure( + { + "artifacts": {"url": share.repo, "push": False}, + "cachedir": str(tmpdir), + "cache": {"pull-buildtrees": False}, + } + ) # A test to capture the integration of the pullbuildtrees @@ -39,50 +38,52 @@ def default_state(cli, tmpdir, share): # directory of an element. @pytest.mark.integration @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_pullbuildtrees(cli2, tmpdir, datafiles): project = str(datafiles) - element_name = 'autotools/amhello.bst' + element_name = "autotools/amhello.bst" cwd = str(tmpdir) # Create artifact shares for pull & push testing - with create_artifact_share(os.path.join(str(tmpdir), 'share1')) as share1,\ - create_artifact_share(os.path.join(str(tmpdir), 'share2')) as share2,\ - create_artifact_share(os.path.join(str(tmpdir), 'share3')) as share3: - cli2.configure({ - 'artifacts': {'url': share1.repo, 'push': True}, - 'cachedir': str(tmpdir), - 'cache': {'cache-buildtrees': 'always'}, - }) + with create_artifact_share(os.path.join(str(tmpdir), "share1")) as share1, create_artifact_share( + os.path.join(str(tmpdir), "share2") + ) as share2, create_artifact_share(os.path.join(str(tmpdir), "share3")) as share3: + cli2.configure( + { + "artifacts": {"url": share1.repo, "push": True}, + "cachedir": str(tmpdir), + "cache": {"cache-buildtrees": "always"}, + } + ) # Build autotools element, checked pushed, delete local - result = cli2.run(project=project, args=['build', element_name]) + result = cli2.run(project=project, args=["build", element_name]) assert result.exit_code == 0 - assert cli2.get_element_state(project, element_name) == 'cached' - assert share1.get_artifact(cli2.get_artifact_name(project, 'test', element_name)) + assert cli2.get_element_state(project, element_name) == "cached" + assert share1.get_artifact(cli2.get_artifact_name(project, "test", element_name)) default_state(cli2, tmpdir, share1) # Pull artifact with default config, assert that pulling again # doesn't create a pull job, then assert with buildtrees user # config set creates a pull job. - result = cli2.run(project=project, args=['artifact', 'pull', element_name]) + result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() - result = cli2.run(project=project, args=['artifact', 'pull', element_name]) + result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name not in result.get_pulled_elements() - cli2.configure({'cache': {'pull-buildtrees': True}}) - result = cli2.run(project=project, args=['artifact', 'pull', element_name]) + cli2.configure({"cache": {"pull-buildtrees": True}}) + result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() default_state(cli2, tmpdir, share1) # Pull artifact with default config, then assert that pulling # with buildtrees cli flag set creates a pull job. # Also assert that the buildtree is added to the local CAS. - result = cli2.run(project=project, args=['artifact', 'pull', element_name]) + result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() - artifact_name = cli2.get_artifact_name(project, 'test', element_name) + artifact_name = cli2.get_artifact_name(project, "test", element_name) with cli2.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert not buildtreedir - result = cli2.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name]) + result = cli2.run(project=project, args=["--pull-buildtrees", "artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() with cli2.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert os.path.isdir(buildtreedir) @@ -91,22 +92,22 @@ def test_pullbuildtrees(cli2, tmpdir, datafiles): # Pull artifact with pullbuildtrees set in user config, then assert # that pulling with the same user config doesn't creates a pull job, # or when buildtrees cli flag is set. - cli2.configure({'cache': {'pull-buildtrees': True}}) - result = cli2.run(project=project, args=['artifact', 'pull', element_name]) + cli2.configure({"cache": {"pull-buildtrees": True}}) + result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() - result = cli2.run(project=project, args=['artifact', 'pull', element_name]) + result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name not in result.get_pulled_elements() - result = cli2.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name]) + result = cli2.run(project=project, args=["--pull-buildtrees", "artifact", "pull", element_name]) assert element_name not in result.get_pulled_elements() default_state(cli2, tmpdir, share1) # Pull artifact with default config and buildtrees cli flag set, then assert # that pulling with pullbuildtrees set in user config doesn't create a pull # job. - result = cli2.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name]) + result = cli2.run(project=project, args=["--pull-buildtrees", "artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() - cli2.configure({'cache': {'pull-buildtrees': True}}) - result = cli2.run(project=project, args=['artifact', 'pull', element_name]) + cli2.configure({"cache": {"pull-buildtrees": True}}) + result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name not in result.get_pulled_elements() default_state(cli2, tmpdir, share1) @@ -114,73 +115,64 @@ def test_pullbuildtrees(cli2, tmpdir, datafiles): # can't be pushed to an artifact share, then assert that a complete build element # can be. This will attempt a partial pull from share1 and then a partial push # to share2 - result = cli2.run(project=project, args=['artifact', 'pull', element_name]) + result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() - cli2.configure({'artifacts': {'url': share2.repo, 'push': True}}) - result = cli2.run(project=project, args=['artifact', 'push', element_name]) + cli2.configure({"artifacts": {"url": share2.repo, "push": True}}) + result = cli2.run(project=project, args=["artifact", "push", element_name]) assert element_name not in result.get_pushed_elements() - assert not share2.get_artifact(cli2.get_artifact_name(project, 'test', element_name)) + assert not share2.get_artifact(cli2.get_artifact_name(project, "test", element_name)) # Assert that after pulling the missing buildtree the element artifact can be # successfully pushed to the remote. This will attempt to pull the buildtree # from share1 and then a 'complete' push to share2 - cli2.configure({'artifacts': {'url': share1.repo, 'push': False}}) - result = cli2.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name]) + cli2.configure({"artifacts": {"url": share1.repo, "push": False}}) + result = cli2.run(project=project, args=["--pull-buildtrees", "artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() - cli2.configure({'artifacts': {'url': share2.repo, 'push': True}}) - result = cli2.run(project=project, args=['artifact', 'push', element_name]) + cli2.configure({"artifacts": {"url": share2.repo, "push": True}}) + result = cli2.run(project=project, args=["artifact", "push", element_name]) assert element_name in result.get_pushed_elements() - assert share2.get_artifact(cli2.get_artifact_name(project, 'test', element_name)) + assert share2.get_artifact(cli2.get_artifact_name(project, "test", element_name)) default_state(cli2, tmpdir, share1) # Assert that bst artifact push will automatically attempt to pull a missing buildtree # if pull-buildtrees is set, however as share3 is the only defined remote and is empty, # assert that no element artifact buildtrees are pulled (no available remote buildtree) and thus the # artifact cannot be pushed. - result = cli2.run(project=project, args=['artifact', 'pull', element_name]) + result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() - cli2.configure({'artifacts': {'url': share3.repo, 'push': True}}) - result = cli2.run(project=project, args=['--pull-buildtrees', 'artifact', 'push', element_name]) + cli2.configure({"artifacts": {"url": share3.repo, "push": True}}) + result = cli2.run(project=project, args=["--pull-buildtrees", "artifact", "push", element_name]) assert "Attempting to fetch missing artifact buildtrees" in result.stderr assert element_name not in result.get_pulled_elements() with cli2.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert not buildtreedir assert element_name not in result.get_pushed_elements() - assert not share3.get_artifact(cli2.get_artifact_name(project, 'test', element_name)) + assert not share3.get_artifact(cli2.get_artifact_name(project, "test", element_name)) # Assert that if we add an extra remote that has the buildtree artfact cached, bst artifact push will # automatically attempt to pull it and will be successful, leading to the full artifact being pushed # to the empty share3. This gives the ability to attempt push currently partial artifacts to a remote, # without exlipictly requiring a bst artifact pull. - cli2.configure({'artifacts': [{'url': share1.repo, 'push': False}, {'url': share3.repo, 'push': True}]}) - result = cli2.run(project=project, args=['--pull-buildtrees', 'artifact', 'push', element_name]) + cli2.configure({"artifacts": [{"url": share1.repo, "push": False}, {"url": share3.repo, "push": True}]}) + result = cli2.run(project=project, args=["--pull-buildtrees", "artifact", "push", element_name]) assert "Attempting to fetch missing artifact buildtrees" in result.stderr assert element_name in result.get_pulled_elements() with cli2.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert os.path.isdir(buildtreedir) assert element_name in result.get_pushed_elements() - assert share3.get_artifact(cli2.get_artifact_name(project, 'test', element_name)) + assert share3.get_artifact(cli2.get_artifact_name(project, "test", element_name)) # Ensure that only valid pull-buildtrees boolean options make it through the loading # process. -@pytest.mark.parametrize("value,success", [ - (True, True), - (False, True), - ("pony", False), - ("1", False) -]) +@pytest.mark.parametrize("value,success", [(True, True), (False, True), ("pony", False), ("1", False)]) @pytest.mark.datafiles(DATA_DIR) def test_invalid_cache_pullbuildtrees(cli, datafiles, value, success): project = str(datafiles) - cli.configure({ - 'cache': { - 'pull-buildtrees': value, - } - }) + cli.configure({"cache": {"pull-buildtrees": value,}}) - res = cli.run(project=project, args=['workspace', 'list']) + res = cli.run(project=project, args=["workspace", "list"]) if success: res.assert_success() else: diff --git a/tests/integration/sandbox-bwrap.py b/tests/integration/sandbox-bwrap.py index 1f410ea92..2908752de 100644 --- a/tests/integration/sandbox-bwrap.py +++ b/tests/integration/sandbox-bwrap.py @@ -13,60 +13,55 @@ from buildstream.testing._utils.site import HAVE_SANDBOX, HAVE_BWRAP_JSON_STATUS pytestmark = pytest.mark.integration -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project" -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") # Bubblewrap sandbox doesn't remove the dirs it created during its execution, # so BuildStream tries to remove them to do good. BuildStream should be extra # careful when those folders already exist and should not touch them, though. -@pytest.mark.skipif(HAVE_SANDBOX != 'bwrap', reason='Only available with bubblewrap') +@pytest.mark.skipif(HAVE_SANDBOX != "bwrap", reason="Only available with bubblewrap") @pytest.mark.datafiles(DATA_DIR) def test_sandbox_bwrap_cleanup_build(cli, datafiles): project = str(datafiles) # This element depends on a base image with non-empty `/tmp` folder. - element_name = 'sandbox-bwrap/test-cleanup.bst' + element_name = "sandbox-bwrap/test-cleanup.bst" # Here, BuildStream should not attempt any rmdir etc. - result = cli.run(project=project, args=['build', element_name]) + result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 -@pytest.mark.skipif(HAVE_SANDBOX != 'bwrap', reason='Only available with bubblewrap') -@pytest.mark.skipif(not HAVE_BWRAP_JSON_STATUS, reason='Only available with bubblewrap supporting --json-status-fd') +@pytest.mark.skipif(HAVE_SANDBOX != "bwrap", reason="Only available with bubblewrap") +@pytest.mark.skipif(not HAVE_BWRAP_JSON_STATUS, reason="Only available with bubblewrap supporting --json-status-fd") @pytest.mark.datafiles(DATA_DIR) def test_sandbox_bwrap_distinguish_setup_error(cli, datafiles): project = str(datafiles) - element_name = 'sandbox-bwrap/non-executable-shell.bst' + element_name = "sandbox-bwrap/non-executable-shell.bst" - result = cli.run(project=project, args=['build', element_name]) + result = cli.run(project=project, args=["build", element_name]) result.assert_task_error(error_domain=ErrorDomain.SANDBOX, error_reason="bwrap-sandbox-fail") -@pytest.mark.skipif(HAVE_SANDBOX != 'bwrap', reason='Only available with bubblewrap') +@pytest.mark.skipif(HAVE_SANDBOX != "bwrap", reason="Only available with bubblewrap") @pytest.mark.datafiles(DATA_DIR) def test_sandbox_bwrap_return_subprocess(cli, datafiles): project = str(datafiles) - element_name = 'sandbox-bwrap/command-exit-42.bst' + element_name = "sandbox-bwrap/command-exit-42.bst" - cli.configure({ - "logging": { - "message-format": "%{element}|%{message}", - }, - }) + cli.configure( + {"logging": {"message-format": "%{element}|%{message}",},} + ) - result = cli.run(project=project, args=['build', element_name]) + result = cli.run(project=project, args=["build", element_name]) result.assert_task_error(error_domain=ErrorDomain.SANDBOX, error_reason="command-failed") assert "sandbox-bwrap/command-exit-42.bst|Command failed with exitcode 42" in result.stderr -@pytest.mark.skipif(HAVE_SANDBOX != 'bwrap', reason='Only available with bubblewrap') +@pytest.mark.skipif(HAVE_SANDBOX != "bwrap", reason="Only available with bubblewrap") @pytest.mark.datafiles(DATA_DIR) def test_sandbox_bwrap_dev_shm(cli, datafiles): project = str(datafiles) - element_name = 'sandbox-bwrap/test-dev-shm.bst' + element_name = "sandbox-bwrap/test-dev-shm.bst" - result = cli.run(project=project, args=['build', element_name]) + result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 diff --git a/tests/integration/script.py b/tests/integration/script.py index 0e88ae53c..b27f1507e 100644 --- a/tests/integration/script.py +++ b/tests/integration/script.py @@ -12,10 +12,7 @@ from buildstream.testing._utils.site import HAVE_SANDBOX pytestmark = pytest.mark.integration -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project" -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") def create_script_element(name, path, config=None, variables=None): @@ -26,218 +23,211 @@ def create_script_element(name, path, config=None, variables=None): variables = {} element = { - 'kind': 'script', - 'depends': [{ - 'filename': 'base.bst', - 'type': 'build' - }], - 'config': config, - 'variables': variables + "kind": "script", + "depends": [{"filename": "base.bst", "type": "build"}], + "config": config, + "variables": variables, } os.makedirs(os.path.dirname(os.path.join(path, name)), exist_ok=True) _yaml.roundtrip_dump(element, os.path.join(path, name)) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_script(cli, datafiles): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') - element_path = os.path.join(project, 'elements') - element_name = 'script/script-layout.bst' + checkout = os.path.join(cli.directory, "checkout") + element_path = os.path.join(project, "elements") + element_name = "script/script-layout.bst" - create_script_element(element_name, element_path, - config={ - 'commands': [ - "mkdir -p %{install-root}", - "echo 'Hi' > %{install-root}/test" - ], - }) + create_script_element( + element_name, + element_path, + config={"commands": ["mkdir -p %{install-root}", "echo 'Hi' > %{install-root}/test"],}, + ) - res = cli.run(project=project, args=['build', element_name]) + res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 - res = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout]) + res = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert res.exit_code == 0 - with open(os.path.join(checkout, 'test')) as f: + with open(os.path.join(checkout, "test")) as f: text = f.read() assert text == "Hi\n" @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_script_root(cli, datafiles): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') - element_path = os.path.join(project, 'elements') - element_name = 'script/script-layout.bst' + checkout = os.path.join(cli.directory, "checkout") + element_path = os.path.join(project, "elements") + element_name = "script/script-layout.bst" - create_script_element(element_name, element_path, - config={ - # Root-read only is False by default, we - # want to check the default here - # 'root-read-only': False, - 'commands': [ - "mkdir -p %{install-root}", - "echo 'I can write to root' > /test", - "cp /test %{install-root}" - ], - }) + create_script_element( + element_name, + element_path, + config={ + # Root-read only is False by default, we + # want to check the default here + # 'root-read-only': False, + "commands": ["mkdir -p %{install-root}", "echo 'I can write to root' > /test", "cp /test %{install-root}"], + }, + ) - res = cli.run(project=project, args=['build', element_name]) + res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 - res = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout]) + res = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert res.exit_code == 0 - with open(os.path.join(checkout, 'test')) as f: + with open(os.path.join(checkout, "test")) as f: text = f.read() assert text == "I can write to root\n" @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') -@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") +@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox") def test_script_no_root(cli, datafiles): project = str(datafiles) - element_path = os.path.join(project, 'elements') - element_name = 'script/script-layout.bst' - - create_script_element(element_name, element_path, - config={ - 'root-read-only': True, - 'commands': [ - "mkdir -p %{install-root}", - "echo 'I can not write to root' > /test", - "cp /test %{install-root}" - ], - }) - - res = cli.run(project=project, args=['build', element_name]) + element_path = os.path.join(project, "elements") + element_name = "script/script-layout.bst" + + create_script_element( + element_name, + element_path, + config={ + "root-read-only": True, + "commands": [ + "mkdir -p %{install-root}", + "echo 'I can not write to root' > /test", + "cp /test %{install-root}", + ], + }, + ) + + res = cli.run(project=project, args=["build", element_name]) assert res.exit_code != 0 assert "/test: Read-only file system" in res.stderr @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') -@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") +@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox") def test_script_cwd(cli, datafiles): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') - element_path = os.path.join(project, 'elements') - element_name = 'script/script-layout.bst' + checkout = os.path.join(cli.directory, "checkout") + element_path = os.path.join(project, "elements") + element_name = "script/script-layout.bst" - create_script_element(element_name, element_path, - config={ - 'commands': [ - "echo 'test' > test", - "cp /buildstream/test %{install-root}" - ], - }, - variables={ - 'cwd': '/buildstream' - }) + create_script_element( + element_name, + element_path, + config={"commands": ["echo 'test' > test", "cp /buildstream/test %{install-root}"],}, + variables={"cwd": "/buildstream"}, + ) - res = cli.run(project=project, args=['build', element_name]) + res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 - res = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout]) + res = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert res.exit_code == 0 - with open(os.path.join(checkout, 'test')) as f: + with open(os.path.join(checkout, "test")) as f: text = f.read() assert text == "test\n" @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_script_layout(cli, datafiles): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') - element_name = 'script/script-layout.bst' + checkout = os.path.join(cli.directory, "checkout") + element_name = "script/script-layout.bst" - res = cli.run(project=project, args=['build', element_name]) + res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 - cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout]) + cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert res.exit_code == 0 - with open(os.path.join(checkout, 'test')) as f: + with open(os.path.join(checkout, "test")) as f: text = f.read() assert text == "Hi\n" @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_regression_cache_corruption(cli, datafiles): project = str(datafiles) - checkout_original = os.path.join(cli.directory, 'checkout-original') - checkout_after = os.path.join(cli.directory, 'checkout-after') - element_name = 'script/corruption.bst' - canary_element_name = 'script/corruption-image.bst' + checkout_original = os.path.join(cli.directory, "checkout-original") + checkout_after = os.path.join(cli.directory, "checkout-after") + element_name = "script/corruption.bst" + canary_element_name = "script/corruption-image.bst" - res = cli.run(project=project, args=['build', canary_element_name]) + res = cli.run(project=project, args=["build", canary_element_name]) assert res.exit_code == 0 - res = cli.run(project=project, args=['artifact', 'checkout', canary_element_name, - '--directory', checkout_original]) + res = cli.run( + project=project, args=["artifact", "checkout", canary_element_name, "--directory", checkout_original] + ) assert res.exit_code == 0 - with open(os.path.join(checkout_original, 'canary')) as f: - assert f.read() == 'alive\n' + with open(os.path.join(checkout_original, "canary")) as f: + assert f.read() == "alive\n" - res = cli.run(project=project, args=['build', element_name]) + res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 - res = cli.run(project=project, args=['artifact', 'checkout', canary_element_name, - '--directory', checkout_after]) + res = cli.run(project=project, args=["artifact", "checkout", canary_element_name, "--directory", checkout_after]) assert res.exit_code == 0 - with open(os.path.join(checkout_after, 'canary')) as f: - assert f.read() == 'alive\n' + with open(os.path.join(checkout_after, "canary")) as f: + assert f.read() == "alive\n" @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_regression_tmpdir(cli, datafiles): project = str(datafiles) - element_name = 'script/tmpdir.bst' + element_name = "script/tmpdir.bst" - res = cli.run(project=project, args=['build', element_name]) + res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_regression_cache_corruption_2(cli, datafiles): project = str(datafiles) - checkout_original = os.path.join(cli.directory, 'checkout-original') - checkout_after = os.path.join(cli.directory, 'checkout-after') - element_name = 'script/corruption-2.bst' - canary_element_name = 'script/corruption-image.bst' + checkout_original = os.path.join(cli.directory, "checkout-original") + checkout_after = os.path.join(cli.directory, "checkout-after") + element_name = "script/corruption-2.bst" + canary_element_name = "script/corruption-image.bst" - res = cli.run(project=project, args=['build', canary_element_name]) + res = cli.run(project=project, args=["build", canary_element_name]) assert res.exit_code == 0 - res = cli.run(project=project, args=['artifact', 'checkout', canary_element_name, - '--directory', checkout_original]) + res = cli.run( + project=project, args=["artifact", "checkout", canary_element_name, "--directory", checkout_original] + ) assert res.exit_code == 0 - with open(os.path.join(checkout_original, 'canary')) as f: - assert f.read() == 'alive\n' + with open(os.path.join(checkout_original, "canary")) as f: + assert f.read() == "alive\n" - res = cli.run(project=project, args=['build', element_name]) + res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 - res = cli.run(project=project, args=['artifact', 'checkout', canary_element_name, - '--directory', checkout_after]) + res = cli.run(project=project, args=["artifact", "checkout", canary_element_name, "--directory", checkout_after]) assert res.exit_code == 0 - with open(os.path.join(checkout_after, 'canary')) as f: - assert f.read() == 'alive\n' + with open(os.path.join(checkout_after, "canary")) as f: + assert f.read() == "alive\n" diff --git a/tests/integration/shell.py b/tests/integration/shell.py index 49e9a5950..e331a5fe0 100644 --- a/tests/integration/shell.py +++ b/tests/integration/shell.py @@ -16,10 +16,7 @@ from tests.testutils import create_artifact_share pytestmark = pytest.mark.integration -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project" -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") # execute_shell() @@ -35,28 +32,26 @@ DATA_DIR = os.path.join( # element (str): The element to build and run a shell with # isolate (bool): Whether to pass --isolate to `bst shell` # -def execute_shell(cli, project, command, *, config=None, mount=None, element='base.bst', isolate=False): +def execute_shell(cli, project, command, *, config=None, mount=None, element="base.bst", isolate=False): # Ensure the element is built - result = cli.run_project_config( - project=project, project_config=config, args=['build', element]) + result = cli.run_project_config(project=project, project_config=config, args=["build", element]) assert result.exit_code == 0 - args = ['shell'] + args = ["shell"] if isolate: - args += ['--isolate'] + args += ["--isolate"] if mount is not None: host_path, target_path = mount - args += ['--mount', host_path, target_path] - args += [element, '--', *command] + args += ["--mount", host_path, target_path] + args += [element, "--", *command] - return cli.run_project_config( - project=project, project_config=config, args=args) + return cli.run_project_config(project=project, project_config=config, args=args) # Test running something through a shell, allowing it to find the # executable @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_shell(cli, datafiles): project = str(datafiles) @@ -67,7 +62,7 @@ def test_shell(cli, datafiles): # Test running an executable directly @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_executable(cli, datafiles): project = str(datafiles) @@ -79,19 +74,15 @@ def test_executable(cli, datafiles): # Test shell environment variable explicit assignments @pytest.mark.parametrize("animal", [("Horse"), ("Pony")]) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") # This test seems to fail or pass depending on if this file is run or the hole test suite def test_env_assign(cli, datafiles, animal): project = str(datafiles) - expected = animal + '\n' + expected = animal + "\n" - result = execute_shell(cli, project, ['/bin/sh', '-c', 'echo ${ANIMAL}'], config={ - 'shell': { - 'environment': { - 'ANIMAL': animal - } - } - }) + result = execute_shell( + cli, project, ["/bin/sh", "-c", "echo ${ANIMAL}"], config={"shell": {"environment": {"ANIMAL": animal}}} + ) assert result.exit_code == 0 assert result.output == expected @@ -100,21 +91,20 @@ def test_env_assign(cli, datafiles, animal): # Test shell environment variable explicit assignments with host env var expansion @pytest.mark.parametrize("animal", [("Horse"), ("Pony")]) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") # This test seems to fail or pass depending on if this file is run or the hole test suite def test_env_assign_expand_host_environ(cli, datafiles, animal): project = str(datafiles) - expected = 'The animal is: {}\n'.format(animal) + expected = "The animal is: {}\n".format(animal) - os.environ['BEAST'] = animal + os.environ["BEAST"] = animal - result = execute_shell(cli, project, ['/bin/sh', '-c', 'echo ${ANIMAL}'], config={ - 'shell': { - 'environment': { - 'ANIMAL': 'The animal is: ${BEAST}' - } - } - }) + result = execute_shell( + cli, + project, + ["/bin/sh", "-c", "echo ${ANIMAL}"], + config={"shell": {"environment": {"ANIMAL": "The animal is: ${BEAST}"}}}, + ) assert result.exit_code == 0 assert result.output == expected @@ -124,51 +114,42 @@ def test_env_assign_expand_host_environ(cli, datafiles, animal): # when running an isolated shell @pytest.mark.parametrize("animal", [("Horse"), ("Pony")]) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") # This test seems to faili or pass depending on if this file is run or the hole test suite def test_env_assign_isolated(cli, datafiles, animal): project = str(datafiles) - result = execute_shell(cli, project, ['/bin/sh', '-c', 'echo ${ANIMAL}'], isolate=True, config={ - 'shell': { - 'environment': { - 'ANIMAL': animal - } - } - }) + result = execute_shell( + cli, + project, + ["/bin/sh", "-c", "echo ${ANIMAL}"], + isolate=True, + config={"shell": {"environment": {"ANIMAL": animal}}}, + ) assert result.exit_code == 0 - assert result.output == '\n' + assert result.output == "\n" # Test running an executable in a runtime with no shell (i.e., no # /bin/sh) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_no_shell(cli, datafiles): project = str(datafiles) - element_path = os.path.join(project, 'elements') - element_name = 'shell/no-shell.bst' + element_path = os.path.join(project, "elements") + element_name = "shell/no-shell.bst" # Create an element that removes /bin/sh from the base runtime element = { - 'kind': 'script', - 'depends': [{ - 'filename': 'base.bst', - 'type': 'build' - }], - 'variables': { - 'install-root': '/' - }, - 'config': { - 'commands': [ - 'rm /bin/sh' - ] - } + "kind": "script", + "depends": [{"filename": "base.bst", "type": "build"}], + "variables": {"install-root": "/"}, + "config": {"commands": ["rm /bin/sh"]}, } os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True) _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) - result = execute_shell(cli, project, ['/bin/echo', 'Pegasissies!'], element=element_name) + result = execute_shell(cli, project, ["/bin/echo", "Pegasissies!"], element=element_name) assert result.exit_code == 0 assert result.output == "Pegasissies!\n" @@ -176,99 +157,82 @@ def test_no_shell(cli, datafiles): # Test that bind mounts defined in project.conf work @pytest.mark.parametrize("path", [("/etc/pony.conf"), ("/usr/share/pony/pony.txt")]) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') -@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") +@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox") def test_host_files(cli, datafiles, path): project = str(datafiles) - ponyfile = os.path.join(project, 'files', 'shell-mount', 'pony.txt') - result = execute_shell(cli, project, ['cat', path], config={ - 'shell': { - 'host-files': [ - { - 'host_path': ponyfile, - 'path': path - } - ] - } - }) + ponyfile = os.path.join(project, "files", "shell-mount", "pony.txt") + result = execute_shell( + cli, project, ["cat", path], config={"shell": {"host-files": [{"host_path": ponyfile, "path": path}]}} + ) assert result.exit_code == 0 - assert result.output == 'pony\n' + assert result.output == "pony\n" # Test that bind mounts defined in project.conf work @pytest.mark.parametrize("path", [("/etc"), ("/usr/share/pony")]) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') -@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") +@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox") def test_host_files_expand_environ(cli, datafiles, path): project = str(datafiles) - hostpath = os.path.join(project, 'files', 'shell-mount') - fullpath = os.path.join(path, 'pony.txt') - - os.environ['BASE_PONY'] = path - os.environ['HOST_PONY_PATH'] = hostpath - - result = execute_shell(cli, project, ['cat', fullpath], config={ - 'shell': { - 'host-files': [ - { - 'host_path': '${HOST_PONY_PATH}/pony.txt', - 'path': '${BASE_PONY}/pony.txt' - } - ] - } - }) + hostpath = os.path.join(project, "files", "shell-mount") + fullpath = os.path.join(path, "pony.txt") + + os.environ["BASE_PONY"] = path + os.environ["HOST_PONY_PATH"] = hostpath + + result = execute_shell( + cli, + project, + ["cat", fullpath], + config={ + "shell": {"host-files": [{"host_path": "${HOST_PONY_PATH}/pony.txt", "path": "${BASE_PONY}/pony.txt"}]} + }, + ) assert result.exit_code == 0 - assert result.output == 'pony\n' + assert result.output == "pony\n" # Test that bind mounts defined in project.conf dont mount in isolation @pytest.mark.parametrize("path", [("/etc/pony.conf"), ("/usr/share/pony/pony.txt")]) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_isolated_no_mount(cli, datafiles, path): project = str(datafiles) - ponyfile = os.path.join(project, 'files', 'shell-mount', 'pony.txt') - result = execute_shell(cli, project, ['cat', path], isolate=True, config={ - 'shell': { - 'host-files': [ - { - 'host_path': ponyfile, - 'path': path - } - ] - } - }) + ponyfile = os.path.join(project, "files", "shell-mount", "pony.txt") + result = execute_shell( + cli, + project, + ["cat", path], + isolate=True, + config={"shell": {"host-files": [{"host_path": ponyfile, "path": path}]}}, + ) assert result.exit_code != 0 assert path in result.stderr - assert 'No such file or directory' in result.stderr + assert "No such file or directory" in result.stderr # Test that we warn about non-existing files on the host if the mount is not # declared as optional, and that there is no warning if it is optional @pytest.mark.parametrize("optional", [("mandatory"), ("optional")]) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_host_files_missing(cli, datafiles, optional): project = str(datafiles) - ponyfile = os.path.join(project, 'files', 'shell-mount', 'horsy.txt') + ponyfile = os.path.join(project, "files", "shell-mount", "horsy.txt") - option = (optional == "optional") + option = optional == "optional" # Assert that we did successfully run something in the shell anyway - result = execute_shell(cli, project, ['echo', 'Hello'], config={ - 'shell': { - 'host-files': [ - { - 'host_path': ponyfile, - 'path': '/etc/pony.conf', - 'optional': option - } - ] - } - }) + result = execute_shell( + cli, + project, + ["echo", "Hello"], + config={"shell": {"host-files": [{"host_path": ponyfile, "path": "/etc/pony.conf", "optional": option}]}}, + ) assert result.exit_code == 0 - assert result.output == 'Hello\n' + assert result.output == "Hello\n" if option: # Assert that there was no warning about the mount @@ -281,92 +245,100 @@ def test_host_files_missing(cli, datafiles, optional): # Test that bind mounts defined in project.conf work @pytest.mark.parametrize("path", [("/etc/pony.conf"), ("/usr/share/pony/pony.txt")]) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') -@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") +@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox") def test_cli_mount(cli, datafiles, path): project = str(datafiles) - ponyfile = os.path.join(project, 'files', 'shell-mount', 'pony.txt') + ponyfile = os.path.join(project, "files", "shell-mount", "pony.txt") - result = execute_shell(cli, project, ['cat', path], mount=(ponyfile, path)) + result = execute_shell(cli, project, ["cat", path], mount=(ponyfile, path)) assert result.exit_code == 0 - assert result.output == 'pony\n' + assert result.output == "pony\n" # Test that we can see the workspace files in a shell @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_workspace_visible(cli, datafiles): project = str(datafiles) - workspace = os.path.join(cli.directory, 'workspace') - element_name = 'workspace/workspace-mount-fail.bst' + workspace = os.path.join(cli.directory, "workspace") + element_name = "workspace/workspace-mount-fail.bst" # Open a workspace on our build failing element # - res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name]) + res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name]) assert res.exit_code == 0 # Ensure the dependencies of our build failing element are built - result = cli.run(project=project, args=['build', 'base.bst']) + result = cli.run(project=project, args=["build", "base.bst"]) assert result.exit_code == 0 # Obtain a copy of the hello.c content from the workspace # - workspace_hello_path = os.path.join(cli.directory, 'workspace', 'hello.c') + workspace_hello_path = os.path.join(cli.directory, "workspace", "hello.c") assert os.path.exists(workspace_hello_path) - with open(workspace_hello_path, 'r') as f: + with open(workspace_hello_path, "r") as f: workspace_hello = f.read() # Cat the hello.c file from a bst shell command, and assert # that we got the same content here # - result = cli.run(project=project, args=[ - 'source', 'fetch', element_name]) + result = cli.run(project=project, args=["source", "fetch", element_name]) assert result.exit_code == 0 - result = cli.run(project=project, args=[ - 'shell', '--build', element_name, '--', 'cat', 'hello.c' - ]) + result = cli.run(project=project, args=["shell", "--build", element_name, "--", "cat", "hello.c"]) assert result.exit_code == 0 assert result.output == workspace_hello # Test that '--sysroot' works @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') -@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") +@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox") def test_sysroot(cli, tmpdir, datafiles): project = str(datafiles) base_element = "base/base-alpine.bst" # test element only needs to be something lightweight for this test test_element = "script/script.bst" - checkout_dir = os.path.join(str(tmpdir), 'alpine-sysroot') - test_file = 'hello' + checkout_dir = os.path.join(str(tmpdir), "alpine-sysroot") + test_file = "hello" # Build and check out a sysroot - res = cli.run(project=project, args=['build', base_element]) + res = cli.run(project=project, args=["build", base_element]) res.assert_success() - res = cli.run(project=project, args=['artifact', 'checkout', base_element, '--directory', checkout_dir]) + res = cli.run(project=project, args=["artifact", "checkout", base_element, "--directory", checkout_dir]) res.assert_success() # Mutate the sysroot test_path = os.path.join(checkout_dir, test_file) - with open(test_path, 'w') as f: - f.write('hello\n') + with open(test_path, "w") as f: + f.write("hello\n") # Shell into the sysroot and check the test file exists - res = cli.run(project=project, args=[ - 'shell', '--build', '--sysroot', checkout_dir, test_element, '--', - 'grep', '-q', 'hello', '/' + test_file - ]) + res = cli.run( + project=project, + args=[ + "shell", + "--build", + "--sysroot", + checkout_dir, + test_element, + "--", + "grep", + "-q", + "hello", + "/" + test_file, + ], + ) res.assert_success() # Test system integration commands can access devices in /dev @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_integration_devices(cli, datafiles): project = str(datafiles) - element_name = 'integration.bst' + element_name = "integration.bst" result = execute_shell(cli, project, ["true"], element=element_name) assert result.exit_code == 0 @@ -376,79 +348,81 @@ def test_integration_devices(cli, datafiles): @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("build_shell", [("build"), ("nobuild")]) @pytest.mark.parametrize("guess_element", [True, False], ids=["guess", "no-guess"]) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_integration_external_workspace(cli, tmpdir_factory, datafiles, build_shell, guess_element): tmpdir = tmpdir_factory.mktemp("") project = str(datafiles) - element_name = 'autotools/amhello.bst' - workspace_dir = os.path.join(str(tmpdir), 'workspace') + element_name = "autotools/amhello.bst" + workspace_dir = os.path.join(str(tmpdir), "workspace") if guess_element: # Mutate the project.conf to use a default shell command - project_file = os.path.join(project, 'project.conf') - config_text = "shell:\n"\ - " command: ['true']\n" - with open(project_file, 'a') as f: + project_file = os.path.join(project, "project.conf") + config_text = "shell:\n" " command: ['true']\n" + with open(project_file, "a") as f: f.write(config_text) - result = cli.run(project=project, args=[ - 'workspace', 'open', '--directory', workspace_dir, element_name - ]) + result = cli.run(project=project, args=["workspace", "open", "--directory", workspace_dir, element_name]) result.assert_success() - result = cli.run(project=project, args=['-C', workspace_dir, 'build', element_name]) + result = cli.run(project=project, args=["-C", workspace_dir, "build", element_name]) result.assert_success() - command = ['-C', workspace_dir, 'shell'] - if build_shell == 'build': - command.append('--build') + command = ["-C", workspace_dir, "shell"] + if build_shell == "build": + command.append("--build") if not guess_element: - command.extend([element_name, '--', 'true']) + command.extend([element_name, "--", "true"]) result = cli.run(project=project, cwd=workspace_dir, args=command) result.assert_success() @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_integration_partial_artifact(cli, datafiles, tmpdir, integration_cache): project = str(datafiles) - element_name = 'autotools/amhello.bst' + element_name = "autotools/amhello.bst" # push to an artifact server so we can pull from it later. - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: - cli.configure({'artifacts': { - 'url': share.repo, - 'push': True - }}) - result = cli.run(project=project, args=['build', element_name]) + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: + cli.configure({"artifacts": {"url": share.repo, "push": True}}) + result = cli.run(project=project, args=["build", element_name]) result.assert_success() # If the build is cached then it might not push to the artifact cache - result = cli.run(project=project, args=['artifact', 'push', element_name]) + result = cli.run(project=project, args=["artifact", "push", element_name]) result.assert_success() - result = cli.run(project=project, args=['shell', element_name]) + result = cli.run(project=project, args=["shell", element_name]) result.assert_success() # do a checkout and get the digest of the hello binary. - result = cli.run(project=project, args=[ - 'artifact', 'checkout', '--deps', 'none', - '--directory', os.path.join(str(tmpdir), 'tmp'), - 'autotools/amhello.bst']) + result = cli.run( + project=project, + args=[ + "artifact", + "checkout", + "--deps", + "none", + "--directory", + os.path.join(str(tmpdir), "tmp"), + "autotools/amhello.bst", + ], + ) result.assert_success() - digest = utils.sha256sum(os.path.join(str(tmpdir), 'tmp', 'usr', 'bin', 'hello')) + digest = utils.sha256sum(os.path.join(str(tmpdir), "tmp", "usr", "bin", "hello")) # Remove the binary from the CAS - cachedir = cli.config['cachedir'] - objpath = os.path.join(cachedir, 'cas', 'objects', digest[:2], digest[2:]) + cachedir = cli.config["cachedir"] + objpath = os.path.join(cachedir, "cas", "objects", digest[:2], digest[2:]) os.unlink(objpath) # check shell doesn't work - result = cli.run(project=project, args=['shell', element_name, '--', 'hello']) + result = cli.run(project=project, args=["shell", element_name, "--", "hello"]) result.assert_main_error(ErrorDomain.APP, None) # check the artifact gets completed with '--pull' specified - result = cli.run(project=project, args=['shell', '--pull', element_name, '--', 'hello']) + result = cli.run(project=project, args=["shell", "--pull", element_name, "--", "hello"]) result.assert_success() - assert 'autotools/amhello.bst' in result.get_pulled_elements() + assert "autotools/amhello.bst" in result.get_pulled_elements() diff --git a/tests/integration/shellbuildtrees.py b/tests/integration/shellbuildtrees.py index 146bc6062..d3191870a 100644 --- a/tests/integration/shellbuildtrees.py +++ b/tests/integration/shellbuildtrees.py @@ -16,321 +16,336 @@ from tests.testutils import create_artifact_share pytestmark = pytest.mark.integration -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project" -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_buildtree_staged(cli_integration, datafiles): # We can only test the non interacitve case # The non interactive case defaults to not using buildtrees # for `bst shell --build` project = str(datafiles) - element_name = 'build-shell/buildtree.bst' + element_name = "build-shell/buildtree.bst" - res = cli_integration.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name]) + res = cli_integration.run(project=project, args=["--cache-buildtrees", "always", "build", element_name]) res.assert_success() - res = cli_integration.run(project=project, args=[ - 'shell', '--build', element_name, '--', 'cat', 'test' - ]) + res = cli_integration.run(project=project, args=["shell", "--build", element_name, "--", "cat", "test"]) res.assert_shell_error() @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_buildtree_staged_forced_true(cli_integration, datafiles): # Test that if we ask for a build tree it is there. project = str(datafiles) - element_name = 'build-shell/buildtree.bst' + element_name = "build-shell/buildtree.bst" - res = cli_integration.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name]) + res = cli_integration.run(project=project, args=["--cache-buildtrees", "always", "build", element_name]) res.assert_success() - res = cli_integration.run(project=project, args=[ - 'shell', '--build', '--use-buildtree', 'always', element_name, '--', 'cat', 'test' - ]) + res = cli_integration.run( + project=project, args=["shell", "--build", "--use-buildtree", "always", element_name, "--", "cat", "test"] + ) res.assert_success() - assert 'Hi' in res.output + assert "Hi" in res.output @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_buildtree_staged_warn_empty_cached(cli_integration, tmpdir, datafiles): # Test that if we stage a cached and empty buildtree, we warn the user. project = str(datafiles) - element_name = 'build-shell/buildtree.bst' + element_name = "build-shell/buildtree.bst" # Switch to a temp artifact cache dir to ensure the artifact is rebuilt, # without caching a buildtree which is the default bst behaviour - cli_integration.configure({ - 'cachedir': str(tmpdir) - }) + cli_integration.configure({"cachedir": str(tmpdir)}) - res = cli_integration.run(project=project, args=['build', element_name]) + res = cli_integration.run(project=project, args=["build", element_name]) res.assert_success() - res = cli_integration.run(project=project, args=[ - 'shell', '--build', '--use-buildtree', 'always', element_name, '--', 'cat', 'test' - ]) + res = cli_integration.run( + project=project, args=["shell", "--build", "--use-buildtree", "always", element_name, "--", "cat", "test"] + ) res.assert_main_error(ErrorDomain.APP, None) assert "Artifact was created without buildtree, unable to launch shell with it" in res.stderr # Now attempt the same with the try option, this should not attempt to find a buildtree # and just launch the shell, however the cat should still fail. - res = cli_integration.run(project=project, args=[ - 'shell', '--build', '--use-buildtree', 'try', element_name, '--', 'cat', 'test' - ]) + res = cli_integration.run( + project=project, args=["shell", "--build", "--use-buildtree", "try", element_name, "--", "cat", "test"] + ) assert "Artifact created without buildtree, shell will be loaded without it" in res.stderr - assert 'Hi' not in res.output + assert "Hi" not in res.output @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_buildtree_staged_if_available(cli_integration, datafiles): # Test that a build tree can be correctly detected. project = str(datafiles) - element_name = 'build-shell/buildtree.bst' + element_name = "build-shell/buildtree.bst" - res = cli_integration.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name]) + res = cli_integration.run(project=project, args=["--cache-buildtrees", "always", "build", element_name]) res.assert_success() - res = cli_integration.run(project=project, args=[ - 'shell', '--build', '--use-buildtree', 'try', element_name, '--', 'cat', 'test' - ]) + res = cli_integration.run( + project=project, args=["shell", "--build", "--use-buildtree", "try", element_name, "--", "cat", "test"] + ) res.assert_success() - assert 'Hi' in res.output + assert "Hi" in res.output @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_buildtree_staged_forced_false(cli_integration, datafiles): # Test that if we ask not to have a build tree it is not there project = str(datafiles) - element_name = 'build-shell/buildtree.bst' + element_name = "build-shell/buildtree.bst" - res = cli_integration.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name]) + res = cli_integration.run(project=project, args=["--cache-buildtrees", "always", "build", element_name]) res.assert_success() - res = cli_integration.run(project=project, args=[ - 'shell', '--build', '--use-buildtree', 'never', element_name, '--', 'cat', 'test' - ]) + res = cli_integration.run( + project=project, args=["shell", "--build", "--use-buildtree", "never", element_name, "--", "cat", "test"] + ) res.assert_shell_error() - assert 'Hi' not in res.output + assert "Hi" not in res.output @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_buildtree_from_failure(cli_integration, datafiles): # Test that we can use a build tree after a failure project = str(datafiles) - element_name = 'build-shell/buildtree-fail.bst' + element_name = "build-shell/buildtree-fail.bst" - res = cli_integration.run(project=project, args=['build', element_name]) + res = cli_integration.run(project=project, args=["build", element_name]) res.assert_main_error(ErrorDomain.STREAM, None) # Assert that file has expected contents - res = cli_integration.run(project=project, args=[ - 'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test' - ]) + res = cli_integration.run( + project=project, args=["shell", "--build", element_name, "--use-buildtree", "always", "--", "cat", "test"] + ) res.assert_success() assert "WARNING: using a buildtree from a failed build" in res.stderr - assert 'Hi' in res.output + assert "Hi" in res.output @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_buildtree_from_failure_option_never(cli_integration, tmpdir, datafiles): project = str(datafiles) - element_name = 'build-shell/buildtree-fail.bst' + element_name = "build-shell/buildtree-fail.bst" # Switch to a temp artifact cache dir to ensure the artifact is rebuilt, # without caching a buildtree explicitly - cli_integration.configure({ - 'cachedir': str(tmpdir) - }) + cli_integration.configure({"cachedir": str(tmpdir)}) - res = cli_integration.run(project=project, args=['--cache-buildtrees', 'never', 'build', element_name]) + res = cli_integration.run(project=project, args=["--cache-buildtrees", "never", "build", element_name]) res.assert_main_error(ErrorDomain.STREAM, None) - res = cli_integration.run(project=project, args=[ - 'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test' - ]) + res = cli_integration.run( + project=project, args=["shell", "--build", element_name, "--use-buildtree", "always", "--", "cat", "test"] + ) res.assert_main_error(ErrorDomain.APP, None) assert "Artifact was created without buildtree, unable to launch shell with it" in res.stderr @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_buildtree_from_failure_option_always(cli_integration, tmpdir, datafiles): project = str(datafiles) - element_name = 'build-shell/buildtree-fail.bst' + element_name = "build-shell/buildtree-fail.bst" # build with --cache-buildtrees set to 'always', behaviour should match # default behaviour (which is always) as the buildtree will explicitly have been # cached with content. - cli_integration.configure({ - 'cachedir': str(tmpdir) - }) + cli_integration.configure({"cachedir": str(tmpdir)}) - res = cli_integration.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name]) + res = cli_integration.run(project=project, args=["--cache-buildtrees", "always", "build", element_name]) res.assert_main_error(ErrorDomain.STREAM, None) - res = cli_integration.run(project=project, args=[ - 'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test' - ]) + res = cli_integration.run( + project=project, args=["shell", "--build", element_name, "--use-buildtree", "always", "--", "cat", "test"] + ) res.assert_success() assert "WARNING: using a buildtree from a failed build" in res.stderr - assert 'Hi' in res.output + assert "Hi" in res.output # Check that build shells work when pulled from a remote cache # This is to roughly simulate remote execution @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_buildtree_pulled(cli, tmpdir, datafiles): project = str(datafiles) - element_name = 'build-shell/buildtree.bst' + element_name = "build-shell/buildtree.bst" - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # Build the element to push it to cache - cli.configure({ - 'artifacts': {'url': share.repo, 'push': True} - }) - result = cli.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name]) + cli.configure({"artifacts": {"url": share.repo, "push": True}}) + result = cli.run(project=project, args=["--cache-buildtrees", "always", "build", element_name]) result.assert_success() - assert cli.get_element_state(project, element_name) == 'cached' + assert cli.get_element_state(project, element_name) == "cached" # Discard the cache - shutil.rmtree(str(os.path.join(str(tmpdir), 'cache', 'cas'))) - shutil.rmtree(str(os.path.join(str(tmpdir), 'cache', 'artifacts'))) - assert cli.get_element_state(project, element_name) != 'cached' + shutil.rmtree(str(os.path.join(str(tmpdir), "cache", "cas"))) + shutil.rmtree(str(os.path.join(str(tmpdir), "cache", "artifacts"))) + assert cli.get_element_state(project, element_name) != "cached" # Pull from cache, ensuring cli options is set to pull the buildtree - result = cli.run(project=project, - args=['--pull-buildtrees', 'artifact', 'pull', '--deps', 'all', element_name]) + result = cli.run( + project=project, args=["--pull-buildtrees", "artifact", "pull", "--deps", "all", element_name] + ) result.assert_success() # Check it's using the cached build tree - res = cli.run(project=project, args=[ - 'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test' - ]) + res = cli.run( + project=project, args=["shell", "--build", element_name, "--use-buildtree", "always", "--", "cat", "test"] + ) res.assert_success() # This test checks for correct behaviour if a buildtree is not present in the local cache. @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_buildtree_options(cli, tmpdir, datafiles): project = str(datafiles) - element_name = 'build-shell/buildtree.bst' + element_name = "build-shell/buildtree.bst" - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # Build the element to push it to cache - cli.configure({ - 'artifacts': {'url': share.repo, 'push': True} - }) - result = cli.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name]) + cli.configure({"artifacts": {"url": share.repo, "push": True}}) + result = cli.run(project=project, args=["--cache-buildtrees", "always", "build", element_name]) result.assert_success() - assert cli.get_element_state(project, element_name) == 'cached' - assert share.get_artifact(cli.get_artifact_name(project, 'test', element_name)) + assert cli.get_element_state(project, element_name) == "cached" + assert share.get_artifact(cli.get_artifact_name(project, "test", element_name)) # Discard the cache - shutil.rmtree(str(os.path.join(str(tmpdir), 'cache', 'cas'))) - shutil.rmtree(str(os.path.join(str(tmpdir), 'cache', 'artifacts'))) - assert cli.get_element_state(project, element_name) != 'cached' + shutil.rmtree(str(os.path.join(str(tmpdir), "cache", "cas"))) + shutil.rmtree(str(os.path.join(str(tmpdir), "cache", "artifacts"))) + assert cli.get_element_state(project, element_name) != "cached" # Pull from cache, but do not include buildtrees. - result = cli.run(project=project, args=['artifact', 'pull', '--deps', 'all', element_name]) + result = cli.run(project=project, args=["artifact", "pull", "--deps", "all", element_name]) result.assert_success() # Check it's not using the cached build tree - res = cli.run(project=project, args=[ - 'shell', '--build', element_name, '--use-buildtree', 'never', '--', 'cat', 'test' - ]) + res = cli.run( + project=project, args=["shell", "--build", element_name, "--use-buildtree", "never", "--", "cat", "test"] + ) res.assert_shell_error() - assert 'Hi' not in res.output + assert "Hi" not in res.output # Check it's not using the cached build tree, default is to ask, and fall back to not # for non interactive behavior - res = cli.run(project=project, args=[ - 'shell', '--build', element_name, '--', 'cat', 'test' - ]) + res = cli.run(project=project, args=["shell", "--build", element_name, "--", "cat", "test"]) res.assert_shell_error() - assert 'Hi' not in res.output + assert "Hi" not in res.output # Check correctly handling the lack of buildtree, with 'try' not attempting to # pull the buildtree as the user context is by default set to not pull them - res = cli.run(project=project, args=[ - 'shell', '--build', element_name, '--use-buildtree', 'try', '--', 'cat', 'test' - ]) - assert 'Hi' not in res.output - assert 'Attempting to fetch missing artifact buildtrees' not in res.stderr + res = cli.run( + project=project, args=["shell", "--build", element_name, "--use-buildtree", "try", "--", "cat", "test"] + ) + assert "Hi" not in res.output + assert "Attempting to fetch missing artifact buildtrees" not in res.stderr # Check correctly handling the lack of buildtree, with 'try' attempting and succeeding # to pull the buildtree as the user context allow the pulling of buildtrees and it is # available in the remote - res = cli.run(project=project, args=[ - '--pull-buildtrees', 'shell', '--build', element_name, '--use-buildtree', 'try', '--', 'cat', 'test' - ]) - assert 'Attempting to fetch missing artifact buildtree' in res.stderr - assert 'Hi' in res.output - shutil.rmtree(os.path.join(os.path.join(str(tmpdir), 'cache', 'cas'))) - shutil.rmtree(os.path.join(os.path.join(str(tmpdir), 'cache', 'artifacts'))) - assert cli.get_element_state(project, element_name) != 'cached' + res = cli.run( + project=project, + args=[ + "--pull-buildtrees", + "shell", + "--build", + element_name, + "--use-buildtree", + "try", + "--", + "cat", + "test", + ], + ) + assert "Attempting to fetch missing artifact buildtree" in res.stderr + assert "Hi" in res.output + shutil.rmtree(os.path.join(os.path.join(str(tmpdir), "cache", "cas"))) + shutil.rmtree(os.path.join(os.path.join(str(tmpdir), "cache", "artifacts"))) + assert cli.get_element_state(project, element_name) != "cached" # Check it's not loading the shell at all with always set for the buildtree, when the # user context does not allow for buildtree pulling - result = cli.run(project=project, args=['artifact', 'pull', '--deps', 'all', element_name]) + result = cli.run(project=project, args=["artifact", "pull", "--deps", "all", element_name]) result.assert_success() - res = cli.run(project=project, args=[ - 'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test' - ]) + res = cli.run( + project=project, args=["shell", "--build", element_name, "--use-buildtree", "always", "--", "cat", "test"] + ) res.assert_main_error(ErrorDomain.APP, None) - assert 'Buildtree is not cached locally or in available remotes' in res.stderr - assert 'Hi' not in res.output - assert 'Attempting to fetch missing artifact buildtree' not in res.stderr + assert "Buildtree is not cached locally or in available remotes" in res.stderr + assert "Hi" not in res.output + assert "Attempting to fetch missing artifact buildtree" not in res.stderr # Check that when user context is set to pull buildtrees and a remote has the buildtree, # 'always' will attempt and succeed at pulling the missing buildtree. - res = cli.run(project=project, args=[ - '--pull-buildtrees', 'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test' - ]) - assert 'Hi' in res.output + res = cli.run( + project=project, + args=[ + "--pull-buildtrees", + "shell", + "--build", + element_name, + "--use-buildtree", + "always", + "--", + "cat", + "test", + ], + ) + assert "Hi" in res.output assert "buildtree is not cached locally, will attempt to pull from available remotes" in res.stderr - assert 'Attempting to fetch missing artifact buildtree' in res.stderr + assert "Attempting to fetch missing artifact buildtree" in res.stderr # Tests running pull and pull-buildtree options at the same time. @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_pull_buildtree_pulled(cli, tmpdir, datafiles): project = str(datafiles) - element_name = 'build-shell/buildtree.bst' + element_name = "build-shell/buildtree.bst" - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # Build the element to push it to cache - cli.configure({ - 'artifacts': {'url': share.repo, 'push': True} - }) - result = cli.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name]) + cli.configure({"artifacts": {"url": share.repo, "push": True}}) + result = cli.run(project=project, args=["--cache-buildtrees", "always", "build", element_name]) result.assert_success() - assert cli.get_element_state(project, element_name) == 'cached' + assert cli.get_element_state(project, element_name) == "cached" # Discard the cache - shutil.rmtree(str(os.path.join(str(tmpdir), 'cache', 'cas'))) - shutil.rmtree(str(os.path.join(str(tmpdir), 'cache', 'artifacts'))) - assert cli.get_element_state(project, element_name) != 'cached' + shutil.rmtree(str(os.path.join(str(tmpdir), "cache", "cas"))) + shutil.rmtree(str(os.path.join(str(tmpdir), "cache", "artifacts"))) + assert cli.get_element_state(project, element_name) != "cached" # Check it's using the cached build tree - res = cli.run(project=project, args=[ - '--pull-buildtrees', 'shell', '--build', element_name, '--pull', - '--use-buildtree', 'always', '--', 'cat', 'test' - ]) + res = cli.run( + project=project, + args=[ + "--pull-buildtrees", + "shell", + "--build", + element_name, + "--pull", + "--use-buildtree", + "always", + "--", + "cat", + "test", + ], + ) res.assert_success() diff --git a/tests/integration/sockets.py b/tests/integration/sockets.py index 763238baf..3fb656e95 100644 --- a/tests/integration/sockets.py +++ b/tests/integration/sockets.py @@ -10,27 +10,24 @@ from buildstream.testing._utils.site import HAVE_SANDBOX pytestmark = pytest.mark.integration -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project" -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_builddir_socket_ignored(cli, datafiles): project = str(datafiles) - element_name = 'sockets/make-builddir-socket.bst' + element_name = "sockets/make-builddir-socket.bst" - result = cli.run(project=project, args=['build', element_name]) + result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_install_root_socket_ignored(cli, datafiles): project = str(datafiles) - element_name = 'sockets/make-install-root-socket.bst' + element_name = "sockets/make-install-root-socket.bst" - result = cli.run(project=project, args=['build', element_name]) + result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 diff --git a/tests/integration/source-determinism.py b/tests/integration/source-determinism.py index 70c4b79de..a349f9eeb 100644 --- a/tests/integration/source-determinism.py +++ b/tests/integration/source-determinism.py @@ -9,79 +9,62 @@ from buildstream.testing import cli_integration as cli # pylint: disable=unused from buildstream.testing._utils.site import HAVE_SANDBOX -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project" -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") -def create_test_file(*path, mode=0o644, content='content\n'): +def create_test_file(*path, mode=0o644, content="content\n"): path = os.path.join(*path) os.makedirs(os.path.dirname(path), exist_ok=True) - with open(path, 'w') as f: + with open(path, "w") as f: f.write(content) os.fchmod(f.fileno(), mode) def create_test_directory(*path, mode=0o644): - create_test_file(*path, '.keep', content='') + create_test_file(*path, ".keep", content="") path = os.path.join(*path) os.chmod(path, mode) @pytest.mark.integration @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_deterministic_source_local(cli, tmpdir, datafiles): """Only user rights should be considered for local source. """ project = str(datafiles) - element_name = 'test.bst' - element_path = os.path.join(project, 'elements', element_name) - sourcedir = os.path.join(project, 'source') + element_name = "test.bst" + element_path = os.path.join(project, "elements", element_name) + sourcedir = os.path.join(project, "source") element = { - 'kind': 'manual', - 'depends': [ - { - 'filename': 'base.bst', - 'type': 'build' - } - ], - 'sources': [ - { - 'kind': 'local', - 'path': 'source' - } - ], - 'config': { - 'install-commands': [ - 'ls -l >"%{install-root}/ls-l"' - ] - } + "kind": "manual", + "depends": [{"filename": "base.bst", "type": "build"}], + "sources": [{"kind": "local", "path": "source"}], + "config": {"install-commands": ['ls -l >"%{install-root}/ls-l"']}, } _yaml.roundtrip_dump(element, element_path) def get_value_for_mask(mask): - checkoutdir = os.path.join(str(tmpdir), 'checkout-{}'.format(mask)) - - create_test_file(sourcedir, 'a.txt', mode=0o644 & mask) - create_test_file(sourcedir, 'b.txt', mode=0o755 & mask) - create_test_file(sourcedir, 'c.txt', mode=0o4755 & mask) - create_test_file(sourcedir, 'd.txt', mode=0o2755 & mask) - create_test_file(sourcedir, 'e.txt', mode=0o1755 & mask) - create_test_directory(sourcedir, 'dir-a', mode=0o0755 & mask) - create_test_directory(sourcedir, 'dir-b', mode=0o4755 & mask) - create_test_directory(sourcedir, 'dir-c', mode=0o2755 & mask) - create_test_directory(sourcedir, 'dir-d', mode=0o1755 & mask) + checkoutdir = os.path.join(str(tmpdir), "checkout-{}".format(mask)) + + create_test_file(sourcedir, "a.txt", mode=0o644 & mask) + create_test_file(sourcedir, "b.txt", mode=0o755 & mask) + create_test_file(sourcedir, "c.txt", mode=0o4755 & mask) + create_test_file(sourcedir, "d.txt", mode=0o2755 & mask) + create_test_file(sourcedir, "e.txt", mode=0o1755 & mask) + create_test_directory(sourcedir, "dir-a", mode=0o0755 & mask) + create_test_directory(sourcedir, "dir-b", mode=0o4755 & mask) + create_test_directory(sourcedir, "dir-c", mode=0o2755 & mask) + create_test_directory(sourcedir, "dir-d", mode=0o1755 & mask) try: - result = cli.run(project=project, args=['build', element_name]) + result = cli.run(project=project, args=["build", element_name]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkoutdir]) result.assert_success() - with open(os.path.join(checkoutdir, 'ls-l'), 'r') as f: + with open(os.path.join(checkoutdir, "ls-l"), "r") as f: return f.read() finally: cli.remove_artifact_from_cache(project, element_name) diff --git a/tests/integration/stack.py b/tests/integration/stack.py index 9d6b38345..bad807fd6 100644 --- a/tests/integration/stack.py +++ b/tests/integration/stack.py @@ -11,29 +11,26 @@ from buildstream.testing._utils.site import HAVE_SANDBOX pytestmark = pytest.mark.integration -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project" -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_stack(cli, datafiles): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') - element_name = 'stack/stack.bst' + checkout = os.path.join(cli.directory, "checkout") + element_name = "stack/stack.bst" - res = cli.run(project=project, args=['build', element_name]) + res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 - cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout]) + cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert res.exit_code == 0 - with open(os.path.join(checkout, 'hi')) as f: + with open(os.path.join(checkout, "hi")) as f: hi = f.read() - with open(os.path.join(checkout, 'another-hi')) as f: + with open(os.path.join(checkout, "another-hi")) as f: another_hi = f.read() assert hi == "Hi\n" diff --git a/tests/integration/symlinks.py b/tests/integration/symlinks.py index 85bbc53fd..f1d044f07 100644 --- a/tests/integration/symlinks.py +++ b/tests/integration/symlinks.py @@ -11,80 +11,77 @@ from buildstream.testing._utils.site import HAVE_SANDBOX pytestmark = pytest.mark.integration -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project" -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_absolute_symlinks(cli, datafiles): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') - element_name = 'symlinks/dangling-symlink.bst' + checkout = os.path.join(cli.directory, "checkout") + element_name = "symlinks/dangling-symlink.bst" - result = cli.run(project=project, args=['build', element_name]) + result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 - result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout]) + result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert result.exit_code == 0 - symlink = os.path.join(checkout, 'opt', 'orgname') + symlink = os.path.join(checkout, "opt", "orgname") assert os.path.islink(symlink) # The symlink is created to point to /usr/orgs/orgname and BuildStream # should not mangle symlinks. - assert os.readlink(symlink) == '/usr/orgs/orgname' + assert os.readlink(symlink) == "/usr/orgs/orgname" @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_disallow_overlaps_inside_symlink_with_dangling_target(cli, datafiles): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') - element_name = 'symlinks/dangling-symlink-overlap.bst' + checkout = os.path.join(cli.directory, "checkout") + element_name = "symlinks/dangling-symlink-overlap.bst" - result = cli.run(project=project, args=['build', element_name]) + result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 - result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout]) + result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert result.exit_code == -1 - assert 'Destination is a symlink, not a directory: /opt/orgname' in result.stderr + assert "Destination is a symlink, not a directory: /opt/orgname" in result.stderr @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_detect_symlink_overlaps_pointing_outside_sandbox(cli, datafiles): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') - element_name = 'symlinks/symlink-to-outside-sandbox-overlap.bst' + checkout = os.path.join(cli.directory, "checkout") + element_name = "symlinks/symlink-to-outside-sandbox-overlap.bst" # Building the two elements should succeed... - result = cli.run(project=project, args=['build', element_name]) + result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 # ...but when we compose them together, the overlaps create paths that # point outside the sandbox which BuildStream needs to detect before it # tries to actually write there. - result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout]) + result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert result.exit_code == -1 - assert 'Destination is a symlink, not a directory: /opt/escape-hatch' in result.stderr + assert "Destination is a symlink, not a directory: /opt/escape-hatch" in result.stderr @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_symlink_in_sandbox_path(cli, datafiles): project = str(datafiles) - element_name = 'symlinks/link-on-path-use.bst' - base_element_name = 'symlinks/link-on-path.bst' + element_name = "symlinks/link-on-path-use.bst" + base_element_name = "symlinks/link-on-path.bst" # This test is inspired by how freedesktop-SDK has /bin -> /usr/bin # Create a element that has sh in altbin and a link from bin to altbin - result1 = cli.run(project=project, args=['build', base_element_name]) + result1 = cli.run(project=project, args=["build", base_element_name]) result1.assert_success() # Build a element that uses the element that has sh in altbin. - result2 = cli.run(project=project, args=['build', element_name]) + result2 = cli.run(project=project, args=["build", element_name]) result2.assert_success() # When this element is built it demonstrates that the virtual sandbox # can detect sh across links and that the sandbox can find sh accross diff --git a/tests/integration/workspace.py b/tests/integration/workspace.py index 0d8a88007..6c6503f96 100644 --- a/tests/integration/workspace.py +++ b/tests/integration/workspace.py @@ -13,113 +13,107 @@ from buildstream._exceptions import ErrorDomain pytestmark = pytest.mark.integration -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project" -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_workspace_stages_once(cli, datafiles): project = str(datafiles) - workspace = os.path.join(cli.directory, 'workspace') - element_name = 'workspace/workspace-mount.bst' + workspace = os.path.join(cli.directory, "workspace") + element_name = "workspace/workspace-mount.bst" - res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name]) + res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name]) assert res.exit_code == 0 - assert cli.get_element_key(project, element_name) != "{:?<64}".format('') - res = cli.run(project=project, args=['build', element_name]) + assert cli.get_element_key(project, element_name) != "{:?<64}".format("") + res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_workspace_mount(cli, datafiles): project = str(datafiles) - workspace = os.path.join(cli.directory, 'workspace') - element_name = 'workspace/workspace-mount.bst' + workspace = os.path.join(cli.directory, "workspace") + element_name = "workspace/workspace-mount.bst" - res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name]) + res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name]) assert res.exit_code == 0 - res = cli.run(project=project, args=['build', element_name]) + res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 - assert os.path.exists(os.path.join(cli.directory, 'workspace')) + assert os.path.exists(os.path.join(cli.directory, "workspace")) @pytest.mark.datafiles(DATA_DIR) def test_workspace_mount_on_read_only_directory(cli, datafiles): project = str(datafiles) - workspace = os.path.join(cli.directory, 'workspace') + workspace = os.path.join(cli.directory, "workspace") os.makedirs(workspace) - element_name = 'workspace/workspace-mount.bst' + element_name = "workspace/workspace-mount.bst" # make directory RO os.chmod(workspace, 0o555) - res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name]) + res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name]) assert res.exit_code == 0 @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') -@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") +@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox") @pytest.mark.xfail(reason="Incremental builds are currently incompatible with workspace source plugin.") def test_workspace_commanddir(cli, datafiles): project = str(datafiles) - workspace = os.path.join(cli.directory, 'workspace') - element_name = 'workspace/workspace-commanddir.bst' + workspace = os.path.join(cli.directory, "workspace") + element_name = "workspace/workspace-commanddir.bst" - res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name]) + res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name]) assert res.exit_code == 0 - res = cli.run(project=project, args=['build', element_name]) + res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 - assert os.path.exists(os.path.join(cli.directory, 'workspace')) - assert os.path.exists(os.path.join(cli.directory, 'workspace', 'build')) + assert os.path.exists(os.path.join(cli.directory, "workspace")) + assert os.path.exists(os.path.join(cli.directory, "workspace", "build")) @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_workspace_updated_dependency(cli, datafiles): project = str(datafiles) - workspace = os.path.join(cli.directory, 'workspace') - element_path = os.path.join(project, 'elements') - element_name = 'workspace/workspace-updated-dependency.bst' - dep_name = 'workspace/dependency.bst' + workspace = os.path.join(cli.directory, "workspace") + element_path = os.path.join(project, "elements") + element_name = "workspace/workspace-updated-dependency.bst" + dep_name = "workspace/dependency.bst" dependency = { - 'kind': 'manual', - 'depends': [{ - 'filename': 'base.bst', - 'type': 'build' - }], - 'config': { - 'build-commands': [ - 'mkdir -p %{install-root}/etc/test/', - 'echo "Hello world!" > %{install-root}/etc/test/hello.txt' + "kind": "manual", + "depends": [{"filename": "base.bst", "type": "build"}], + "config": { + "build-commands": [ + "mkdir -p %{install-root}/etc/test/", + 'echo "Hello world!" > %{install-root}/etc/test/hello.txt', ] - } + }, } os.makedirs(os.path.dirname(os.path.join(element_path, dep_name)), exist_ok=True) _yaml.roundtrip_dump(dependency, os.path.join(element_path, dep_name)) # First open the workspace - res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name]) + res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name]) assert res.exit_code == 0 # We build the workspaced element, so that we have an artifact # with specific built dependencies - res = cli.run(project=project, args=['build', element_name]) + res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 # Now we update a dependency of our element. - dependency['config']['build-commands'] = [ - 'mkdir -p %{install-root}/etc/test/', - 'echo "Hello china!" > %{install-root}/etc/test/hello.txt' + dependency["config"]["build-commands"] = [ + "mkdir -p %{install-root}/etc/test/", + 'echo "Hello china!" > %{install-root}/etc/test/hello.txt', ] _yaml.roundtrip_dump(dependency, os.path.join(element_path, dep_name)) @@ -128,185 +122,169 @@ def test_workspace_updated_dependency(cli, datafiles): # therefore ensure that we change the mtimes of any files touched # since the last successful build of this element, otherwise this # build will fail. - res = cli.run(project=project, args=['build', element_name]) + res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 - res = cli.run(project=project, args=['shell', element_name, '/usr/bin/test.sh']) + res = cli.run(project=project, args=["shell", element_name, "/usr/bin/test.sh"]) assert res.exit_code == 0 - assert res.output == 'Hello china!\n\n' + assert res.output == "Hello china!\n\n" @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_workspace_update_dependency_failed(cli, datafiles): project = str(datafiles) - workspace = os.path.join(cli.directory, 'workspace') - element_path = os.path.join(project, 'elements') - element_name = 'workspace/workspace-updated-dependency-failed.bst' - dep_name = 'workspace/dependency.bst' + workspace = os.path.join(cli.directory, "workspace") + element_path = os.path.join(project, "elements") + element_name = "workspace/workspace-updated-dependency-failed.bst" + dep_name = "workspace/dependency.bst" dependency = { - 'kind': 'manual', - 'depends': [{ - 'filename': 'base.bst', - 'type': 'build' - }], - 'config': { - 'build-commands': [ - 'mkdir -p %{install-root}/etc/test/', + "kind": "manual", + "depends": [{"filename": "base.bst", "type": "build"}], + "config": { + "build-commands": [ + "mkdir -p %{install-root}/etc/test/", 'echo "Hello world!" > %{install-root}/etc/test/hello.txt', - 'echo "Hello brazil!" > %{install-root}/etc/test/brazil.txt' + 'echo "Hello brazil!" > %{install-root}/etc/test/brazil.txt', ] - } + }, } os.makedirs(os.path.dirname(os.path.join(element_path, dep_name)), exist_ok=True) _yaml.roundtrip_dump(dependency, os.path.join(element_path, dep_name)) # First open the workspace - res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name]) + res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name]) assert res.exit_code == 0 # We build the workspaced element, so that we have an artifact # with specific built dependencies - res = cli.run(project=project, args=['build', element_name]) + res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 # Now we update a dependency of our element. - dependency['config']['build-commands'] = [ - 'mkdir -p %{install-root}/etc/test/', + dependency["config"]["build-commands"] = [ + "mkdir -p %{install-root}/etc/test/", 'echo "Hello china!" > %{install-root}/etc/test/hello.txt', - 'echo "Hello brazil!" > %{install-root}/etc/test/brazil.txt' + 'echo "Hello brazil!" > %{install-root}/etc/test/brazil.txt', ] _yaml.roundtrip_dump(dependency, os.path.join(element_path, dep_name)) # And our build fails! - with open(os.path.join(workspace, 'Makefile'), 'a') as f: + with open(os.path.join(workspace, "Makefile"), "a") as f: f.write("\texit 1") - res = cli.run(project=project, args=['build', element_name]) + res = cli.run(project=project, args=["build", element_name]) assert res.exit_code != 0 # We update our dependency again... - dependency['config']['build-commands'] = [ - 'mkdir -p %{install-root}/etc/test/', + dependency["config"]["build-commands"] = [ + "mkdir -p %{install-root}/etc/test/", 'echo "Hello world!" > %{install-root}/etc/test/hello.txt', - 'echo "Hello spain!" > %{install-root}/etc/test/brazil.txt' + 'echo "Hello spain!" > %{install-root}/etc/test/brazil.txt', ] _yaml.roundtrip_dump(dependency, os.path.join(element_path, dep_name)) # And fix the source - with open(os.path.join(workspace, 'Makefile'), 'r') as f: + with open(os.path.join(workspace, "Makefile"), "r") as f: makefile = f.readlines() - with open(os.path.join(workspace, 'Makefile'), 'w') as f: + with open(os.path.join(workspace, "Makefile"), "w") as f: f.write("\n".join(makefile[:-1])) # Since buildstream thinks hello.txt did not change, we could end # up not rebuilding a file! We need to make sure that a case like # this can't blind-side us. - res = cli.run(project=project, args=['build', element_name]) + res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 - res = cli.run(project=project, args=['shell', element_name, '/usr/bin/test.sh']) + res = cli.run(project=project, args=["shell", element_name, "/usr/bin/test.sh"]) assert res.exit_code == 0 - assert res.output == 'Hello world!\nHello spain!\n\n' + assert res.output == "Hello world!\nHello spain!\n\n" @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_updated_dependency_nested(cli, datafiles): project = str(datafiles) - workspace = os.path.join(cli.directory, 'workspace') - element_path = os.path.join(project, 'elements') - element_name = 'workspace/workspace-updated-dependency-nested.bst' - dep_name = 'workspace/dependency.bst' + workspace = os.path.join(cli.directory, "workspace") + element_path = os.path.join(project, "elements") + element_name = "workspace/workspace-updated-dependency-nested.bst" + dep_name = "workspace/dependency.bst" dependency = { - 'kind': 'manual', - 'depends': [{ - 'filename': 'base.bst', - 'type': 'build' - }], - 'config': { - 'build-commands': [ - 'mkdir -p %{install-root}/etc/test/tests/', + "kind": "manual", + "depends": [{"filename": "base.bst", "type": "build"}], + "config": { + "build-commands": [ + "mkdir -p %{install-root}/etc/test/tests/", 'echo "Hello world!" > %{install-root}/etc/test/hello.txt', - 'echo "Hello brazil!" > %{install-root}/etc/test/tests/brazil.txt' + 'echo "Hello brazil!" > %{install-root}/etc/test/tests/brazil.txt', ] - } + }, } os.makedirs(os.path.dirname(os.path.join(element_path, dep_name)), exist_ok=True) _yaml.roundtrip_dump(dependency, os.path.join(element_path, dep_name)) # First open the workspace - res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name]) + res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name]) assert res.exit_code == 0 # We build the workspaced element, so that we have an artifact # with specific built dependencies - res = cli.run(project=project, args=['build', element_name]) + res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 # Now we update a dependency of our element. - dependency['config']['build-commands'] = [ - 'mkdir -p %{install-root}/etc/test/tests/', + dependency["config"]["build-commands"] = [ + "mkdir -p %{install-root}/etc/test/tests/", 'echo "Hello world!" > %{install-root}/etc/test/hello.txt', - 'echo "Hello test!" > %{install-root}/etc/test/tests/tests.txt' + 'echo "Hello test!" > %{install-root}/etc/test/tests/tests.txt', ] _yaml.roundtrip_dump(dependency, os.path.join(element_path, dep_name)) - res = cli.run(project=project, args=['build', element_name]) + res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 # Buildstream should pick up the newly added element, and pick up # the lack of the newly removed element - res = cli.run(project=project, args=['shell', element_name, '/usr/bin/test.sh']) + res = cli.run(project=project, args=["shell", element_name, "/usr/bin/test.sh"]) assert res.exit_code == 0 - assert res.output == 'Hello world!\nHello test!\n\n' + assert res.output == "Hello world!\nHello test!\n\n" @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') -@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") +@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox") @pytest.mark.xfail(reason="Incremental builds are currently incompatible with workspace source plugin.") def test_incremental_configure_commands_run_only_once(cli, datafiles): project = str(datafiles) - workspace = os.path.join(cli.directory, 'workspace') - element_path = os.path.join(project, 'elements') - element_name = 'workspace/incremental.bst' + workspace = os.path.join(cli.directory, "workspace") + element_path = os.path.join(project, "elements") + element_name = "workspace/incremental.bst" element = { - 'kind': 'manual', - 'depends': [{ - 'filename': 'base.bst', - 'type': 'build' - }], - 'sources': [{ - 'kind': 'local', - 'path': 'files/workspace-configure-only-once' - }], - 'config': { - 'configure-commands': [ - '$SHELL configure' - ] - } + "kind": "manual", + "depends": [{"filename": "base.bst", "type": "build"}], + "sources": [{"kind": "local", "path": "files/workspace-configure-only-once"}], + "config": {"configure-commands": ["$SHELL configure"]}, } _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) # We open a workspace on the above element - res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name]) + res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name]) res.assert_success() # Then we build, and check whether the configure step succeeded - res = cli.run(project=project, args=['build', element_name]) + res = cli.run(project=project, args=["build", element_name]) res.assert_success() - assert os.path.exists(os.path.join(workspace, 'prepared')) + assert os.path.exists(os.path.join(workspace, "prepared")) # When we build again, the configure commands should not be # called, and we should therefore exit cleanly (the configure # commands are set to always fail after the first run) - res = cli.run(project=project, args=['build', element_name]) + res = cli.run(project=project, args=["build", element_name]) res.assert_success() - assert not os.path.exists(os.path.join(workspace, 'prepared-again')) + assert not os.path.exists(os.path.join(workspace, "prepared-again")) # Test that rebuilding an already built workspaced element does @@ -319,48 +297,48 @@ def test_incremental_configure_commands_run_only_once(cli, datafiles): # part of a cleanup job. # @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') -@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") +@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox") def test_workspace_missing_last_successful(cli, datafiles): project = str(datafiles) - workspace = os.path.join(cli.directory, 'workspace') - element_name = 'workspace/workspace-commanddir.bst' + workspace = os.path.join(cli.directory, "workspace") + element_name = "workspace/workspace-commanddir.bst" # Open workspace - res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name]) + res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name]) assert res.exit_code == 0 # Build first, this will record the last successful build in local state - res = cli.run(project=project, args=['build', element_name]) + res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 # Remove the artifact from the cache, invalidating the last successful build - res = cli.run(project=project, args=['artifact', 'delete', element_name]) + res = cli.run(project=project, args=["artifact", "delete", element_name]) assert res.exit_code == 0 # Build again, ensure we dont crash just because the artifact went missing - res = cli.run(project=project, args=['build', element_name]) + res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 # Check that we can still read failed workspace logs @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_workspace_failed_logs(cli, datafiles): project = str(datafiles) - workspace = os.path.join(cli.directory, 'failing_amhello') - element_name = 'autotools/amhello-failure.bst' + workspace = os.path.join(cli.directory, "failing_amhello") + element_name = "autotools/amhello-failure.bst" # Open workspace - res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name]) + res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name]) res.assert_success() # Try to build and ensure the build fails - res = cli.run(project=project, args=['build', element_name]) + res = cli.run(project=project, args=["build", element_name]) res.assert_main_error(ErrorDomain.STREAM, None) - assert cli.get_element_state(project, element_name) == 'failed' + assert cli.get_element_state(project, element_name) == "failed" - res = cli.run(project=project, args=['artifact', 'log', element_name]) + res = cli.run(project=project, args=["artifact", "log", element_name]) res.assert_success() log = res.output diff --git a/tests/internals/context.py b/tests/internals/context.py index ddd558b6c..c2ee1efb5 100644 --- a/tests/internals/context.py +++ b/tests/internals/context.py @@ -7,32 +7,26 @@ import pytest from buildstream._context import Context from buildstream._exceptions import LoadError, LoadErrorReason -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - 'context', -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "context",) # Simple fixture to create a Context object. @pytest.fixture() def context_fixture(): - if os.environ.get('XDG_CACHE_HOME'): - cache_home = os.environ['XDG_CACHE_HOME'] + if os.environ.get("XDG_CACHE_HOME"): + cache_home = os.environ["XDG_CACHE_HOME"] else: - cache_home = os.path.expanduser('~/.cache') + cache_home = os.path.expanduser("~/.cache") with Context() as context: - yield { - 'xdg-cache': cache_home, - 'context': context - } + yield {"xdg-cache": cache_home, "context": context} ####################################### # Test instantiation # ####################################### def test_context_create(context_fixture): - context = context_fixture['context'] + context = context_fixture["context"] assert isinstance(context, Context) @@ -40,51 +34,49 @@ def test_context_create(context_fixture): # Test configuration loading # ####################################### def test_context_load(context_fixture): - context = context_fixture['context'] - cache_home = os.path.normpath(context_fixture['xdg-cache']) + context = context_fixture["context"] + cache_home = os.path.normpath(context_fixture["xdg-cache"]) assert isinstance(context, Context) context.load(config=os.devnull) - assert context.sourcedir == os.path.join(cache_home, 'buildstream', 'sources') - assert context.builddir == os.path.join(cache_home, 'buildstream', 'build') - assert context.cachedir == os.path.join(cache_home, 'buildstream') - assert context.logdir == os.path.join(cache_home, 'buildstream', 'logs') + assert context.sourcedir == os.path.join(cache_home, "buildstream", "sources") + assert context.builddir == os.path.join(cache_home, "buildstream", "build") + assert context.cachedir == os.path.join(cache_home, "buildstream") + assert context.logdir == os.path.join(cache_home, "buildstream", "logs") # Assert that a changed XDG_CACHE_HOME doesn't cause issues def test_context_load_envvar(context_fixture): - os.environ['XDG_CACHE_HOME'] = '/some/path/' + os.environ["XDG_CACHE_HOME"] = "/some/path/" - context = context_fixture['context'] + context = context_fixture["context"] assert isinstance(context, Context) context.load(config=os.devnull) - assert context.sourcedir == os.path.join('/', 'some', 'path', 'buildstream', 'sources') - assert context.builddir == os.path.join('/', 'some', 'path', 'buildstream', 'build') - assert context.cachedir == os.path.join('/', 'some', 'path', 'buildstream') - assert context.logdir == os.path.join('/', 'some', 'path', 'buildstream', 'logs') + assert context.sourcedir == os.path.join("/", "some", "path", "buildstream", "sources") + assert context.builddir == os.path.join("/", "some", "path", "buildstream", "build") + assert context.cachedir == os.path.join("/", "some", "path", "buildstream") + assert context.logdir == os.path.join("/", "some", "path", "buildstream", "logs") # Reset the environment variable - del os.environ['XDG_CACHE_HOME'] + del os.environ["XDG_CACHE_HOME"] # Test that values in a user specified config file # override the defaults @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_context_load_user_config(context_fixture, datafiles): - context = context_fixture['context'] - cache_home = context_fixture['xdg-cache'] + context = context_fixture["context"] + cache_home = context_fixture["xdg-cache"] assert isinstance(context, Context) - conf_file = os.path.join(datafiles.dirname, - datafiles.basename, - 'userconf.yaml') + conf_file = os.path.join(datafiles.dirname, datafiles.basename, "userconf.yaml") context.load(conf_file) - assert context.sourcedir == os.path.expanduser('~/pony') - assert context.builddir == os.path.join(cache_home, 'buildstream', 'build') - assert context.cachedir == os.path.join(cache_home, 'buildstream') - assert context.logdir == os.path.join(cache_home, 'buildstream', 'logs') + assert context.sourcedir == os.path.expanduser("~/pony") + assert context.builddir == os.path.join(cache_home, "buildstream", "build") + assert context.cachedir == os.path.join(cache_home, "buildstream") + assert context.logdir == os.path.join(cache_home, "buildstream", "logs") ####################################### @@ -92,12 +84,10 @@ def test_context_load_user_config(context_fixture, datafiles): ####################################### @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_context_load_missing_config(context_fixture, datafiles): - context = context_fixture['context'] + context = context_fixture["context"] assert isinstance(context, Context) - conf_file = os.path.join(datafiles.dirname, - datafiles.basename, - 'nonexistant.yaml') + conf_file = os.path.join(datafiles.dirname, datafiles.basename, "nonexistant.yaml") with pytest.raises(LoadError) as exc: context.load(conf_file) @@ -107,12 +97,10 @@ def test_context_load_missing_config(context_fixture, datafiles): @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_context_load_malformed_config(context_fixture, datafiles): - context = context_fixture['context'] + context = context_fixture["context"] assert isinstance(context, Context) - conf_file = os.path.join(datafiles.dirname, - datafiles.basename, - 'malformed.yaml') + conf_file = os.path.join(datafiles.dirname, datafiles.basename, "malformed.yaml") with pytest.raises(LoadError) as exc: context.load(conf_file) @@ -122,12 +110,10 @@ def test_context_load_malformed_config(context_fixture, datafiles): @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_context_load_notdict_config(context_fixture, datafiles): - context = context_fixture['context'] + context = context_fixture["context"] assert isinstance(context, Context) - conf_file = os.path.join(datafiles.dirname, - datafiles.basename, - 'notdict.yaml') + conf_file = os.path.join(datafiles.dirname, datafiles.basename, "notdict.yaml") with pytest.raises(LoadError) as exc: context.load(conf_file) diff --git a/tests/internals/loader.py b/tests/internals/loader.py index 39ef8ac99..781d144ae 100644 --- a/tests/internals/loader.py +++ b/tests/internals/loader.py @@ -10,10 +10,7 @@ from buildstream._loader.loader import _NO_PROGRESS from tests.testutils import dummy_context -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - 'loader', -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "loader",) @contextmanager @@ -26,52 +23,52 @@ def make_loader(basedir): ############################################################## # Basics: Test behavior loading the simplest of projects # ############################################################## -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'onefile')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "onefile")) def test_one_file(datafiles): basedir = str(datafiles) with make_loader(basedir) as loader: - element = loader.load(['elements/onefile.bst'], _NO_PROGRESS)[0] + element = loader.load(["elements/onefile.bst"], _NO_PROGRESS)[0] assert isinstance(element, MetaElement) - assert element.kind == 'pony' + assert element.kind == "pony" -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'onefile')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "onefile")) def test_missing_file(datafiles): basedir = str(datafiles) with make_loader(basedir) as loader, pytest.raises(LoadError) as exc: - loader.load(['elements/missing.bst'], _NO_PROGRESS) + loader.load(["elements/missing.bst"], _NO_PROGRESS) assert exc.value.reason == LoadErrorReason.MISSING_FILE -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'onefile')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "onefile")) def test_invalid_reference(datafiles): basedir = str(datafiles) with make_loader(basedir) as loader, pytest.raises(LoadError) as exc: - loader.load(['elements/badreference.bst'], _NO_PROGRESS) + loader.load(["elements/badreference.bst"], _NO_PROGRESS) assert exc.value.reason == LoadErrorReason.INVALID_YAML -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'onefile')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "onefile")) def test_invalid_yaml(datafiles): basedir = str(datafiles) with make_loader(basedir) as loader, pytest.raises(LoadError) as exc: - loader.load(['elements/badfile.bst'], _NO_PROGRESS) + loader.load(["elements/badfile.bst"], _NO_PROGRESS) assert exc.value.reason == LoadErrorReason.INVALID_YAML -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'onefile')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "onefile")) def test_fail_fullpath_target(datafiles): basedir = str(datafiles) - fullpath = os.path.join(basedir, 'elements', 'onefile.bst') + fullpath = os.path.join(basedir, "elements", "onefile.bst") with make_loader(basedir) as loader, pytest.raises(LoadError) as exc: loader.load([fullpath], _NO_PROGRESS) @@ -79,21 +76,21 @@ def test_fail_fullpath_target(datafiles): assert exc.value.reason == LoadErrorReason.INVALID_DATA -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'onefile')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "onefile")) def test_invalid_key(datafiles): basedir = str(datafiles) with make_loader(basedir) as loader, pytest.raises(LoadError) as exc: - loader.load(['elements/invalidkey.bst'], _NO_PROGRESS) + loader.load(["elements/invalidkey.bst"], _NO_PROGRESS) assert exc.value.reason == LoadErrorReason.INVALID_DATA -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'onefile')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "onefile")) def test_invalid_directory_load(datafiles): basedir = str(datafiles) with make_loader(basedir) as loader, pytest.raises(LoadError) as exc: - loader.load(['elements/'], _NO_PROGRESS) + loader.load(["elements/"], _NO_PROGRESS) assert exc.value.reason == LoadErrorReason.LOADING_DIRECTORY diff --git a/tests/internals/pluginfactory.py b/tests/internals/pluginfactory.py index b3f77c8b1..13b9d3aae 100644 --- a/tests/internals/pluginfactory.py +++ b/tests/internals/pluginfactory.py @@ -10,236 +10,219 @@ from buildstream._elementfactory import ElementFactory from buildstream._sourcefactory import SourceFactory from buildstream._exceptions import PluginError -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - 'pluginfactory', -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "pluginfactory",) # Simple fixture to create a PluginBase object that # we use for loading plugins. @pytest.fixture() def plugin_fixture(): - return { - 'base': PluginBase(package='buildstream.plugins') - } + return {"base": PluginBase(package="buildstream.plugins")} ############################################################## # Basics: test the fixture, test we can create the factories # ############################################################## def test_fixture(plugin_fixture): - assert isinstance(plugin_fixture['base'], PluginBase) + assert isinstance(plugin_fixture["base"], PluginBase) def test_source_factory(plugin_fixture): - factory = SourceFactory(plugin_fixture['base']) + factory = SourceFactory(plugin_fixture["base"]) assert isinstance(factory, SourceFactory) def test_element_factory(plugin_fixture): - factory = ElementFactory(plugin_fixture['base']) + factory = ElementFactory(plugin_fixture["base"]) assert isinstance(factory, ElementFactory) ############################################################## # Check that we can load custom sources & elements # ############################################################## -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'customsource')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "customsource")) def test_custom_source(plugin_fixture, datafiles): - plugins = [Node.from_dict({ - 'origin': 'local', - 'path': os.path.join(datafiles.dirname, - datafiles.basename), - 'plugins': ['foo'] - })] - factory = SourceFactory(plugin_fixture['base'], plugin_origins=plugins) + plugins = [ + Node.from_dict( + {"origin": "local", "path": os.path.join(datafiles.dirname, datafiles.basename), "plugins": ["foo"]} + ) + ] + factory = SourceFactory(plugin_fixture["base"], plugin_origins=plugins) assert isinstance(factory, SourceFactory) - foo_type, _ = factory.lookup('foo') - assert foo_type.__name__ == 'FooSource' + foo_type, _ = factory.lookup("foo") + assert foo_type.__name__ == "FooSource" -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'customelement')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "customelement")) def test_custom_element(plugin_fixture, datafiles): - plugins = [Node.from_dict({ - 'origin': 'local', - 'path': os.path.join(datafiles.dirname, - datafiles.basename), - 'plugins': ['foo'] - })] - factory = ElementFactory(plugin_fixture['base'], plugin_origins=plugins) + plugins = [ + Node.from_dict( + {"origin": "local", "path": os.path.join(datafiles.dirname, datafiles.basename), "plugins": ["foo"]} + ) + ] + factory = ElementFactory(plugin_fixture["base"], plugin_origins=plugins) assert isinstance(factory, ElementFactory) - foo_type, _ = factory.lookup('foo') - assert foo_type.__name__ == 'FooElement' + foo_type, _ = factory.lookup("foo") + assert foo_type.__name__ == "FooElement" ############################################################## # Check plugin loading failure modes # ############################################################## def test_missing_source(plugin_fixture): - factory = SourceFactory(plugin_fixture['base']) + factory = SourceFactory(plugin_fixture["base"]) assert isinstance(factory, SourceFactory) # Test fails if PluginError is not raised with pytest.raises(PluginError): - factory.lookup('foo') + factory.lookup("foo") def test_missing_element(plugin_fixture): - factory = ElementFactory(plugin_fixture['base']) + factory = ElementFactory(plugin_fixture["base"]) assert isinstance(factory, ElementFactory) # Test fails if PluginError is not raised with pytest.raises(PluginError): - factory.lookup('foo') + factory.lookup("foo") # Load a factory with a plugin that returns a value instead of Source subclass -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'notatype')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "notatype")) def test_source_notatype(plugin_fixture, datafiles): - plugins = [Node.from_dict({ - 'origin': 'local', - 'path': os.path.join(datafiles.dirname, - datafiles.basename), - 'plugins': ['foo'] - })] - factory = SourceFactory(plugin_fixture['base'], plugin_origins=plugins) + plugins = [ + Node.from_dict( + {"origin": "local", "path": os.path.join(datafiles.dirname, datafiles.basename), "plugins": ["foo"]} + ) + ] + factory = SourceFactory(plugin_fixture["base"], plugin_origins=plugins) with pytest.raises(PluginError): - factory.lookup('foo') + factory.lookup("foo") # Load a factory with a plugin that returns a value instead of Element subclass -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'notatype')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "notatype")) def test_element_notatype(plugin_fixture, datafiles): - plugins = [Node.from_dict({ - 'origin': 'local', - 'path': os.path.join(datafiles.dirname, - datafiles.basename), - 'plugins': ['foo'] - })] - factory = ElementFactory(plugin_fixture['base'], plugin_origins=plugins) + plugins = [ + Node.from_dict( + {"origin": "local", "path": os.path.join(datafiles.dirname, datafiles.basename), "plugins": ["foo"]} + ) + ] + factory = ElementFactory(plugin_fixture["base"], plugin_origins=plugins) with pytest.raises(PluginError): - factory.lookup('foo') + factory.lookup("foo") # Load a factory with a plugin that returns a type # which is not a Source subclass -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'wrongtype')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "wrongtype")) def test_source_wrongtype(plugin_fixture, datafiles): - plugins = [Node.from_dict({ - 'origin': 'local', - 'path': os.path.join(datafiles.dirname, - datafiles.basename), - 'plugins': ['foo'] - })] - factory = SourceFactory(plugin_fixture['base'], plugin_origins=plugins) + plugins = [ + Node.from_dict( + {"origin": "local", "path": os.path.join(datafiles.dirname, datafiles.basename), "plugins": ["foo"]} + ) + ] + factory = SourceFactory(plugin_fixture["base"], plugin_origins=plugins) with pytest.raises(PluginError): - factory.lookup('foo') + factory.lookup("foo") # Load a factory with a plugin that returns a type # which is not a Element subclass -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'wrongtype')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "wrongtype")) def test_element_wrongtype(plugin_fixture, datafiles): - plugins = [Node.from_dict({ - 'origin': 'local', - 'path': os.path.join(datafiles.dirname, - datafiles.basename), - 'plugins': ['foo'] - })] - factory = ElementFactory(plugin_fixture['base'], plugin_origins=plugins) + plugins = [ + Node.from_dict( + {"origin": "local", "path": os.path.join(datafiles.dirname, datafiles.basename), "plugins": ["foo"]} + ) + ] + factory = ElementFactory(plugin_fixture["base"], plugin_origins=plugins) with pytest.raises(PluginError): - factory.lookup('foo') + factory.lookup("foo") # Load a factory with a plugin which fails to provide a setup() function -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'nosetup')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "nosetup")) def test_source_missing_setup(plugin_fixture, datafiles): - plugins = [Node.from_dict({ - 'origin': 'local', - 'path': os.path.join(datafiles.dirname, - datafiles.basename), - 'plugins': ['foo'] - })] - factory = SourceFactory(plugin_fixture['base'], plugin_origins=plugins) + plugins = [ + Node.from_dict( + {"origin": "local", "path": os.path.join(datafiles.dirname, datafiles.basename), "plugins": ["foo"]} + ) + ] + factory = SourceFactory(plugin_fixture["base"], plugin_origins=plugins) with pytest.raises(PluginError): - factory.lookup('foo') + factory.lookup("foo") # Load a factory with a plugin which fails to provide a setup() function -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'nosetup')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "nosetup")) def test_element_missing_setup(plugin_fixture, datafiles): - plugins = [Node.from_dict({ - 'origin': 'local', - 'path': os.path.join(datafiles.dirname, - datafiles.basename), - 'plugins': ['foo'] - })] - factory = ElementFactory(plugin_fixture['base'], plugin_origins=plugins) + plugins = [ + Node.from_dict( + {"origin": "local", "path": os.path.join(datafiles.dirname, datafiles.basename), "plugins": ["foo"]} + ) + ] + factory = ElementFactory(plugin_fixture["base"], plugin_origins=plugins) with pytest.raises(PluginError): - factory.lookup('foo') + factory.lookup("foo") # Load a factory with a plugin which provides a setup symbol # that is not a function -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'badsetup')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "badsetup")) def test_source_bad_setup(plugin_fixture, datafiles): - plugins = [Node.from_dict({ - 'origin': 'local', - 'path': os.path.join(datafiles.dirname, - datafiles.basename), - 'plugins': ['foo'] - })] - factory = SourceFactory(plugin_fixture['base'], plugin_origins=plugins) + plugins = [ + Node.from_dict( + {"origin": "local", "path": os.path.join(datafiles.dirname, datafiles.basename), "plugins": ["foo"]} + ) + ] + factory = SourceFactory(plugin_fixture["base"], plugin_origins=plugins) with pytest.raises(PluginError): - factory.lookup('foo') + factory.lookup("foo") # Load a factory with a plugin which provides a setup symbol # that is not a function -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'badsetup')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "badsetup")) def test_element_bad_setup(plugin_fixture, datafiles): - plugins = [Node.from_dict({ - 'origin': 'local', - 'path': os.path.join(datafiles.dirname, - datafiles.basename), - 'plugins': ['foo'] - })] - factory = ElementFactory(plugin_fixture['base'], plugin_origins=plugins) + plugins = [ + Node.from_dict( + {"origin": "local", "path": os.path.join(datafiles.dirname, datafiles.basename), "plugins": ["foo"]} + ) + ] + factory = ElementFactory(plugin_fixture["base"], plugin_origins=plugins) with pytest.raises(PluginError): - factory.lookup('foo') + factory.lookup("foo") # Load a factory with a plugin which requires an absurdly # high version of buildstream -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'badversionsource')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "badversionsource")) def test_source_badversion(plugin_fixture, datafiles): - plugins = [Node.from_dict({ - 'origin': 'local', - 'path': os.path.join(datafiles.dirname, - datafiles.basename), - 'plugins': ['foo'] - })] - factory = SourceFactory(plugin_fixture['base'], plugin_origins=plugins) + plugins = [ + Node.from_dict( + {"origin": "local", "path": os.path.join(datafiles.dirname, datafiles.basename), "plugins": ["foo"]} + ) + ] + factory = SourceFactory(plugin_fixture["base"], plugin_origins=plugins) with pytest.raises(PluginError): - factory.lookup('foo') + factory.lookup("foo") # Load a factory with a plugin which requires an absurdly # high version of buildstream -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'badversionelement')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "badversionelement")) def test_element_badversion(plugin_fixture, datafiles): - plugins = [Node.from_dict({ - 'origin': 'local', - 'path': os.path.join(datafiles.dirname, - datafiles.basename), - 'plugins': ['foo'] - })] - factory = ElementFactory(plugin_fixture['base'], plugin_origins=plugins) + plugins = [ + Node.from_dict( + {"origin": "local", "path": os.path.join(datafiles.dirname, datafiles.basename), "plugins": ["foo"]} + ) + ] + factory = ElementFactory(plugin_fixture["base"], plugin_origins=plugins) with pytest.raises(PluginError): - factory.lookup('foo') + factory.lookup("foo") ############################################################## @@ -249,56 +232,56 @@ def test_element_badversion(plugin_fixture, datafiles): # Load two factories, both of which define a different 'foo' plugin @pytest.mark.datafiles(DATA_DIR) def test_source_multicontext(plugin_fixture, datafiles): - plugins1 = Node.from_dict({ - 'origin': 'local', - 'path': os.path.join(datafiles.dirname, - datafiles.basename, - 'customsource'), - 'plugins': ['foo'] - }) - plugins2 = Node.from_dict({ - 'origin': 'local', - 'path': os.path.join(datafiles.dirname, - datafiles.basename, - 'anothersource'), - 'plugins': ['foo'] - }) - - factory1 = SourceFactory(plugin_fixture['base'], plugin_origins=[plugins1]) - factory2 = SourceFactory(plugin_fixture['base'], plugin_origins=[plugins2]) + plugins1 = Node.from_dict( + { + "origin": "local", + "path": os.path.join(datafiles.dirname, datafiles.basename, "customsource"), + "plugins": ["foo"], + } + ) + plugins2 = Node.from_dict( + { + "origin": "local", + "path": os.path.join(datafiles.dirname, datafiles.basename, "anothersource"), + "plugins": ["foo"], + } + ) + + factory1 = SourceFactory(plugin_fixture["base"], plugin_origins=[plugins1]) + factory2 = SourceFactory(plugin_fixture["base"], plugin_origins=[plugins2]) assert isinstance(factory1, SourceFactory) assert isinstance(factory2, SourceFactory) - foo_type1, _ = factory1.lookup('foo') - foo_type2, _ = factory2.lookup('foo') - assert foo_type1.__name__ == 'FooSource' - assert foo_type2.__name__ == 'AnotherFooSource' + foo_type1, _ = factory1.lookup("foo") + foo_type2, _ = factory2.lookup("foo") + assert foo_type1.__name__ == "FooSource" + assert foo_type2.__name__ == "AnotherFooSource" # Load two factories, both of which define a different 'foo' plugin @pytest.mark.datafiles(DATA_DIR) def test_element_multicontext(plugin_fixture, datafiles): - plugins1 = Node.from_dict({ - 'origin': 'local', - 'path': os.path.join(datafiles.dirname, - datafiles.basename, - 'customelement'), - 'plugins': ['foo'] - }) - plugins2 = Node.from_dict({ - 'origin': 'local', - 'path': os.path.join(datafiles.dirname, - datafiles.basename, - 'anotherelement'), - 'plugins': ['foo'] - }) - - factory1 = ElementFactory(plugin_fixture['base'], plugin_origins=[plugins1]) - factory2 = ElementFactory(plugin_fixture['base'], plugin_origins=[plugins2]) + plugins1 = Node.from_dict( + { + "origin": "local", + "path": os.path.join(datafiles.dirname, datafiles.basename, "customelement"), + "plugins": ["foo"], + } + ) + plugins2 = Node.from_dict( + { + "origin": "local", + "path": os.path.join(datafiles.dirname, datafiles.basename, "anotherelement"), + "plugins": ["foo"], + } + ) + + factory1 = ElementFactory(plugin_fixture["base"], plugin_origins=[plugins1]) + factory2 = ElementFactory(plugin_fixture["base"], plugin_origins=[plugins2]) assert isinstance(factory1, ElementFactory) assert isinstance(factory2, ElementFactory) - foo_type1, _ = factory1.lookup('foo') - foo_type2, _ = factory2.lookup('foo') - assert foo_type1.__name__ == 'FooElement' - assert foo_type2.__name__ == 'AnotherFooElement' + foo_type1, _ = factory1.lookup("foo") + foo_type2, _ = factory2.lookup("foo") + assert foo_type1.__name__ == "FooElement" + assert foo_type2.__name__ == "AnotherFooElement" diff --git a/tests/internals/pluginfactory/wrongtype/foo.py b/tests/internals/pluginfactory/wrongtype/foo.py index 3fe9a1a62..37d9f6bfe 100644 --- a/tests/internals/pluginfactory/wrongtype/foo.py +++ b/tests/internals/pluginfactory/wrongtype/foo.py @@ -4,7 +4,7 @@ # This one fails the requirement -class Foo(): +class Foo: pass diff --git a/tests/internals/pluginloading.py b/tests/internals/pluginloading.py index 4b6baf229..83944bbd9 100644 --- a/tests/internals/pluginloading.py +++ b/tests/internals/pluginloading.py @@ -8,55 +8,52 @@ from buildstream._pipeline import Pipeline from tests.testutils import dummy_context -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - 'pluginloading', -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "pluginloading",) @contextmanager def create_pipeline(tmpdir, basedir, target): with dummy_context() as context: - context.deploydir = os.path.join(str(tmpdir), 'deploy') - context.casdir = os.path.join(str(tmpdir), 'cas') + context.deploydir = os.path.join(str(tmpdir), "deploy") + context.casdir = os.path.join(str(tmpdir), "cas") project = Project(basedir, context) pipeline = Pipeline(context, project, None) - targets, = pipeline.load([(target,)]) + (targets,) = pipeline.load([(target,)]) yield targets -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'customsource')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "customsource")) def test_customsource(datafiles, tmpdir): basedir = str(datafiles) - with create_pipeline(tmpdir, basedir, 'simple.bst') as targets: + with create_pipeline(tmpdir, basedir, "simple.bst") as targets: assert targets[0].get_kind() == "autotools" -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'customelement')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "customelement")) def test_customelement(datafiles, tmpdir): basedir = str(datafiles) - with create_pipeline(tmpdir, basedir, 'simple.bst') as targets: + with create_pipeline(tmpdir, basedir, "simple.bst") as targets: assert targets[0].get_kind() == "foo" -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'badversionsource')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "badversionsource")) def test_badversionsource(datafiles, tmpdir): basedir = str(datafiles) - with pytest.raises(LoadError) as exc, create_pipeline(tmpdir, basedir, 'simple.bst'): + with pytest.raises(LoadError) as exc, create_pipeline(tmpdir, basedir, "simple.bst"): pass assert exc.value.reason == LoadErrorReason.UNSUPPORTED_PLUGIN -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'badversionelement')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "badversionelement")) def test_badversionelement(datafiles, tmpdir): basedir = str(datafiles) - with pytest.raises(LoadError) as exc, create_pipeline(tmpdir, basedir, 'simple.bst'): + with pytest.raises(LoadError) as exc, create_pipeline(tmpdir, basedir, "simple.bst"): pass assert exc.value.reason == LoadErrorReason.UNSUPPORTED_PLUGIN diff --git a/tests/internals/pluginloading/customelement/pluginelements/foo.py b/tests/internals/pluginloading/customelement/pluginelements/foo.py index 823306ebc..c6a85a5b1 100644 --- a/tests/internals/pluginloading/customelement/pluginelements/foo.py +++ b/tests/internals/pluginloading/customelement/pluginelements/foo.py @@ -2,7 +2,6 @@ from buildstream import Element class FooElement(Element): - def preflight(self): pass diff --git a/tests/internals/pluginloading/customsource/pluginsources/foo.py b/tests/internals/pluginloading/customsource/pluginsources/foo.py index 8dd16801c..706c96f3b 100644 --- a/tests/internals/pluginloading/customsource/pluginsources/foo.py +++ b/tests/internals/pluginloading/customsource/pluginsources/foo.py @@ -2,7 +2,6 @@ from buildstream import Source, Consistency class FooSource(Source): - def preflight(self): pass diff --git a/tests/internals/storage.py b/tests/internals/storage.py index a26ca4858..8aa7f4a17 100644 --- a/tests/internals/storage.py +++ b/tests/internals/storage.py @@ -7,10 +7,7 @@ from buildstream._cas import CASCache from buildstream.storage._casbaseddirectory import CasBasedDirectory from buildstream.storage._filebaseddirectory import FileBasedDirectory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "storage" -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "storage") @contextmanager @@ -25,8 +22,7 @@ def setup_backend(backend_class, tmpdir): cas_cache.release_resources() -@pytest.mark.parametrize("backend", [ - FileBasedDirectory, CasBasedDirectory]) +@pytest.mark.parametrize("backend", [FileBasedDirectory, CasBasedDirectory]) @pytest.mark.datafiles(DATA_DIR) def test_import(tmpdir, datafiles, backend): original = os.path.join(str(datafiles), "original") @@ -38,8 +34,7 @@ def test_import(tmpdir, datafiles, backend): assert "bin/hello" in c.list_relative_paths() -@pytest.mark.parametrize("backend", [ - FileBasedDirectory, CasBasedDirectory]) +@pytest.mark.parametrize("backend", [FileBasedDirectory, CasBasedDirectory]) @pytest.mark.datafiles(DATA_DIR) def test_modified_file_list(tmpdir, datafiles, backend): original = os.path.join(str(datafiles), "original") diff --git a/tests/internals/storage_vdir_import.py b/tests/internals/storage_vdir_import.py index 808e1be9a..6b70f92b5 100644 --- a/tests/internals/storage_vdir_import.py +++ b/tests/internals/storage_vdir_import.py @@ -38,11 +38,11 @@ from buildstream.storage.directory import VirtualDirectoryError # (directory) with content being the contents for a file or the # destination for a symlink. root_filesets = [ - [('a/b/c/textfile1', 'F', 'This is textfile 1\n')], - [('a/b/c/textfile1', 'F', 'This is the replacement textfile 1\n')], - [('a/b/f', 'S', '/a/b/c')], - [('a/b/c', 'D', ''), ('a/b/f', 'S', '/a/b/c')], - [('a/b/f', 'F', 'This is textfile 1\n')], + [("a/b/c/textfile1", "F", "This is textfile 1\n")], + [("a/b/c/textfile1", "F", "This is the replacement textfile 1\n")], + [("a/b/f", "S", "/a/b/c")], + [("a/b/c", "D", ""), ("a/b/f", "S", "/a/b/c")], + [("a/b/f", "F", "This is textfile 1\n")], ] empty_hash_ref = sha256().hexdigest() @@ -60,14 +60,14 @@ def generate_import_root(rootdir, filelist): if os.path.exists(rootdir): return for (path, typesymbol, content) in filelist: - if typesymbol == 'F': + if typesymbol == "F": (dirnames, filename) = os.path.split(path) os.makedirs(os.path.join(rootdir, dirnames), exist_ok=True) with open(os.path.join(rootdir, dirnames, filename), "wt") as f: f.write(content) - elif typesymbol == 'D': + elif typesymbol == "D": os.makedirs(os.path.join(rootdir, path), exist_ok=True) - elif typesymbol == 'S': + elif typesymbol == "S": (dirnames, filename) = os.path.split(path) os.makedirs(os.path.join(rootdir, dirnames), exist_ok=True) os.symlink(content, os.path.join(rootdir, path)) @@ -83,26 +83,26 @@ def generate_random_root(rootno, directory): if os.path.exists(rootdir): return things = [] - locations = ['.'] + locations = ["."] os.makedirs(rootdir) for i in range(0, 100): location = random.choice(locations) thingname = "node{}".format(i) - thing = random.choice(['dir', 'link', 'file']) - if thing == 'dir': + thing = random.choice(["dir", "link", "file"]) + if thing == "dir": thingname = "dir" + thingname target = os.path.join(rootdir, location, thingname) - if thing == 'dir': + if thing == "dir": os.makedirs(target) locations.append(os.path.join(location, thingname)) - elif thing == 'file': + elif thing == "file": with open(target, "wt") as f: f.write("This is node {}\n".format(i)) - elif thing == 'link': - symlink_type = random.choice(['absolute', 'relative', 'broken']) - if symlink_type == 'broken' or not things: + elif thing == "link": + symlink_type = random.choice(["absolute", "relative", "broken"]) + if symlink_type == "broken" or not things: os.symlink("/broken", target) - elif symlink_type == 'absolute': + elif symlink_type == "absolute": symlink_destination = random.choice(things) os.symlink(symlink_destination, target) else: @@ -159,7 +159,7 @@ def resolve_symlinks(path, root): if os.path.islink(location): # Resolve the link, add on all the remaining components target = os.path.join(os.readlink(location)) - tail = os.path.sep.join(components[i + 1:]) + tail = os.path.sep.join(components[i + 1 :]) if target.startswith(os.path.sep): # Absolute link - relative to root @@ -202,22 +202,23 @@ def _import_test(tmpdir, original, overlay, generator_function, verify_contents= for item in root_filesets[overlay - 1]: (path, typename, content) = item realpath = resolve_symlinks(path, export_dir) - if typename == 'F': + if typename == "F": if os.path.isdir(realpath) and directory_not_empty(realpath): # The file should not have overwritten the directory in this case. pass else: - assert os.path.isfile(realpath), \ - "{} did not exist in the combined virtual directory".format(path) + assert os.path.isfile(realpath), "{} did not exist in the combined virtual directory".format( + path + ) assert file_contents_are(realpath, content) - elif typename == 'S': + elif typename == "S": if os.path.isdir(realpath) and directory_not_empty(realpath): # The symlink should not have overwritten the directory in this case. pass else: assert os.path.islink(realpath) assert os.readlink(realpath) == content - elif typename == 'D': + elif typename == "D": # We can't do any more tests than this because it # depends on things present in the original. Blank # directories here will be ignored and the original @@ -274,22 +275,18 @@ def test_fixed_directory_listing(tmpdir, root): # Check that the vdir is decending and readable def test_descend(tmpdir): - cas_dir = os.path.join(str(tmpdir), 'cas') + cas_dir = os.path.join(str(tmpdir), "cas") cas_cache = CASCache(cas_dir) try: d = CasBasedDirectory(cas_cache) - Content_to_check = 'You got me' - test_dir = os.path.join(str(tmpdir), 'importfrom') - filesys_discription = [ - ('a', 'D', ''), - ('a/l', 'D', ''), - ('a/l/g', 'F', Content_to_check) - ] + Content_to_check = "You got me" + test_dir = os.path.join(str(tmpdir), "importfrom") + filesys_discription = [("a", "D", ""), ("a/l", "D", ""), ("a/l/g", "F", Content_to_check)] generate_import_root(test_dir, filesys_discription) d.import_files(test_dir) - digest = d.descend('a', 'l').index['g'].get_digest() + digest = d.descend("a", "l").index["g"].get_digest() assert Content_to_check == open(cas_cache.objpath(digest)).read() finally: @@ -300,31 +297,27 @@ def test_descend(tmpdir): # Make sure the correct erros are raised when trying # to decend in to files or links to files def test_bad_symlinks(tmpdir): - cas_dir = os.path.join(str(tmpdir), 'cas') + cas_dir = os.path.join(str(tmpdir), "cas") cas_cache = CASCache(cas_dir) try: d = CasBasedDirectory(cas_cache) - test_dir = os.path.join(str(tmpdir), 'importfrom') - filesys_discription = [ - ('a', 'D', ''), - ('a/l', 'S', '../target'), - ('target', 'F', 'You got me') - ] + test_dir = os.path.join(str(tmpdir), "importfrom") + filesys_discription = [("a", "D", ""), ("a/l", "S", "../target"), ("target", "F", "You got me")] generate_import_root(test_dir, filesys_discription) d.import_files(test_dir) exp_reason = "not-a-directory" with pytest.raises(VirtualDirectoryError) as error: - d.descend('a', 'l', follow_symlinks=True) + d.descend("a", "l", follow_symlinks=True) assert error.reason == exp_reason with pytest.raises(VirtualDirectoryError) as error: - d.descend('a', 'l') + d.descend("a", "l") assert error.reason == exp_reason with pytest.raises(VirtualDirectoryError) as error: - d.descend('a', 'f') + d.descend("a", "f") assert error.reason == exp_reason finally: cas_cache.release_resources() @@ -333,23 +326,23 @@ def test_bad_symlinks(tmpdir): # Check symlink logic for edgecases # Check decend accross relitive link def test_relative_symlink(tmpdir): - cas_dir = os.path.join(str(tmpdir), 'cas') + cas_dir = os.path.join(str(tmpdir), "cas") cas_cache = CASCache(cas_dir) try: d = CasBasedDirectory(cas_cache) - Content_to_check = 'You got me' - test_dir = os.path.join(str(tmpdir), 'importfrom') + Content_to_check = "You got me" + test_dir = os.path.join(str(tmpdir), "importfrom") filesys_discription = [ - ('a', 'D', ''), - ('a/l', 'S', '../target'), - ('target', 'D', ''), - ('target/file', 'F', Content_to_check) + ("a", "D", ""), + ("a/l", "S", "../target"), + ("target", "D", ""), + ("target/file", "F", Content_to_check), ] generate_import_root(test_dir, filesys_discription) d.import_files(test_dir) - digest = d.descend('a', 'l', follow_symlinks=True).index['file'].get_digest() + digest = d.descend("a", "l", follow_symlinks=True).index["file"].get_digest() assert Content_to_check == open(cas_cache.objpath(digest)).read() finally: cas_cache.release_resources() @@ -358,23 +351,23 @@ def test_relative_symlink(tmpdir): # Check symlink logic for edgecases # Check deccend accross abs link def test_abs_symlink(tmpdir): - cas_dir = os.path.join(str(tmpdir), 'cas') + cas_dir = os.path.join(str(tmpdir), "cas") cas_cache = CASCache(cas_dir) try: d = CasBasedDirectory(cas_cache) - Content_to_check = 'two step file' - test_dir = os.path.join(str(tmpdir), 'importfrom') + Content_to_check = "two step file" + test_dir = os.path.join(str(tmpdir), "importfrom") filesys_discription = [ - ('a', 'D', ''), - ('a/l', 'S', '/target'), - ('target', 'D', ''), - ('target/file', 'F', Content_to_check) + ("a", "D", ""), + ("a/l", "S", "/target"), + ("target", "D", ""), + ("target/file", "F", Content_to_check), ] generate_import_root(test_dir, filesys_discription) d.import_files(test_dir) - digest = d.descend('a', 'l', follow_symlinks=True).index['file'].get_digest() + digest = d.descend("a", "l", follow_symlinks=True).index["file"].get_digest() assert Content_to_check == open(cas_cache.objpath(digest)).read() finally: @@ -384,24 +377,24 @@ def test_abs_symlink(tmpdir): # Check symlink logic for edgecases # Check symlink can not escape root def test_bad_sym_escape(tmpdir): - cas_dir = os.path.join(str(tmpdir), 'cas') + cas_dir = os.path.join(str(tmpdir), "cas") cas_cache = CASCache(cas_dir) try: d = CasBasedDirectory(cas_cache) - test_dir = os.path.join(str(tmpdir), 'importfrom') + test_dir = os.path.join(str(tmpdir), "importfrom") filesys_discription = [ - ('jail', 'D', ''), - ('jail/a', 'D', ''), - ('jail/a/l', 'S', '../../target'), - ('target', 'D', ''), - ('target/file', 'F', 'two step file') + ("jail", "D", ""), + ("jail/a", "D", ""), + ("jail/a/l", "S", "../../target"), + ("target", "D", ""), + ("target/file", "F", "two step file"), ] generate_import_root(test_dir, filesys_discription) - d.import_files(os.path.join(test_dir, 'jail')) + d.import_files(os.path.join(test_dir, "jail")) with pytest.raises(VirtualDirectoryError) as error: - d.descend('a', 'l', follow_symlinks=True) + d.descend("a", "l", follow_symlinks=True) assert error.reason == "directory-not-found" finally: cas_cache.release_resources() diff --git a/tests/internals/utils_save_atomic.py b/tests/internals/utils_save_atomic.py index 0731f7bea..898286076 100644 --- a/tests/internals/utils_save_atomic.py +++ b/tests/internals/utils_save_atomic.py @@ -5,58 +5,58 @@ from buildstream.utils import save_file_atomic def test_save_new_file(tmpdir): - filename = os.path.join(str(tmpdir), 'savefile-success.test') - with save_file_atomic(filename, 'w') as f: - f.write('foo\n') + filename = os.path.join(str(tmpdir), "savefile-success.test") + with save_file_atomic(filename, "w") as f: + f.write("foo\n") - assert os.listdir(str(tmpdir)) == ['savefile-success.test'] + assert os.listdir(str(tmpdir)) == ["savefile-success.test"] with open(filename) as f: - assert f.read() == 'foo\n' + assert f.read() == "foo\n" def test_save_over_existing_file(tmpdir): - filename = os.path.join(str(tmpdir), 'savefile-overwrite.test') + filename = os.path.join(str(tmpdir), "savefile-overwrite.test") - with open(filename, 'w') as f: - f.write('existing contents\n') + with open(filename, "w") as f: + f.write("existing contents\n") - with save_file_atomic(filename, 'w') as f: - f.write('overwritten contents\n') + with save_file_atomic(filename, "w") as f: + f.write("overwritten contents\n") - assert os.listdir(str(tmpdir)) == ['savefile-overwrite.test'] + assert os.listdir(str(tmpdir)) == ["savefile-overwrite.test"] with open(filename) as f: - assert f.read() == 'overwritten contents\n' + assert f.read() == "overwritten contents\n" def test_exception_new_file(tmpdir): - filename = os.path.join(str(tmpdir), 'savefile-exception.test') + filename = os.path.join(str(tmpdir), "savefile-exception.test") with pytest.raises(RuntimeError): - with save_file_atomic(filename, 'w') as f: - f.write('Some junk\n') + with save_file_atomic(filename, "w") as f: + f.write("Some junk\n") raise RuntimeError("Something goes wrong") assert os.listdir(str(tmpdir)) == [] def test_exception_existing_file(tmpdir): - filename = os.path.join(str(tmpdir), 'savefile-existing.test') + filename = os.path.join(str(tmpdir), "savefile-existing.test") - with open(filename, 'w') as f: - f.write('existing contents\n') + with open(filename, "w") as f: + f.write("existing contents\n") with pytest.raises(RuntimeError): - with save_file_atomic(filename, 'w') as f: - f.write('Some junk\n') + with save_file_atomic(filename, "w") as f: + f.write("Some junk\n") raise RuntimeError("Something goes wrong") - assert os.listdir(str(tmpdir)) == ['savefile-existing.test'] + assert os.listdir(str(tmpdir)) == ["savefile-existing.test"] with open(filename) as f: - assert f.read() == 'existing contents\n' + assert f.read() == "existing contents\n" def test_attributes(tmpdir): - filename = os.path.join(str(tmpdir), 'savefile-attributes.test') - with save_file_atomic(filename, 'w') as f: + filename = os.path.join(str(tmpdir), "savefile-attributes.test") + with save_file_atomic(filename, "w") as f: assert f.real_filename == filename assert f.name != filename diff --git a/tests/internals/yaml.py b/tests/internals/yaml.py index 93619dc4c..7b711575c 100644 --- a/tests/internals/yaml.py +++ b/tests/internals/yaml.py @@ -7,21 +7,16 @@ from buildstream import _yaml, Node, ProvenanceInformation, SequenceNode from buildstream._exceptions import LoadError, LoadErrorReason -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - 'yaml', -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "yaml",) @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_load_yaml(datafiles): - filename = os.path.join(datafiles.dirname, - datafiles.basename, - 'basics.yaml') + filename = os.path.join(datafiles.dirname, datafiles.basename, "basics.yaml") loaded = _yaml.load(filename) - assert loaded.get_str('kind') == 'pony' + assert loaded.get_str("kind") == "pony" def assert_provenance(filename, line, col, node): @@ -37,12 +32,10 @@ def assert_provenance(filename, line, col, node): @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_basic_provenance(datafiles): - filename = os.path.join(datafiles.dirname, - datafiles.basename, - 'basics.yaml') + filename = os.path.join(datafiles.dirname, datafiles.basename, "basics.yaml") loaded = _yaml.load(filename) - assert loaded.get_str('kind') == 'pony' + assert loaded.get_str("kind") == "pony" assert_provenance(filename, 1, 0, loaded) @@ -50,45 +43,37 @@ def test_basic_provenance(datafiles): @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_member_provenance(datafiles): - filename = os.path.join(datafiles.dirname, - datafiles.basename, - 'basics.yaml') + filename = os.path.join(datafiles.dirname, datafiles.basename, "basics.yaml") loaded = _yaml.load(filename) - assert loaded.get_str('kind') == 'pony' - assert_provenance(filename, 2, 13, loaded.get_scalar('description')) + assert loaded.get_str("kind") == "pony" + assert_provenance(filename, 2, 13, loaded.get_scalar("description")) @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_element_provenance(datafiles): - filename = os.path.join(datafiles.dirname, - datafiles.basename, - 'basics.yaml') + filename = os.path.join(datafiles.dirname, datafiles.basename, "basics.yaml") loaded = _yaml.load(filename) - assert loaded.get_str('kind') == 'pony' - assert_provenance(filename, 5, 2, loaded.get_sequence('moods').scalar_at(1)) + assert loaded.get_str("kind") == "pony" + assert_provenance(filename, 5, 2, loaded.get_sequence("moods").scalar_at(1)) @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_mapping_validate_keys(datafiles): - valid = os.path.join(datafiles.dirname, - datafiles.basename, - 'basics.yaml') - invalid = os.path.join(datafiles.dirname, - datafiles.basename, - 'invalid.yaml') + valid = os.path.join(datafiles.dirname, datafiles.basename, "basics.yaml") + invalid = os.path.join(datafiles.dirname, datafiles.basename, "invalid.yaml") base = _yaml.load(valid) - base.validate_keys(['kind', 'description', 'moods', 'children', 'extra']) + base.validate_keys(["kind", "description", "moods", "children", "extra"]) base = _yaml.load(invalid) with pytest.raises(LoadError) as exc: - base.validate_keys(['kind', 'description', 'moods', 'children', 'extra']) + base.validate_keys(["kind", "description", "moods", "children", "extra"]) assert exc.value.reason == LoadErrorReason.INVALID_DATA @@ -96,23 +81,21 @@ def test_mapping_validate_keys(datafiles): @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_node_get(datafiles): - filename = os.path.join(datafiles.dirname, - datafiles.basename, - 'basics.yaml') + filename = os.path.join(datafiles.dirname, datafiles.basename, "basics.yaml") base = _yaml.load(filename) - assert base.get_str('kind') == 'pony' + assert base.get_str("kind") == "pony" - children = base.get_sequence('children') + children = base.get_sequence("children") assert isinstance(children, SequenceNode) assert len(children) == 7 - child = base.get_sequence('children').mapping_at(6) - assert_provenance(filename, 20, 8, child.get_scalar('mood')) + child = base.get_sequence("children").mapping_at(6) + assert_provenance(filename, 20, 8, child.get_scalar("mood")) - extra = base.get_mapping('extra') + extra = base.get_mapping("extra") with pytest.raises(LoadError) as exc: - extra.get_mapping('old') + extra.get_mapping("old") assert exc.value.reason == LoadErrorReason.INVALID_DATA @@ -120,50 +103,44 @@ def test_node_get(datafiles): @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_node_set(datafiles): - filename = os.path.join(datafiles.dirname, - datafiles.basename, - 'basics.yaml') + filename = os.path.join(datafiles.dirname, datafiles.basename, "basics.yaml") base = _yaml.load(filename) - assert 'mother' not in base - base['mother'] = 'snow white' - assert base.get_str('mother') == 'snow white' + assert "mother" not in base + base["mother"] = "snow white" + assert base.get_str("mother") == "snow white" @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_node_set_overwrite(datafiles): - filename = os.path.join(datafiles.dirname, - datafiles.basename, - 'basics.yaml') + filename = os.path.join(datafiles.dirname, datafiles.basename, "basics.yaml") base = _yaml.load(filename) # Overwrite a string - assert base.get_str('kind') == 'pony' - base['kind'] = 'cow' - assert base.get_str('kind') == 'cow' + assert base.get_str("kind") == "pony" + base["kind"] = "cow" + assert base.get_str("kind") == "cow" # Overwrite a list as a string - assert base.get_str_list('moods') == ['happy', 'sad'] - base['moods'] = 'unemotional' - assert base.get_str('moods') == 'unemotional' + assert base.get_str_list("moods") == ["happy", "sad"] + base["moods"] = "unemotional" + assert base.get_str("moods") == "unemotional" @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_node_set_list_element(datafiles): - filename = os.path.join(datafiles.dirname, - datafiles.basename, - 'basics.yaml') + filename = os.path.join(datafiles.dirname, datafiles.basename, "basics.yaml") base = _yaml.load(filename) - assert base.get_str_list('moods') == ['happy', 'sad'] - base.get_sequence('moods')[0] = 'confused' + assert base.get_str_list("moods") == ["happy", "sad"] + base.get_sequence("moods")[0] = "confused" - assert base.get_str_list('moods') == ['confused', 'sad'] + assert base.get_str_list("moods") == ["confused", "sad"] # Really this is testing _yaml.node_copy(), we want to @@ -173,26 +150,22 @@ def test_node_set_list_element(datafiles): @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_composite_preserve_originals(datafiles): - filename = os.path.join(datafiles.dirname, - datafiles.basename, - 'basics.yaml') - overlayfile = os.path.join(datafiles.dirname, - datafiles.basename, - 'composite.yaml') + filename = os.path.join(datafiles.dirname, datafiles.basename, "basics.yaml") + overlayfile = os.path.join(datafiles.dirname, datafiles.basename, "composite.yaml") base = _yaml.load(filename) overlay = _yaml.load(overlayfile) base_copy = base.clone() overlay._composite(base_copy) - copy_extra = base_copy.get_mapping('extra') - orig_extra = base.get_mapping('extra') + copy_extra = base_copy.get_mapping("extra") + orig_extra = base.get_mapping("extra") # Test that the node copy has the overridden value... - assert copy_extra.get_str('old') == 'override' + assert copy_extra.get_str("old") == "override" # But the original node is not effected by the override. - assert orig_extra.get_str('old') == 'new' + assert orig_extra.get_str("old") == "new" # Tests for list composition @@ -210,66 +183,62 @@ def test_composite_preserve_originals(datafiles): # prov_col: The expected provenance column of "mood" # @pytest.mark.datafiles(os.path.join(DATA_DIR)) -@pytest.mark.parametrize("filename,index,length,mood,prov_file,prov_line,prov_col", [ - - # Test results of compositing with the (<) prepend directive - ('listprepend.yaml', 0, 9, 'prepended1', 'listprepend.yaml', 5, 10), - ('listprepend.yaml', 1, 9, 'prepended2', 'listprepend.yaml', 7, 10), - ('listprepend.yaml', 2, 9, 'silly', 'basics.yaml', 8, 8), - ('listprepend.yaml', 8, 9, 'sleepy', 'basics.yaml', 20, 8), - - # Test results of compositing with the (>) append directive - ('listappend.yaml', 7, 9, 'appended1', 'listappend.yaml', 5, 10), - ('listappend.yaml', 8, 9, 'appended2', 'listappend.yaml', 7, 10), - ('listappend.yaml', 0, 9, 'silly', 'basics.yaml', 8, 8), - ('listappend.yaml', 6, 9, 'sleepy', 'basics.yaml', 20, 8), - - # Test results of compositing with both (<) and (>) directives - ('listappendprepend.yaml', 0, 11, 'prepended1', 'listappendprepend.yaml', 5, 10), - ('listappendprepend.yaml', 1, 11, 'prepended2', 'listappendprepend.yaml', 7, 10), - ('listappendprepend.yaml', 2, 11, 'silly', 'basics.yaml', 8, 8), - ('listappendprepend.yaml', 8, 11, 'sleepy', 'basics.yaml', 20, 8), - ('listappendprepend.yaml', 9, 11, 'appended1', 'listappendprepend.yaml', 10, 10), - ('listappendprepend.yaml', 10, 11, 'appended2', 'listappendprepend.yaml', 12, 10), - - # Test results of compositing with the (=) overwrite directive - ('listoverwrite.yaml', 0, 2, 'overwrite1', 'listoverwrite.yaml', 5, 10), - ('listoverwrite.yaml', 1, 2, 'overwrite2', 'listoverwrite.yaml', 7, 10), - - # Test results of compositing without any directive, implicitly overwriting - ('implicitoverwrite.yaml', 0, 2, 'overwrite1', 'implicitoverwrite.yaml', 4, 8), - ('implicitoverwrite.yaml', 1, 2, 'overwrite2', 'implicitoverwrite.yaml', 6, 8), -]) -def test_list_composition(datafiles, filename, tmpdir, - index, length, mood, - prov_file, prov_line, prov_col): - base_file = os.path.join(datafiles.dirname, datafiles.basename, 'basics.yaml') +@pytest.mark.parametrize( + "filename,index,length,mood,prov_file,prov_line,prov_col", + [ + # Test results of compositing with the (<) prepend directive + ("listprepend.yaml", 0, 9, "prepended1", "listprepend.yaml", 5, 10), + ("listprepend.yaml", 1, 9, "prepended2", "listprepend.yaml", 7, 10), + ("listprepend.yaml", 2, 9, "silly", "basics.yaml", 8, 8), + ("listprepend.yaml", 8, 9, "sleepy", "basics.yaml", 20, 8), + # Test results of compositing with the (>) append directive + ("listappend.yaml", 7, 9, "appended1", "listappend.yaml", 5, 10), + ("listappend.yaml", 8, 9, "appended2", "listappend.yaml", 7, 10), + ("listappend.yaml", 0, 9, "silly", "basics.yaml", 8, 8), + ("listappend.yaml", 6, 9, "sleepy", "basics.yaml", 20, 8), + # Test results of compositing with both (<) and (>) directives + ("listappendprepend.yaml", 0, 11, "prepended1", "listappendprepend.yaml", 5, 10), + ("listappendprepend.yaml", 1, 11, "prepended2", "listappendprepend.yaml", 7, 10), + ("listappendprepend.yaml", 2, 11, "silly", "basics.yaml", 8, 8), + ("listappendprepend.yaml", 8, 11, "sleepy", "basics.yaml", 20, 8), + ("listappendprepend.yaml", 9, 11, "appended1", "listappendprepend.yaml", 10, 10), + ("listappendprepend.yaml", 10, 11, "appended2", "listappendprepend.yaml", 12, 10), + # Test results of compositing with the (=) overwrite directive + ("listoverwrite.yaml", 0, 2, "overwrite1", "listoverwrite.yaml", 5, 10), + ("listoverwrite.yaml", 1, 2, "overwrite2", "listoverwrite.yaml", 7, 10), + # Test results of compositing without any directive, implicitly overwriting + ("implicitoverwrite.yaml", 0, 2, "overwrite1", "implicitoverwrite.yaml", 4, 8), + ("implicitoverwrite.yaml", 1, 2, "overwrite2", "implicitoverwrite.yaml", 6, 8), + ], +) +def test_list_composition(datafiles, filename, tmpdir, index, length, mood, prov_file, prov_line, prov_col): + base_file = os.path.join(datafiles.dirname, datafiles.basename, "basics.yaml") overlay_file = os.path.join(datafiles.dirname, datafiles.basename, filename) - base = _yaml.load(base_file, 'basics.yaml') + base = _yaml.load(base_file, "basics.yaml") overlay = _yaml.load(overlay_file, shortname=filename) overlay._composite(base) - children = base.get_sequence('children') + children = base.get_sequence("children") assert len(children) == length child = children.mapping_at(index) - assert child.get_str('mood') == mood - assert_provenance(prov_file, prov_line, prov_col, child.get_node('mood')) + assert child.get_str("mood") == mood + assert_provenance(prov_file, prov_line, prov_col, child.get_node("mood")) # Test that overwriting a list with an empty list works as expected. @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_list_deletion(datafiles): - base = os.path.join(datafiles.dirname, datafiles.basename, 'basics.yaml') - overlay = os.path.join(datafiles.dirname, datafiles.basename, 'listoverwriteempty.yaml') + base = os.path.join(datafiles.dirname, datafiles.basename, "basics.yaml") + overlay = os.path.join(datafiles.dirname, datafiles.basename, "listoverwriteempty.yaml") - base = _yaml.load(base, shortname='basics.yaml') - overlay = _yaml.load(overlay, shortname='listoverwriteempty.yaml') + base = _yaml.load(base, shortname="basics.yaml") + overlay = _yaml.load(overlay, shortname="listoverwriteempty.yaml") overlay._composite(base) - children = base.get_sequence('children') + children = base.get_sequence("children") assert not children @@ -300,109 +269,103 @@ def test_list_deletion(datafiles): # prov_col: The expected provenance column of "mood" # @pytest.mark.datafiles(os.path.join(DATA_DIR)) -@pytest.mark.parametrize("filename1,filename2,index,length,mood,prov_file,prov_line,prov_col", [ - - # Test results of compositing literal list with (>) and then (<) - ('listprepend.yaml', 'listappend.yaml', 0, 11, 'prepended1', 'listprepend.yaml', 5, 10), - ('listprepend.yaml', 'listappend.yaml', 1, 11, 'prepended2', 'listprepend.yaml', 7, 10), - ('listprepend.yaml', 'listappend.yaml', 2, 11, 'silly', 'basics.yaml', 8, 8), - ('listprepend.yaml', 'listappend.yaml', 8, 11, 'sleepy', 'basics.yaml', 20, 8), - ('listprepend.yaml', 'listappend.yaml', 9, 11, 'appended1', 'listappend.yaml', 5, 10), - ('listprepend.yaml', 'listappend.yaml', 10, 11, 'appended2', 'listappend.yaml', 7, 10), - - # Test results of compositing literal list with (<) and then (>) - ('listappend.yaml', 'listprepend.yaml', 0, 11, 'prepended1', 'listprepend.yaml', 5, 10), - ('listappend.yaml', 'listprepend.yaml', 1, 11, 'prepended2', 'listprepend.yaml', 7, 10), - ('listappend.yaml', 'listprepend.yaml', 2, 11, 'silly', 'basics.yaml', 8, 8), - ('listappend.yaml', 'listprepend.yaml', 8, 11, 'sleepy', 'basics.yaml', 20, 8), - ('listappend.yaml', 'listprepend.yaml', 9, 11, 'appended1', 'listappend.yaml', 5, 10), - ('listappend.yaml', 'listprepend.yaml', 10, 11, 'appended2', 'listappend.yaml', 7, 10), - - # Test results of compositing literal list with (>) and then (>) - ('listappend.yaml', 'secondappend.yaml', 0, 11, 'silly', 'basics.yaml', 8, 8), - ('listappend.yaml', 'secondappend.yaml', 6, 11, 'sleepy', 'basics.yaml', 20, 8), - ('listappend.yaml', 'secondappend.yaml', 7, 11, 'appended1', 'listappend.yaml', 5, 10), - ('listappend.yaml', 'secondappend.yaml', 8, 11, 'appended2', 'listappend.yaml', 7, 10), - ('listappend.yaml', 'secondappend.yaml', 9, 11, 'secondappend1', 'secondappend.yaml', 5, 10), - ('listappend.yaml', 'secondappend.yaml', 10, 11, 'secondappend2', 'secondappend.yaml', 7, 10), - - # Test results of compositing literal list with (>) and then (>) - ('listprepend.yaml', 'secondprepend.yaml', 0, 11, 'secondprepend1', 'secondprepend.yaml', 5, 10), - ('listprepend.yaml', 'secondprepend.yaml', 1, 11, 'secondprepend2', 'secondprepend.yaml', 7, 10), - ('listprepend.yaml', 'secondprepend.yaml', 2, 11, 'prepended1', 'listprepend.yaml', 5, 10), - ('listprepend.yaml', 'secondprepend.yaml', 3, 11, 'prepended2', 'listprepend.yaml', 7, 10), - ('listprepend.yaml', 'secondprepend.yaml', 4, 11, 'silly', 'basics.yaml', 8, 8), - ('listprepend.yaml', 'secondprepend.yaml', 10, 11, 'sleepy', 'basics.yaml', 20, 8), - - # Test results of compositing literal list with (>) or (<) and then another literal list - ('listappend.yaml', 'implicitoverwrite.yaml', 0, 2, 'overwrite1', 'implicitoverwrite.yaml', 4, 8), - ('listappend.yaml', 'implicitoverwrite.yaml', 1, 2, 'overwrite2', 'implicitoverwrite.yaml', 6, 8), - ('listprepend.yaml', 'implicitoverwrite.yaml', 0, 2, 'overwrite1', 'implicitoverwrite.yaml', 4, 8), - ('listprepend.yaml', 'implicitoverwrite.yaml', 1, 2, 'overwrite2', 'implicitoverwrite.yaml', 6, 8), - - # Test results of compositing literal list with (>) or (<) and then an explicit (=) overwrite - ('listappend.yaml', 'listoverwrite.yaml', 0, 2, 'overwrite1', 'listoverwrite.yaml', 5, 10), - ('listappend.yaml', 'listoverwrite.yaml', 1, 2, 'overwrite2', 'listoverwrite.yaml', 7, 10), - ('listprepend.yaml', 'listoverwrite.yaml', 0, 2, 'overwrite1', 'listoverwrite.yaml', 5, 10), - ('listprepend.yaml', 'listoverwrite.yaml', 1, 2, 'overwrite2', 'listoverwrite.yaml', 7, 10), - - # Test results of compositing literal list an explicit overwrite (=) and then with (>) or (<) - ('listoverwrite.yaml', 'listappend.yaml', 0, 4, 'overwrite1', 'listoverwrite.yaml', 5, 10), - ('listoverwrite.yaml', 'listappend.yaml', 1, 4, 'overwrite2', 'listoverwrite.yaml', 7, 10), - ('listoverwrite.yaml', 'listappend.yaml', 2, 4, 'appended1', 'listappend.yaml', 5, 10), - ('listoverwrite.yaml', 'listappend.yaml', 3, 4, 'appended2', 'listappend.yaml', 7, 10), - ('listoverwrite.yaml', 'listprepend.yaml', 0, 4, 'prepended1', 'listprepend.yaml', 5, 10), - ('listoverwrite.yaml', 'listprepend.yaml', 1, 4, 'prepended2', 'listprepend.yaml', 7, 10), - ('listoverwrite.yaml', 'listprepend.yaml', 2, 4, 'overwrite1', 'listoverwrite.yaml', 5, 10), - ('listoverwrite.yaml', 'listprepend.yaml', 3, 4, 'overwrite2', 'listoverwrite.yaml', 7, 10), -]) -def test_list_composition_twice(datafiles, tmpdir, filename1, filename2, - index, length, mood, - prov_file, prov_line, prov_col): - file_base = os.path.join(datafiles.dirname, datafiles.basename, 'basics.yaml') +@pytest.mark.parametrize( + "filename1,filename2,index,length,mood,prov_file,prov_line,prov_col", + [ + # Test results of compositing literal list with (>) and then (<) + ("listprepend.yaml", "listappend.yaml", 0, 11, "prepended1", "listprepend.yaml", 5, 10), + ("listprepend.yaml", "listappend.yaml", 1, 11, "prepended2", "listprepend.yaml", 7, 10), + ("listprepend.yaml", "listappend.yaml", 2, 11, "silly", "basics.yaml", 8, 8), + ("listprepend.yaml", "listappend.yaml", 8, 11, "sleepy", "basics.yaml", 20, 8), + ("listprepend.yaml", "listappend.yaml", 9, 11, "appended1", "listappend.yaml", 5, 10), + ("listprepend.yaml", "listappend.yaml", 10, 11, "appended2", "listappend.yaml", 7, 10), + # Test results of compositing literal list with (<) and then (>) + ("listappend.yaml", "listprepend.yaml", 0, 11, "prepended1", "listprepend.yaml", 5, 10), + ("listappend.yaml", "listprepend.yaml", 1, 11, "prepended2", "listprepend.yaml", 7, 10), + ("listappend.yaml", "listprepend.yaml", 2, 11, "silly", "basics.yaml", 8, 8), + ("listappend.yaml", "listprepend.yaml", 8, 11, "sleepy", "basics.yaml", 20, 8), + ("listappend.yaml", "listprepend.yaml", 9, 11, "appended1", "listappend.yaml", 5, 10), + ("listappend.yaml", "listprepend.yaml", 10, 11, "appended2", "listappend.yaml", 7, 10), + # Test results of compositing literal list with (>) and then (>) + ("listappend.yaml", "secondappend.yaml", 0, 11, "silly", "basics.yaml", 8, 8), + ("listappend.yaml", "secondappend.yaml", 6, 11, "sleepy", "basics.yaml", 20, 8), + ("listappend.yaml", "secondappend.yaml", 7, 11, "appended1", "listappend.yaml", 5, 10), + ("listappend.yaml", "secondappend.yaml", 8, 11, "appended2", "listappend.yaml", 7, 10), + ("listappend.yaml", "secondappend.yaml", 9, 11, "secondappend1", "secondappend.yaml", 5, 10), + ("listappend.yaml", "secondappend.yaml", 10, 11, "secondappend2", "secondappend.yaml", 7, 10), + # Test results of compositing literal list with (>) and then (>) + ("listprepend.yaml", "secondprepend.yaml", 0, 11, "secondprepend1", "secondprepend.yaml", 5, 10), + ("listprepend.yaml", "secondprepend.yaml", 1, 11, "secondprepend2", "secondprepend.yaml", 7, 10), + ("listprepend.yaml", "secondprepend.yaml", 2, 11, "prepended1", "listprepend.yaml", 5, 10), + ("listprepend.yaml", "secondprepend.yaml", 3, 11, "prepended2", "listprepend.yaml", 7, 10), + ("listprepend.yaml", "secondprepend.yaml", 4, 11, "silly", "basics.yaml", 8, 8), + ("listprepend.yaml", "secondprepend.yaml", 10, 11, "sleepy", "basics.yaml", 20, 8), + # Test results of compositing literal list with (>) or (<) and then another literal list + ("listappend.yaml", "implicitoverwrite.yaml", 0, 2, "overwrite1", "implicitoverwrite.yaml", 4, 8), + ("listappend.yaml", "implicitoverwrite.yaml", 1, 2, "overwrite2", "implicitoverwrite.yaml", 6, 8), + ("listprepend.yaml", "implicitoverwrite.yaml", 0, 2, "overwrite1", "implicitoverwrite.yaml", 4, 8), + ("listprepend.yaml", "implicitoverwrite.yaml", 1, 2, "overwrite2", "implicitoverwrite.yaml", 6, 8), + # Test results of compositing literal list with (>) or (<) and then an explicit (=) overwrite + ("listappend.yaml", "listoverwrite.yaml", 0, 2, "overwrite1", "listoverwrite.yaml", 5, 10), + ("listappend.yaml", "listoverwrite.yaml", 1, 2, "overwrite2", "listoverwrite.yaml", 7, 10), + ("listprepend.yaml", "listoverwrite.yaml", 0, 2, "overwrite1", "listoverwrite.yaml", 5, 10), + ("listprepend.yaml", "listoverwrite.yaml", 1, 2, "overwrite2", "listoverwrite.yaml", 7, 10), + # Test results of compositing literal list an explicit overwrite (=) and then with (>) or (<) + ("listoverwrite.yaml", "listappend.yaml", 0, 4, "overwrite1", "listoverwrite.yaml", 5, 10), + ("listoverwrite.yaml", "listappend.yaml", 1, 4, "overwrite2", "listoverwrite.yaml", 7, 10), + ("listoverwrite.yaml", "listappend.yaml", 2, 4, "appended1", "listappend.yaml", 5, 10), + ("listoverwrite.yaml", "listappend.yaml", 3, 4, "appended2", "listappend.yaml", 7, 10), + ("listoverwrite.yaml", "listprepend.yaml", 0, 4, "prepended1", "listprepend.yaml", 5, 10), + ("listoverwrite.yaml", "listprepend.yaml", 1, 4, "prepended2", "listprepend.yaml", 7, 10), + ("listoverwrite.yaml", "listprepend.yaml", 2, 4, "overwrite1", "listoverwrite.yaml", 5, 10), + ("listoverwrite.yaml", "listprepend.yaml", 3, 4, "overwrite2", "listoverwrite.yaml", 7, 10), + ], +) +def test_list_composition_twice( + datafiles, tmpdir, filename1, filename2, index, length, mood, prov_file, prov_line, prov_col +): + file_base = os.path.join(datafiles.dirname, datafiles.basename, "basics.yaml") file1 = os.path.join(datafiles.dirname, datafiles.basename, filename1) file2 = os.path.join(datafiles.dirname, datafiles.basename, filename2) ##################### # Round 1 - Fight ! ##################### - base = _yaml.load(file_base, shortname='basics.yaml') + base = _yaml.load(file_base, shortname="basics.yaml") overlay1 = _yaml.load(file1, shortname=filename1) overlay2 = _yaml.load(file2, shortname=filename2) overlay1._composite(base) overlay2._composite(base) - children = base.get_sequence('children') + children = base.get_sequence("children") assert len(children) == length child = children.mapping_at(index) - assert child.get_str('mood') == mood - assert_provenance(prov_file, prov_line, prov_col, child.get_node('mood')) + assert child.get_str("mood") == mood + assert_provenance(prov_file, prov_line, prov_col, child.get_node("mood")) ##################### # Round 2 - Fight ! ##################### - base = _yaml.load(file_base, shortname='basics.yaml') + base = _yaml.load(file_base, shortname="basics.yaml") overlay1 = _yaml.load(file1, shortname=filename1) overlay2 = _yaml.load(file2, shortname=filename2) overlay2._composite(overlay1) overlay1._composite(base) - children = base.get_sequence('children') + children = base.get_sequence("children") assert len(children) == length child = children.mapping_at(index) - assert child.get_str('mood') == mood - assert_provenance(prov_file, prov_line, prov_col, child.get_node('mood')) + assert child.get_str("mood") == mood + assert_provenance(prov_file, prov_line, prov_col, child.get_node("mood")) @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_convert_value_to_string(datafiles): - conf_file = os.path.join(datafiles.dirname, - datafiles.basename, - 'convert_value_to_str.yaml') + conf_file = os.path.join(datafiles.dirname, datafiles.basename, "convert_value_to_str.yaml") # Run file through yaml to convert it test_dict = _yaml.load(conf_file) @@ -426,9 +389,7 @@ def test_convert_value_to_string(datafiles): @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_value_doesnt_match_expected(datafiles): - conf_file = os.path.join(datafiles.dirname, - datafiles.basename, - 'convert_value_to_str.yaml') + conf_file = os.path.join(datafiles.dirname, datafiles.basename, "convert_value_to_str.yaml") # Run file through yaml to convert it test_dict = _yaml.load(conf_file) @@ -439,11 +400,9 @@ def test_value_doesnt_match_expected(datafiles): @pytest.mark.datafiles(os.path.join(DATA_DIR)) -@pytest.mark.parametrize('fromdisk', [(True), (False)]) +@pytest.mark.parametrize("fromdisk", [(True), (False)]) def test_roundtrip_dump(datafiles, fromdisk): - filename = os.path.join(datafiles.dirname, - datafiles.basename, - "roundtrip-test.yaml") + filename = os.path.join(datafiles.dirname, datafiles.basename, "roundtrip-test.yaml") with open(filename, "r") as fh: rt_raw = fh.read() if fromdisk: @@ -480,16 +439,9 @@ def test_roundtrip_dump(datafiles, fromdisk): @pytest.mark.datafiles(os.path.join(DATA_DIR)) -@pytest.mark.parametrize('case', [ - ['a', 'b', 'c'], - ['foo', 1], - ['stuff', 0, 'colour'], - ['bird', 0, 1], -]) +@pytest.mark.parametrize("case", [["a", "b", "c"], ["foo", 1], ["stuff", 0, "colour"], ["bird", 0, 1],]) def test_node_find_target(datafiles, case): - filename = os.path.join(datafiles.dirname, - datafiles.basename, - "traversal.yaml") + filename = os.path.join(datafiles.dirname, datafiles.basename, "traversal.yaml") # We set copy_tree in order to ensure that the nodes in `loaded` # are not the same nodes as in `prov.toplevel` loaded = _yaml.load(filename, copy_tree=True) @@ -523,9 +475,7 @@ def test_node_find_target(datafiles, case): @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_node_find_target_fails(datafiles): - filename = os.path.join(datafiles.dirname, - datafiles.basename, - "traversal.yaml") + filename = os.path.join(datafiles.dirname, datafiles.basename, "traversal.yaml") loaded = _yaml.load(filename, copy_tree=True) brand_new = Node.from_dict({}) diff --git a/tests/plugins/deprecationwarnings/deprecationwarnings.py b/tests/plugins/deprecationwarnings/deprecationwarnings.py index 4d2d22c05..628faea68 100644 --- a/tests/plugins/deprecationwarnings/deprecationwarnings.py +++ b/tests/plugins/deprecationwarnings/deprecationwarnings.py @@ -8,10 +8,7 @@ import pytest from buildstream.testing import cli # pylint: disable=unused-import -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project" -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") _DEPRECATION_MESSAGE = "Here is some detail." _DEPRECATION_WARNING = "Using deprecated plugin deprecated_plugin: {}".format(_DEPRECATION_MESSAGE) @@ -20,7 +17,7 @@ _DEPRECATION_WARNING = "Using deprecated plugin deprecated_plugin: {}".format(_D @pytest.mark.datafiles(DATA_DIR) def test_deprecation_warning_present(cli, datafiles): project = str(datafiles) - result = cli.run(project=project, args=['show', 'deprecated.bst']) + result = cli.run(project=project, args=["show", "deprecated.bst"]) result.assert_success() assert _DEPRECATION_WARNING in result.stderr @@ -28,16 +25,14 @@ def test_deprecation_warning_present(cli, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_suppress_deprecation_warning(cli, datafiles): project = str(datafiles) - cli.run(project=project, args=['show', 'manual.bst']) + cli.run(project=project, args=["show", "manual.bst"]) - element_overrides = "elements:\n" \ - " deprecated_plugin:\n" \ - " suppress-deprecation-warnings : True\n" + element_overrides = "elements:\n" " deprecated_plugin:\n" " suppress-deprecation-warnings : True\n" - project_conf = os.path.join(project, 'project.conf') - with open(project_conf, 'a') as f: + project_conf = os.path.join(project, "project.conf") + with open(project_conf, "a") as f: f.write(element_overrides) - result = cli.run(project=project, args=['show', 'deprecated.bst']) + result = cli.run(project=project, args=["show", "deprecated.bst"]) result.assert_success() assert _DEPRECATION_WARNING not in result.stderr diff --git a/tests/remoteexecution/buildfail.py b/tests/remoteexecution/buildfail.py index 0fb4cdb95..22d9c825f 100644 --- a/tests/remoteexecution/buildfail.py +++ b/tests/remoteexecution/buildfail.py @@ -28,48 +28,33 @@ from buildstream.testing import cli_remote_execution as cli # pylint: disable=u pytestmark = pytest.mark.remoteexecution # Project directory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project", -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",) @pytest.mark.datafiles(DATA_DIR) def test_build_remote_failure(cli, datafiles): project = str(datafiles) - element_path = os.path.join(project, 'elements', 'element.bst') - checkout_path = os.path.join(cli.directory, 'checkout') + element_path = os.path.join(project, "elements", "element.bst") + checkout_path = os.path.join(cli.directory, "checkout") # Write out our test target element = { - 'kind': 'script', - 'depends': [ - { - 'filename': 'base.bst', - 'type': 'build', - }, - ], - 'config': { - 'commands': [ - 'touch %{install-root}/foo', - 'false', - ], - }, + "kind": "script", + "depends": [{"filename": "base.bst", "type": "build",},], + "config": {"commands": ["touch %{install-root}/foo", "false",],}, } _yaml.roundtrip_dump(element, element_path) services = cli.ensure_services() - assert set(services) == set(['action-cache', 'execution', 'storage']) + assert set(services) == set(["action-cache", "execution", "storage"]) # Try to build it, this should result in a failure that contains the content - result = cli.run(project=project, args=['build', 'element.bst']) + result = cli.run(project=project, args=["build", "element.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) - result = cli.run(project=project, args=[ - 'artifact', 'checkout', 'element.bst', '--directory', checkout_path - ]) + result = cli.run(project=project, args=["artifact", "checkout", "element.bst", "--directory", checkout_path]) result.assert_success() # check that the file created before the failure exists - filename = os.path.join(checkout_path, 'foo') + filename = os.path.join(checkout_path, "foo") assert os.path.isfile(filename) diff --git a/tests/remoteexecution/buildtree.py b/tests/remoteexecution/buildtree.py index a64b8716c..57e25cd14 100644 --- a/tests/remoteexecution/buildtree.py +++ b/tests/remoteexecution/buildtree.py @@ -28,54 +28,43 @@ from tests.testutils import create_artifact_share pytestmark = pytest.mark.remoteexecution # Project directory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project", -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",) @pytest.mark.datafiles(DATA_DIR) def test_buildtree_remote(cli, tmpdir, datafiles): project = str(datafiles) - element_name = 'build-shell/buildtree.bst' - share_path = os.path.join(str(tmpdir), 'share') + element_name = "build-shell/buildtree.bst" + share_path = os.path.join(str(tmpdir), "share") services = cli.ensure_services() - assert set(services) == set(['action-cache', 'execution', 'storage']) + assert set(services) == set(["action-cache", "execution", "storage"]) with create_artifact_share(share_path) as share: - cli.configure({ - 'artifacts': {'url': share.repo, 'push': True}, - 'cache': {'pull-buildtrees': False} - }) + cli.configure({"artifacts": {"url": share.repo, "push": True}, "cache": {"pull-buildtrees": False}}) - res = cli.run(project=project, args=[ - '--cache-buildtrees', 'always', 'build', element_name]) + res = cli.run(project=project, args=["--cache-buildtrees", "always", "build", element_name]) res.assert_success() # remove local cache - shutil.rmtree(os.path.join(str(tmpdir), 'cache', 'cas')) - shutil.rmtree(os.path.join(str(tmpdir), 'cache', 'artifacts')) + shutil.rmtree(os.path.join(str(tmpdir), "cache", "cas")) + shutil.rmtree(os.path.join(str(tmpdir), "cache", "artifacts")) # pull without buildtree - res = cli.run(project=project, args=[ - 'artifact', 'pull', '--deps', 'all', element_name]) + res = cli.run(project=project, args=["artifact", "pull", "--deps", "all", element_name]) res.assert_success() # check shell doesn't work - res = cli.run(project=project, args=[ - 'shell', '--build', element_name, '--', 'cat', 'test' - ]) + res = cli.run(project=project, args=["shell", "--build", element_name, "--", "cat", "test"]) res.assert_shell_error() # pull with buildtree - res = cli.run(project=project, args=[ - '--pull-buildtrees', 'artifact', 'pull', '--deps', 'all', element_name]) + res = cli.run(project=project, args=["--pull-buildtrees", "artifact", "pull", "--deps", "all", element_name]) res.assert_success() # check it works this time - res = cli.run(project=project, args=[ - 'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test' - ]) + res = cli.run( + project=project, args=["shell", "--build", element_name, "--use-buildtree", "always", "--", "cat", "test"] + ) res.assert_success() assert "Hi" in res.output diff --git a/tests/remoteexecution/junction.py b/tests/remoteexecution/junction.py index db087fd90..2b0261612 100644 --- a/tests/remoteexecution/junction.py +++ b/tests/remoteexecution/junction.py @@ -28,92 +28,70 @@ from tests.testutils import generate_junction pytestmark = pytest.mark.remoteexecution # Project directory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project", -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",) def configure_project(path, config): - config['name'] = 'test' - config['element-path'] = 'elements' - _yaml.roundtrip_dump(config, os.path.join(path, 'project.conf')) + config["name"] = "test" + config["element-path"] = "elements" + _yaml.roundtrip_dump(config, os.path.join(path, "project.conf")) def create_element(repo, name, path, dependencies, ref=None): - element = { - 'kind': 'import', - 'sources': [ - repo.source_config(ref=ref) - ], - 'depends': dependencies - } + element = {"kind": "import", "sources": [repo.source_config(ref=ref)], "depends": dependencies} _yaml.roundtrip_dump(element, os.path.join(path, name)) @pytest.mark.datafiles(DATA_DIR) def test_junction_build_remote(cli, tmpdir, datafiles): project = str(datafiles) - subproject_path = os.path.join(project, 'files', 'sub-project') - subproject_element_path = os.path.join(subproject_path, 'elements') - amhello_files_path = os.path.join(subproject_path, 'files') - element_path = os.path.join(project, 'elements') - junction_path = os.path.join(element_path, 'junction.bst') + subproject_path = os.path.join(project, "files", "sub-project") + subproject_element_path = os.path.join(subproject_path, "elements") + amhello_files_path = os.path.join(subproject_path, "files") + element_path = os.path.join(project, "elements") + junction_path = os.path.join(element_path, "junction.bst") # We need a repo for real trackable elements - repo = create_repo('git', str(tmpdir)) + repo = create_repo("git", str(tmpdir)) ref = repo.create(amhello_files_path) # ensure that the correct project directory is also listed in the junction - subproject_conf = os.path.join(subproject_path, 'project.conf') + subproject_conf = os.path.join(subproject_path, "project.conf") with open(subproject_conf) as f: config = f.read() config = config.format(project_dir=subproject_path) - with open(subproject_conf, 'w') as f: + with open(subproject_conf, "w") as f: f.write(config) # Create a trackable element to depend on the cross junction element, # this one has it's ref resolved already - create_element(repo, 'sub-target.bst', subproject_element_path, ['autotools/amhello.bst'], ref=ref) + create_element(repo, "sub-target.bst", subproject_element_path, ["autotools/amhello.bst"], ref=ref) # Create a trackable element to depend on the cross junction element - create_element(repo, 'target.bst', element_path, [ - { - 'junction': 'junction.bst', - 'filename': 'sub-target.bst' - } - ]) + create_element(repo, "target.bst", element_path, [{"junction": "junction.bst", "filename": "sub-target.bst"}]) # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path, store_ref=False) # Now create a compose element at the top level - element = { - 'kind': 'compose', - 'depends': [ - { - 'filename': 'target.bst', - 'type': 'build' - } - ] - } - _yaml.roundtrip_dump(element, os.path.join(element_path, 'composed.bst')) + element = {"kind": "compose", "depends": [{"filename": "target.bst", "type": "build"}]} + _yaml.roundtrip_dump(element, os.path.join(element_path, "composed.bst")) # We're doing remote execution so ensure services are available services = cli.ensure_services() - assert set(services) == set(['action-cache', 'execution', 'storage']) + assert set(services) == set(["action-cache", "execution", "storage"]) # track the junction first to ensure we have refs - result = cli.run(project=project, args=['source', 'track', 'junction.bst']) + result = cli.run(project=project, args=["source", "track", "junction.bst"]) result.assert_success() # track target to ensure we have refs - result = cli.run(project=project, args=['source', 'track', '--deps', 'all', 'composed.bst']) + result = cli.run(project=project, args=["source", "track", "--deps", "all", "composed.bst"]) result.assert_success() # build - result = cli.run(project=project, silent=True, args=['build', 'composed.bst']) + result = cli.run(project=project, silent=True, args=["build", "composed.bst"]) result.assert_success() # Assert that the main target is cached as a result - assert cli.get_element_state(project, 'composed.bst') == 'cached' + assert cli.get_element_state(project, "composed.bst") == "cached" diff --git a/tests/remoteexecution/partial.py b/tests/remoteexecution/partial.py index a640f27d5..a452d6613 100644 --- a/tests/remoteexecution/partial.py +++ b/tests/remoteexecution/partial.py @@ -14,75 +14,68 @@ from tests.testutils.artifactshare import create_artifact_share pytestmark = pytest.mark.remoteexecution -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project" -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") # Test that `bst build` does not download file blobs of a build-only dependency # to the local cache. @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize('pull_artifact_files', [True, False]) -@pytest.mark.parametrize('build_all', [True, False]) +@pytest.mark.parametrize("pull_artifact_files", [True, False]) +@pytest.mark.parametrize("build_all", [True, False]) def test_build_dependency_partial_local_cas(cli, datafiles, pull_artifact_files, build_all): project = str(datafiles) - element_name = 'no-runtime-deps.bst' - builddep_element_name = 'autotools/amhello.bst' - checkout = os.path.join(cli.directory, 'checkout') - builddep_checkout = os.path.join(cli.directory, 'builddep-checkout') + element_name = "no-runtime-deps.bst" + builddep_element_name = "autotools/amhello.bst" + checkout = os.path.join(cli.directory, "checkout") + builddep_checkout = os.path.join(cli.directory, "builddep-checkout") services = cli.ensure_services() - assert set(services) == set(['action-cache', 'execution', 'storage']) + assert set(services) == set(["action-cache", "execution", "storage"]) # configure pull blobs if build_all: - cli.configure({ - 'build': { - 'dependencies': 'all' - } - }) - cli.config['remote-execution']['pull-artifact-files'] = pull_artifact_files - - result = cli.run(project=project, args=['build', element_name]) + cli.configure({"build": {"dependencies": "all"}}) + cli.config["remote-execution"]["pull-artifact-files"] = pull_artifact_files + + result = cli.run(project=project, args=["build", element_name]) result.assert_success() # Verify artifact is pulled bar files when ensure artifact files is set - result = cli.run(project=project, args=['artifact', 'checkout', element_name, - '--directory', checkout]) + result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) if pull_artifact_files: result.assert_success() - assert_contains(checkout, ['/test']) + assert_contains(checkout, ["/test"]) else: - result.assert_main_error(ErrorDomain.STREAM, 'uncached-checkout-attempt') + result.assert_main_error(ErrorDomain.STREAM, "uncached-checkout-attempt") # Verify build dependencies is pulled for ALL and BUILD - result = cli.run(project=project, args=['artifact', 'checkout', builddep_element_name, - '--directory', builddep_checkout]) + result = cli.run( + project=project, args=["artifact", "checkout", builddep_element_name, "--directory", builddep_checkout] + ) if build_all and pull_artifact_files: result.assert_success() else: - result.assert_main_error(ErrorDomain.STREAM, 'uncached-checkout-attempt') + result.assert_main_error(ErrorDomain.STREAM, "uncached-checkout-attempt") @pytest.mark.datafiles(DATA_DIR) def test_build_partial_push(cli, tmpdir, datafiles): project = str(datafiles) share_dir = os.path.join(str(tmpdir), "artifactshare") - element_name = 'no-runtime-deps.bst' - builddep_element_name = 'autotools/amhello.bst' + element_name = "no-runtime-deps.bst" + builddep_element_name = "autotools/amhello.bst" with create_artifact_share(share_dir) as share: services = cli.ensure_services() - assert set(services) == set(['action-cache', 'execution', 'storage']) + assert set(services) == set(["action-cache", "execution", "storage"]) - cli.config['artifacts'] = { - 'url': share.repo, - 'push': True, + cli.config["artifacts"] = { + "url": share.repo, + "push": True, } - res = cli.run(project=project, args=['build', element_name]) + res = cli.run(project=project, args=["build", element_name]) res.assert_success() assert builddep_element_name in res.get_pushed_elements() diff --git a/tests/remoteexecution/simple.py b/tests/remoteexecution/simple.py index 1b7f7818a..cb8f80930 100644 --- a/tests/remoteexecution/simple.py +++ b/tests/remoteexecution/simple.py @@ -11,49 +11,54 @@ from buildstream.testing.integration import assert_contains pytestmark = pytest.mark.remoteexecution -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project" -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") # Test building an executable with remote-execution: @pytest.mark.datafiles(DATA_DIR) def test_remote_autotools_build(cli, datafiles): project = str(datafiles) - checkout = os.path.join(cli.directory, 'checkout') - element_name = 'autotools/amhello.bst' + checkout = os.path.join(cli.directory, "checkout") + element_name = "autotools/amhello.bst" services = cli.ensure_services() - assert set(services) == set(['action-cache', 'execution', 'storage']) + assert set(services) == set(["action-cache", "execution", "storage"]) - result = cli.run(project=project, args=['build', element_name]) + result = cli.run(project=project, args=["build", element_name]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout]) + result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) result.assert_success() - assert_contains(checkout, ['/usr', '/usr/lib', '/usr/bin', - '/usr/share', - '/usr/bin/hello', '/usr/share/doc', - '/usr/share/doc/amhello', - '/usr/share/doc/amhello/README']) + assert_contains( + checkout, + [ + "/usr", + "/usr/lib", + "/usr/bin", + "/usr/share", + "/usr/bin/hello", + "/usr/share/doc", + "/usr/share/doc/amhello", + "/usr/share/doc/amhello/README", + ], + ) # Test running an executable built with remote-execution: @pytest.mark.datafiles(DATA_DIR) def test_remote_autotools_run(cli, datafiles): project = str(datafiles) - element_name = 'autotools/amhello.bst' + element_name = "autotools/amhello.bst" services = cli.ensure_services() - assert set(services) == set(['action-cache', 'execution', 'storage']) + assert set(services) == set(["action-cache", "execution", "storage"]) services = cli.ensure_services() - result = cli.run(project=project, args=['build', element_name]) + result = cli.run(project=project, args=["build", element_name]) result.assert_success() - result = cli.run(project=project, args=['shell', element_name, '/usr/bin/hello']) + result = cli.run(project=project, args=["shell", element_name, "/usr/bin/hello"]) result.assert_success() - assert result.output == 'Hello World!\nThis is amhello 1.0.\n' + assert result.output == "Hello World!\nThis is amhello 1.0.\n" diff --git a/tests/sandboxes/fallback.py b/tests/sandboxes/fallback.py index f2f585e70..948e3a6de 100644 --- a/tests/sandboxes/fallback.py +++ b/tests/sandboxes/fallback.py @@ -26,51 +26,43 @@ from buildstream.testing import cli # pylint: disable=unused-import pytestmark = pytest.mark.integration -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project" -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") @pytest.mark.datafiles(DATA_DIR) def test_fallback_platform_fails(cli, datafiles): project = str(datafiles) - element_path = os.path.join(project, 'elements', 'element.bst') + element_path = os.path.join(project, "elements", "element.bst") # Write out our test target element = { - 'kind': 'script', - 'depends': [ - { - 'filename': 'base.bst', - 'type': 'build', - }, - ], - 'config': { - 'commands': [ - 'true', - ], - }, + "kind": "script", + "depends": [{"filename": "base.bst", "type": "build",},], + "config": {"commands": ["true",],}, } _yaml.roundtrip_dump(element, element_path) - result = cli.run(project=project, args=['build', 'element.bst'], - env={'BST_FORCE_BACKEND': 'fallback', - 'BST_FORCE_SANDBOX': None}) + result = cli.run( + project=project, + args=["build", "element.bst"], + env={"BST_FORCE_BACKEND": "fallback", "BST_FORCE_SANDBOX": None}, + ) result.assert_main_error(ErrorDomain.STREAM, None) assert "FallBack platform only implements dummy sandbox" in result.stderr # The dummy sandbox can not build the element but it can get the element read # There for the element should be `buildable` rather than `waiting` - assert cli.get_element_state(project, 'element.bst') == 'buildable' + assert cli.get_element_state(project, "element.bst") == "buildable" @pytest.mark.datafiles(DATA_DIR) def test_fallback_platform_can_use_dummy(cli, datafiles): project = str(datafiles) - result = cli.run(project=project, args=['build', 'import-file1.bst'], - env={'BST_FORCE_BACKEND': 'fallback', - 'BST_FORCE_SANDBOX': None}) + result = cli.run( + project=project, + args=["build", "import-file1.bst"], + env={"BST_FORCE_BACKEND": "fallback", "BST_FORCE_SANDBOX": None}, + ) result.assert_success() # The fallback platform can still provide a dummy sandbox that alows simple elemnts that do not need # a full sandbox to still be built on new platforms. diff --git a/tests/sandboxes/missing-command.py b/tests/sandboxes/missing-command.py index 171e855f7..87e668966 100644 --- a/tests/sandboxes/missing-command.py +++ b/tests/sandboxes/missing-command.py @@ -9,15 +9,12 @@ from buildstream._exceptions import ErrorDomain from buildstream.testing import cli # pylint: disable=unused-import -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "missing-command" -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "missing-command") @pytest.mark.datafiles(DATA_DIR) def test_missing_command(cli, datafiles): project = str(datafiles) - result = cli.run(project=project, args=['build', 'no-runtime.bst']) - result.assert_task_error(ErrorDomain.SANDBOX, 'missing-command') - assert cli.get_element_state(project, 'no-runtime.bst') == 'failed' + result = cli.run(project=project, args=["build", "no-runtime.bst"]) + result.assert_task_error(ErrorDomain.SANDBOX, "missing-command") + assert cli.get_element_state(project, "no-runtime.bst") == "failed" diff --git a/tests/sandboxes/missing_dependencies.py b/tests/sandboxes/missing_dependencies.py index a5bf31e76..722cfc647 100644 --- a/tests/sandboxes/missing_dependencies.py +++ b/tests/sandboxes/missing_dependencies.py @@ -12,10 +12,7 @@ from buildstream.testing import cli # pylint: disable=unused-import # Project directory -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "missing-dependencies", -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "missing-dependencies",) def _symlink_host_tools_to_dir(host_tools, dir_): @@ -25,85 +22,67 @@ def _symlink_host_tools_to_dir(host_tools, dir_): os.symlink(utils.get_host_tool(tool), str(target_path)) -@pytest.mark.skipif(not IS_LINUX, reason='Only available on Linux') +@pytest.mark.skipif(not IS_LINUX, reason="Only available on Linux") @pytest.mark.datafiles(DATA_DIR) def test_missing_bwrap_has_nice_error_message(cli, datafiles, tmp_path): # Create symlink to buildbox-casd and git to work with custom PATH bin_dir = tmp_path / "bin" - _symlink_host_tools_to_dir(['buildbox-casd', 'git'], bin_dir) + _symlink_host_tools_to_dir(["buildbox-casd", "git"], bin_dir) project = str(datafiles) - element_path = os.path.join(project, 'elements', 'element.bst') + element_path = os.path.join(project, "elements", "element.bst") # Write out our test target element = { - 'kind': 'script', - 'depends': [ - { - 'filename': 'base.bst', - 'type': 'build', - }, - ], - 'config': { - 'commands': [ - 'false', - ], - }, + "kind": "script", + "depends": [{"filename": "base.bst", "type": "build",},], + "config": {"commands": ["false",],}, } _yaml.roundtrip_dump(element, element_path) # Build without access to host tools, this should fail with a nice error result = cli.run( - project=project, - args=['build', 'element.bst'], - env={'PATH': str(bin_dir), - 'BST_FORCE_SANDBOX': None}) - result.assert_task_error(ErrorDomain.SANDBOX, 'unavailable-local-sandbox') + project=project, args=["build", "element.bst"], env={"PATH": str(bin_dir), "BST_FORCE_SANDBOX": None} + ) + result.assert_task_error(ErrorDomain.SANDBOX, "unavailable-local-sandbox") assert "not found" in result.stderr -@pytest.mark.skipif(not IS_LINUX, reason='Only available on Linux') +@pytest.mark.skipif(not IS_LINUX, reason="Only available on Linux") @pytest.mark.datafiles(DATA_DIR) def test_old_brwap_has_nice_error_message(cli, datafiles, tmp_path): - bwrap = tmp_path.joinpath('bin/bwrap') + bwrap = tmp_path.joinpath("bin/bwrap") bwrap.parent.mkdir() - with bwrap.open('w') as fp: - fp.write(''' + with bwrap.open("w") as fp: + fp.write( + """ #!/bin/sh echo bubblewrap 0.0.1 - '''.strip()) + """.strip() + ) bwrap.chmod(0o755) # Create symlink to buildbox-casd and git to work with custom PATH bin_dir = tmp_path / "bin" - _symlink_host_tools_to_dir(['buildbox-casd', 'git'], bin_dir) + _symlink_host_tools_to_dir(["buildbox-casd", "git"], bin_dir) project = str(datafiles) - element_path = os.path.join(project, 'elements', 'element3.bst') + element_path = os.path.join(project, "elements", "element3.bst") # Write out our test target element = { - 'kind': 'script', - 'depends': [ - { - 'filename': 'base.bst', - 'type': 'build', - }, - ], - 'config': { - 'commands': [ - 'false', - ], - }, + "kind": "script", + "depends": [{"filename": "base.bst", "type": "build",},], + "config": {"commands": ["false",],}, } _yaml.roundtrip_dump(element, element_path) # Build without access to host tools, this should fail with a nice error result = cli.run( project=project, - args=['--debug', '--verbose', 'build', 'element3.bst'], - env={'PATH': str(bin_dir), - 'BST_FORCE_SANDBOX': None}) - result.assert_task_error(ErrorDomain.SANDBOX, 'unavailable-local-sandbox') + args=["--debug", "--verbose", "build", "element3.bst"], + env={"PATH": str(bin_dir), "BST_FORCE_SANDBOX": None}, + ) + result.assert_task_error(ErrorDomain.SANDBOX, "unavailable-local-sandbox") assert "too old" in result.stderr diff --git a/tests/sandboxes/mounting/mount_simple.py b/tests/sandboxes/mounting/mount_simple.py index 65aaf209d..0e78a5603 100644 --- a/tests/sandboxes/mounting/mount_simple.py +++ b/tests/sandboxes/mounting/mount_simple.py @@ -13,35 +13,35 @@ def test_bind_mount(): src = stack.enter_context(tempfile.TemporaryDirectory()) target = stack.enter_context(tempfile.TemporaryDirectory()) - with open(os.path.join(src, 'test'), 'a') as test: - test.write('Test') + with open(os.path.join(src, "test"), "a") as test: + test.write("Test") with Mounter.bind_mount(target, src) as dest: # Ensure we get the correct path back assert dest == target # Ensure we can access files from src from target - with open(os.path.join(target, 'test'), 'r') as test: - assert test.read() == 'Test' + with open(os.path.join(target, "test"), "r") as test: + assert test.read() == "Test" # Ensure the files from src are gone from target with pytest.raises(FileNotFoundError): - with open(os.path.join(target, 'test'), 'r'): + with open(os.path.join(target, "test"), "r"): # Actual contents don't matter pass # Ensure the files in src are still in src - with open(os.path.join(src, 'test'), 'r') as test: - assert test.read() == 'Test' + with open(os.path.join(src, "test"), "r") as test: + assert test.read() == "Test" @pytest.mark.skipif(not os.geteuid() == 0, reason="requires root permissions") def test_mount_proc(): with ExitStack() as stack: - src = '/proc' + src = "/proc" target = stack.enter_context(tempfile.TemporaryDirectory()) - with Mounter.mount(target, src, mount_type='proc', ro=True) as dest: + with Mounter.mount(target, src, mount_type="proc", ro=True) as dest: # Ensure we get the correct path back assert dest == target diff --git a/tests/sandboxes/remote-exec-config.py b/tests/sandboxes/remote-exec-config.py index a6aeeb7ab..7066ddeab 100644 --- a/tests/sandboxes/remote-exec-config.py +++ b/tests/sandboxes/remote-exec-config.py @@ -9,10 +9,7 @@ from buildstream import _yaml from buildstream._exceptions import ErrorDomain, LoadErrorReason from buildstream.testing.runcli import cli # pylint: disable=unused-import -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "remote-exec-config" -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "remote-exec-config") # Tests that we get a useful error message when supplying invalid # remote execution configurations. @@ -22,28 +19,23 @@ DATA_DIR = os.path.join( # are used at once, a LoadError results. @pytest.mark.datafiles(DATA_DIR) def test_old_and_new_configs(cli, datafiles): - project = os.path.join(datafiles.dirname, datafiles.basename, 'missing-certs') + project = os.path.join(datafiles.dirname, datafiles.basename, "missing-certs") project_conf = { - 'name': 'test', - - 'remote-execution': { - 'url': 'https://cache.example.com:12345', - 'execution-service': { - 'url': 'http://localhost:8088' - }, - 'storage-service': { - 'url': 'http://charactron:11001', - } - } + "name": "test", + "remote-execution": { + "url": "https://cache.example.com:12345", + "execution-service": {"url": "http://localhost:8088"}, + "storage-service": {"url": "http://charactron:11001",}, + }, } - project_conf_file = os.path.join(project, 'project.conf') + project_conf_file = os.path.join(project, "project.conf") _yaml.roundtrip_dump(project_conf, project_conf_file) # Use `pull` here to ensure we try to initialize the remotes, triggering the error # # This does not happen for a simple `bst show`. - result = cli.run(project=project, args=['artifact', 'pull', 'element.bst']) + result = cli.run(project=project, args=["artifact", "pull", "element.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA, "specify one") @@ -51,52 +43,38 @@ def test_old_and_new_configs(cli, datafiles): # without specifying its counterpart, we get a comprehensive LoadError # instead of an unhandled exception. @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize('config_key, config_value', [ - ('client-cert', 'client.crt'), - ('client-key', 'client.key') -]) +@pytest.mark.parametrize("config_key, config_value", [("client-cert", "client.crt"), ("client-key", "client.key")]) def test_missing_certs(cli, datafiles, config_key, config_value): - project = os.path.join(datafiles.dirname, datafiles.basename, 'missing-certs') + project = os.path.join(datafiles.dirname, datafiles.basename, "missing-certs") project_conf = { - 'name': 'test', - - 'remote-execution': { - 'execution-service': { - 'url': 'http://localhost:8088' - }, - 'storage-service': { - 'url': 'http://charactron:11001', - config_key: config_value, - } - } + "name": "test", + "remote-execution": { + "execution-service": {"url": "http://localhost:8088"}, + "storage-service": {"url": "http://charactron:11001", config_key: config_value,}, + }, } - project_conf_file = os.path.join(project, 'project.conf') + project_conf_file = os.path.join(project, "project.conf") _yaml.roundtrip_dump(project_conf, project_conf_file) # Use `pull` here to ensure we try to initialize the remotes, triggering the error # # This does not happen for a simple `bst show`. - result = cli.run(project=project, args=['show', 'element.bst']) + result = cli.run(project=project, args=["show", "element.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA, "Your config is missing") # Assert that if incomplete information is supplied we get a sensible error message. @pytest.mark.datafiles(DATA_DIR) def test_empty_config(cli, datafiles): - project = os.path.join(datafiles.dirname, datafiles.basename, 'missing-certs') + project = os.path.join(datafiles.dirname, datafiles.basename, "missing-certs") - project_conf = { - 'name': 'test', - - 'remote-execution': { - } - } - project_conf_file = os.path.join(project, 'project.conf') + project_conf = {"name": "test", "remote-execution": {}} + project_conf_file = os.path.join(project, "project.conf") _yaml.roundtrip_dump(project_conf, project_conf_file) # Use `pull` here to ensure we try to initialize the remotes, triggering the error # # This does not happen for a simple `bst show`. - result = cli.run(project=project, args=['artifact', 'pull', 'element.bst']) + result = cli.run(project=project, args=["artifact", "pull", "element.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA, "specify one") diff --git a/tests/sandboxes/selection.py b/tests/sandboxes/selection.py index b4bbb1b00..a338fe0bb 100644 --- a/tests/sandboxes/selection.py +++ b/tests/sandboxes/selection.py @@ -26,79 +26,58 @@ from buildstream.testing import cli # pylint: disable=unused-import pytestmark = pytest.mark.integration -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project" -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") @pytest.mark.datafiles(DATA_DIR) def test_force_sandbox(cli, datafiles): project = str(datafiles) - element_path = os.path.join(project, 'elements', 'element.bst') + element_path = os.path.join(project, "elements", "element.bst") # Write out our test target element = { - 'kind': 'script', - 'depends': [ - { - 'filename': 'base.bst', - 'type': 'build', - }, - ], - 'config': { - 'commands': [ - 'true', - ], - }, + "kind": "script", + "depends": [{"filename": "base.bst", "type": "build",},], + "config": {"commands": ["true",],}, } _yaml.roundtrip_dump(element, element_path) # Build without access to host tools, this will fail - result = cli.run(project=project, args=['build', 'element.bst'], env={'PATH': '', 'BST_FORCE_SANDBOX': 'bwrap'}) + result = cli.run(project=project, args=["build", "element.bst"], env={"PATH": "", "BST_FORCE_SANDBOX": "bwrap"}) result.assert_main_error(ErrorDomain.PLATFORM, None) assert "Bubblewrap not found" in result.stderr # we have asked for a spesific sand box, but it is not avalble so # bst should fail early and the element should be waiting - assert cli.get_element_state(project, 'element.bst') == 'waiting' + assert cli.get_element_state(project, "element.bst") == "waiting" @pytest.mark.datafiles(DATA_DIR) def test_dummy_sandbox_fallback(cli, datafiles, tmp_path): # Create symlink to buildbox-casd to work with custom PATH - buildbox_casd = tmp_path.joinpath('bin/buildbox-casd') + buildbox_casd = tmp_path.joinpath("bin/buildbox-casd") buildbox_casd.parent.mkdir() - os.symlink(utils.get_host_tool('buildbox-casd'), str(buildbox_casd)) + os.symlink(utils.get_host_tool("buildbox-casd"), str(buildbox_casd)) project = str(datafiles) - element_path = os.path.join(project, 'elements', 'element.bst') + element_path = os.path.join(project, "elements", "element.bst") # Write out our test target element = { - 'kind': 'script', - 'depends': [ - { - 'filename': 'base.bst', - 'type': 'build', - }, - ], - 'config': { - 'commands': [ - 'true', - ], - }, + "kind": "script", + "depends": [{"filename": "base.bst", "type": "build",},], + "config": {"commands": ["true",],}, } _yaml.roundtrip_dump(element, element_path) # Build without access to host tools, this will fail result = cli.run( project=project, - args=['build', 'element.bst'], - env={'PATH': str(tmp_path.joinpath('bin')), - 'BST_FORCE_SANDBOX': None}) + args=["build", "element.bst"], + env={"PATH": str(tmp_path.joinpath("bin")), "BST_FORCE_SANDBOX": None}, + ) # But if we dont spesify a sandbox then we fall back to dummy, we still # fail early but only once we know we need a facny sandbox and that # dumy is not enough, there for element gets fetched and so is buildable - result.assert_task_error(ErrorDomain.SANDBOX, 'unavailable-local-sandbox') - assert cli.get_element_state(project, 'element.bst') == 'buildable' + result.assert_task_error(ErrorDomain.SANDBOX, "unavailable-local-sandbox") + assert cli.get_element_state(project, "element.bst") == "buildable" diff --git a/tests/sourcecache/cache.py b/tests/sourcecache/cache.py index 9aa2c67ac..bbc3d8329 100644 --- a/tests/sourcecache/cache.py +++ b/tests/sourcecache/cache.py @@ -39,10 +39,10 @@ def test_patch_sources_cached_1(cli, datafiles): # as we have a local, patch, local config, the first local and patch should # be cached together, and the last local on it's own - source_protos = os.path.join(project_dir, 'cache', 'source_protos') + source_protos = os.path.join(project_dir, "cache", "source_protos") - assert len(os.listdir(os.path.join(source_protos, 'patch'))) == 1 - assert len(os.listdir(os.path.join(source_protos, 'local'))) == 2 + assert len(os.listdir(os.path.join(source_protos, "patch"))) == 1 + assert len(os.listdir(os.path.join(source_protos, "local"))) == 2 @pytest.mark.datafiles(DATA_DIR) @@ -53,9 +53,9 @@ def test_patch_sources_cached_2(cli, datafiles): res.assert_success() # As everything is before the patch it should all be cached together - source_protos = os.path.join(project_dir, 'cache', 'source_protos') + source_protos = os.path.join(project_dir, "cache", "source_protos") - assert len(os.listdir(os.path.join(source_protos, 'patch'))) == 1 + assert len(os.listdir(os.path.join(source_protos, "patch"))) == 1 @pytest.mark.datafiles(DATA_DIR) @@ -66,35 +66,34 @@ def test_sources_without_patch(cli, datafiles): res.assert_success() # No patches so everything should be cached seperately - source_protos = os.path.join(project_dir, 'cache', 'source_protos') + source_protos = os.path.join(project_dir, "cache", "source_protos") - assert len(os.listdir(os.path.join(source_protos, 'local'))) == 3 + assert len(os.listdir(os.path.join(source_protos, "local"))) == 3 @pytest.mark.datafiles(DATA_DIR) def test_source_cache_key(cli, datafiles): project_dir = str(datafiles) - file_path = os.path.join(project_dir, 'files') - file_url = 'file://' + file_path - element_path = os.path.join(project_dir, 'elements') - element_name = 'key_check.bst' + file_path = os.path.join(project_dir, "files") + file_url = "file://" + file_path + element_path = os.path.join(project_dir, "elements") + element_name = "key_check.bst" element = { - 'kind': 'import', - 'sources': [ + "kind": "import", + "sources": [ { - 'kind': 'remote', - 'url': os.path.join(file_url, 'bin-files', 'usr', 'bin', 'hello'), - 'directory': 'usr/bin' - }, { - 'kind': 'remote', - 'url': os.path.join(file_url, 'dev-files', 'usr', 'include', 'pony.h'), - 'directory': 'usr/include' - }, { - 'kind': 'patch', - 'path': 'files/hello-patch.diff' - } - ] + "kind": "remote", + "url": os.path.join(file_url, "bin-files", "usr", "bin", "hello"), + "directory": "usr/bin", + }, + { + "kind": "remote", + "url": os.path.join(file_url, "dev-files", "usr", "include", "pony.h"), + "directory": "usr/include", + }, + {"kind": "patch", "path": "files/hello-patch.diff"}, + ], } _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) @@ -105,11 +104,11 @@ def test_source_cache_key(cli, datafiles): res.assert_success() # Should have one source ref - patch_protos = os.path.join(project_dir, 'cache', 'source_protos', 'patch') + patch_protos = os.path.join(project_dir, "cache", "source_protos", "patch") assert len(os.listdir(patch_protos)) == 1 # modify hello-patch file and check tracking updates refs - with open(os.path.join(file_path, 'dev-files', 'usr', 'include', 'pony.h'), 'a') as f: + with open(os.path.join(file_path, "dev-files", "usr", "include", "pony.h"), "a") as f: f.write("\nappending nonsense") res = cli.run(project=project_dir, args=["source", "track", element_name]) diff --git a/tests/sourcecache/capabilities.py b/tests/sourcecache/capabilities.py index 18dfddbab..9d41eba11 100644 --- a/tests/sourcecache/capabilities.py +++ b/tests/sourcecache/capabilities.py @@ -13,10 +13,7 @@ from tests.testutils import dummy_context from tests.testutils.artifactshare import create_dummy_artifact_share -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - "project", -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",) @pytest.mark.datafiles(DATA_DIR) @@ -26,17 +23,9 @@ def test_artifact_cache_with_missing_capabilities_is_skipped(cli, tmpdir, datafi # Set up an artifact cache. with create_dummy_artifact_share() as share: # Configure artifact share - cache_dir = os.path.join(str(tmpdir), 'cache') - user_config_file = str(tmpdir.join('buildstream.conf')) - user_config = { - 'scheduler': { - 'pushers': 1 - }, - 'source-caches': { - 'url': share.repo, - }, - 'cachedir': cache_dir - } + cache_dir = os.path.join(str(tmpdir), "cache") + user_config_file = str(tmpdir.join("buildstream.conf")) + user_config = {"scheduler": {"pushers": 1}, "source-caches": {"url": share.repo,}, "cachedir": cache_dir} _yaml.roundtrip_dump(user_config, file=user_config_file) with dummy_context(config=user_config_file) as context: @@ -50,5 +39,6 @@ def test_artifact_cache_with_missing_capabilities_is_skipped(cli, tmpdir, datafi # Manually setup the CAS remote sourcecache.setup_remotes(use_config=True) - assert not sourcecache.has_fetch_remotes(), \ - "System didn't realize the source cache didn't support BuildStream" + assert ( + not sourcecache.has_fetch_remotes() + ), "System didn't realize the source cache didn't support BuildStream" diff --git a/tests/sourcecache/config.py b/tests/sourcecache/config.py index 2ab11e9f9..9233e9f44 100644 --- a/tests/sourcecache/config.py +++ b/tests/sourcecache/config.py @@ -36,27 +36,19 @@ DATA_DIR = os.path.dirname(os.path.realpath(__file__)) # without specifying its counterpart, we get a comprehensive LoadError # instead of an unhandled exception. @pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize('config_key, config_value', [ - ('client-cert', 'client.crt'), - ('client-key', 'client.key') -]) +@pytest.mark.parametrize("config_key, config_value", [("client-cert", "client.crt"), ("client-key", "client.key")]) def test_missing_certs(cli, datafiles, config_key, config_value): - project = os.path.join(datafiles.dirname, datafiles.basename, 'missing-certs') + project = os.path.join(datafiles.dirname, datafiles.basename, "missing-certs") project_conf = { - 'name': 'test', - - 'source-caches': { - 'url': 'https://cache.example.com:12345', - 'push': 'true', - config_key: config_value - } + "name": "test", + "source-caches": {"url": "https://cache.example.com:12345", "push": "true", config_key: config_value}, } - project_conf_file = os.path.join(project, 'project.conf') + project_conf_file = os.path.join(project, "project.conf") _yaml.roundtrip_dump(project_conf, project_conf_file) # Use `pull` here to ensure we try to initialize the remotes, triggering the error # # This does not happen for a simple `bst show`. - result = cli.run(project=project, args=['source', 'fetch', 'element.bst']) + result = cli.run(project=project, args=["source", "fetch", "element.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) diff --git a/tests/sourcecache/fetch.py b/tests/sourcecache/fetch.py index a5863b867..0c347ebbf 100644 --- a/tests/sourcecache/fetch.py +++ b/tests/sourcecache/fetch.py @@ -36,22 +36,19 @@ DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") def move_local_cas_to_remote_source_share(local, remote): - shutil.rmtree(os.path.join(remote, 'repo', 'cas')) - shutil.move(os.path.join(local, 'source_protos'), os.path.join(remote, 'repo')) - shutil.move(os.path.join(local, 'cas'), os.path.join(remote, 'repo')) - shutil.rmtree(os.path.join(local, 'sources')) - shutil.rmtree(os.path.join(local, 'artifacts')) + shutil.rmtree(os.path.join(remote, "repo", "cas")) + shutil.move(os.path.join(local, "source_protos"), os.path.join(remote, "repo")) + shutil.move(os.path.join(local, "cas"), os.path.join(remote, "repo")) + shutil.rmtree(os.path.join(local, "sources")) + shutil.rmtree(os.path.join(local, "artifacts")) def create_test_element(tmpdir, project_dir): - repo = create_repo('git', str(tmpdir)) - ref = repo.create(os.path.join(project_dir, 'files')) - element_path = os.path.join(project_dir, 'elements') - element_name = 'fetch.bst' - element = { - 'kind': 'import', - 'sources': [repo.source_config(ref=ref)] - } + repo = create_repo("git", str(tmpdir)) + ref = repo.create(os.path.join(project_dir, "files")) + element_path = os.path.join(project_dir, "elements") + element_name = "fetch.bst" + element = {"kind": "import", "sources": [repo.source_config(ref=ref)]} _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) return element_name, repo, ref @@ -59,15 +56,11 @@ def create_test_element(tmpdir, project_dir): @contextmanager def context_with_source_cache(cli, cache, share, tmpdir): - user_config_file = str(tmpdir.join('buildstream.conf')) + user_config_file = str(tmpdir.join("buildstream.conf")) user_config = { - 'scheduler': { - 'pushers': 1 - }, - 'source-caches': { - 'url': share.repo, - }, - 'cachedir': cache, + "scheduler": {"pushers": 1}, + "source-caches": {"url": share.repo,}, + "cachedir": cache, } _yaml.roundtrip_dump(user_config, file=user_config_file) cli.configure(user_config) @@ -80,10 +73,10 @@ def context_with_source_cache(cli, cache, share, tmpdir): def test_source_fetch(cli, tmpdir, datafiles): project_dir = str(datafiles) element_name, _repo, _ref = create_test_element(tmpdir, project_dir) - cache_dir = os.path.join(str(tmpdir), 'cache') + cache_dir = os.path.join(str(tmpdir), "cache") # use artifact cache for sources for now, they should work the same - with create_artifact_share(os.path.join(str(tmpdir), 'sourceshare')) as share: + with create_artifact_share(os.path.join(str(tmpdir), "sourceshare")) as share: with context_with_source_cache(cli, cache_dir, share, tmpdir) as context: project = Project(project_dir, context) project.ensure_fully_loaded() @@ -96,10 +89,10 @@ def test_source_fetch(cli, tmpdir, datafiles): assert not cas.contains(source._get_source_name()) # Just check that we sensibly fetch and build the element - res = cli.run(project=project_dir, args=['build', element_name]) + res = cli.run(project=project_dir, args=["build", element_name]) res.assert_success() - assert os.listdir(os.path.join(str(tmpdir), 'cache', 'sources', 'git')) != [] + assert os.listdir(os.path.join(str(tmpdir), "cache", "sources", "git")) != [] # get root digest of source sourcecache = context.sourcecache @@ -111,26 +104,26 @@ def test_source_fetch(cli, tmpdir, datafiles): assert share.has_object(digest) state = cli.get_element_state(project_dir, element_name) - assert state == 'fetch needed' + assert state == "fetch needed" # Now fetch the source and check - res = cli.run(project=project_dir, args=['source', 'fetch', element_name]) + res = cli.run(project=project_dir, args=["source", "fetch", element_name]) res.assert_success() assert "Pulled source" in res.stderr # check that we have the source in the cas now and it's not fetched assert element._source_cached() - assert os.listdir(os.path.join(str(tmpdir), 'cache', 'sources', 'git')) == [] + assert os.listdir(os.path.join(str(tmpdir), "cache", "sources", "git")) == [] @pytest.mark.datafiles(DATA_DIR) def test_fetch_fallback(cli, tmpdir, datafiles): project_dir = str(datafiles) element_name, repo, ref = create_test_element(tmpdir, project_dir) - cache_dir = os.path.join(str(tmpdir), 'cache') + cache_dir = os.path.join(str(tmpdir), "cache") # use artifact cache for sources for now, they should work the same - with create_artifact_share(os.path.join(str(tmpdir), 'sourceshare')) as share: + with create_artifact_share(os.path.join(str(tmpdir), "sourceshare")) as share: with context_with_source_cache(cli, cache_dir, share, tmpdir) as context: project = Project(project_dir, context) project.ensure_fully_loaded() @@ -141,16 +134,16 @@ def test_fetch_fallback(cli, tmpdir, datafiles): cas = context.get_cascache() assert not cas.contains(source._get_source_name()) - assert not os.path.exists(os.path.join(cache_dir, 'sources')) + assert not os.path.exists(os.path.join(cache_dir, "sources")) # Now check if it falls back to the source fetch method. - res = cli.run(project=project_dir, args=['source', 'fetch', element_name]) + res = cli.run(project=project_dir, args=["source", "fetch", element_name]) res.assert_success() brief_key = source._get_brief_display_key() - assert ("Remote source service ({}) does not have source {} cached" - .format(share.repo, brief_key)) in res.stderr - assert ("SUCCESS Fetching from {}" - .format(repo.source_config(ref=ref)['url'])) in res.stderr + assert ( + "Remote source service ({}) does not have source {} cached".format(share.repo, brief_key) + ) in res.stderr + assert ("SUCCESS Fetching from {}".format(repo.source_config(ref=ref)["url"])) in res.stderr # Check that the source in both in the source dir and the local CAS assert element._source_cached() @@ -160,9 +153,9 @@ def test_fetch_fallback(cli, tmpdir, datafiles): def test_pull_fail(cli, tmpdir, datafiles): project_dir = str(datafiles) element_name, repo, _ref = create_test_element(tmpdir, project_dir) - cache_dir = os.path.join(str(tmpdir), 'cache') + cache_dir = os.path.join(str(tmpdir), "cache") - with create_artifact_share(os.path.join(str(tmpdir), 'sourceshare')) as share: + with create_artifact_share(os.path.join(str(tmpdir), "sourceshare")) as share: with context_with_source_cache(cli, cache_dir, share, tmpdir) as context: project = Project(project_dir, context) project.ensure_fully_loaded() @@ -175,21 +168,25 @@ def test_pull_fail(cli, tmpdir, datafiles): shutil.rmtree(repo.repo) # Should fail in stream, with a plugin task causing the error - res = cli.run(project=project_dir, args=['build', element_name]) + res = cli.run(project=project_dir, args=["build", element_name]) res.assert_main_error(ErrorDomain.STREAM, None) res.assert_task_error(ErrorDomain.PLUGIN, None) - assert "Remote source service ({}) does not have source {} cached".format( - share.repo, source._get_brief_display_key()) in res.stderr + assert ( + "Remote source service ({}) does not have source {} cached".format( + share.repo, source._get_brief_display_key() + ) + in res.stderr + ) @pytest.mark.datafiles(DATA_DIR) def test_source_pull_partial_fallback_fetch(cli, tmpdir, datafiles): project_dir = str(datafiles) element_name, repo, ref = create_test_element(tmpdir, project_dir) - cache_dir = os.path.join(str(tmpdir), 'cache') + cache_dir = os.path.join(str(tmpdir), "cache") # use artifact cache for sources for now, they should work the same - with create_artifact_share(os.path.join(str(tmpdir), 'sourceshare')) as share: + with create_artifact_share(os.path.join(str(tmpdir), "sourceshare")) as share: with context_with_source_cache(cli, cache_dir, share, tmpdir) as context: project = Project(project_dir, context) project.ensure_fully_loaded() @@ -202,10 +199,10 @@ def test_source_pull_partial_fallback_fetch(cli, tmpdir, datafiles): assert not cas.contains(source._get_source_name()) # Just check that we sensibly fetch and build the element - res = cli.run(project=project_dir, args=['build', element_name]) + res = cli.run(project=project_dir, args=["build", element_name]) res.assert_success() - assert os.listdir(os.path.join(str(tmpdir), 'cache', 'sources', 'git')) != [] + assert os.listdir(os.path.join(str(tmpdir), "cache", "sources", "git")) != [] # get root digest of source sourcecache = context.sourcecache @@ -219,11 +216,10 @@ def test_source_pull_partial_fallback_fetch(cli, tmpdir, datafiles): assert not share.has_object(digest) state = cli.get_element_state(project_dir, element_name) - assert state == 'fetch needed' + assert state == "fetch needed" # Now fetch the source and check - res = cli.run(project=project_dir, args=['source', 'fetch', element_name]) + res = cli.run(project=project_dir, args=["source", "fetch", element_name]) res.assert_success() - assert ("SUCCESS Fetching from {}" - .format(repo.source_config(ref=ref)['url'])) in res.stderr + assert ("SUCCESS Fetching from {}".format(repo.source_config(ref=ref)["url"])) in res.stderr diff --git a/tests/sourcecache/project/plugins/elements/always_fail.py b/tests/sourcecache/project/plugins/elements/always_fail.py index 99ef0d7de..43dba5626 100644 --- a/tests/sourcecache/project/plugins/elements/always_fail.py +++ b/tests/sourcecache/project/plugins/elements/always_fail.py @@ -23,7 +23,6 @@ from buildstream.buildelement import BuildElement class AlwaysFail(BuildElement): - def assemble(self, sandbox): raise ElementError("Always fails") diff --git a/tests/sourcecache/push.py b/tests/sourcecache/push.py index 406aeba9f..719860425 100644 --- a/tests/sourcecache/push.py +++ b/tests/sourcecache/push.py @@ -47,6 +47,7 @@ def message_handler(message, is_silenced): @contextmanager def _configure_caches(tmpdir, *directories): with ExitStack() as stack: + def create_share(directory): return create_artifact_share(os.path.join(str(tmpdir), directory)) @@ -55,37 +56,27 @@ def _configure_caches(tmpdir, *directories): @pytest.mark.datafiles(DATA_DIR) def test_source_push_split(cli, tmpdir, datafiles): - cache_dir = os.path.join(str(tmpdir), 'cache') + cache_dir = os.path.join(str(tmpdir), "cache") project_dir = str(datafiles) - with _configure_caches(tmpdir, 'indexshare', 'storageshare') as (index, storage): - user_config_file = str(tmpdir.join('buildstream.conf')) + with _configure_caches(tmpdir, "indexshare", "storageshare") as (index, storage): + user_config_file = str(tmpdir.join("buildstream.conf")) user_config = { - 'scheduler': { - 'pushers': 1 - }, - 'source-caches': [{ - 'url': index.repo, - 'push': True, - 'type': 'index' - }, { - 'url': storage.repo, - 'push': True, - 'type': 'storage' - }], - 'cachedir': cache_dir + "scheduler": {"pushers": 1}, + "source-caches": [ + {"url": index.repo, "push": True, "type": "index"}, + {"url": storage.repo, "push": True, "type": "storage"}, + ], + "cachedir": cache_dir, } _yaml.roundtrip_dump(user_config, file=user_config_file) cli.configure(user_config) - repo = create_repo('git', str(tmpdir)) - ref = repo.create(os.path.join(project_dir, 'files')) - element_path = os.path.join(project_dir, 'elements') - element_name = 'push.bst' - element = { - 'kind': 'import', - 'sources': [repo.source_config(ref=ref)] - } + repo = create_repo("git", str(tmpdir)) + ref = repo.create(os.path.join(project_dir, "files")) + element_path = os.path.join(project_dir, "elements") + element_name = "push.bst" + element = {"kind": "import", "sources": [repo.source_config(ref=ref)]} _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) # get the source object @@ -93,7 +84,7 @@ def test_source_push_split(cli, tmpdir, datafiles): project = Project(project_dir, context) project.ensure_fully_loaded() - element = project.load_elements(['push.bst'])[0] + element = project.load_elements(["push.bst"])[0] assert not element._source_cached() source = list(element.sources())[0] @@ -103,7 +94,7 @@ def test_source_push_split(cli, tmpdir, datafiles): # build the element, this should fetch and then push the source to the # remote - res = cli.run(project=project_dir, args=['build', 'push.bst']) + res = cli.run(project=project_dir, args=["build", "push.bst"]) res.assert_success() assert "Pushed source" in res.stderr @@ -118,32 +109,24 @@ def test_source_push_split(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_source_push(cli, tmpdir, datafiles): - cache_dir = os.path.join(str(tmpdir), 'cache') + cache_dir = os.path.join(str(tmpdir), "cache") project_dir = str(datafiles) - with create_artifact_share(os.path.join(str(tmpdir), 'sourceshare')) as share: - user_config_file = str(tmpdir.join('buildstream.conf')) + with create_artifact_share(os.path.join(str(tmpdir), "sourceshare")) as share: + user_config_file = str(tmpdir.join("buildstream.conf")) user_config = { - 'scheduler': { - 'pushers': 1 - }, - 'source-caches': { - 'url': share.repo, - 'push': True, - }, - 'cachedir': cache_dir, + "scheduler": {"pushers": 1}, + "source-caches": {"url": share.repo, "push": True,}, + "cachedir": cache_dir, } _yaml.roundtrip_dump(user_config, file=user_config_file) cli.configure(user_config) - repo = create_repo('git', str(tmpdir)) - ref = repo.create(os.path.join(project_dir, 'files')) - element_path = os.path.join(project_dir, 'elements') - element_name = 'push.bst' - element = { - 'kind': 'import', - 'sources': [repo.source_config(ref=ref)] - } + repo = create_repo("git", str(tmpdir)) + ref = repo.create(os.path.join(project_dir, "files")) + element_path = os.path.join(project_dir, "elements") + element_name = "push.bst" + element = {"kind": "import", "sources": [repo.source_config(ref=ref)]} _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) # get the source object @@ -151,7 +134,7 @@ def test_source_push(cli, tmpdir, datafiles): project = Project(project_dir, context) project.ensure_fully_loaded() - element = project.load_elements(['push.bst'])[0] + element = project.load_elements(["push.bst"])[0] assert not element._source_cached() source = list(element.sources())[0] @@ -161,7 +144,7 @@ def test_source_push(cli, tmpdir, datafiles): # build the element, this should fetch and then push the source to the # remote - res = cli.run(project=project_dir, args=['build', 'push.bst']) + res = cli.run(project=project_dir, args=["build", "push.bst"]) res.assert_success() assert "Pushed source" in res.stderr @@ -177,35 +160,27 @@ def test_source_push(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_push_pull(cli, datafiles, tmpdir): project_dir = str(datafiles) - cache_dir = os.path.join(str(tmpdir), 'cache') + cache_dir = os.path.join(str(tmpdir), "cache") - with create_artifact_share(os.path.join(str(tmpdir), 'sourceshare')) as share: - user_config_file = str(tmpdir.join('buildstream.conf')) + with create_artifact_share(os.path.join(str(tmpdir), "sourceshare")) as share: + user_config_file = str(tmpdir.join("buildstream.conf")) user_config = { - 'scheduler': { - 'pushers': 1 - }, - 'source-caches': { - 'url': share.repo, - 'push': True, - }, - 'cachedir': cache_dir, + "scheduler": {"pushers": 1}, + "source-caches": {"url": share.repo, "push": True,}, + "cachedir": cache_dir, } _yaml.roundtrip_dump(user_config, file=user_config_file) cli.configure(user_config) # create repo to pull from - repo = create_repo('git', str(tmpdir)) - ref = repo.create(os.path.join(project_dir, 'files')) - element_path = os.path.join(project_dir, 'elements') - element_name = 'push.bst' - element = { - 'kind': 'import', - 'sources': [repo.source_config(ref=ref)] - } + repo = create_repo("git", str(tmpdir)) + ref = repo.create(os.path.join(project_dir, "files")) + element_path = os.path.join(project_dir, "elements") + element_name = "push.bst" + element = {"kind": "import", "sources": [repo.source_config(ref=ref)]} _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) - res = cli.run(project=project_dir, args=['build', 'push.bst']) + res = cli.run(project=project_dir, args=["build", "push.bst"]) res.assert_success() # remove local cache dir, and repo files and check it all works @@ -214,45 +189,37 @@ def test_push_pull(cli, datafiles, tmpdir): shutil.rmtree(repo.repo) # check it's pulls from the share - res = cli.run(project=project_dir, args=['build', 'push.bst']) + res = cli.run(project=project_dir, args=["build", "push.bst"]) res.assert_success() @pytest.mark.datafiles(DATA_DIR) def test_push_fail(cli, tmpdir, datafiles): project_dir = str(datafiles) - cache_dir = os.path.join(str(tmpdir), 'cache') + cache_dir = os.path.join(str(tmpdir), "cache") # set up config with remote that we'll take down - with create_artifact_share(os.path.join(str(tmpdir), 'sourceshare')) as share: + with create_artifact_share(os.path.join(str(tmpdir), "sourceshare")) as share: remote = share.repo - user_config_file = str(tmpdir.join('buildstream.conf')) + user_config_file = str(tmpdir.join("buildstream.conf")) user_config = { - 'scheduler': { - 'pushers': 1 - }, - 'source-caches': { - 'url': share.repo, - 'push': True, - }, - 'cachedir': cache_dir, + "scheduler": {"pushers": 1}, + "source-caches": {"url": share.repo, "push": True,}, + "cachedir": cache_dir, } _yaml.roundtrip_dump(user_config, file=user_config_file) cli.configure(user_config) # create repo to pull from - repo = create_repo('git', str(tmpdir)) - ref = repo.create(os.path.join(project_dir, 'files')) - element_path = os.path.join(project_dir, 'elements') - element_name = 'push.bst' - element = { - 'kind': 'import', - 'sources': [repo.source_config(ref=ref)] - } + repo = create_repo("git", str(tmpdir)) + ref = repo.create(os.path.join(project_dir, "files")) + element_path = os.path.join(project_dir, "elements") + element_name = "push.bst" + element = {"kind": "import", "sources": [repo.source_config(ref=ref)]} _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) # build and check that it fails to set up the remote - res = cli.run(project=project_dir, args=['build', 'push.bst']) + res = cli.run(project=project_dir, args=["build", "push.bst"]) res.assert_success() assert "Failed to initialize remote {}".format(remote) in res.stderr @@ -260,37 +227,29 @@ def test_push_fail(cli, tmpdir, datafiles): assert "Pushed" not in res.stderr -@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox') +@pytest.mark.xfail(HAVE_SANDBOX == "buildbox", reason="Not working with BuildBox") @pytest.mark.datafiles(DATA_DIR) def test_source_push_build_fail(cli, tmpdir, datafiles): project_dir = str(datafiles) - cache_dir = os.path.join(str(tmpdir), 'cache') + cache_dir = os.path.join(str(tmpdir), "cache") - with create_artifact_share(os.path.join(str(tmpdir), 'share')) as share: + with create_artifact_share(os.path.join(str(tmpdir), "share")) as share: user_config = { - 'scheduler': { - 'pushers': 1 - }, - 'source-caches': { - 'url': share.repo, - 'push': True, - }, - 'cachedir': cache_dir, + "scheduler": {"pushers": 1}, + "source-caches": {"url": share.repo, "push": True,}, + "cachedir": cache_dir, } cli.configure(user_config) - repo = create_repo('git', str(tmpdir)) - ref = repo.create(os.path.join(project_dir, 'files')) - element_path = os.path.join(project_dir, 'elements') + repo = create_repo("git", str(tmpdir)) + ref = repo.create(os.path.join(project_dir, "files")) + element_path = os.path.join(project_dir, "elements") - element_name = 'always-fail.bst' - element = { - 'kind': 'always_fail', - 'sources': [repo.source_config(ref=ref)] - } + element_name = "always-fail.bst" + element = {"kind": "always_fail", "sources": [repo.source_config(ref=ref)]} _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) - res = cli.run(project=project_dir, args=['build', 'always-fail.bst']) + res = cli.run(project=project_dir, args=["build", "always-fail.bst"]) res.assert_main_error(ErrorDomain.STREAM, None) res.assert_task_error(ErrorDomain.ELEMENT, None) diff --git a/tests/sourcecache/source-checkout.py b/tests/sourcecache/source-checkout.py index 4e3391c12..f1e0706a7 100644 --- a/tests/sourcecache/source-checkout.py +++ b/tests/sourcecache/source-checkout.py @@ -36,20 +36,20 @@ DATA_DIR = os.path.dirname(os.path.realpath(__file__)) @pytest.mark.datafiles(DATA_DIR) def test_source_checkout(tmpdir, datafiles, cli): - project_dir = os.path.join(str(tmpdir), 'project') - element_path = 'elements' - cache_dir = os.path.join(str(tmpdir), 'cache') - source_dir = os.path.join(cache_dir, 'sources') + project_dir = os.path.join(str(tmpdir), "project") + element_path = "elements" + cache_dir = os.path.join(str(tmpdir), "cache") + source_dir = os.path.join(cache_dir, "sources") - cli.configure({ - 'cachedir': cache_dir, - }) - target_dir = os.path.join(str(tmpdir), 'target') + cli.configure( + {"cachedir": cache_dir,} + ) + target_dir = os.path.join(str(tmpdir), "target") - repo = create_element_size('target.bst', project_dir, element_path, [], 100000) + repo = create_element_size("target.bst", project_dir, element_path, [], 100000) # check implicit fetching - res = cli.run(project=project_dir, args=['source', 'checkout', '--directory', target_dir, 'target.bst']) + res = cli.run(project=project_dir, args=["source", "checkout", "--directory", target_dir, "target.bst"]) res.assert_success() assert "Fetching from" in res.stderr @@ -59,15 +59,13 @@ def test_source_checkout(tmpdir, datafiles, cli): shutil.rmtree(target_dir) shutil.rmtree(source_dir) - res = cli.run(project=project_dir, - args=['source', 'checkout', '--directory', target_dir, 'target.bst']) + res = cli.run(project=project_dir, args=["source", "checkout", "--directory", target_dir, "target.bst"]) res.assert_success() assert "Fetching from" not in res.stderr # remove the CAS and check it doesn't work again shutil.rmtree(target_dir) - shutil.rmtree(os.path.join(cache_dir, 'cas')) + shutil.rmtree(os.path.join(cache_dir, "cas")) - res = cli.run(project=project_dir, - args=['source', 'checkout', '--directory', target_dir, 'target.bst']) + res = cli.run(project=project_dir, args=["source", "checkout", "--directory", target_dir, "target.bst"]) res.assert_task_error(ErrorDomain.PLUGIN, None) diff --git a/tests/sourcecache/staging.py b/tests/sourcecache/staging.py index 186a4bd9f..994adb32a 100644 --- a/tests/sourcecache/staging.py +++ b/tests/sourcecache/staging.py @@ -45,12 +45,10 @@ def relative_walk(rootdir): @pytest.mark.datafiles(DATA_DIR) def test_source_staged(tmpdir, cli, datafiles): - project_dir = os.path.join(datafiles.dirname, datafiles.basename, 'project') - cachedir = os.path.join(str(tmpdir), 'cache') + project_dir = os.path.join(datafiles.dirname, datafiles.basename, "project") + cachedir = os.path.join(str(tmpdir), "cache") - cli.configure({ - 'cachedir': cachedir - }) + cli.configure({"cachedir": cachedir}) res = cli.run(project=project_dir, args=["build", "import-bin.bst"]) res.assert_success() @@ -83,12 +81,10 @@ def test_source_staged(tmpdir, cli, datafiles): # Check sources are staged during a fetch @pytest.mark.datafiles(DATA_DIR) def test_source_fetch(tmpdir, cli, datafiles): - project_dir = os.path.join(datafiles.dirname, datafiles.basename, 'project') - cachedir = os.path.join(str(tmpdir), 'cache') + project_dir = os.path.join(datafiles.dirname, datafiles.basename, "project") + cachedir = os.path.join(str(tmpdir), "cache") - cli.configure({ - 'cachedir': cachedir - }) + cli.configure({"cachedir": cachedir}) res = cli.run(project=project_dir, args=["source", "fetch", "import-dev.bst"]) res.assert_success() @@ -118,17 +114,15 @@ def test_source_fetch(tmpdir, cli, datafiles): # Check that with sources only in the CAS build successfully completes @pytest.mark.datafiles(DATA_DIR) def test_staged_source_build(tmpdir, datafiles, cli): - project_dir = os.path.join(datafiles.dirname, datafiles.basename, 'project') - cachedir = os.path.join(str(tmpdir), 'cache') - element_path = 'elements' - source_protos = os.path.join(str(tmpdir), 'cache', 'source_protos') - source_dir = os.path.join(str(tmpdir), 'cache', 'sources') + project_dir = os.path.join(datafiles.dirname, datafiles.basename, "project") + cachedir = os.path.join(str(tmpdir), "cache") + element_path = "elements" + source_protos = os.path.join(str(tmpdir), "cache", "source_protos") + source_dir = os.path.join(str(tmpdir), "cache", "sources") - cli.configure({ - 'cachedir': cachedir - }) + cli.configure({"cachedir": cachedir}) - create_element_size('target.bst', project_dir, element_path, [], 10000) + create_element_size("target.bst", project_dir, element_path, [], 10000) with dummy_context() as context: context.cachedir = cachedir @@ -140,23 +134,23 @@ def test_staged_source_build(tmpdir, datafiles, cli): # check consistency of the source assert not element._source_cached() - res = cli.run(project=project_dir, args=['build', 'target.bst']) + res = cli.run(project=project_dir, args=["build", "target.bst"]) res.assert_success() # delete artifacts check state is buildable - cli.remove_artifact_from_cache(project_dir, 'target.bst') - states = cli.get_element_states(project_dir, ['target.bst']) - assert states['target.bst'] == 'buildable' + cli.remove_artifact_from_cache(project_dir, "target.bst") + states = cli.get_element_states(project_dir, ["target.bst"]) + assert states["target.bst"] == "buildable" # delete source dir and check that state is still buildable shutil.rmtree(source_dir) - states = cli.get_element_states(project_dir, ['target.bst']) - assert states['target.bst'] == 'buildable' + states = cli.get_element_states(project_dir, ["target.bst"]) + assert states["target.bst"] == "buildable" # build and check that no fetching was done. - res = cli.run(project=project_dir, args=['build', 'target.bst']) + res = cli.run(project=project_dir, args=["build", "target.bst"]) res.assert_success() - assert 'Fetching from' not in res.stderr + assert "Fetching from" not in res.stderr # assert the source directory is still empty (though there may be # directories from staging etc.) @@ -167,11 +161,11 @@ def test_staged_source_build(tmpdir, datafiles, cli): # Now remove the source refs and check the state shutil.rmtree(source_protos) - cli.remove_artifact_from_cache(project_dir, 'target.bst') - states = cli.get_element_states(project_dir, ['target.bst']) - assert states['target.bst'] == 'fetch needed' + cli.remove_artifact_from_cache(project_dir, "target.bst") + states = cli.get_element_states(project_dir, ["target.bst"]) + assert states["target.bst"] == "fetch needed" # Check that it now fetches from when building the target - res = cli.run(project=project_dir, args=['build', 'target.bst']) + res = cli.run(project=project_dir, args=["build", "target.bst"]) res.assert_success() - assert 'Fetching from' in res.stderr + assert "Fetching from" in res.stderr diff --git a/tests/sourcecache/workspace.py b/tests/sourcecache/workspace.py index 22316c4fe..bb1ea505a 100644 --- a/tests/sourcecache/workspace.py +++ b/tests/sourcecache/workspace.py @@ -39,66 +39,55 @@ DATA_DIR = os.path.dirname(os.path.realpath(__file__)) # for opening a workspace @pytest.mark.datafiles(DATA_DIR) def test_workspace_source_fetch(tmpdir, datafiles, cli): - project_dir = os.path.join(str(tmpdir), 'project') - element_path = 'elements' - source_dir = os.path.join(str(tmpdir), 'cache', 'sources') - workspace = os.path.join(cli.directory, 'workspace') + project_dir = os.path.join(str(tmpdir), "project") + element_path = "elements" + source_dir = os.path.join(str(tmpdir), "cache", "sources") + workspace = os.path.join(cli.directory, "workspace") - cli.configure({ - 'cachedir': os.path.join(str(tmpdir), 'cache') - }) + cli.configure({"cachedir": os.path.join(str(tmpdir), "cache")}) - create_element_size('target.bst', project_dir, element_path, [], 10000) - res = cli.run(project=project_dir, args=['build', 'target.bst']) + create_element_size("target.bst", project_dir, element_path, [], 10000) + res = cli.run(project=project_dir, args=["build", "target.bst"]) res.assert_success() - assert 'Fetching from' in res.stderr + assert "Fetching from" in res.stderr # remove the original sources shutil.rmtree(source_dir) # Open a workspace and check that fetches the original sources - res = cli.run(project=project_dir, - args=['workspace', 'open', 'target.bst', '--directory', workspace]) + res = cli.run(project=project_dir, args=["workspace", "open", "target.bst", "--directory", workspace]) res.assert_success() - assert 'Fetching from' in res.stderr + assert "Fetching from" in res.stderr assert os.listdir(workspace) != [] @pytest.mark.datafiles(DATA_DIR) def test_workspace_open_no_source_push(tmpdir, datafiles, cli): - project_dir = os.path.join(str(tmpdir), 'project') - element_path = 'elements' - cache_dir = os.path.join(str(tmpdir), 'cache') - share_dir = os.path.join(str(tmpdir), 'share') - workspace = os.path.join(cli.directory, 'workspace') + project_dir = os.path.join(str(tmpdir), "project") + element_path = "elements" + cache_dir = os.path.join(str(tmpdir), "cache") + share_dir = os.path.join(str(tmpdir), "share") + workspace = os.path.join(cli.directory, "workspace") with create_artifact_share(share_dir) as share: - cli.configure({ - 'cachedir': cache_dir, - 'scheduler': { - 'pushers': 1 - }, - 'source-caches': { - 'url': share.repo, - 'push': True, - }, - }) + cli.configure( + {"cachedir": cache_dir, "scheduler": {"pushers": 1}, "source-caches": {"url": share.repo, "push": True,},} + ) # Fetch as in previous test and check it pushes the source - create_element_size('target.bst', project_dir, element_path, [], 10000) - res = cli.run(project=project_dir, args=['build', 'target.bst']) + create_element_size("target.bst", project_dir, element_path, [], 10000) + res = cli.run(project=project_dir, args=["build", "target.bst"]) res.assert_success() - assert 'Fetching from' in res.stderr - assert 'Pushed source' in res.stderr + assert "Fetching from" in res.stderr + assert "Pushed source" in res.stderr # clear the cas and open a workspace - shutil.rmtree(os.path.join(cache_dir, 'cas')) - res = cli.run(project=project_dir, - args=['workspace', 'open', 'target.bst', '--directory', workspace]) + shutil.rmtree(os.path.join(cache_dir, "cas")) + res = cli.run(project=project_dir, args=["workspace", "open", "target.bst", "--directory", workspace]) res.assert_success() # Check that this time it does not push the sources - res = cli.run(project=project_dir, args=['build', 'target.bst']) + res = cli.run(project=project_dir, args=["build", "target.bst"]) res.assert_success() assert "Pushed source" not in res.stderr diff --git a/tests/sources/bzr.py b/tests/sources/bzr.py index c6e78f8c1..ca727c793 100644 --- a/tests/sources/bzr.py +++ b/tests/sources/bzr.py @@ -10,40 +10,32 @@ from buildstream.testing import cli # pylint: disable=unused-import from buildstream.testing import create_repo from buildstream.testing._utils.site import HAVE_BZR -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - 'bzr' -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "bzr") @pytest.mark.skipif(HAVE_BZR is False, reason="bzr is not available") @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_fetch_checkout(cli, tmpdir, datafiles): project = str(datafiles) - checkoutdir = os.path.join(str(tmpdir), 'checkout') + checkoutdir = os.path.join(str(tmpdir), "checkout") - repo = create_repo('bzr', str(tmpdir)) - ref = repo.create(os.path.join(project, 'basic')) + repo = create_repo("bzr", str(tmpdir)) + ref = repo.create(os.path.join(project, "basic")) # Write out our test target - element = { - 'kind': 'import', - 'sources': [ - repo.source_config(ref=ref) - ] - } - _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) + element = {"kind": "import", "sources": [repo.source_config(ref=ref)]} + _yaml.roundtrip_dump(element, os.path.join(project, "target.bst")) # Fetch, build, checkout - result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) assert result.exit_code == 0 - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) assert result.exit_code == 0 - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) assert result.exit_code == 0 # Assert we checked out the file as it was commited - with open(os.path.join(checkoutdir, 'test')) as f: + with open(os.path.join(checkoutdir, "test")) as f: text = f.read() - assert text == 'test\n' + assert text == "test\n" diff --git a/tests/sources/deb.py b/tests/sources/deb.py index e536e522a..96060a1a4 100644 --- a/tests/sources/deb.py +++ b/tests/sources/deb.py @@ -12,22 +12,14 @@ from buildstream.testing import cli # pylint: disable=unused-import from buildstream.testing._utils.site import HAVE_ARPY from . import list_dir_contents -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - 'deb', -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "deb",) deb_name = "a_deb.deb" def generate_project(project_dir, tmpdir): project_file = os.path.join(project_dir, "project.conf") - _yaml.roundtrip_dump({ - 'name': 'foo', - 'aliases': { - 'tmpdir': "file:///" + str(tmpdir) - } - }, project_file) + _yaml.roundtrip_dump({"name": "foo", "aliases": {"tmpdir": "file:///" + str(tmpdir)}}, project_file) def _copy_deb(start_location, tmpdir): @@ -38,31 +30,29 @@ def _copy_deb(start_location, tmpdir): # Test that without ref, consistency is set appropriately. @pytest.mark.skipif(HAVE_ARPY is False, reason="arpy is not available") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-ref')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "no-ref")) def test_no_ref(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, tmpdir) - assert cli.get_element_state(project, 'target.bst') == 'no reference' + assert cli.get_element_state(project, "target.bst") == "no reference" # Test that when I fetch a nonexistent URL, errors are handled gracefully and a retry is performed. @pytest.mark.skipif(HAVE_ARPY is False, reason="arpy is not available") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) def test_fetch_bad_url(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, tmpdir) # Try to fetch it - result = cli.run(project=project, args=[ - 'source', 'fetch', 'target.bst' - ]) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) assert "FAILURE Try #" in result.stderr result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.SOURCE, None) @pytest.mark.skipif(HAVE_ARPY is False, reason="arpy is not available") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) def test_fetch_bad_ref(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, tmpdir) @@ -71,16 +61,14 @@ def test_fetch_bad_ref(cli, tmpdir, datafiles): _copy_deb(DATA_DIR, tmpdir) # Try to fetch it - result = cli.run(project=project, args=[ - 'source', 'fetch', 'target.bst' - ]) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.SOURCE, None) # Test that when tracking with a ref set, there is a warning @pytest.mark.skipif(HAVE_ARPY is False, reason="arpy is not available") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) def test_track_warning(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, tmpdir) @@ -89,16 +77,14 @@ def test_track_warning(cli, tmpdir, datafiles): _copy_deb(DATA_DIR, tmpdir) # Track it - result = cli.run(project=project, args=[ - 'source', 'track', 'target.bst' - ]) + result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() assert "Potential man-in-the-middle attack!" in result.stderr # Test that a staged checkout matches what was tarred up, with the default first subdir @pytest.mark.skipif(HAVE_ARPY is False, reason="arpy is not available") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) def test_stage_default_basedir(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, tmpdir) @@ -108,13 +94,13 @@ def test_stage_default_basedir(cli, tmpdir, datafiles): _copy_deb(DATA_DIR, tmpdir) # Track, fetch, build, checkout - result = cli.run(project=project, args=['source', 'track', 'target.bst']) + result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the content of the first directory is checked out (base-dir: '') @@ -126,7 +112,7 @@ def test_stage_default_basedir(cli, tmpdir, datafiles): # Test that a staged checkout matches what was tarred up, with an empty base-dir @pytest.mark.skipif(HAVE_ARPY is False, reason="arpy is not available") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-basedir')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "no-basedir")) def test_stage_no_basedir(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, tmpdir) @@ -136,13 +122,13 @@ def test_stage_no_basedir(cli, tmpdir, datafiles): _copy_deb(DATA_DIR, tmpdir) # Track, fetch, build, checkout - result = cli.run(project=project, args=['source', 'track', 'target.bst']) + result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the full content of the tarball is checked out (base-dir: '') @@ -154,7 +140,7 @@ def test_stage_no_basedir(cli, tmpdir, datafiles): # Test that a staged checkout matches what was tarred up, with an explicit basedir @pytest.mark.skipif(HAVE_ARPY is False, reason="arpy is not available") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'explicit-basedir')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "explicit-basedir")) def test_stage_explicit_basedir(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, tmpdir) @@ -164,13 +150,13 @@ def test_stage_explicit_basedir(cli, tmpdir, datafiles): _copy_deb(DATA_DIR, tmpdir) # Track, fetch, build, checkout - result = cli.run(project=project, args=['source', 'track', 'target.bst']) + result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the content of the first directory is checked out (base-dir: '') diff --git a/tests/sources/git.py b/tests/sources/git.py index 245f90131..fbb1394ec 100644 --- a/tests/sources/git.py +++ b/tests/sources/git.py @@ -36,354 +36,304 @@ from buildstream.testing import cli # pylint: disable=unused-import from buildstream.testing import create_repo from buildstream.testing._utils.site import HAVE_GIT, HAVE_OLD_GIT -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - 'git', -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "git",) @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "template")) def test_fetch_bad_ref(cli, tmpdir, datafiles): project = str(datafiles) # Create the repo from 'repofiles' subdir - repo = create_repo('git', str(tmpdir)) - repo.create(os.path.join(project, 'repofiles')) + repo = create_repo("git", str(tmpdir)) + repo.create(os.path.join(project, "repofiles")) # Write out our test target with a bad ref - element = { - 'kind': 'import', - 'sources': [ - repo.source_config(ref='5') - ] - } - _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) + element = {"kind": "import", "sources": [repo.source_config(ref="5")]} + _yaml.roundtrip_dump(element, os.path.join(project, "target.bst")) # Assert that fetch raises an error here - result = cli.run(project=project, args=[ - 'source', 'fetch', 'target.bst' - ]) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.SOURCE, None) @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "template")) def test_submodule_fetch_checkout(cli, tmpdir, datafiles): project = str(datafiles) checkoutdir = os.path.join(str(tmpdir), "checkout") # Create the submodule first from the 'subrepofiles' subdir - subrepo = create_repo('git', str(tmpdir), 'subrepo') - subrepo.create(os.path.join(project, 'subrepofiles')) + subrepo = create_repo("git", str(tmpdir), "subrepo") + subrepo.create(os.path.join(project, "subrepofiles")) # Create the repo from 'repofiles' subdir - repo = create_repo('git', str(tmpdir)) - repo.create(os.path.join(project, 'repofiles')) + repo = create_repo("git", str(tmpdir)) + repo.create(os.path.join(project, "repofiles")) # Add a submodule pointing to the one we created - ref = repo.add_submodule('subdir', 'file://' + subrepo.repo) + ref = repo.add_submodule("subdir", "file://" + subrepo.repo) # Write out our test target - element = { - 'kind': 'import', - 'sources': [ - repo.source_config(ref=ref) - ] - } - _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) + element = {"kind": "import", "sources": [repo.source_config(ref=ref)]} + _yaml.roundtrip_dump(element, os.path.join(project, "target.bst")) # Fetch, build, checkout - result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Assert we checked out both files at their expected location - assert os.path.exists(os.path.join(checkoutdir, 'file.txt')) - assert os.path.exists(os.path.join(checkoutdir, 'subdir', 'ponyfile.txt')) + assert os.path.exists(os.path.join(checkoutdir, "file.txt")) + assert os.path.exists(os.path.join(checkoutdir, "subdir", "ponyfile.txt")) @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "template")) def test_submodule_fetch_source_enable_explicit(cli, tmpdir, datafiles): project = str(datafiles) checkoutdir = os.path.join(str(tmpdir), "checkout") # Create the submodule first from the 'subrepofiles' subdir - subrepo = create_repo('git', str(tmpdir), 'subrepo') - subrepo.create(os.path.join(project, 'subrepofiles')) + subrepo = create_repo("git", str(tmpdir), "subrepo") + subrepo.create(os.path.join(project, "subrepofiles")) # Create the repo from 'repofiles' subdir - repo = create_repo('git', str(tmpdir)) - repo.create(os.path.join(project, 'repofiles')) + repo = create_repo("git", str(tmpdir)) + repo.create(os.path.join(project, "repofiles")) # Add a submodule pointing to the one we created - ref = repo.add_submodule('subdir', 'file://' + subrepo.repo) + ref = repo.add_submodule("subdir", "file://" + subrepo.repo) # Write out our test target - element = { - 'kind': 'import', - 'sources': [ - repo.source_config_extra(ref=ref, checkout_submodules=True) - ] - } - _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) + element = {"kind": "import", "sources": [repo.source_config_extra(ref=ref, checkout_submodules=True)]} + _yaml.roundtrip_dump(element, os.path.join(project, "target.bst")) # Fetch, build, checkout - result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Assert we checked out both files at their expected location - assert os.path.exists(os.path.join(checkoutdir, 'file.txt')) - assert os.path.exists(os.path.join(checkoutdir, 'subdir', 'ponyfile.txt')) + assert os.path.exists(os.path.join(checkoutdir, "file.txt")) + assert os.path.exists(os.path.join(checkoutdir, "subdir", "ponyfile.txt")) @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "template")) def test_submodule_fetch_source_disable(cli, tmpdir, datafiles): project = str(datafiles) checkoutdir = os.path.join(str(tmpdir), "checkout") # Create the submodule first from the 'subrepofiles' subdir - subrepo = create_repo('git', str(tmpdir), 'subrepo') - subrepo.create(os.path.join(project, 'subrepofiles')) + subrepo = create_repo("git", str(tmpdir), "subrepo") + subrepo.create(os.path.join(project, "subrepofiles")) # Create the repo from 'repofiles' subdir - repo = create_repo('git', str(tmpdir)) - repo.create(os.path.join(project, 'repofiles')) + repo = create_repo("git", str(tmpdir)) + repo.create(os.path.join(project, "repofiles")) # Add a submodule pointing to the one we created - ref = repo.add_submodule('subdir', 'file://' + subrepo.repo) + ref = repo.add_submodule("subdir", "file://" + subrepo.repo) # Write out our test target - element = { - 'kind': 'import', - 'sources': [ - repo.source_config_extra(ref=ref, checkout_submodules=False) - ] - } - _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) + element = {"kind": "import", "sources": [repo.source_config_extra(ref=ref, checkout_submodules=False)]} + _yaml.roundtrip_dump(element, os.path.join(project, "target.bst")) # Fetch, build, checkout - result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Assert we checked out both files at their expected location - assert os.path.exists(os.path.join(checkoutdir, 'file.txt')) - assert not os.path.exists(os.path.join(checkoutdir, 'subdir', 'ponyfile.txt')) + assert os.path.exists(os.path.join(checkoutdir, "file.txt")) + assert not os.path.exists(os.path.join(checkoutdir, "subdir", "ponyfile.txt")) @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "template")) def test_submodule_fetch_submodule_does_override(cli, tmpdir, datafiles): project = str(datafiles) checkoutdir = os.path.join(str(tmpdir), "checkout") # Create the submodule first from the 'subrepofiles' subdir - subrepo = create_repo('git', str(tmpdir), 'subrepo') - subrepo.create(os.path.join(project, 'subrepofiles')) + subrepo = create_repo("git", str(tmpdir), "subrepo") + subrepo.create(os.path.join(project, "subrepofiles")) # Create the repo from 'repofiles' subdir - repo = create_repo('git', str(tmpdir)) - repo.create(os.path.join(project, 'repofiles')) + repo = create_repo("git", str(tmpdir)) + repo.create(os.path.join(project, "repofiles")) # Add a submodule pointing to the one we created - ref = repo.add_submodule('subdir', 'file://' + subrepo.repo, checkout=True) + ref = repo.add_submodule("subdir", "file://" + subrepo.repo, checkout=True) # Write out our test target - element = { - 'kind': 'import', - 'sources': [ - repo.source_config_extra(ref=ref, checkout_submodules=False) - ] - } - _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) + element = {"kind": "import", "sources": [repo.source_config_extra(ref=ref, checkout_submodules=False)]} + _yaml.roundtrip_dump(element, os.path.join(project, "target.bst")) # Fetch, build, checkout - result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Assert we checked out both files at their expected location - assert os.path.exists(os.path.join(checkoutdir, 'file.txt')) - assert os.path.exists(os.path.join(checkoutdir, 'subdir', 'ponyfile.txt')) + assert os.path.exists(os.path.join(checkoutdir, "file.txt")) + assert os.path.exists(os.path.join(checkoutdir, "subdir", "ponyfile.txt")) @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "template")) def test_submodule_fetch_submodule_individual_checkout(cli, tmpdir, datafiles): project = str(datafiles) checkoutdir = os.path.join(str(tmpdir), "checkout") # Create the submodule first from the 'subrepofiles' subdir - subrepo = create_repo('git', str(tmpdir), 'subrepo') - subrepo.create(os.path.join(project, 'subrepofiles')) + subrepo = create_repo("git", str(tmpdir), "subrepo") + subrepo.create(os.path.join(project, "subrepofiles")) # Create another submodule from the 'othersubrepofiles' subdir - other_subrepo = create_repo('git', str(tmpdir), 'othersubrepo') - other_subrepo.create(os.path.join(project, 'othersubrepofiles')) + other_subrepo = create_repo("git", str(tmpdir), "othersubrepo") + other_subrepo.create(os.path.join(project, "othersubrepofiles")) # Create the repo from 'repofiles' subdir - repo = create_repo('git', str(tmpdir)) - repo.create(os.path.join(project, 'repofiles')) + repo = create_repo("git", str(tmpdir)) + repo.create(os.path.join(project, "repofiles")) # Add a submodule pointing to the one we created - repo.add_submodule('subdir', 'file://' + subrepo.repo, checkout=False) - ref = repo.add_submodule('othersubdir', 'file://' + other_subrepo.repo) + repo.add_submodule("subdir", "file://" + subrepo.repo, checkout=False) + ref = repo.add_submodule("othersubdir", "file://" + other_subrepo.repo) # Write out our test target - element = { - 'kind': 'import', - 'sources': [ - repo.source_config_extra(ref=ref, checkout_submodules=True) - ] - } - _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) + element = {"kind": "import", "sources": [repo.source_config_extra(ref=ref, checkout_submodules=True)]} + _yaml.roundtrip_dump(element, os.path.join(project, "target.bst")) # Fetch, build, checkout - result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Assert we checked out files at their expected location - assert os.path.exists(os.path.join(checkoutdir, 'file.txt')) - assert not os.path.exists(os.path.join(checkoutdir, 'subdir', 'ponyfile.txt')) - assert os.path.exists(os.path.join(checkoutdir, 'othersubdir', 'unicornfile.txt')) + assert os.path.exists(os.path.join(checkoutdir, "file.txt")) + assert not os.path.exists(os.path.join(checkoutdir, "subdir", "ponyfile.txt")) + assert os.path.exists(os.path.join(checkoutdir, "othersubdir", "unicornfile.txt")) @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "template")) def test_submodule_fetch_submodule_individual_checkout_explicit(cli, tmpdir, datafiles): project = str(datafiles) checkoutdir = os.path.join(str(tmpdir), "checkout") # Create the submodule first from the 'subrepofiles' subdir - subrepo = create_repo('git', str(tmpdir), 'subrepo') - subrepo.create(os.path.join(project, 'subrepofiles')) + subrepo = create_repo("git", str(tmpdir), "subrepo") + subrepo.create(os.path.join(project, "subrepofiles")) # Create another submodule from the 'othersubrepofiles' subdir - other_subrepo = create_repo('git', str(tmpdir), 'othersubrepo') - other_subrepo.create(os.path.join(project, 'othersubrepofiles')) + other_subrepo = create_repo("git", str(tmpdir), "othersubrepo") + other_subrepo.create(os.path.join(project, "othersubrepofiles")) # Create the repo from 'repofiles' subdir - repo = create_repo('git', str(tmpdir)) - repo.create(os.path.join(project, 'repofiles')) + repo = create_repo("git", str(tmpdir)) + repo.create(os.path.join(project, "repofiles")) # Add a submodule pointing to the one we created - repo.add_submodule('subdir', 'file://' + subrepo.repo, checkout=False) - ref = repo.add_submodule('othersubdir', 'file://' + other_subrepo.repo, checkout=True) + repo.add_submodule("subdir", "file://" + subrepo.repo, checkout=False) + ref = repo.add_submodule("othersubdir", "file://" + other_subrepo.repo, checkout=True) # Write out our test target - element = { - 'kind': 'import', - 'sources': [ - repo.source_config_extra(ref=ref, checkout_submodules=True) - ] - } - _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) + element = {"kind": "import", "sources": [repo.source_config_extra(ref=ref, checkout_submodules=True)]} + _yaml.roundtrip_dump(element, os.path.join(project, "target.bst")) # Fetch, build, checkout - result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Assert we checked out files at their expected location - assert os.path.exists(os.path.join(checkoutdir, 'file.txt')) - assert not os.path.exists(os.path.join(checkoutdir, 'subdir', 'ponyfile.txt')) - assert os.path.exists(os.path.join(checkoutdir, 'othersubdir', 'unicornfile.txt')) + assert os.path.exists(os.path.join(checkoutdir, "file.txt")) + assert not os.path.exists(os.path.join(checkoutdir, "subdir", "ponyfile.txt")) + assert os.path.exists(os.path.join(checkoutdir, "othersubdir", "unicornfile.txt")) @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project-override')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "project-override")) def test_submodule_fetch_project_override(cli, tmpdir, datafiles): project = str(datafiles) checkoutdir = os.path.join(str(tmpdir), "checkout") # Create the submodule first from the 'subrepofiles' subdir - subrepo = create_repo('git', str(tmpdir), 'subrepo') - subrepo.create(os.path.join(project, 'subrepofiles')) + subrepo = create_repo("git", str(tmpdir), "subrepo") + subrepo.create(os.path.join(project, "subrepofiles")) # Create the repo from 'repofiles' subdir - repo = create_repo('git', str(tmpdir)) - repo.create(os.path.join(project, 'repofiles')) + repo = create_repo("git", str(tmpdir)) + repo.create(os.path.join(project, "repofiles")) # Add a submodule pointing to the one we created - ref = repo.add_submodule('subdir', 'file://' + subrepo.repo) + ref = repo.add_submodule("subdir", "file://" + subrepo.repo) # Write out our test target - element = { - 'kind': 'import', - 'sources': [ - repo.source_config(ref=ref) - ] - } - _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) + element = {"kind": "import", "sources": [repo.source_config(ref=ref)]} + _yaml.roundtrip_dump(element, os.path.join(project, "target.bst")) # Fetch, build, checkout - result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Assert we checked out both files at their expected location - assert os.path.exists(os.path.join(checkoutdir, 'file.txt')) - assert not os.path.exists(os.path.join(checkoutdir, 'subdir', 'ponyfile.txt')) + assert os.path.exists(os.path.join(checkoutdir, "file.txt")) + assert not os.path.exists(os.path.join(checkoutdir, "subdir", "ponyfile.txt")) @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "template")) def test_submodule_track_ignore_inconsistent(cli, tmpdir, datafiles): project = str(datafiles) # Create the repo from 'repofiles' subdir - repo = create_repo('git', str(tmpdir)) - ref = repo.create(os.path.join(project, 'repofiles')) + repo = create_repo("git", str(tmpdir)) + ref = repo.create(os.path.join(project, "repofiles")) # Write out our test target - element = { - 'kind': 'import', - 'sources': [ - repo.source_config(ref=ref) - ] - } - _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) + element = {"kind": "import", "sources": [repo.source_config(ref=ref)]} + _yaml.roundtrip_dump(element, os.path.join(project, "target.bst")) # Now add a .gitmodules file with an inconsistent submodule, # we are calling this inconsistent because the file was created # but `git submodule add` was never called, so there is no reference # associated to the submodule. # - repo.add_file(os.path.join(project, 'inconsistent-submodule', '.gitmodules')) + repo.add_file(os.path.join(project, "inconsistent-submodule", ".gitmodules")) # Fetch should work, we're not yet at the offending ref - result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() # Track will encounter an inconsistent submodule without any ref - result = cli.run(project=project, args=['source', 'track', 'target.bst']) + result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() # Assert that we are just fine without it, and emit a warning to the user. @@ -391,68 +341,55 @@ def test_submodule_track_ignore_inconsistent(cli, tmpdir, datafiles): @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "template")) def test_submodule_track_no_ref_or_track(cli, tmpdir, datafiles): project = str(datafiles) # Create the repo from 'repofiles' subdir - repo = create_repo('git', str(tmpdir)) - repo.create(os.path.join(project, 'repofiles')) + repo = create_repo("git", str(tmpdir)) + repo.create(os.path.join(project, "repofiles")) # Write out our test target gitsource = repo.source_config(ref=None) - gitsource.pop('track') - element = { - 'kind': 'import', - 'sources': [ - gitsource - ] - } + gitsource.pop("track") + element = {"kind": "import", "sources": [gitsource]} - _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) + _yaml.roundtrip_dump(element, os.path.join(project, "target.bst")) # Track will encounter an inconsistent submodule without any ref - result = cli.run(project=project, args=['show', 'target.bst']) + result = cli.run(project=project, args=["show", "target.bst"]) result.assert_main_error(ErrorDomain.SOURCE, "missing-track-and-ref") result.assert_task_error(None, None) @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) -@pytest.mark.parametrize("fail", ['warn', 'error']) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "template")) +@pytest.mark.parametrize("fail", ["warn", "error"]) def test_ref_not_in_track(cli, tmpdir, datafiles, fail): project = str(datafiles) # Make the warning an error if we're testing errors - if fail == 'error': - project_template = { - "name": "foo", - "fatal-warnings": [CoreWarnings.REF_NOT_IN_TRACK] - } - _yaml.roundtrip_dump(project_template, os.path.join(project, 'project.conf')) + if fail == "error": + project_template = {"name": "foo", "fatal-warnings": [CoreWarnings.REF_NOT_IN_TRACK]} + _yaml.roundtrip_dump(project_template, os.path.join(project, "project.conf")) # Create the repo from 'repofiles', create a branch without latest commit - repo = create_repo('git', str(tmpdir)) - ref = repo.create(os.path.join(project, 'repofiles')) + repo = create_repo("git", str(tmpdir)) + ref = repo.create(os.path.join(project, "repofiles")) gitsource = repo.source_config(ref=ref) # Overwrite the track value to the added branch - gitsource['track'] = 'foo' + gitsource["track"] = "foo" # Write out our test target - element = { - 'kind': 'import', - 'sources': [ - gitsource - ] - } - _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) + element = {"kind": "import", "sources": [gitsource]} + _yaml.roundtrip_dump(element, os.path.join(project, "target.bst")) - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) # Assert a warning or an error depending on what we're checking - if fail == 'error': + if fail == "error": result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.REF_NOT_IN_TRACK) else: @@ -461,29 +398,26 @@ def test_ref_not_in_track(cli, tmpdir, datafiles, fail): @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) -@pytest.mark.parametrize("fail", ['warn', 'error']) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "template")) +@pytest.mark.parametrize("fail", ["warn", "error"]) def test_unlisted_submodule(cli, tmpdir, datafiles, fail): project = str(datafiles) # Make the warning an error if we're testing errors - if fail == 'error': - project_template = { - "name": "foo", - "fatal-warnings": ['git:unlisted-submodule'] - } - _yaml.roundtrip_dump(project_template, os.path.join(project, 'project.conf')) + if fail == "error": + project_template = {"name": "foo", "fatal-warnings": ["git:unlisted-submodule"]} + _yaml.roundtrip_dump(project_template, os.path.join(project, "project.conf")) # Create the submodule first from the 'subrepofiles' subdir - subrepo = create_repo('git', str(tmpdir), 'subrepo') - subrepo.create(os.path.join(project, 'subrepofiles')) + subrepo = create_repo("git", str(tmpdir), "subrepo") + subrepo.create(os.path.join(project, "subrepofiles")) # Create the repo from 'repofiles' subdir - repo = create_repo('git', str(tmpdir)) - repo.create(os.path.join(project, 'repofiles')) + repo = create_repo("git", str(tmpdir)) + repo.create(os.path.join(project, "repofiles")) # Add a submodule pointing to the one we created - ref = repo.add_submodule('subdir', 'file://' + subrepo.repo) + ref = repo.add_submodule("subdir", "file://" + subrepo.repo) # Create the source, and delete the explicit configuration # of the submodules. @@ -492,127 +426,111 @@ def test_unlisted_submodule(cli, tmpdir, datafiles, fail): # after the source has been fetched. # gitsource = repo.source_config(ref=ref) - del gitsource['submodules'] + del gitsource["submodules"] # Write out our test target - element = { - 'kind': 'import', - 'sources': [ - gitsource - ] - } - _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) + element = {"kind": "import", "sources": [gitsource]} + _yaml.roundtrip_dump(element, os.path.join(project, "target.bst")) # We will not see the warning or error before the first fetch, because # we don't have the repository yet and so we have no knowledge of # the unlisted submodule. - result = cli.run(project=project, args=['show', 'target.bst']) + result = cli.run(project=project, args=["show", "target.bst"]) result.assert_success() assert "git:unlisted-submodule" not in result.stderr # We will notice this directly in fetch, as it will try to fetch # the submodules it discovers as a result of fetching the primary repo. - result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) # Assert a warning or an error depending on what we're checking - if fail == 'error': + if fail == "error": result.assert_main_error(ErrorDomain.STREAM, None) - result.assert_task_error(ErrorDomain.PLUGIN, 'git:unlisted-submodule') + result.assert_task_error(ErrorDomain.PLUGIN, "git:unlisted-submodule") else: result.assert_success() assert "git:unlisted-submodule" in result.stderr # Now that we've fetched it, `bst show` will discover the unlisted submodule too - result = cli.run(project=project, args=['show', 'target.bst']) + result = cli.run(project=project, args=["show", "target.bst"]) # Assert a warning or an error depending on what we're checking - if fail == 'error': - result.assert_main_error(ErrorDomain.PLUGIN, 'git:unlisted-submodule') + if fail == "error": + result.assert_main_error(ErrorDomain.PLUGIN, "git:unlisted-submodule") else: result.assert_success() assert "git:unlisted-submodule" in result.stderr @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) -@pytest.mark.parametrize("fail", ['warn', 'error']) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "template")) +@pytest.mark.parametrize("fail", ["warn", "error"]) def test_track_unlisted_submodule(cli, tmpdir, datafiles, fail): project = str(datafiles) # Make the warning an error if we're testing errors - if fail == 'error': - project_template = { - "name": "foo", - "fatal-warnings": ['git:unlisted-submodule'] - } - _yaml.roundtrip_dump(project_template, os.path.join(project, 'project.conf')) + if fail == "error": + project_template = {"name": "foo", "fatal-warnings": ["git:unlisted-submodule"]} + _yaml.roundtrip_dump(project_template, os.path.join(project, "project.conf")) # Create the submodule first from the 'subrepofiles' subdir - subrepo = create_repo('git', str(tmpdir), 'subrepo') - subrepo.create(os.path.join(project, 'subrepofiles')) + subrepo = create_repo("git", str(tmpdir), "subrepo") + subrepo.create(os.path.join(project, "subrepofiles")) # Create the repo from 'repofiles' subdir - repo = create_repo('git', str(tmpdir)) - ref = repo.create(os.path.join(project, 'repofiles')) + repo = create_repo("git", str(tmpdir)) + ref = repo.create(os.path.join(project, "repofiles")) # Add a submodule pointing to the one we created, but use # the original ref, let the submodules appear after tracking - repo.add_submodule('subdir', 'file://' + subrepo.repo) + repo.add_submodule("subdir", "file://" + subrepo.repo) # Create the source, and delete the explicit configuration # of the submodules. gitsource = repo.source_config(ref=ref) - del gitsource['submodules'] + del gitsource["submodules"] # Write out our test target - element = { - 'kind': 'import', - 'sources': [ - gitsource - ] - } - _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) + element = {"kind": "import", "sources": [gitsource]} + _yaml.roundtrip_dump(element, os.path.join(project, "target.bst")) # Fetch the repo, we will not see the warning because we # are still pointing to a ref which predates the submodules - result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() assert "git:unlisted-submodule" not in result.stderr # We won't get a warning/error when tracking either, the source # has not become Consistency.CACHED so the opportunity to check # for the warning has not yet arisen. - result = cli.run(project=project, args=['source', 'track', 'target.bst']) + result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() assert "git:unlisted-submodule" not in result.stderr # Fetching the repo at the new ref will finally reveal the warning - result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) - if fail == 'error': + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) + if fail == "error": result.assert_main_error(ErrorDomain.STREAM, None) - result.assert_task_error(ErrorDomain.PLUGIN, 'git:unlisted-submodule') + result.assert_task_error(ErrorDomain.PLUGIN, "git:unlisted-submodule") else: result.assert_success() assert "git:unlisted-submodule" in result.stderr @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) -@pytest.mark.parametrize("fail", ['warn', 'error']) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "template")) +@pytest.mark.parametrize("fail", ["warn", "error"]) def test_invalid_submodule(cli, tmpdir, datafiles, fail): project = str(datafiles) # Make the warning an error if we're testing errors - if fail == 'error': - project_template = { - "name": "foo", - "fatal-warnings": ['git:invalid-submodule'] - } - _yaml.roundtrip_dump(project_template, os.path.join(project, 'project.conf')) + if fail == "error": + project_template = {"name": "foo", "fatal-warnings": ["git:invalid-submodule"]} + _yaml.roundtrip_dump(project_template, os.path.join(project, "project.conf")) # Create the repo from 'repofiles' subdir - repo = create_repo('git', str(tmpdir)) - ref = repo.create(os.path.join(project, 'repofiles')) + repo = create_repo("git", str(tmpdir)) + ref = repo.create(os.path.join(project, "repofiles")) # Create the source without any submodules, and add # an invalid submodule configuration to it. @@ -622,46 +540,37 @@ def test_invalid_submodule(cli, tmpdir, datafiles, fail): # the real submodules actually are. # gitsource = repo.source_config(ref=ref) - gitsource['submodules'] = { - 'subdir': { - 'url': 'https://pony.org/repo.git' - } - } + gitsource["submodules"] = {"subdir": {"url": "https://pony.org/repo.git"}} # Write out our test target - element = { - 'kind': 'import', - 'sources': [ - gitsource - ] - } - _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) + element = {"kind": "import", "sources": [gitsource]} + _yaml.roundtrip_dump(element, os.path.join(project, "target.bst")) # We will not see the warning or error before the first fetch, because # we don't have the repository yet and so we have no knowledge of # the unlisted submodule. - result = cli.run(project=project, args=['show', 'target.bst']) + result = cli.run(project=project, args=["show", "target.bst"]) result.assert_success() assert "git:invalid-submodule" not in result.stderr # We will notice this directly in fetch, as it will try to fetch # the submodules it discovers as a result of fetching the primary repo. - result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) # Assert a warning or an error depending on what we're checking - if fail == 'error': + if fail == "error": result.assert_main_error(ErrorDomain.STREAM, None) - result.assert_task_error(ErrorDomain.PLUGIN, 'git:invalid-submodule') + result.assert_task_error(ErrorDomain.PLUGIN, "git:invalid-submodule") else: result.assert_success() assert "git:invalid-submodule" in result.stderr # Now that we've fetched it, `bst show` will discover the unlisted submodule too - result = cli.run(project=project, args=['show', 'target.bst']) + result = cli.run(project=project, args=["show", "target.bst"]) # Assert a warning or an error depending on what we're checking - if fail == 'error': - result.assert_main_error(ErrorDomain.PLUGIN, 'git:invalid-submodule') + if fail == "error": + result.assert_main_error(ErrorDomain.PLUGIN, "git:invalid-submodule") else: result.assert_success() assert "git:invalid-submodule" in result.stderr @@ -669,49 +578,41 @@ def test_invalid_submodule(cli, tmpdir, datafiles, fail): @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") @pytest.mark.skipif(HAVE_OLD_GIT, reason="old git rm does not update .gitmodules") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) -@pytest.mark.parametrize("fail", ['warn', 'error']) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "template")) +@pytest.mark.parametrize("fail", ["warn", "error"]) def test_track_invalid_submodule(cli, tmpdir, datafiles, fail): project = str(datafiles) # Make the warning an error if we're testing errors - if fail == 'error': - project_template = { - "name": "foo", - "fatal-warnings": ['git:invalid-submodule'] - } - _yaml.roundtrip_dump(project_template, os.path.join(project, 'project.conf')) + if fail == "error": + project_template = {"name": "foo", "fatal-warnings": ["git:invalid-submodule"]} + _yaml.roundtrip_dump(project_template, os.path.join(project, "project.conf")) # Create the submodule first from the 'subrepofiles' subdir - subrepo = create_repo('git', str(tmpdir), 'subrepo') - subrepo.create(os.path.join(project, 'subrepofiles')) + subrepo = create_repo("git", str(tmpdir), "subrepo") + subrepo.create(os.path.join(project, "subrepofiles")) # Create the repo from 'repofiles' subdir - repo = create_repo('git', str(tmpdir)) - repo.create(os.path.join(project, 'repofiles')) + repo = create_repo("git", str(tmpdir)) + repo.create(os.path.join(project, "repofiles")) # Add a submodule pointing to the one we created - ref = repo.add_submodule('subdir', 'file://' + subrepo.repo) + ref = repo.add_submodule("subdir", "file://" + subrepo.repo) # Add a commit beyond the ref which *removes* the submodule we've added - repo.remove_path('subdir') + repo.remove_path("subdir") # Create the source, this will keep the submodules so initially # the configuration is valid for the ref we're using gitsource = repo.source_config(ref=ref) # Write out our test target - element = { - 'kind': 'import', - 'sources': [ - gitsource - ] - } - _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) + element = {"kind": "import", "sources": [gitsource]} + _yaml.roundtrip_dump(element, os.path.join(project, "target.bst")) # Fetch the repo, we will not see the warning because we # are still pointing to a ref which predates the submodules - result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() assert "git:invalid-submodule" not in result.stderr @@ -720,417 +621,392 @@ def test_track_invalid_submodule(cli, tmpdir, datafiles, fail): # not locally cached, the Source will be CACHED directly after # tracking and the validations will occur as a result. # - result = cli.run(project=project, args=['source', 'track', 'target.bst']) - if fail == 'error': + result = cli.run(project=project, args=["source", "track", "target.bst"]) + if fail == "error": result.assert_main_error(ErrorDomain.STREAM, None) - result.assert_task_error(ErrorDomain.PLUGIN, 'git:invalid-submodule') + result.assert_task_error(ErrorDomain.PLUGIN, "git:invalid-submodule") else: result.assert_success() assert "git:invalid-submodule" in result.stderr @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) -@pytest.mark.parametrize("ref_format", ['sha1', 'git-describe']) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "template")) +@pytest.mark.parametrize("ref_format", ["sha1", "git-describe"]) @pytest.mark.parametrize("tag,extra_commit", [(False, False), (True, False), (True, True)]) def test_track_fetch(cli, tmpdir, datafiles, ref_format, tag, extra_commit): project = str(datafiles) # Create the repo from 'repofiles' subdir - repo = create_repo('git', str(tmpdir)) - repo.create(os.path.join(project, 'repofiles')) + repo = create_repo("git", str(tmpdir)) + repo.create(os.path.join(project, "repofiles")) if tag: - repo.add_tag('tag') + repo.add_tag("tag") if extra_commit: repo.add_commit() # Write out our test target - element = { - 'kind': 'import', - 'sources': [ - repo.source_config() - ] - } - element['sources'][0]['ref-format'] = ref_format - element_path = os.path.join(project, 'target.bst') + element = {"kind": "import", "sources": [repo.source_config()]} + element["sources"][0]["ref-format"] = ref_format + element_path = os.path.join(project, "target.bst") _yaml.roundtrip_dump(element, element_path) # Track it - result = cli.run(project=project, args=['source', 'track', 'target.bst']) + result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() element = _yaml.load(element_path) - new_ref = element.get_sequence('sources').mapping_at(0).get_str('ref') + new_ref = element.get_sequence("sources").mapping_at(0).get_str("ref") - if ref_format == 'git-describe' and tag: + if ref_format == "git-describe" and tag: # Check and strip prefix - prefix = 'tag-{}-g'.format(0 if not extra_commit else 1) + prefix = "tag-{}-g".format(0 if not extra_commit else 1) assert new_ref.startswith(prefix) - new_ref = new_ref[len(prefix):] + new_ref = new_ref[len(prefix) :] # 40 chars for SHA-1 assert len(new_ref) == 40 # Fetch it - result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") @pytest.mark.skipif(HAVE_OLD_GIT, reason="old git describe lacks --first-parent") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) -@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')]) -@pytest.mark.parametrize("tag_type", [('annotated'), ('lightweight')]) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "template")) +@pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")]) +@pytest.mark.parametrize("tag_type", [("annotated"), ("lightweight")]) def test_git_describe(cli, tmpdir, datafiles, ref_storage, tag_type): project = str(datafiles) - project_config = _yaml.load(os.path.join(project, 'project.conf')) - project_config['ref-storage'] = ref_storage - _yaml.roundtrip_dump(project_config, os.path.join(project, 'project.conf')) + project_config = _yaml.load(os.path.join(project, "project.conf")) + project_config["ref-storage"] = ref_storage + _yaml.roundtrip_dump(project_config, os.path.join(project, "project.conf")) - repofiles = os.path.join(str(tmpdir), 'repofiles') + repofiles = os.path.join(str(tmpdir), "repofiles") os.makedirs(repofiles, exist_ok=True) - file0 = os.path.join(repofiles, 'file0') - with open(file0, 'w') as f: - f.write('test\n') + file0 = os.path.join(repofiles, "file0") + with open(file0, "w") as f: + f.write("test\n") - repo = create_repo('git', str(tmpdir)) + repo = create_repo("git", str(tmpdir)) def tag(name): - if tag_type == 'annotated': + if tag_type == "annotated": repo.add_annotated_tag(name, name) else: repo.add_tag(name) repo.create(repofiles) - tag('uselesstag') + tag("uselesstag") - file1 = os.path.join(str(tmpdir), 'file1') - with open(file1, 'w') as f: - f.write('test\n') + file1 = os.path.join(str(tmpdir), "file1") + with open(file1, "w") as f: + f.write("test\n") repo.add_file(file1) - tag('tag1') + tag("tag1") - file2 = os.path.join(str(tmpdir), 'file2') - with open(file2, 'w') as f: - f.write('test\n') - repo.branch('branch2') + file2 = os.path.join(str(tmpdir), "file2") + with open(file2, "w") as f: + f.write("test\n") + repo.branch("branch2") repo.add_file(file2) - tag('tag2') + tag("tag2") - repo.checkout('master') - file3 = os.path.join(str(tmpdir), 'file3') - with open(file3, 'w') as f: - f.write('test\n') + repo.checkout("master") + file3 = os.path.join(str(tmpdir), "file3") + with open(file3, "w") as f: + f.write("test\n") repo.add_file(file3) - repo.merge('branch2') + repo.merge("branch2") config = repo.source_config() - config['track'] = repo.latest_commit() - config['track-tags'] = True + config["track"] = repo.latest_commit() + config["track-tags"] = True # Write out our test target element = { - 'kind': 'import', - 'sources': [ - config - ], + "kind": "import", + "sources": [config], } - element_path = os.path.join(project, 'target.bst') + element_path = os.path.join(project, "target.bst") _yaml.roundtrip_dump(element, element_path) - if ref_storage == 'inline': - result = cli.run(project=project, args=['source', 'track', 'target.bst']) + if ref_storage == "inline": + result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() else: - result = cli.run(project=project, args=['source', 'track', 'target.bst', '--deps', 'all']) + result = cli.run(project=project, args=["source", "track", "target.bst", "--deps", "all"]) result.assert_success() - if ref_storage == 'inline': + if ref_storage == "inline": element = _yaml.load(element_path) - tags = element.get_sequence('sources').mapping_at(0).get_sequence('tags') + tags = element.get_sequence("sources").mapping_at(0).get_sequence("tags") assert len(tags) == 2 for tag in tags: - assert 'tag' in tag - assert 'commit' in tag - assert 'annotated' in tag - assert tag.get_bool('annotated') == (tag_type == 'annotated') - - assert {(tag.get_str('tag'), - tag.get_str('commit')) - for tag in tags} == {('tag1', repo.rev_parse('tag1^{commit}')), - ('tag2', repo.rev_parse('tag2^{commit}'))} + assert "tag" in tag + assert "commit" in tag + assert "annotated" in tag + assert tag.get_bool("annotated") == (tag_type == "annotated") + + assert {(tag.get_str("tag"), tag.get_str("commit")) for tag in tags} == { + ("tag1", repo.rev_parse("tag1^{commit}")), + ("tag2", repo.rev_parse("tag2^{commit}")), + } - checkout = os.path.join(str(tmpdir), 'checkout') + checkout = os.path.join(str(tmpdir), "checkout") - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkout]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkout]) result.assert_success() - if tag_type == 'annotated': + if tag_type == "annotated": options = [] else: - options = ['--tags'] - describe = subprocess.check_output(['git', 'describe', *options], - cwd=checkout, universal_newlines=True) - assert describe.startswith('tag2-2-') + options = ["--tags"] + describe = subprocess.check_output(["git", "describe", *options], cwd=checkout, universal_newlines=True) + assert describe.startswith("tag2-2-") - describe_fp = subprocess.check_output(['git', 'describe', '--first-parent', *options], - cwd=checkout, universal_newlines=True) - assert describe_fp.startswith('tag1-2-') + describe_fp = subprocess.check_output( + ["git", "describe", "--first-parent", *options], cwd=checkout, universal_newlines=True + ) + assert describe_fp.startswith("tag1-2-") - tags = subprocess.check_output(['git', 'tag'], - cwd=checkout, universal_newlines=True) + tags = subprocess.check_output(["git", "tag"], cwd=checkout, universal_newlines=True) tags = set(tags.splitlines()) - assert tags == set(['tag1', 'tag2']) + assert tags == set(["tag1", "tag2"]) - p = subprocess.run(['git', 'log', repo.rev_parse('uselesstag')], - cwd=checkout) + p = subprocess.run(["git", "log", repo.rev_parse("uselesstag")], cwd=checkout) assert p.returncode != 0 @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) -@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')]) -@pytest.mark.parametrize("tag_type", [('annotated'), ('lightweight')]) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "template")) +@pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")]) +@pytest.mark.parametrize("tag_type", [("annotated"), ("lightweight")]) def test_git_describe_head_is_tagged(cli, tmpdir, datafiles, ref_storage, tag_type): project = str(datafiles) - project_config = _yaml.load(os.path.join(project, 'project.conf')) - project_config['ref-storage'] = ref_storage - _yaml.roundtrip_dump(project_config, os.path.join(project, 'project.conf')) + project_config = _yaml.load(os.path.join(project, "project.conf")) + project_config["ref-storage"] = ref_storage + _yaml.roundtrip_dump(project_config, os.path.join(project, "project.conf")) - repofiles = os.path.join(str(tmpdir), 'repofiles') + repofiles = os.path.join(str(tmpdir), "repofiles") os.makedirs(repofiles, exist_ok=True) - file0 = os.path.join(repofiles, 'file0') - with open(file0, 'w') as f: - f.write('test\n') + file0 = os.path.join(repofiles, "file0") + with open(file0, "w") as f: + f.write("test\n") - repo = create_repo('git', str(tmpdir)) + repo = create_repo("git", str(tmpdir)) def tag(name): - if tag_type == 'annotated': + if tag_type == "annotated": repo.add_annotated_tag(name, name) else: repo.add_tag(name) repo.create(repofiles) - tag('uselesstag') + tag("uselesstag") - file1 = os.path.join(str(tmpdir), 'file1') - with open(file1, 'w') as f: - f.write('test\n') + file1 = os.path.join(str(tmpdir), "file1") + with open(file1, "w") as f: + f.write("test\n") repo.add_file(file1) - file2 = os.path.join(str(tmpdir), 'file2') - with open(file2, 'w') as f: - f.write('test\n') - repo.branch('branch2') + file2 = os.path.join(str(tmpdir), "file2") + with open(file2, "w") as f: + f.write("test\n") + repo.branch("branch2") repo.add_file(file2) - repo.checkout('master') - file3 = os.path.join(str(tmpdir), 'file3') - with open(file3, 'w') as f: - f.write('test\n') + repo.checkout("master") + file3 = os.path.join(str(tmpdir), "file3") + with open(file3, "w") as f: + f.write("test\n") repo.add_file(file3) - tagged_ref = repo.merge('branch2') - tag('tag') + tagged_ref = repo.merge("branch2") + tag("tag") config = repo.source_config() - config['track'] = repo.latest_commit() - config['track-tags'] = True + config["track"] = repo.latest_commit() + config["track-tags"] = True # Write out our test target element = { - 'kind': 'import', - 'sources': [ - config - ], + "kind": "import", + "sources": [config], } - element_path = os.path.join(project, 'target.bst') + element_path = os.path.join(project, "target.bst") _yaml.roundtrip_dump(element, element_path) - if ref_storage == 'inline': - result = cli.run(project=project, args=['source', 'track', 'target.bst']) + if ref_storage == "inline": + result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() else: - result = cli.run(project=project, args=['source', 'track', 'target.bst', '--deps', 'all']) + result = cli.run(project=project, args=["source", "track", "target.bst", "--deps", "all"]) result.assert_success() - if ref_storage == 'inline': + if ref_storage == "inline": element = _yaml.load(element_path) - source = element.get_sequence('sources').mapping_at(0) - tags = source.get_sequence('tags') + source = element.get_sequence("sources").mapping_at(0) + tags = source.get_sequence("tags") assert len(tags) == 1 - tag = source.get_sequence('tags').mapping_at(0) - assert 'tag' in tag - assert 'commit' in tag - assert 'annotated' in tag - assert tag.get_bool('annotated') == (tag_type == 'annotated') + tag = source.get_sequence("tags").mapping_at(0) + assert "tag" in tag + assert "commit" in tag + assert "annotated" in tag + assert tag.get_bool("annotated") == (tag_type == "annotated") - tag_name = tag.get_str('tag') - commit = tag.get_str('commit') - assert (tag_name, commit) == ('tag', repo.rev_parse('tag^{commit}')) + tag_name = tag.get_str("tag") + commit = tag.get_str("commit") + assert (tag_name, commit) == ("tag", repo.rev_parse("tag^{commit}")) - checkout = os.path.join(str(tmpdir), 'checkout') + checkout = os.path.join(str(tmpdir), "checkout") - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkout]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkout]) result.assert_success() - if tag_type == 'annotated': + if tag_type == "annotated": options = [] else: - options = ['--tags'] - describe = subprocess.check_output(['git', 'describe', *options], - cwd=checkout, universal_newlines=True) - assert describe.startswith('tag') - - tags = subprocess.check_output(['git', 'tag'], - cwd=checkout, - universal_newlines=True) + options = ["--tags"] + describe = subprocess.check_output(["git", "describe", *options], cwd=checkout, universal_newlines=True) + assert describe.startswith("tag") + + tags = subprocess.check_output(["git", "tag"], cwd=checkout, universal_newlines=True) tags = set(tags.splitlines()) - assert tags == set(['tag']) + assert tags == set(["tag"]) - rev_list = subprocess.check_output(['git', 'rev-list', '--all'], - cwd=checkout, - universal_newlines=True) + rev_list = subprocess.check_output(["git", "rev-list", "--all"], cwd=checkout, universal_newlines=True) assert set(rev_list.splitlines()) == set([tagged_ref]) - p = subprocess.run(['git', 'log', repo.rev_parse('uselesstag')], - cwd=checkout) + p = subprocess.run(["git", "log", repo.rev_parse("uselesstag")], cwd=checkout) assert p.returncode != 0 @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "template")) def test_git_describe_relevant_history(cli, tmpdir, datafiles): project = str(datafiles) - project_config = _yaml.load(os.path.join(project, 'project.conf')) - project_config['ref-storage'] = 'project.refs' - _yaml.roundtrip_dump(project_config, os.path.join(project, 'project.conf')) + project_config = _yaml.load(os.path.join(project, "project.conf")) + project_config["ref-storage"] = "project.refs" + _yaml.roundtrip_dump(project_config, os.path.join(project, "project.conf")) - repofiles = os.path.join(str(tmpdir), 'repofiles') + repofiles = os.path.join(str(tmpdir), "repofiles") os.makedirs(repofiles, exist_ok=True) - file0 = os.path.join(repofiles, 'file0') - with open(file0, 'w') as f: - f.write('test\n') + file0 = os.path.join(repofiles, "file0") + with open(file0, "w") as f: + f.write("test\n") - repo = create_repo('git', str(tmpdir)) + repo = create_repo("git", str(tmpdir)) repo.create(repofiles) - file1 = os.path.join(str(tmpdir), 'file1') - with open(file1, 'w') as f: - f.write('test\n') + file1 = os.path.join(str(tmpdir), "file1") + with open(file1, "w") as f: + f.write("test\n") repo.add_file(file1) - repo.branch('branch') - repo.checkout('master') + repo.branch("branch") + repo.checkout("master") - file2 = os.path.join(str(tmpdir), 'file2') - with open(file2, 'w') as f: - f.write('test\n') + file2 = os.path.join(str(tmpdir), "file2") + with open(file2, "w") as f: + f.write("test\n") repo.add_file(file2) - file3 = os.path.join(str(tmpdir), 'file3') - with open(file3, 'w') as f: - f.write('test\n') + file3 = os.path.join(str(tmpdir), "file3") + with open(file3, "w") as f: + f.write("test\n") branch_boundary = repo.add_file(file3) - repo.checkout('branch') - file4 = os.path.join(str(tmpdir), 'file4') - with open(file4, 'w') as f: - f.write('test\n') + repo.checkout("branch") + file4 = os.path.join(str(tmpdir), "file4") + with open(file4, "w") as f: + f.write("test\n") tagged_ref = repo.add_file(file4) - repo.add_annotated_tag('tag1', 'tag1') + repo.add_annotated_tag("tag1", "tag1") - head = repo.merge('master') + head = repo.merge("master") config = repo.source_config() - config['track'] = head - config['track-tags'] = True + config["track"] = head + config["track-tags"] = True # Write out our test target element = { - 'kind': 'import', - 'sources': [ - config - ], + "kind": "import", + "sources": [config], } - element_path = os.path.join(project, 'target.bst') + element_path = os.path.join(project, "target.bst") _yaml.roundtrip_dump(element, element_path) - result = cli.run(project=project, args=['source', 'track', 'target.bst', '--deps', 'all']) + result = cli.run(project=project, args=["source", "track", "target.bst", "--deps", "all"]) result.assert_success() - checkout = os.path.join(str(tmpdir), 'checkout') + checkout = os.path.join(str(tmpdir), "checkout") - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkout]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkout]) result.assert_success() - describe = subprocess.check_output(['git', 'describe'], - cwd=checkout, - universal_newlines=True) - assert describe.startswith('tag1-2-') + describe = subprocess.check_output(["git", "describe"], cwd=checkout, universal_newlines=True) + assert describe.startswith("tag1-2-") - rev_list = subprocess.check_output(['git', 'rev-list', '--all'], - cwd=checkout, - universal_newlines=True) + rev_list = subprocess.check_output(["git", "rev-list", "--all"], cwd=checkout, universal_newlines=True) assert set(rev_list.splitlines()) == set([head, tagged_ref, branch_boundary]) @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "template")) def test_default_do_not_track_tags(cli, tmpdir, datafiles): project = str(datafiles) - project_config = _yaml.load(os.path.join(project, 'project.conf')) - project_config['ref-storage'] = 'inline' - _yaml.roundtrip_dump(project_config, os.path.join(project, 'project.conf')) + project_config = _yaml.load(os.path.join(project, "project.conf")) + project_config["ref-storage"] = "inline" + _yaml.roundtrip_dump(project_config, os.path.join(project, "project.conf")) - repofiles = os.path.join(str(tmpdir), 'repofiles') + repofiles = os.path.join(str(tmpdir), "repofiles") os.makedirs(repofiles, exist_ok=True) - file0 = os.path.join(repofiles, 'file0') - with open(file0, 'w') as f: - f.write('test\n') + file0 = os.path.join(repofiles, "file0") + with open(file0, "w") as f: + f.write("test\n") - repo = create_repo('git', str(tmpdir)) + repo = create_repo("git", str(tmpdir)) repo.create(repofiles) - repo.add_tag('tag') + repo.add_tag("tag") config = repo.source_config() - config['track'] = repo.latest_commit() + config["track"] = repo.latest_commit() # Write out our test target element = { - 'kind': 'import', - 'sources': [ - config - ], + "kind": "import", + "sources": [config], } - element_path = os.path.join(project, 'target.bst') + element_path = os.path.join(project, "target.bst") _yaml.roundtrip_dump(element, element_path) - result = cli.run(project=project, args=['source', 'track', 'target.bst']) + result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() element = _yaml.load(element_path) - source = element.get_sequence('sources').mapping_at(0) - assert 'tags' not in source + source = element.get_sequence("sources").mapping_at(0) + assert "tags" not in source @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "template")) def test_overwrite_rogue_tag_multiple_remotes(cli, tmpdir, datafiles): """When using multiple remotes in cache (i.e. when using aliases), we need to make sure we override tags. This is not allowed to fetch @@ -1139,88 +1015,74 @@ def test_overwrite_rogue_tag_multiple_remotes(cli, tmpdir, datafiles): project = str(datafiles) - repofiles = os.path.join(str(tmpdir), 'repofiles') + repofiles = os.path.join(str(tmpdir), "repofiles") os.makedirs(repofiles, exist_ok=True) - file0 = os.path.join(repofiles, 'file0') - with open(file0, 'w') as f: - f.write('test\n') + file0 = os.path.join(repofiles, "file0") + with open(file0, "w") as f: + f.write("test\n") - repo = create_repo('git', str(tmpdir)) + repo = create_repo("git", str(tmpdir)) top_commit = repo.create(repofiles) repodir, reponame = os.path.split(repo.repo) - project_config = _yaml.load(os.path.join(project, 'project.conf')) - project_config['aliases'] = Node.from_dict({ - 'repo': 'http://example.com/' - }) - project_config['mirrors'] = [ - { - 'name': 'middle-earth', - 'aliases': { - 'repo': ['file://{}/'.format(repodir)] - } - } - ] - _yaml.roundtrip_dump(project_config, os.path.join(project, 'project.conf')) + project_config = _yaml.load(os.path.join(project, "project.conf")) + project_config["aliases"] = Node.from_dict({"repo": "http://example.com/"}) + project_config["mirrors"] = [{"name": "middle-earth", "aliases": {"repo": ["file://{}/".format(repodir)]}}] + _yaml.roundtrip_dump(project_config, os.path.join(project, "project.conf")) - repo.add_annotated_tag('tag', 'tag') + repo.add_annotated_tag("tag", "tag") - file1 = os.path.join(repofiles, 'file1') - with open(file1, 'w') as f: - f.write('test\n') + file1 = os.path.join(repofiles, "file1") + with open(file1, "w") as f: + f.write("test\n") ref = repo.add_file(file1) config = repo.source_config(ref=ref) - del config['track'] - config['url'] = 'repo:{}'.format(reponame) + del config["track"] + config["url"] = "repo:{}".format(reponame) # Write out our test target element = { - 'kind': 'import', - 'sources': [ - config - ], + "kind": "import", + "sources": [config], } - element_path = os.path.join(project, 'target.bst') + element_path = os.path.join(project, "target.bst") _yaml.roundtrip_dump(element, element_path) - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() repo.checkout(top_commit) - file2 = os.path.join(repofiles, 'file2') - with open(file2, 'w') as f: - f.write('test\n') + file2 = os.path.join(repofiles, "file2") + with open(file2, "w") as f: + f.write("test\n") new_ref = repo.add_file(file2) - repo.delete_tag('tag') - repo.add_annotated_tag('tag', 'tag') - repo.checkout('master') + repo.delete_tag("tag") + repo.add_annotated_tag("tag", "tag") + repo.checkout("master") - otherpath = os.path.join(str(tmpdir), 'other_path') - shutil.copytree(repo.repo, - os.path.join(otherpath, 'repo')) - create_repo('git', otherpath) + otherpath = os.path.join(str(tmpdir), "other_path") + shutil.copytree(repo.repo, os.path.join(otherpath, "repo")) + create_repo("git", otherpath) repodir, reponame = os.path.split(repo.repo) - _yaml.roundtrip_dump(project_config, os.path.join(project, 'project.conf')) + _yaml.roundtrip_dump(project_config, os.path.join(project, "project.conf")) config = repo.source_config(ref=new_ref) - del config['track'] - config['url'] = 'repo:{}'.format(reponame) + del config["track"] + config["url"] = "repo:{}".format(reponame) element = { - 'kind': 'import', - 'sources': [ - config - ], + "kind": "import", + "sources": [config], } _yaml.roundtrip_dump(element, element_path) - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() diff --git a/tests/sources/keytest.py b/tests/sources/keytest.py index d3eab8d6b..46d0d07fe 100644 --- a/tests/sources/keytest.py +++ b/tests/sources/keytest.py @@ -27,8 +27,7 @@ import pytest from buildstream._exceptions import ErrorDomain from buildstream.testing import cli # pylint: disable=unused-import -DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), - "project_key_test") +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project_key_test") # using the key-test plugin to ensure get_unique_key is never called before diff --git a/tests/sources/local.py b/tests/sources/local.py index 4b72a4343..92313ca7f 100644 --- a/tests/sources/local.py +++ b/tests/sources/local.py @@ -10,103 +10,93 @@ from buildstream.testing import cli # pylint: disable=unused-import from buildstream.testing._utils.site import HAVE_SANDBOX from tests.testutils import filetypegenerator -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - 'local', -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "local",) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_missing_path(cli, datafiles): project = str(datafiles) # Removing the local file causes preflight to fail - localfile = os.path.join(project, 'file.txt') + localfile = os.path.join(project, "file.txt") os.remove(localfile) - result = cli.run(project=project, args=[ - 'show', 'target.bst' - ]) + result = cli.run(project=project, args=["show", "target.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_non_regular_file_or_directory(cli, datafiles): project = str(datafiles) - localfile = os.path.join(project, 'file.txt') + localfile = os.path.join(project, "file.txt") for _file_type in filetypegenerator.generate_file_types(localfile): - result = cli.run(project=project, args=[ - 'show', 'target.bst' - ]) + result = cli.run(project=project, args=["show", "target.bst"]) if os.path.isdir(localfile) and not os.path.islink(localfile): result.assert_success() elif os.path.isfile(localfile) and not os.path.islink(localfile): result.assert_success() else: - result.assert_main_error(ErrorDomain.LOAD, - LoadErrorReason.PROJ_PATH_INVALID_KIND) + result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.PROJ_PATH_INVALID_KIND) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_invalid_absolute_path(cli, datafiles): project = str(datafiles) - with open(os.path.join(project, "target.bst"), 'r') as f: + with open(os.path.join(project, "target.bst"), "r") as f: old_yaml = f.read() new_yaml = old_yaml.replace("file.txt", os.path.join(project, "file.txt")) assert old_yaml != new_yaml - with open(os.path.join(project, "target.bst"), 'w') as f: + with open(os.path.join(project, "target.bst"), "w") as f: f.write(new_yaml) - result = cli.run(project=project, args=['show', 'target.bst']) - result.assert_main_error(ErrorDomain.LOAD, - LoadErrorReason.PROJ_PATH_INVALID) + result = cli.run(project=project, args=["show", "target.bst"]) + result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.PROJ_PATH_INVALID) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'invalid-relative-path')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "invalid-relative-path")) def test_invalid_relative_path(cli, datafiles): project = str(datafiles) - result = cli.run(project=project, args=['show', 'target.bst']) - result.assert_main_error(ErrorDomain.LOAD, - LoadErrorReason.PROJ_PATH_INVALID) + result = cli.run(project=project, args=["show", "target.bst"]) + result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.PROJ_PATH_INVALID) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_stage_file(cli, tmpdir, datafiles): project = str(datafiles) checkoutdir = os.path.join(str(tmpdir), "checkout") # Build, checkout - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the checkout contains the expected file - assert os.path.exists(os.path.join(checkoutdir, 'file.txt')) + assert os.path.exists(os.path.join(checkoutdir, "file.txt")) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'directory')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "directory")) def test_stage_directory(cli, tmpdir, datafiles): project = str(datafiles) checkoutdir = os.path.join(str(tmpdir), "checkout") # Build, checkout - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the checkout contains the expected file and directory and other file - assert os.path.exists(os.path.join(checkoutdir, 'file.txt')) - assert os.path.exists(os.path.join(checkoutdir, 'subdir', 'anotherfile.txt')) + assert os.path.exists(os.path.join(checkoutdir, "file.txt")) + assert os.path.exists(os.path.join(checkoutdir, "subdir", "anotherfile.txt")) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'symlink')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "symlink")) def test_stage_symlink(cli, tmpdir, datafiles): project = str(datafiles) @@ -117,104 +107,91 @@ def test_stage_symlink(cli, tmpdir, datafiles): # https://github.com/omarkohl/pytest-datafiles/issues/1 # # Create the symlink by hand. - symlink = os.path.join(project, 'files', 'symlink-to-file.txt') - os.symlink('file.txt', symlink) + symlink = os.path.join(project, "files", "symlink-to-file.txt") + os.symlink("file.txt", symlink) # Build, checkout - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the checkout contains the expected file and directory and other file - assert os.path.exists(os.path.join(checkoutdir, 'file.txt')) - assert os.path.exists(os.path.join(checkoutdir, 'symlink-to-file.txt')) - assert os.path.islink(os.path.join(checkoutdir, 'symlink-to-file.txt')) + assert os.path.exists(os.path.join(checkoutdir, "file.txt")) + assert os.path.exists(os.path.join(checkoutdir, "symlink-to-file.txt")) + assert os.path.islink(os.path.join(checkoutdir, "symlink-to-file.txt")) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'file-exists')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "file-exists")) def test_stage_file_exists(cli, datafiles): project = str(datafiles) # Build, checkout - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.ELEMENT, "import-source-files-fail") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'directory')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "directory")) def test_stage_directory_symlink(cli, tmpdir, datafiles): project = str(datafiles) checkoutdir = os.path.join(str(tmpdir), "checkout") - symlink = os.path.join(project, 'files', 'symlink-to-subdir') - os.symlink('subdir', symlink) + symlink = os.path.join(project, "files", "symlink-to-subdir") + os.symlink("subdir", symlink) # Build, checkout - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the checkout contains the expected directory and directory symlink - assert os.path.exists(os.path.join(checkoutdir, 'subdir', 'anotherfile.txt')) - assert os.path.exists(os.path.join(checkoutdir, 'symlink-to-subdir', 'anotherfile.txt')) - assert os.path.islink(os.path.join(checkoutdir, 'symlink-to-subdir')) + assert os.path.exists(os.path.join(checkoutdir, "subdir", "anotherfile.txt")) + assert os.path.exists(os.path.join(checkoutdir, "symlink-to-subdir", "anotherfile.txt")) + assert os.path.islink(os.path.join(checkoutdir, "symlink-to-subdir")) @pytest.mark.integration -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'deterministic-umask')) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +@pytest.mark.datafiles(os.path.join(DATA_DIR, "deterministic-umask")) +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_deterministic_source_umask(cli, tmpdir, datafiles): - - def create_test_file(*path, mode=0o644, content='content\n'): + def create_test_file(*path, mode=0o644, content="content\n"): path = os.path.join(*path) os.makedirs(os.path.dirname(path), exist_ok=True) - with open(path, 'w') as f: + with open(path, "w") as f: f.write(content) os.fchmod(f.fileno(), mode) def create_test_directory(*path, mode=0o644): - create_test_file(*path, '.keep', content='') + create_test_file(*path, ".keep", content="") path = os.path.join(*path) os.chmod(path, mode) project = str(datafiles) - element_name = 'list.bst' - element_path = os.path.join(project, 'elements', element_name) - sourcedir = os.path.join(project, 'source') - - create_test_file(sourcedir, 'a.txt', mode=0o700) - create_test_file(sourcedir, 'b.txt', mode=0o755) - create_test_file(sourcedir, 'c.txt', mode=0o600) - create_test_file(sourcedir, 'd.txt', mode=0o400) - create_test_file(sourcedir, 'e.txt', mode=0o644) - create_test_file(sourcedir, 'f.txt', mode=0o4755) - create_test_file(sourcedir, 'g.txt', mode=0o2755) - create_test_file(sourcedir, 'h.txt', mode=0o1755) - create_test_directory(sourcedir, 'dir-a', mode=0o0700) - create_test_directory(sourcedir, 'dir-c', mode=0o0755) - create_test_directory(sourcedir, 'dir-d', mode=0o4755) - create_test_directory(sourcedir, 'dir-e', mode=0o2755) - create_test_directory(sourcedir, 'dir-f', mode=0o1755) - - source = {'kind': 'local', - 'path': 'source'} + element_name = "list.bst" + element_path = os.path.join(project, "elements", element_name) + sourcedir = os.path.join(project, "source") + + create_test_file(sourcedir, "a.txt", mode=0o700) + create_test_file(sourcedir, "b.txt", mode=0o755) + create_test_file(sourcedir, "c.txt", mode=0o600) + create_test_file(sourcedir, "d.txt", mode=0o400) + create_test_file(sourcedir, "e.txt", mode=0o644) + create_test_file(sourcedir, "f.txt", mode=0o4755) + create_test_file(sourcedir, "g.txt", mode=0o2755) + create_test_file(sourcedir, "h.txt", mode=0o1755) + create_test_directory(sourcedir, "dir-a", mode=0o0700) + create_test_directory(sourcedir, "dir-c", mode=0o0755) + create_test_directory(sourcedir, "dir-d", mode=0o4755) + create_test_directory(sourcedir, "dir-e", mode=0o2755) + create_test_directory(sourcedir, "dir-f", mode=0o1755) + + source = {"kind": "local", "path": "source"} element = { - 'kind': 'manual', - 'depends': [ - { - 'filename': 'base.bst', - 'type': 'build' - } - ], - 'sources': [ - source - ], - 'config': { - 'install-commands': [ - 'ls -l >"%{install-root}/ls-l"' - ] - } + "kind": "manual", + "depends": [{"filename": "base.bst", "type": "build"}], + "sources": [source], + "config": {"install-commands": ['ls -l >"%{install-root}/ls-l"']}, } _yaml.roundtrip_dump(element, element_path) diff --git a/tests/sources/no-fetch-cached/plugins/sources/always_cached.py b/tests/sources/no-fetch-cached/plugins/sources/always_cached.py index fa143a020..623ab19ab 100644 --- a/tests/sources/no-fetch-cached/plugins/sources/always_cached.py +++ b/tests/sources/no-fetch-cached/plugins/sources/always_cached.py @@ -11,7 +11,6 @@ from buildstream import Consistency, Source class AlwaysCachedSource(Source): - def configure(self, node): pass diff --git a/tests/sources/no_fetch_cached.py b/tests/sources/no_fetch_cached.py index 81032881c..1ee3dd7bd 100644 --- a/tests/sources/no_fetch_cached.py +++ b/tests/sources/no_fetch_cached.py @@ -10,10 +10,7 @@ from buildstream.testing import cli # pylint: disable=unused-import from buildstream.testing import create_repo from buildstream.testing._utils.site import HAVE_GIT -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - 'no-fetch-cached' -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "no-fetch-cached") ################################################################## @@ -26,23 +23,13 @@ def test_no_fetch_cached(cli, tmpdir, datafiles): project = str(datafiles) # Create the repo from 'files' subdir - repo = create_repo('git', str(tmpdir)) - ref = repo.create(os.path.join(project, 'files')) + repo = create_repo("git", str(tmpdir)) + ref = repo.create(os.path.join(project, "files")) # Write out test target with a cached and a non-cached source - element = { - 'kind': 'import', - 'sources': [ - repo.source_config(ref=ref), - { - 'kind': 'always_cached' - } - ] - } - _yaml.roundtrip_dump(element, os.path.join(project, 'target.bst')) + element = {"kind": "import", "sources": [repo.source_config(ref=ref), {"kind": "always_cached"}]} + _yaml.roundtrip_dump(element, os.path.join(project, "target.bst")) # Test fetch of target with a cached and a non-cached source - result = cli.run(project=project, args=[ - 'source', 'fetch', 'target.bst' - ]) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() diff --git a/tests/sources/patch.py b/tests/sources/patch.py index 4f9db815b..2b80a7055 100644 --- a/tests/sources/patch.py +++ b/tests/sources/patch.py @@ -8,148 +8,137 @@ from buildstream._exceptions import ErrorDomain, LoadErrorReason from buildstream.testing import cli # pylint: disable=unused-import from tests.testutils import filetypegenerator -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - 'patch', -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "patch",) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_missing_patch(cli, datafiles): project = str(datafiles) # Removing the local file causes preflight to fail - localfile = os.path.join(project, 'file_1.patch') + localfile = os.path.join(project, "file_1.patch") os.remove(localfile) - result = cli.run(project=project, args=[ - 'show', 'target.bst' - ]) + result = cli.run(project=project, args=["show", "target.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_non_regular_file_patch(cli, datafiles): project = str(datafiles) - patch_path = os.path.join(project, 'irregular_file.patch') + patch_path = os.path.join(project, "irregular_file.patch") for _file_type in filetypegenerator.generate_file_types(patch_path): - result = cli.run(project=project, args=[ - 'show', 'irregular.bst' - ]) + result = cli.run(project=project, args=["show", "irregular.bst"]) if os.path.isfile(patch_path) and not os.path.islink(patch_path): result.assert_success() else: - result.assert_main_error(ErrorDomain.LOAD, - LoadErrorReason.PROJ_PATH_INVALID_KIND) + result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.PROJ_PATH_INVALID_KIND) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_invalid_absolute_path(cli, datafiles): project = str(datafiles) - with open(os.path.join(project, "target.bst"), 'r') as f: + with open(os.path.join(project, "target.bst"), "r") as f: old_yaml = f.read() - new_yaml = old_yaml.replace("file_1.patch", - os.path.join(project, "file_1.patch")) + new_yaml = old_yaml.replace("file_1.patch", os.path.join(project, "file_1.patch")) assert old_yaml != new_yaml - with open(os.path.join(project, "target.bst"), 'w') as f: + with open(os.path.join(project, "target.bst"), "w") as f: f.write(new_yaml) - result = cli.run(project=project, args=['show', 'target.bst']) - result.assert_main_error(ErrorDomain.LOAD, - LoadErrorReason.PROJ_PATH_INVALID) + result = cli.run(project=project, args=["show", "target.bst"]) + result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.PROJ_PATH_INVALID) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'invalid-relative-path')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "invalid-relative-path")) def test_invalid_relative_path(cli, datafiles): project = str(datafiles) - result = cli.run(project=project, args=['show', 'irregular.bst']) - result.assert_main_error(ErrorDomain.LOAD, - LoadErrorReason.PROJ_PATH_INVALID) + result = cli.run(project=project, args=["show", "irregular.bst"]) + result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.PROJ_PATH_INVALID) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_stage_and_patch(cli, tmpdir, datafiles): project = str(datafiles) checkoutdir = os.path.join(str(tmpdir), "checkout") # Build, checkout - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Test the file.txt was patched and changed - with open(os.path.join(checkoutdir, 'file.txt')) as f: - assert f.read() == 'This is text file with superpowers\n' + with open(os.path.join(checkoutdir, "file.txt")) as f: + assert f.read() == "This is text file with superpowers\n" -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_stage_file_nonexistent_dir(cli, datafiles): project = str(datafiles) # Fails at build time because it tries to patch into a non-existing directory - result = cli.run(project=project, args=['build', 'failure-nonexistent-dir.bst']) + result = cli.run(project=project, args=["build", "failure-nonexistent-dir.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.SOURCE, "patch-no-files") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_stage_file_empty_dir(cli, datafiles): project = str(datafiles) # Fails at build time because it tries to patch with nothing else staged - result = cli.run(project=project, args=['build', 'failure-empty-dir.bst']) + result = cli.run(project=project, args=["build", "failure-empty-dir.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.SOURCE, "patch-no-files") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'separate-patch-dir')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "separate-patch-dir")) def test_stage_separate_patch_dir(cli, tmpdir, datafiles): project = str(datafiles) checkoutdir = os.path.join(str(tmpdir), "checkout") # Track, fetch, build, checkout - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Test the file.txt was patched and changed - with open(os.path.join(checkoutdir, 'test-dir', 'file.txt')) as f: - assert f.read() == 'This is text file in a directory with superpowers\n' + with open(os.path.join(checkoutdir, "test-dir", "file.txt")) as f: + assert f.read() == "This is text file in a directory with superpowers\n" -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'multiple-patches')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "multiple-patches")) def test_stage_multiple_patches(cli, tmpdir, datafiles): project = str(datafiles) checkoutdir = os.path.join(str(tmpdir), "checkout") # Track, fetch, build, checkout - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Test the file.txt was patched and changed - with open(os.path.join(checkoutdir, 'file.txt')) as f: - assert f.read() == 'This is text file with more superpowers\n' + with open(os.path.join(checkoutdir, "file.txt")) as f: + assert f.read() == "This is text file with more superpowers\n" -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'different-strip-level')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "different-strip-level")) def test_patch_strip_level(cli, tmpdir, datafiles): project = str(datafiles) checkoutdir = os.path.join(str(tmpdir), "checkout") # Track, fetch, build, checkout - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Test the file.txt was patched and changed - with open(os.path.join(checkoutdir, 'file.txt')) as f: - assert f.read() == 'This is text file with superpowers\n' + with open(os.path.join(checkoutdir, "file.txt")) as f: + assert f.read() == "This is text file with superpowers\n" diff --git a/tests/sources/pip.py b/tests/sources/pip.py index 7f91ba701..aafdfaf1c 100644 --- a/tests/sources/pip.py +++ b/tests/sources/pip.py @@ -9,62 +9,56 @@ from buildstream import _yaml from buildstream.plugins.sources.pip import _match_package_name from buildstream.testing import cli # pylint: disable=unused-import -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - 'pip', -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "pip",) def generate_project(project_dir): project_file = os.path.join(project_dir, "project.conf") - _yaml.roundtrip_dump({'name': 'foo'}, project_file) + _yaml.roundtrip_dump({"name": "foo"}, project_file) # Test that without ref, consistency is set appropriately. -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-ref')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "no-ref")) def test_no_ref(cli, datafiles): project = str(datafiles) generate_project(project) - assert cli.get_element_state(project, 'target.bst') == 'no reference' + assert cli.get_element_state(project, "target.bst") == "no reference" # Test that pip is not allowed to be the first source -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'first-source-pip')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "first-source-pip")) def test_first_source(cli, datafiles): project = str(datafiles) generate_project(project) - result = cli.run(project=project, args=[ - 'show', 'target.bst' - ]) + result = cli.run(project=project, args=["show", "target.bst"]) result.assert_main_error(ErrorDomain.ELEMENT, None) # Test that error is raised when neither packges nor requirements files # have been specified -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-packages')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "no-packages")) def test_no_packages(cli, datafiles): project = str(datafiles) generate_project(project) - result = cli.run(project=project, args=[ - 'show', 'target.bst' - ]) + result = cli.run(project=project, args=["show", "target.bst"]) result.assert_main_error(ErrorDomain.SOURCE, None) # Test that pip source parses tar ball names correctly for the ref @pytest.mark.parametrize( - 'tarball, expected_name, expected_version', + "tarball, expected_name, expected_version", [ - ('dotted.package-0.9.8.tar.gz', 'dotted.package', '0.9.8'), - ('hyphenated-package-2.6.0.tar.gz', 'hyphenated-package', '2.6.0'), - ('underscore_pkg-3.1.0.tar.gz', 'underscore_pkg', '3.1.0'), - ('numbers2and5-1.0.1.tar.gz', 'numbers2and5', '1.0.1'), - ('multiple.dots.package-5.6.7.tar.gz', 'multiple.dots.package', '5.6.7'), - ('multiple-hyphens-package-1.2.3.tar.gz', 'multiple-hyphens-package', '1.2.3'), - ('multiple_underscore_pkg-3.4.5.tar.gz', 'multiple_underscore_pkg', '3.4.5'), - ('shortversion-1.0.tar.gz', 'shortversion', '1.0'), - ('longversion-1.2.3.4.tar.gz', 'longversion', '1.2.3.4') - ]) + ("dotted.package-0.9.8.tar.gz", "dotted.package", "0.9.8"), + ("hyphenated-package-2.6.0.tar.gz", "hyphenated-package", "2.6.0"), + ("underscore_pkg-3.1.0.tar.gz", "underscore_pkg", "3.1.0"), + ("numbers2and5-1.0.1.tar.gz", "numbers2and5", "1.0.1"), + ("multiple.dots.package-5.6.7.tar.gz", "multiple.dots.package", "5.6.7"), + ("multiple-hyphens-package-1.2.3.tar.gz", "multiple-hyphens-package", "1.2.3"), + ("multiple_underscore_pkg-3.4.5.tar.gz", "multiple_underscore_pkg", "3.4.5"), + ("shortversion-1.0.tar.gz", "shortversion", "1.0"), + ("longversion-1.2.3.4.tar.gz", "longversion", "1.2.3.4"), + ], +) def test_match_package_name(tarball, expected_name, expected_version): name, version = _match_package_name(tarball) assert (expected_name, expected_version) == (name, version) diff --git a/tests/sources/previous_source_access.py b/tests/sources/previous_source_access.py index 750b94381..fadf6710c 100644 --- a/tests/sources/previous_source_access.py +++ b/tests/sources/previous_source_access.py @@ -7,10 +7,7 @@ import pytest from buildstream import _yaml from buildstream.testing import cli # pylint: disable=unused-import -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - 'previous_source_access' -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "previous_source_access") ################################################################## @@ -29,30 +26,22 @@ def test_custom_transform_source(cli, datafiles): _yaml.roundtrip_dump(project_config, project_config_path) # Ensure we can track - result = cli.run(project=project, args=[ - 'source', 'track', 'target.bst' - ]) + result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() # Ensure we can fetch - result = cli.run(project=project, args=[ - 'source', 'fetch', 'target.bst' - ]) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() # Ensure we get correct output from foo_transform - cli.run(project=project, args=[ - 'build', 'target.bst' - ]) - destpath = os.path.join(cli.directory, 'checkout') - result = cli.run(project=project, args=[ - 'artifact', 'checkout', 'target.bst', '--directory', destpath - ]) + cli.run(project=project, args=["build", "target.bst"]) + destpath = os.path.join(cli.directory, "checkout") + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", destpath]) result.assert_success() # Assert that files from both sources exist, and that they have # the same content - assert os.path.exists(os.path.join(destpath, 'file')) - assert os.path.exists(os.path.join(destpath, 'filetransform')) - with open(os.path.join(destpath, 'file')) as file1: - with open(os.path.join(destpath, 'filetransform')) as file2: + assert os.path.exists(os.path.join(destpath, "file")) + assert os.path.exists(os.path.join(destpath, "filetransform")) + with open(os.path.join(destpath, "file")) as file1: + with open(os.path.join(destpath, "filetransform")) as file2: assert file1.read() == file2.read() diff --git a/tests/sources/previous_source_access/plugins/sources/foo_transform.py b/tests/sources/previous_source_access/plugins/sources/foo_transform.py index 4b423a1b3..906a8f6be 100644 --- a/tests/sources/previous_source_access/plugins/sources/foo_transform.py +++ b/tests/sources/previous_source_access/plugins/sources/foo_transform.py @@ -25,14 +25,13 @@ class FooTransformSource(Source): """Directory where this source should stage its files """ - path = os.path.join(self.get_mirror_directory(), self.name, - self.ref.strip()) + path = os.path.join(self.get_mirror_directory(), self.name, self.ref.strip()) os.makedirs(path, exist_ok=True) return path def configure(self, node): - node.validate_keys(['ref', *Source.COMMON_CONFIG_KEYS]) - self.ref = node.get_str('ref', None) + node.validate_keys(["ref", *Source.COMMON_CONFIG_KEYS]) + self.ref = node.get_str("ref", None) def preflight(self): pass @@ -45,9 +44,9 @@ class FooTransformSource(Source): return Consistency.INCONSISTENT # If we have a file called "filetransform", verify that its checksum # matches our ref. Otherwise, it resolved but not cached. - fpath = os.path.join(self.mirror, 'filetransform') + fpath = os.path.join(self.mirror, "filetransform") try: - with open(fpath, 'rb') as f: + with open(fpath, "rb") as f: if hashlib.sha256(f.read()).hexdigest() == self.ref.strip(): return Consistency.CACHED except Exception: @@ -58,30 +57,29 @@ class FooTransformSource(Source): return self.ref def set_ref(self, ref, node): - self.ref = node['ref'] = ref + self.ref = node["ref"] = ref def track(self, previous_sources_dir): # Store the checksum of the file from previous source as our ref - fpath = os.path.join(previous_sources_dir, 'file') - with open(fpath, 'rb') as f: + fpath = os.path.join(previous_sources_dir, "file") + with open(fpath, "rb") as f: return hashlib.sha256(f.read()).hexdigest() def fetch(self, previous_sources_dir): - fpath = os.path.join(previous_sources_dir, 'file') + fpath = os.path.join(previous_sources_dir, "file") # Verify that the checksum of the file from previous source matches # our ref - with open(fpath, 'rb') as f: + with open(fpath, "rb") as f: if hashlib.sha256(f.read()).hexdigest() != self.ref.strip(): raise SourceError("Element references do not match") # Copy "file" as "filetransform" - newfpath = os.path.join(self.mirror, 'filetransform') + newfpath = os.path.join(self.mirror, "filetransform") utils.safe_copy(fpath, newfpath) def stage(self, directory): # Simply stage the "filetransform" file - utils.safe_copy(os.path.join(self.mirror, 'filetransform'), - os.path.join(directory, 'filetransform')) + utils.safe_copy(os.path.join(self.mirror, "filetransform"), os.path.join(directory, "filetransform")) def setup(): diff --git a/tests/sources/remote.py b/tests/sources/remote.py index 5b818b960..76e469c60 100644 --- a/tests/sources/remote.py +++ b/tests/sources/remote.py @@ -10,94 +10,71 @@ from buildstream import _yaml from buildstream.testing import cli # pylint: disable=unused-import from tests.testutils.file_server import create_file_server -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - 'remote', -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "remote",) def generate_project(project_dir, tmpdir): project_file = os.path.join(project_dir, "project.conf") - _yaml.roundtrip_dump({ - 'name': 'foo', - 'aliases': { - 'tmpdir': "file:///" + str(tmpdir) - } - }, project_file) + _yaml.roundtrip_dump({"name": "foo", "aliases": {"tmpdir": "file:///" + str(tmpdir)}}, project_file) def generate_project_file_server(server, project_dir): project_file = os.path.join(project_dir, "project.conf") - _yaml.roundtrip_dump({ - 'name': 'foo', - 'aliases': { - 'tmpdir': server.base_url() - } - }, project_file) + _yaml.roundtrip_dump({"name": "foo", "aliases": {"tmpdir": server.base_url()}}, project_file) # Test that without ref, consistency is set appropriately. -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-ref')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "no-ref")) def test_no_ref(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, tmpdir) - assert cli.get_element_state(project, 'target.bst') == 'no reference' + assert cli.get_element_state(project, "target.bst") == "no reference" # Here we are doing a fetch on a file that doesn't exist. target.bst # refers to 'file' but that file is not present. -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'missing-file')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "missing-file")) def test_missing_file(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, tmpdir) # Try to fetch it - result = cli.run(project=project, args=[ - 'source', 'fetch', 'target.bst' - ]) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.SOURCE, None) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'path-in-filename')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "path-in-filename")) def test_path_in_filename(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, tmpdir) # Try to fetch it - result = cli.run(project=project, args=[ - 'source', 'fetch', 'target.bst' - ]) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) # The bst file has a / in the filename param result.assert_main_error(ErrorDomain.SOURCE, "filename-contains-directory") -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'single-file')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "single-file")) def test_simple_file_build(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, tmpdir) checkoutdir = os.path.join(str(tmpdir), "checkout") # Try to fetch it - result = cli.run(project=project, args=[ - 'source', 'fetch', 'target.bst' - ]) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=[ - 'build', 'target.bst' - ]) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=[ - 'artifact', 'checkout', 'target.bst', '--directory', checkoutdir - ]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Note that the url of the file in target.bst is actually /dir/file # but this tests confirms we take the basename - checkout_file = os.path.join(checkoutdir, 'file') + checkout_file = os.path.join(checkoutdir, "file") assert os.path.exists(checkout_file) mode = os.stat(checkout_file).st_mode @@ -107,113 +84,99 @@ def test_simple_file_build(cli, tmpdir, datafiles): assert not mode & (stat.S_IWGRP | stat.S_IWOTH) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'single-file-custom-name')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "single-file-custom-name")) def test_simple_file_custom_name_build(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, tmpdir) checkoutdir = os.path.join(str(tmpdir), "checkout") # Try to fetch it - result = cli.run(project=project, args=[ - 'source', 'fetch', 'target.bst' - ]) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=[ - 'build', 'target.bst' - ]) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=[ - 'artifact', 'checkout', 'target.bst', '--directory', checkoutdir - ]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() - assert not os.path.exists(os.path.join(checkoutdir, 'file')) - assert os.path.exists(os.path.join(checkoutdir, 'custom-file')) + assert not os.path.exists(os.path.join(checkoutdir, "file")) + assert os.path.exists(os.path.join(checkoutdir, "custom-file")) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'unique-keys')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "unique-keys")) def test_unique_key(cli, tmpdir, datafiles): - '''This test confirms that the 'filename' parameter is honoured when it comes + """This test confirms that the 'filename' parameter is honoured when it comes to generating a cache key for the source. - ''' + """ project = str(datafiles) generate_project(project, tmpdir) - states = cli.get_element_states(project, [ - 'target.bst', 'target-custom.bst', 'target-custom-executable.bst' - ]) - assert states['target.bst'] == "fetch needed" - assert states['target-custom.bst'] == "fetch needed" - assert states['target-custom-executable.bst'] == "fetch needed" + states = cli.get_element_states(project, ["target.bst", "target-custom.bst", "target-custom-executable.bst"]) + assert states["target.bst"] == "fetch needed" + assert states["target-custom.bst"] == "fetch needed" + assert states["target-custom-executable.bst"] == "fetch needed" # Try to fetch it - cli.run(project=project, args=[ - 'source', 'fetch', 'target.bst' - ]) + cli.run(project=project, args=["source", "fetch", "target.bst"]) # We should download the file only once - states = cli.get_element_states(project, [ - 'target.bst', 'target-custom.bst', 'target-custom-executable.bst' - ]) - assert states['target.bst'] == 'buildable' - assert states['target-custom.bst'] == 'buildable' - assert states['target-custom-executable.bst'] == 'buildable' + states = cli.get_element_states(project, ["target.bst", "target-custom.bst", "target-custom-executable.bst"]) + assert states["target.bst"] == "buildable" + assert states["target-custom.bst"] == "buildable" + assert states["target-custom-executable.bst"] == "buildable" # But the cache key is different because the 'filename' is different. - assert cli.get_element_key(project, 'target.bst') != \ - cli.get_element_key(project, 'target-custom.bst') != \ - cli.get_element_key(project, 'target-custom-executable.bst') + assert ( + cli.get_element_key(project, "target.bst") + != cli.get_element_key(project, "target-custom.bst") + != cli.get_element_key(project, "target-custom-executable.bst") + ) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'unique-keys')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "unique-keys")) def test_executable(cli, tmpdir, datafiles): - '''This test confirms that the 'ecxecutable' parameter is honoured. - ''' + """This test confirms that the 'ecxecutable' parameter is honoured. + """ project = str(datafiles) generate_project(project, tmpdir) checkoutdir = os.path.join(str(tmpdir), "checkout") - assert cli.get_element_state(project, 'target-custom-executable.bst') == "fetch needed" + assert cli.get_element_state(project, "target-custom-executable.bst") == "fetch needed" # Try to fetch it - cli.run(project=project, args=[ - 'build', 'target-custom-executable.bst' - ]) - - cli.run(project=project, args=[ - 'artifact', 'checkout', 'target-custom-executable.bst', '--directory', checkoutdir - ]) - mode = os.stat(os.path.join(checkoutdir, 'some-custom-file')).st_mode + cli.run(project=project, args=["build", "target-custom-executable.bst"]) + + cli.run(project=project, args=["artifact", "checkout", "target-custom-executable.bst", "--directory", checkoutdir]) + mode = os.stat(os.path.join(checkoutdir, "some-custom-file")).st_mode assert mode & stat.S_IEXEC # Assert executable by anyone assert mode & (stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) -@pytest.mark.parametrize('server_type', ('FTP', 'HTTP')) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'single-file')) +@pytest.mark.parametrize("server_type", ("FTP", "HTTP")) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "single-file")) def test_use_netrc(cli, datafiles, server_type, tmpdir): - fake_home = os.path.join(str(tmpdir), 'fake_home') + fake_home = os.path.join(str(tmpdir), "fake_home") os.makedirs(fake_home, exist_ok=True) project = str(datafiles) - checkoutdir = os.path.join(str(tmpdir), 'checkout') + checkoutdir = os.path.join(str(tmpdir), "checkout") - os.environ['HOME'] = fake_home - with open(os.path.join(fake_home, '.netrc'), 'wb') as f: + os.environ["HOME"] = fake_home + with open(os.path.join(fake_home, ".netrc"), "wb") as f: os.fchmod(f.fileno(), 0o700) - f.write(b'machine 127.0.0.1\n') - f.write(b'login testuser\n') - f.write(b'password 12345\n') + f.write(b"machine 127.0.0.1\n") + f.write(b"login testuser\n") + f.write(b"password 12345\n") with create_file_server(server_type) as server: - server.add_user('testuser', '12345', project) + server.add_user("testuser", "12345", project) generate_project_file_server(server, project) server.start() - result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() - checkout_file = os.path.join(checkoutdir, 'file') + checkout_file = os.path.join(checkoutdir, "file") assert os.path.exists(checkout_file) diff --git a/tests/sources/tar.py b/tests/sources/tar.py index fac6f3f8b..f5f3dde2d 100644 --- a/tests/sources/tar.py +++ b/tests/sources/tar.py @@ -17,10 +17,7 @@ from buildstream.testing._utils.site import HAVE_LZIP from tests.testutils.file_server import create_file_server from . import list_dir_contents -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - 'tar', -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "tar",) def _assemble_tar(workingdir, srcdir, dstfile): @@ -38,58 +35,44 @@ def _assemble_tar_lz(workingdir, srcdir, dstfile): with tarfile.open(fileobj=uncompressed, mode="w:") as tar: tar.add(srcdir) uncompressed.seek(0, 0) - with open(dstfile, 'wb') as dst: - subprocess.call(['lzip'], - stdin=uncompressed, - stdout=dst) + with open(dstfile, "wb") as dst: + subprocess.call(["lzip"], stdin=uncompressed, stdout=dst) os.chdir(old_dir) def generate_project(project_dir, tmpdir): project_file = os.path.join(project_dir, "project.conf") - _yaml.roundtrip_dump({ - 'name': 'foo', - 'aliases': { - 'tmpdir': "file:///" + str(tmpdir) - } - }, project_file) + _yaml.roundtrip_dump({"name": "foo", "aliases": {"tmpdir": "file:///" + str(tmpdir)}}, project_file) def generate_project_file_server(base_url, project_dir): project_file = os.path.join(project_dir, "project.conf") - _yaml.roundtrip_dump({ - 'name': 'foo', - 'aliases': { - 'tmpdir': base_url - } - }, project_file) + _yaml.roundtrip_dump({"name": "foo", "aliases": {"tmpdir": base_url}}, project_file) # Test that without ref, consistency is set appropriately. -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-ref')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "no-ref")) def test_no_ref(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, tmpdir) - assert cli.get_element_state(project, 'target.bst') == 'no reference' + assert cli.get_element_state(project, "target.bst") == "no reference" # Test that when I fetch a nonexistent URL, errors are handled gracefully and a retry is performed. -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) def test_fetch_bad_url(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, tmpdir) # Try to fetch it - result = cli.run(project=project, args=[ - 'source', 'fetch', 'target.bst' - ]) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) assert "FAILURE Try #" in result.stderr result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.SOURCE, None) # Test that when I fetch with an invalid ref, it fails. -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) def test_fetch_bad_ref(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, tmpdir) @@ -99,15 +82,13 @@ def test_fetch_bad_ref(cli, tmpdir, datafiles): _assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar) # Try to fetch it - result = cli.run(project=project, args=[ - 'source', 'fetch', 'target.bst' - ]) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.SOURCE, None) # Test that when tracking with a ref set, there is a warning -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) def test_track_warning(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, tmpdir) @@ -117,15 +98,13 @@ def test_track_warning(cli, tmpdir, datafiles): _assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar) # Track it - result = cli.run(project=project, args=[ - 'source', 'track', 'target.bst' - ]) + result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() assert "Potential man-in-the-middle attack!" in result.stderr # Test that a staged checkout matches what was tarred up, with the default first subdir -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) @pytest.mark.parametrize("srcdir", ["a", "./a"]) def test_stage_default_basedir(cli, tmpdir, datafiles, srcdir): project = str(datafiles) @@ -137,13 +116,13 @@ def test_stage_default_basedir(cli, tmpdir, datafiles, srcdir): _assemble_tar(os.path.join(str(datafiles), "content"), srcdir, src_tar) # Track, fetch, build, checkout - result = cli.run(project=project, args=['source', 'track', 'target.bst']) + result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the content of the first directory is checked out (base-dir: '*') @@ -154,7 +133,7 @@ def test_stage_default_basedir(cli, tmpdir, datafiles, srcdir): # Test that a staged checkout matches what was tarred up, with an empty base-dir -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-basedir')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "no-basedir")) @pytest.mark.parametrize("srcdir", ["a", "./a"]) def test_stage_no_basedir(cli, tmpdir, datafiles, srcdir): project = str(datafiles) @@ -166,13 +145,13 @@ def test_stage_no_basedir(cli, tmpdir, datafiles, srcdir): _assemble_tar(os.path.join(str(datafiles), "content"), srcdir, src_tar) # Track, fetch, build, checkout - result = cli.run(project=project, args=['source', 'track', 'target.bst']) + result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the full content of the tarball is checked out (base-dir: '') @@ -183,7 +162,7 @@ def test_stage_no_basedir(cli, tmpdir, datafiles, srcdir): # Test that a staged checkout matches what was tarred up, with an explicit basedir -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'explicit-basedir')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "explicit-basedir")) @pytest.mark.parametrize("srcdir", ["a", "./a"]) def test_stage_explicit_basedir(cli, tmpdir, datafiles, srcdir): project = str(datafiles) @@ -195,13 +174,13 @@ def test_stage_explicit_basedir(cli, tmpdir, datafiles, srcdir): _assemble_tar(os.path.join(str(datafiles), "content"), srcdir, src_tar) # Track, fetch, build, checkout - result = cli.run(project=project, args=['source', 'track', 'target.bst']) + result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the content of the first directory is checked out (base-dir: '*') @@ -213,7 +192,7 @@ def test_stage_explicit_basedir(cli, tmpdir, datafiles, srcdir): # Test that we succeed to extract tarballs with hardlinks when stripping the # leading paths -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'contains-links')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "contains-links")) def test_stage_contains_links(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, tmpdir) @@ -231,13 +210,13 @@ def test_stage_contains_links(cli, tmpdir, datafiles): _assemble_tar(os.path.join(str(datafiles), "content"), "base-directory", src_tar) # Track, fetch, build, checkout - result = cli.run(project=project, args=['source', 'track', 'target.bst']) + result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the content of the first directory is checked out (base-dir: '*') @@ -247,8 +226,8 @@ def test_stage_contains_links(cli, tmpdir, datafiles): assert checkout_contents == original_contents -@pytest.mark.skipif(not HAVE_LZIP, reason='lzip is not available') -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch')) +@pytest.mark.skipif(not HAVE_LZIP, reason="lzip is not available") +@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) @pytest.mark.parametrize("srcdir", ["a", "./a"]) def test_stage_default_basedir_lzip(cli, tmpdir, datafiles, srcdir): project = str(datafiles) @@ -260,13 +239,13 @@ def test_stage_default_basedir_lzip(cli, tmpdir, datafiles, srcdir): _assemble_tar_lz(os.path.join(str(datafiles), "content"), srcdir, src_tar) # Track, fetch, build, checkout - result = cli.run(project=project, args=['source', 'track', 'target-lz.bst']) + result = cli.run(project=project, args=["source", "track", "target-lz.bst"]) result.assert_success() - result = cli.run(project=project, args=['source', 'fetch', 'target-lz.bst']) + result = cli.run(project=project, args=["source", "fetch", "target-lz.bst"]) result.assert_success() - result = cli.run(project=project, args=['build', 'target-lz.bst']) + result = cli.run(project=project, args=["build", "target-lz.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target-lz.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target-lz.bst", "--directory", checkoutdir]) result.assert_success() # Check that the content of the first directory is checked out (base-dir: '*') @@ -280,8 +259,8 @@ def test_stage_default_basedir_lzip(cli, tmpdir, datafiles, srcdir): # a - contains read-only files in a writable directory # b - root directory has read-only permission # c - contains one file that has no read nor write permissions. Base-dir set to '' to extract root of tarball -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'read-only')) -@pytest.mark.parametrize("tar_name, base_dir", [("a", "*"), ("b", '*'), ("c", '')]) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "read-only")) +@pytest.mark.parametrize("tar_name, base_dir", [("a", "*"), ("b", "*"), ("c", "")]) def test_read_only_dir(cli, tmpdir, datafiles, tar_name, base_dir): try: project = str(datafiles) @@ -290,25 +269,21 @@ def test_read_only_dir(cli, tmpdir, datafiles, tar_name, base_dir): bst_path = os.path.join(project, "target.bst") tar_file = "{}.tar.gz".format(tar_name) - _yaml.roundtrip_dump({ - 'kind': 'import', - 'sources': [ - { - 'kind': 'tar', - 'url': 'tmpdir:/{}'.format(tar_file), - 'ref': 'foo', - 'base-dir': base_dir - } - ] - }, bst_path) + _yaml.roundtrip_dump( + { + "kind": "import", + "sources": [{"kind": "tar", "url": "tmpdir:/{}".format(tar_file), "ref": "foo", "base-dir": base_dir}], + }, + bst_path, + ) # Get the tarball in tests/sources/tar/read-only/content # # NOTE that we need to do this because tarfile.open and tar.add() # are packing the tar up with writeable files and dirs - tarball = os.path.join(str(datafiles), 'content', tar_file) + tarball = os.path.join(str(datafiles), "content", tar_file) if not os.path.exists(tarball): - raise FileNotFoundError('{} does not exist'.format(tarball)) + raise FileNotFoundError("{} does not exist".format(tarball)) copyfile(tarball, os.path.join(str(tmpdir), tar_file)) # Because this test can potentially leave directories behind @@ -320,11 +295,11 @@ def test_read_only_dir(cli, tmpdir, datafiles, tar_name, base_dir): env = {"TMP": tmpdir_str} # Track, fetch, build, checkout - result = cli.run(project=project, args=['source', 'track', 'target.bst'], env=env) + result = cli.run(project=project, args=["source", "track", "target.bst"], env=env) result.assert_success() - result = cli.run(project=project, args=['source', 'fetch', 'target.bst'], env=env) + result = cli.run(project=project, args=["source", "fetch", "target.bst"], env=env) result.assert_success() - result = cli.run(project=project, args=['build', 'target.bst'], env=env) + result = cli.run(project=project, args=["build", "target.bst"], env=env) result.assert_success() finally: @@ -336,85 +311,86 @@ def test_read_only_dir(cli, tmpdir, datafiles, tar_name, base_dir): os.rmdir(path) else: os.remove(path) + rmtree(str(tmpdir), onerror=make_dir_writable) -@pytest.mark.parametrize('server_type', ('FTP', 'HTTP')) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch')) +@pytest.mark.parametrize("server_type", ("FTP", "HTTP")) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) def test_use_netrc(cli, datafiles, server_type, tmpdir): - file_server_files = os.path.join(str(tmpdir), 'file_server') - fake_home = os.path.join(str(tmpdir), 'fake_home') + file_server_files = os.path.join(str(tmpdir), "file_server") + fake_home = os.path.join(str(tmpdir), "fake_home") os.makedirs(file_server_files, exist_ok=True) os.makedirs(fake_home, exist_ok=True) project = str(datafiles) - checkoutdir = os.path.join(str(tmpdir), 'checkout') + checkoutdir = os.path.join(str(tmpdir), "checkout") - os.environ['HOME'] = fake_home - with open(os.path.join(fake_home, '.netrc'), 'wb') as f: + os.environ["HOME"] = fake_home + with open(os.path.join(fake_home, ".netrc"), "wb") as f: os.fchmod(f.fileno(), 0o700) - f.write(b'machine 127.0.0.1\n') - f.write(b'login testuser\n') - f.write(b'password 12345\n') + f.write(b"machine 127.0.0.1\n") + f.write(b"login testuser\n") + f.write(b"password 12345\n") with create_file_server(server_type) as server: - server.add_user('testuser', '12345', file_server_files) + server.add_user("testuser", "12345", file_server_files) generate_project_file_server(server.base_url(), project) - src_tar = os.path.join(file_server_files, 'a.tar.gz') - _assemble_tar(os.path.join(str(datafiles), 'content'), 'a', src_tar) + src_tar = os.path.join(file_server_files, "a.tar.gz") + _assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar) server.start() - result = cli.run(project=project, args=['source', 'track', 'target.bst']) + result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() - original_dir = os.path.join(str(datafiles), 'content', 'a') + original_dir = os.path.join(str(datafiles), "content", "a") original_contents = list_dir_contents(original_dir) checkout_contents = list_dir_contents(checkoutdir) assert checkout_contents == original_contents -@pytest.mark.parametrize('server_type', ('FTP', 'HTTP')) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch')) +@pytest.mark.parametrize("server_type", ("FTP", "HTTP")) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) def test_netrc_already_specified_user(cli, datafiles, server_type, tmpdir): - file_server_files = os.path.join(str(tmpdir), 'file_server') - fake_home = os.path.join(str(tmpdir), 'fake_home') + file_server_files = os.path.join(str(tmpdir), "file_server") + fake_home = os.path.join(str(tmpdir), "fake_home") os.makedirs(file_server_files, exist_ok=True) os.makedirs(fake_home, exist_ok=True) project = str(datafiles) - os.environ['HOME'] = fake_home - with open(os.path.join(fake_home, '.netrc'), 'wb') as f: + os.environ["HOME"] = fake_home + with open(os.path.join(fake_home, ".netrc"), "wb") as f: os.fchmod(f.fileno(), 0o700) - f.write(b'machine 127.0.0.1\n') - f.write(b'login testuser\n') - f.write(b'password 12345\n') + f.write(b"machine 127.0.0.1\n") + f.write(b"login testuser\n") + f.write(b"password 12345\n") with create_file_server(server_type) as server: - server.add_user('otheruser', '12345', file_server_files) + server.add_user("otheruser", "12345", file_server_files) parts = urllib.parse.urlsplit(server.base_url()) - base_url = urllib.parse.urlunsplit([parts[0], 'otheruser@{}'.format(parts[1]), *parts[2:]]) + base_url = urllib.parse.urlunsplit([parts[0], "otheruser@{}".format(parts[1]), *parts[2:]]) generate_project_file_server(base_url, project) - src_tar = os.path.join(file_server_files, 'a.tar.gz') - _assemble_tar(os.path.join(str(datafiles), 'content'), 'a', src_tar) + src_tar = os.path.join(file_server_files, "a.tar.gz") + _assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar) server.start() - result = cli.run(project=project, args=['source', 'track', 'target.bst']) + result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.SOURCE, None) # Test that BuildStream doesnt crash if HOME is unset while # the netrc module is trying to find it's ~/.netrc file. -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) def test_homeless_environment(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, tmpdir) @@ -424,11 +400,11 @@ def test_homeless_environment(cli, tmpdir, datafiles): _assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar) # Use a track, make sure the plugin tries to find a ~/.netrc - result = cli.run(project=project, args=['source', 'track', 'target.bst'], env={'HOME': None}) + result = cli.run(project=project, args=["source", "track", "target.bst"], env={"HOME": None}) result.assert_success() -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'out-of-basedir-hardlinks')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "out-of-basedir-hardlinks")) def test_out_of_basedir_hardlinks(cli, tmpdir, datafiles): def ensure_link(member): # By default, python will simply duplicate files - we want @@ -453,28 +429,28 @@ def test_out_of_basedir_hardlinks(cli, tmpdir, datafiles): # Make sure our tarfile is actually created with the desired # attributes set with tarfile.open(src_tar, "r:gz") as tar: - assert any(member.islnk() and - member.path == "contents/to_extract/a" and - member.linkname == "contents/elsewhere/a" - for member in tar.getmembers()) + assert any( + member.islnk() and member.path == "contents/to_extract/a" and member.linkname == "contents/elsewhere/a" + for member in tar.getmembers() + ) # Assert that we will actually create a singular copy of the file - result = cli.run(project=project, args=['source', 'track', 'target.bst']) + result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() - original_dir = os.path.join(str(datafiles), 'contents', 'to_extract') + original_dir = os.path.join(str(datafiles), "contents", "to_extract") original_contents = list_dir_contents(original_dir) checkout_contents = list_dir_contents(checkoutdir) assert checkout_contents == original_contents -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'out-of-basedir-hardlinks')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "out-of-basedir-hardlinks")) def test_malicious_out_of_basedir_hardlinks(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, tmpdir) @@ -499,13 +475,15 @@ def test_malicious_out_of_basedir_hardlinks(cli, tmpdir, datafiles): # Make sure our tarfile is actually created with the desired # attributes set with tarfile.open(src_tar, "r:gz") as tar: - assert any(member.islnk() and - member.path == "contents/elsewhere/malicious" and - member.linkname == "../../../malicious_target.bst" - for member in tar.getmembers()) + assert any( + member.islnk() + and member.path == "contents/elsewhere/malicious" + and member.linkname == "../../../malicious_target.bst" + for member in tar.getmembers() + ) # Try to execute the exploit - result = cli.run(project=project, args=['source', 'track', 'malicious_target.bst']) + result = cli.run(project=project, args=["source", "track", "malicious_target.bst"]) result.assert_success() - result = cli.run(project=project, args=['source', 'fetch', 'malicious_target.bst']) + result = cli.run(project=project, args=["source", "fetch", "malicious_target.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) diff --git a/tests/sources/zip.py b/tests/sources/zip.py index 3fd43b4bb..dcb1e2637 100644 --- a/tests/sources/zip.py +++ b/tests/sources/zip.py @@ -12,17 +12,14 @@ from buildstream.testing import cli # pylint: disable=unused-import from tests.testutils.file_server import create_file_server from . import list_dir_contents -DATA_DIR = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - 'zip', -) +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "zip",) def _assemble_zip(workingdir, dstfile): old_dir = os.getcwd() os.chdir(workingdir) with zipfile.ZipFile(dstfile, "w") as zipfp: - for root, dirs, files in os.walk('.'): + for root, dirs, files in os.walk("."): names = dirs + files names = [os.path.join(root, name) for name in names] for name in names: @@ -32,49 +29,37 @@ def _assemble_zip(workingdir, dstfile): def generate_project(project_dir, tmpdir): project_file = os.path.join(project_dir, "project.conf") - _yaml.roundtrip_dump({ - 'name': 'foo', - 'aliases': { - 'tmpdir': "file:///" + str(tmpdir) - } - }, project_file) + _yaml.roundtrip_dump({"name": "foo", "aliases": {"tmpdir": "file:///" + str(tmpdir)}}, project_file) def generate_project_file_server(server, project_dir): project_file = os.path.join(project_dir, "project.conf") - _yaml.roundtrip_dump({ - 'name': 'foo', - 'aliases': { - 'tmpdir': server.base_url() - } - }, project_file) + _yaml.roundtrip_dump({"name": "foo", "aliases": {"tmpdir": server.base_url()}}, project_file) # Test that without ref, consistency is set appropriately. -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-ref')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "no-ref")) def test_no_ref(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, tmpdir) - assert cli.get_element_state(project, 'target.bst') == 'no reference' + assert cli.get_element_state(project, "target.bst") == "no reference" # Test that when I fetch a nonexistent URL, errors are handled gracefully and a retry is performed. -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) def test_fetch_bad_url(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, tmpdir) # Try to fetch it - result = cli.run(project=project, args=[ - 'source', 'fetch', 'target.bst' - ]) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) assert "FAILURE Try #" in result.stderr result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.SOURCE, None) # Test that when I fetch with an invalid ref, it fails. -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) def test_fetch_bad_ref(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, tmpdir) @@ -84,15 +69,13 @@ def test_fetch_bad_ref(cli, tmpdir, datafiles): _assemble_zip(os.path.join(str(datafiles), "content"), src_zip) # Try to fetch it - result = cli.run(project=project, args=[ - 'source', 'fetch', 'target.bst' - ]) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.SOURCE, None) # Test that when tracking with a ref set, there is a warning -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) def test_track_warning(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, tmpdir) @@ -102,15 +85,13 @@ def test_track_warning(cli, tmpdir, datafiles): _assemble_zip(os.path.join(str(datafiles), "content"), src_zip) # Track it - result = cli.run(project=project, args=[ - 'source', 'track', 'target.bst' - ]) + result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() assert "Potential man-in-the-middle attack!" in result.stderr # Test that a staged checkout matches what was tarred up, with the default first subdir -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) def test_stage_default_basedir(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, tmpdir) @@ -121,13 +102,13 @@ def test_stage_default_basedir(cli, tmpdir, datafiles): _assemble_zip(os.path.join(str(datafiles), "content"), src_zip) # Track, fetch, build, checkout - result = cli.run(project=project, args=['source', 'track', 'target.bst']) + result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the content of the first directory is checked out (base-dir: '*') @@ -138,7 +119,7 @@ def test_stage_default_basedir(cli, tmpdir, datafiles): # Test that a staged checkout matches what was tarred up, with an empty base-dir -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-basedir')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "no-basedir")) def test_stage_no_basedir(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, tmpdir) @@ -149,13 +130,13 @@ def test_stage_no_basedir(cli, tmpdir, datafiles): _assemble_zip(os.path.join(str(datafiles), "content"), src_zip) # Track, fetch, build, checkout - result = cli.run(project=project, args=['source', 'track', 'target.bst']) + result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the full content of the tarball is checked out (base-dir: '') @@ -166,7 +147,7 @@ def test_stage_no_basedir(cli, tmpdir, datafiles): # Test that a staged checkout matches what was tarred up, with an explicit basedir -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'explicit-basedir')) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "explicit-basedir")) def test_stage_explicit_basedir(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, tmpdir) @@ -177,13 +158,13 @@ def test_stage_explicit_basedir(cli, tmpdir, datafiles): _assemble_zip(os.path.join(str(datafiles), "content"), src_zip) # Track, fetch, build, checkout - result = cli.run(project=project, args=['source', 'track', 'target.bst']) + result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the content of the first directory is checked out (base-dir: '*') @@ -193,42 +174,42 @@ def test_stage_explicit_basedir(cli, tmpdir, datafiles): assert checkout_contents == original_contents -@pytest.mark.parametrize('server_type', ('FTP', 'HTTP')) -@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch')) +@pytest.mark.parametrize("server_type", ("FTP", "HTTP")) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) def test_use_netrc(cli, datafiles, server_type, tmpdir): - file_server_files = os.path.join(str(tmpdir), 'file_server') - fake_home = os.path.join(str(tmpdir), 'fake_home') + file_server_files = os.path.join(str(tmpdir), "file_server") + fake_home = os.path.join(str(tmpdir), "fake_home") os.makedirs(file_server_files, exist_ok=True) os.makedirs(fake_home, exist_ok=True) project = str(datafiles) - checkoutdir = os.path.join(str(tmpdir), 'checkout') + checkoutdir = os.path.join(str(tmpdir), "checkout") - os.environ['HOME'] = fake_home - with open(os.path.join(fake_home, '.netrc'), 'wb') as f: + os.environ["HOME"] = fake_home + with open(os.path.join(fake_home, ".netrc"), "wb") as f: os.fchmod(f.fileno(), 0o700) - f.write(b'machine 127.0.0.1\n') - f.write(b'login testuser\n') - f.write(b'password 12345\n') + f.write(b"machine 127.0.0.1\n") + f.write(b"login testuser\n") + f.write(b"password 12345\n") with create_file_server(server_type) as server: - server.add_user('testuser', '12345', file_server_files) + server.add_user("testuser", "12345", file_server_files) generate_project_file_server(server, project) - src_zip = os.path.join(file_server_files, 'a.zip') - _assemble_zip(os.path.join(str(datafiles), 'content'), src_zip) + src_zip = os.path.join(file_server_files, "a.zip") + _assemble_zip(os.path.join(str(datafiles), "content"), src_zip) server.start() - result = cli.run(project=project, args=['source', 'track', 'target.bst']) + result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['source', 'fetch', 'target.bst']) + result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() - result = cli.run(project=project, args=['artifact', 'checkout', 'target.bst', '--directory', checkoutdir]) + result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() - original_dir = os.path.join(str(datafiles), 'content', 'a') + original_dir = os.path.join(str(datafiles), "content", "a") original_contents = list_dir_contents(original_dir) checkout_contents = list_dir_contents(checkoutdir) assert checkout_contents == original_contents diff --git a/tests/testutils/artifactshare.py b/tests/testutils/artifactshare.py index d96612686..8d0448f8a 100644 --- a/tests/testutils/artifactshare.py +++ b/tests/testutils/artifactshare.py @@ -29,7 +29,7 @@ class BaseArtifactShare: if port is None: raise Exception("Error occurred when starting artifact server.") - self.repo = 'http://localhost:{}'.format(port) + self.repo = "http://localhost:{}".format(port) # run(): # @@ -51,7 +51,7 @@ class BaseArtifactShare: cleanup_on_sigterm() server = stack.enter_context(self._create_server()) - port = server.add_insecure_port('localhost:0') + port = server.add_insecure_port("localhost:0") server.start() except Exception: q.put(None) @@ -104,7 +104,6 @@ class DummyArtifactShare(BaseArtifactShare): # enable_push (bool): Whether the share should allow pushes # class ArtifactShare(BaseArtifactShare): - def __init__(self, directory, *, quota=None, casd=False, index_only=False): # The working directory for the artifact share (in case it @@ -117,9 +116,9 @@ class ArtifactShare(BaseArtifactShare): # Unless this gets more complicated, just use this directly # in tests as a remote artifact push/pull configuration # - self.repodir = os.path.join(self.directory, 'repo') + self.repodir = os.path.join(self.directory, "repo") os.makedirs(self.repodir) - self.artifactdir = os.path.join(self.repodir, 'artifacts', 'refs') + self.artifactdir = os.path.join(self.repodir, "artifacts", "refs") os.makedirs(self.artifactdir) self.cas = CASCache(self.repodir, casd=casd) @@ -130,12 +129,7 @@ class ArtifactShare(BaseArtifactShare): super().__init__() def _create_server(self): - return create_server( - self.repodir, - quota=self.quota, - enable_push=True, - index_only=self.index_only, - ) + return create_server(self.repodir, quota=self.quota, enable_push=True, index_only=self.index_only,) # has_object(): # @@ -159,7 +153,7 @@ class ArtifactShare(BaseArtifactShare): artifact_path = os.path.join(self.artifactdir, artifact_name) try: - with open(artifact_path, 'rb') as f: + with open(artifact_path, "rb") as f: artifact_proto.ParseFromString(f.read()) except FileNotFoundError: return None @@ -171,8 +165,7 @@ class ArtifactShare(BaseArtifactShare): reachable = set() def reachable_dir(digest): - self.cas._reachable_refs_dir( - reachable, digest, update_mtime=False, check_exists=True) + self.cas._reachable_refs_dir(reachable, digest, update_mtime=False, check_exists=True) try: if str(artifact_proto.files): @@ -262,20 +255,22 @@ def create_dummy_artifact_share(): share.close() -statvfs_result = namedtuple('statvfs_result', 'f_blocks f_bfree f_bsize f_bavail') +statvfs_result = namedtuple("statvfs_result", "f_blocks f_bfree f_bsize f_bavail") # Assert that a given artifact is in the share # -def assert_shared(cli, share, project, element_name, *, project_name='test'): +def assert_shared(cli, share, project, element_name, *, project_name="test"): if not share.get_artifact(cli.get_artifact_name(project, project_name, element_name)): - raise AssertionError("Artifact share at {} does not contain the expected element {}" - .format(share.repo, element_name)) + raise AssertionError( + "Artifact share at {} does not contain the expected element {}".format(share.repo, element_name) + ) # Assert that a given artifact is not in the share # -def assert_not_shared(cli, share, project, element_name, *, project_name='test'): +def assert_not_shared(cli, share, project, element_name, *, project_name="test"): if share.get_artifact(cli.get_artifact_name(project, project_name, element_name)): - raise AssertionError("Artifact share at {} unexpectedly contains the element {}" - .format(share.repo, element_name)) + raise AssertionError( + "Artifact share at {} unexpectedly contains the element {}".format(share.repo, element_name) + ) diff --git a/tests/testutils/element_generators.py b/tests/testutils/element_generators.py index 0fbca7f3e..6da465ab7 100644 --- a/tests/testutils/element_generators.py +++ b/tests/testutils/element_generators.py @@ -28,8 +28,8 @@ def create_element_size(name, project_dir, elements_path, dependencies, size): os.makedirs(full_elements_path, exist_ok=True) # Create a git repo - repodir = os.path.join(project_dir, 'repos') - repo = create_repo('git', repodir, subdir=name) + repodir = os.path.join(project_dir, "repos") + repo = create_repo("git", repodir, subdir=name) with utils._tempdir(dir=project_dir) as tmp: @@ -38,26 +38,24 @@ def create_element_size(name, project_dir, elements_path, dependencies, size): # part; this ensures we never include a .git/ directory # in the cached artifacts for these sized elements. # - datadir = os.path.join(tmp, 'data') + datadir = os.path.join(tmp, "data") os.makedirs(datadir) # Use /dev/urandom to create the sized file in the datadir - with open(os.path.join(datadir, name), 'wb+') as f: + with open(os.path.join(datadir, name), "wb+") as f: f.write(os.urandom(size)) # Create the git repo from the temp directory ref = repo.create(tmp) element = { - 'kind': 'import', - 'sources': [ - repo.source_config(ref=ref) - ], - 'config': { + "kind": "import", + "sources": [repo.source_config(ref=ref)], + "config": { # Extract only the data directory - 'source': 'data' + "source": "data" }, - 'depends': dependencies + "depends": dependencies, } _yaml.roundtrip_dump(element, os.path.join(project_dir, elements_path, name)) @@ -91,9 +89,9 @@ def update_element_size(name, project_dir, repo, size): new_file = os.path.join(tmp, name) # Use /dev/urandom to create the sized file in the datadir - with open(new_file, 'wb+') as f: + with open(new_file, "wb+") as f: f.write(os.urandom(size)) # Modify the git repo with a new commit to the same path, # replacing the original file with a new one. - repo.modify_file(new_file, os.path.join('data', name)) + repo.modify_file(new_file, os.path.join("data", name)) diff --git a/tests/testutils/file_server.py b/tests/testutils/file_server.py index 05f896013..3e6e41954 100644 --- a/tests/testutils/file_server.py +++ b/tests/testutils/file_server.py @@ -6,9 +6,9 @@ from .http_server import SimpleHttpServer @contextmanager def create_file_server(file_server_type): - if file_server_type == 'FTP': + if file_server_type == "FTP": server = SimpleFtpServer() - elif file_server_type == 'HTTP': + elif file_server_type == "HTTP": server = SimpleHttpServer() else: assert False diff --git a/tests/testutils/filetypegenerator.py b/tests/testutils/filetypegenerator.py index 8b7d818d8..41c502be2 100644 --- a/tests/testutils/filetypegenerator.py +++ b/tests/testutils/filetypegenerator.py @@ -39,7 +39,7 @@ def generate_file_types(path): clean() - with open(path, 'w'): + with open(path, "w"): pass yield clean() diff --git a/tests/testutils/ftp_server.py b/tests/testutils/ftp_server.py index 52c05f8ba..7eda90b70 100644 --- a/tests/testutils/ftp_server.py +++ b/tests/testutils/ftp_server.py @@ -11,7 +11,7 @@ class SimpleFtpServer(multiprocessing.Process): self.authorizer = DummyAuthorizer() handler = FTPHandler handler.authorizer = self.authorizer - self.server = FTPServer(('127.0.0.1', 0), handler) + self.server = FTPServer(("127.0.0.1", 0), handler) def run(self): self.server.serve_forever() @@ -26,7 +26,7 @@ class SimpleFtpServer(multiprocessing.Process): self.authorizer.add_anonymous(cwd) def add_user(self, user, password, cwd): - self.authorizer.add_user(user, password, cwd, perm='elradfmwMT') + self.authorizer.add_user(user, password, cwd, perm="elradfmwMT") def base_url(self): - return 'ftp://127.0.0.1:{}'.format(self.server.address[1]) + return "ftp://127.0.0.1:{}".format(self.server.address[1]) diff --git a/tests/testutils/http_server.py b/tests/testutils/http_server.py index b72e745c5..8591159f8 100644 --- a/tests/testutils/http_server.py +++ b/tests/testutils/http_server.py @@ -11,45 +11,44 @@ class Unauthorized(Exception): class RequestHandler(SimpleHTTPRequestHandler): - def get_root_dir(self): - authorization = self.headers.get('authorization') + authorization = self.headers.get("authorization") if not authorization: if not self.server.anonymous_dir: - raise Unauthorized('unauthorized') + raise Unauthorized("unauthorized") return self.server.anonymous_dir else: authorization = authorization.split() - if len(authorization) != 2 or authorization[0].lower() != 'basic': - raise Unauthorized('unauthorized') + if len(authorization) != 2 or authorization[0].lower() != "basic": + raise Unauthorized("unauthorized") try: - decoded = base64.decodebytes(authorization[1].encode('ascii')) - user, password = decoded.decode('ascii').split(':') + decoded = base64.decodebytes(authorization[1].encode("ascii")) + user, password = decoded.decode("ascii").split(":") expected_password, directory = self.server.users[user] if password == expected_password: return directory - except: # noqa - raise Unauthorized('unauthorized') + except: # noqa + raise Unauthorized("unauthorized") return None def unauthorized(self): shortmsg, longmsg = self.responses[HTTPStatus.UNAUTHORIZED] self.send_response(HTTPStatus.UNAUTHORIZED, shortmsg) - self.send_header('Connection', 'close') - - content = (self.error_message_format % { - 'code': HTTPStatus.UNAUTHORIZED, - 'message': html.escape(longmsg, quote=False), - 'explain': html.escape(longmsg, quote=False) - }) - body = content.encode('UTF-8', 'replace') - self.send_header('Content-Type', self.error_content_type) - self.send_header('Content-Length', str(len(body))) - self.send_header('WWW-Authenticate', 'Basic realm="{}"'.format(self.server.realm)) + self.send_header("Connection", "close") + + content = self.error_message_format % { + "code": HTTPStatus.UNAUTHORIZED, + "message": html.escape(longmsg, quote=False), + "explain": html.escape(longmsg, quote=False), + } + body = content.encode("UTF-8", "replace") + self.send_header("Content-Type", self.error_content_type) + self.send_header("Content-Length", str(len(body))) + self.send_header("WWW-Authenticate", 'Basic realm="{}"'.format(self.server.realm)) self.end_headers() self.end_headers() - if self.command != 'HEAD' and body: + if self.command != "HEAD" and body: self.wfile.write(body) def do_GET(self): @@ -65,11 +64,11 @@ class RequestHandler(SimpleHTTPRequestHandler): self.unauthorized() def translate_path(self, path): - path = path.split('?', 1)[0] - path = path.split('#', 1)[0] + path = path.split("?", 1)[0] + path = path.split("#", 1)[0] path = posixpath.normpath(path) assert posixpath.isabs(path) - path = posixpath.relpath(path, '/') + path = posixpath.relpath(path, "/") return os.path.join(self.get_root_dir(), path) @@ -77,14 +76,14 @@ class AuthHTTPServer(HTTPServer): def __init__(self, *args, **kwargs): self.users = {} self.anonymous_dir = None - self.realm = 'Realm' + self.realm = "Realm" super().__init__(*args, **kwargs) class SimpleHttpServer(multiprocessing.Process): def __init__(self): super().__init__() - self.server = AuthHTTPServer(('127.0.0.1', 0), RequestHandler) + self.server = AuthHTTPServer(("127.0.0.1", 0), RequestHandler) self.started = False def start(self): @@ -107,4 +106,4 @@ class SimpleHttpServer(multiprocessing.Process): self.server.users[user] = (password, cwd) def base_url(self): - return 'http://127.0.0.1:{}'.format(self.server.server_port) + return "http://127.0.0.1:{}".format(self.server.server_port) diff --git a/tests/testutils/junction.py b/tests/testutils/junction.py index e867695c4..c086f6f17 100644 --- a/tests/testutils/junction.py +++ b/tests/testutils/junction.py @@ -19,17 +19,12 @@ def generate_junction(tmpdir, subproject_path, junction_path, *, store_ref=True) # Create a repo to hold the subproject and generate # a junction element for it # - repo = create_repo('git', str(tmpdir)) + repo = create_repo("git", str(tmpdir)) source_ref = ref = repo.create(subproject_path) if not store_ref: source_ref = None - element = { - 'kind': 'junction', - 'sources': [ - repo.source_config(ref=source_ref) - ] - } + element = {"kind": "junction", "sources": [repo.source_config(ref=source_ref)]} _yaml.roundtrip_dump(element, junction_path) return ref diff --git a/tests/testutils/patch.py b/tests/testutils/patch.py index df287f374..85b38def8 100644 --- a/tests/testutils/patch.py +++ b/tests/testutils/patch.py @@ -3,19 +3,17 @@ import subprocess def apply(file, patch): try: - subprocess.check_output(['patch', file, patch]) + subprocess.check_output(["patch", file, patch]) except subprocess.CalledProcessError as e: - message = "Patch failed with exit code {}\n Output:\n {}".format( - e.returncode, e.output) + message = "Patch failed with exit code {}\n Output:\n {}".format(e.returncode, e.output) print(message) raise def remove(file, patch): try: - subprocess.check_output(['patch', '--reverse', file, patch]) + subprocess.check_output(["patch", "--reverse", file, patch]) except subprocess.CalledProcessError as e: - message = "patch --reverse failed with exit code {}\n Output:\n {}".format( - e.returncode, e.output) + message = "patch --reverse failed with exit code {}\n Output:\n {}".format(e.returncode, e.output) print(message) raise diff --git a/tests/testutils/python_repo.py b/tests/testutils/python_repo.py index c8e5bf343..7d9ae4e47 100644 --- a/tests/testutils/python_repo.py +++ b/tests/testutils/python_repo.py @@ -7,7 +7,7 @@ import sys import pytest -SETUP_TEMPLATE = '''\ +SETUP_TEMPLATE = """\ from setuptools import setup setup( @@ -22,18 +22,18 @@ setup( ] }} ) -''' +""" # All packages generated via generate_pip_package will have the functions below -INIT_TEMPLATE = '''\ +INIT_TEMPLATE = """\ def main(): print('This is {name}') def hello(actor='world'): print('Hello {{}}! This is {name}'.format(actor)) -''' +""" -HTML_TEMPLATE = '''\ +HTML_TEMPLATE = """\ <html> <head> <title>Links for {name}</title> @@ -42,7 +42,7 @@ HTML_TEMPLATE = '''\ <a href='{name}-{version}.tar.gz'>{name}-{version}.tar.gz</a><br /> </body> </html> -''' +""" # Creates a simple python source distribution and copies this into a specified @@ -57,11 +57,11 @@ HTML_TEMPLATE = '''\ # Returns: # None # -def generate_pip_package(tmpdir, pypi, name, version='0.1', dependencies=None): +def generate_pip_package(tmpdir, pypi, name, version="0.1", dependencies=None): if dependencies is None: dependencies = [] # check if package already exists in pypi - pypi_package = os.path.join(pypi, re.sub('[^0-9a-zA-Z]+', '-', name)) + pypi_package = os.path.join(pypi, re.sub("[^0-9a-zA-Z]+", "-", name)) if os.path.exists(pypi_package): return @@ -73,29 +73,22 @@ def generate_pip_package(tmpdir, pypi, name, version='0.1', dependencies=None): # `-- package # `-- __init__.py # - setup_file = os.path.join(tmpdir, 'setup.py') - pkgdirname = re.sub('[^0-9a-zA-Z]+', '', name) - with open(setup_file, 'w') as f: - f.write( - SETUP_TEMPLATE.format( - name=name, - version=version, - pkgdirname=pkgdirname, - pkgdeps=dependencies - ) - ) + setup_file = os.path.join(tmpdir, "setup.py") + pkgdirname = re.sub("[^0-9a-zA-Z]+", "", name) + with open(setup_file, "w") as f: + f.write(SETUP_TEMPLATE.format(name=name, version=version, pkgdirname=pkgdirname, pkgdeps=dependencies)) os.chmod(setup_file, 0o755) package = os.path.join(tmpdir, pkgdirname) os.makedirs(package) - main_file = os.path.join(package, '__init__.py') - with open(main_file, 'w') as f: + main_file = os.path.join(package, "__init__.py") + with open(main_file, "w") as f: f.write(INIT_TEMPLATE.format(name=name)) os.chmod(main_file, 0o644) # Run sdist with a fresh process - p = subprocess.run([sys.executable, 'setup.py', 'sdist'], cwd=tmpdir) + p = subprocess.run([sys.executable, "setup.py", "sdist"], cwd=tmpdir) assert p.returncode == 0 # create directory for this package in pypi resulting in a directory @@ -109,12 +102,12 @@ def generate_pip_package(tmpdir, pypi, name, version='0.1', dependencies=None): os.makedirs(pypi_package) # add an index html page - index_html = os.path.join(pypi_package, 'index.html') - with open(index_html, 'w') as f: + index_html = os.path.join(pypi_package, "index.html") + with open(index_html, "w") as f: f.write(HTML_TEMPLATE.format(name=name, version=version)) # copy generated tarfile to pypi package - dist_dir = os.path.join(tmpdir, 'dist') + dist_dir = os.path.join(tmpdir, "dist") for tar in os.listdir(dist_dir): tarpath = os.path.join(dist_dir, tar) shutil.copy(tarpath, pypi_package) @@ -123,7 +116,7 @@ def generate_pip_package(tmpdir, pypi, name, version='0.1', dependencies=None): @pytest.fixture def setup_pypi_repo(tmpdir): def create_pkgdir(package): - pkgdirname = re.sub('[^0-9a-zA-Z]+', '', package) + pkgdirname = re.sub("[^0-9a-zA-Z]+", "", package) pkgdir = os.path.join(str(tmpdir), pkgdirname) os.makedirs(pkgdir) return pkgdir diff --git a/tests/testutils/repo/bzr.py b/tests/testutils/repo/bzr.py index 074712af1..b6983416e 100644 --- a/tests/testutils/repo/bzr.py +++ b/tests/testutils/repo/bzr.py @@ -7,7 +7,6 @@ from buildstream.testing._utils.site import BZR, BZR_ENV, HAVE_BZR class Bzr(Repo): - def __init__(self, directory, subdir): if not HAVE_BZR: pytest.skip("bzr is not available") @@ -20,34 +19,29 @@ class Bzr(Repo): def create(self, directory): # Work around race condition in bzr's creation of ~/.bazaar in # ensure_config_dir_exists() when running tests in parallel. - bazaar_config_dir = os.path.expanduser('~/.bazaar') + bazaar_config_dir = os.path.expanduser("~/.bazaar") os.makedirs(bazaar_config_dir, exist_ok=True) - branch_dir = os.path.join(self.repo, 'trunk') + branch_dir = os.path.join(self.repo, "trunk") - subprocess.call([self.bzr, 'init-repo', self.repo], env=self.env) - subprocess.call([self.bzr, 'init', branch_dir], env=self.env) + subprocess.call([self.bzr, "init-repo", self.repo], env=self.env) + subprocess.call([self.bzr, "init", branch_dir], env=self.env) self.copy_directory(directory, branch_dir) - subprocess.call([self.bzr, 'add', '.'], env=self.env, cwd=branch_dir) - subprocess.call([self.bzr, 'commit', '--message="Initial commit"'], - env=self.env, cwd=branch_dir) + subprocess.call([self.bzr, "add", "."], env=self.env, cwd=branch_dir) + subprocess.call([self.bzr, "commit", '--message="Initial commit"'], env=self.env, cwd=branch_dir) return self.latest_commit() def source_config(self, ref=None): - config = { - 'kind': 'bzr', - 'url': 'file://' + self.repo, - 'track': 'trunk' - } + config = {"kind": "bzr", "url": "file://" + self.repo, "track": "trunk"} if ref is not None: - config['ref'] = ref + config["ref"] = ref return config def latest_commit(self): - return subprocess.check_output([ - self.bzr, 'version-info', - '--custom', '--template={revno}', - os.path.join(self.repo, 'trunk') - ], env=self.env, universal_newlines=True).strip() + return subprocess.check_output( + [self.bzr, "version-info", "--custom", "--template={revno}", os.path.join(self.repo, "trunk")], + env=self.env, + universal_newlines=True, + ).strip() diff --git a/tests/testutils/repo/git.py b/tests/testutils/repo/git.py index 46694fcf2..b9360e9cd 100644 --- a/tests/testutils/repo/git.py +++ b/tests/testutils/repo/git.py @@ -9,7 +9,6 @@ from buildstream.testing._utils.site import GIT, GIT_ENV, HAVE_GIT class Git(Repo): - def __init__(self, directory, subdir): if not HAVE_GIT: pytest.skip("git is not available") @@ -24,99 +23,87 @@ class Git(Repo): def _run_git(self, *args, **kwargs): argv = [GIT] argv.extend(args) - if 'env' not in kwargs: - kwargs['env'] = dict(self.env, PWD=self.repo) - kwargs.setdefault('cwd', self.repo) - kwargs.setdefault('check', True) + if "env" not in kwargs: + kwargs["env"] = dict(self.env, PWD=self.repo) + kwargs.setdefault("cwd", self.repo) + kwargs.setdefault("check", True) return subprocess.run(argv, **kwargs) def create(self, directory): self.copy_directory(directory, self.repo) - self._run_git('init', '.') - self._run_git('add', '.') - self._run_git('commit', '-m', 'Initial commit') + self._run_git("init", ".") + self._run_git("add", ".") + self._run_git("commit", "-m", "Initial commit") return self.latest_commit() def add_tag(self, tag): - self._run_git('tag', tag) + self._run_git("tag", tag) def add_annotated_tag(self, tag, message): - self._run_git('tag', '-a', tag, '-m', message) + self._run_git("tag", "-a", tag, "-m", message) def add_commit(self): - self._run_git('commit', '--allow-empty', '-m', 'Additional commit') + self._run_git("commit", "--allow-empty", "-m", "Additional commit") return self.latest_commit() def add_file(self, filename): shutil.copy(filename, self.repo) - self._run_git('add', os.path.basename(filename)) - self._run_git('commit', '-m', 'Added {}'.format(os.path.basename(filename))) + self._run_git("add", os.path.basename(filename)) + self._run_git("commit", "-m", "Added {}".format(os.path.basename(filename))) return self.latest_commit() def modify_file(self, new_file, path): shutil.copy(new_file, os.path.join(self.repo, path)) - self._run_git('commit', path, '-m', 'Modified {}'.format(os.path.basename(path))) + self._run_git("commit", path, "-m", "Modified {}".format(os.path.basename(path))) return self.latest_commit() def add_submodule(self, subdir, url=None, checkout=None): submodule = {} if checkout is not None: - submodule['checkout'] = checkout + submodule["checkout"] = checkout if url is not None: - submodule['url'] = url + submodule["url"] = url self.submodules[subdir] = submodule - self._run_git('submodule', 'add', url, subdir) - self._run_git('commit', '-m', 'Added the submodule') + self._run_git("submodule", "add", url, subdir) + self._run_git("commit", "-m", "Added the submodule") return self.latest_commit() # This can also be used to a file or a submodule def remove_path(self, path): - self._run_git('rm', path) - self._run_git('commit', '-m', 'Removing {}'.format(path)) + self._run_git("rm", path) + self._run_git("commit", "-m", "Removing {}".format(path)) return self.latest_commit() def source_config(self, ref=None): return self.source_config_extra(ref) def source_config_extra(self, ref=None, checkout_submodules=None): - config = { - 'kind': 'git', - 'url': 'file://' + self.repo, - 'track': 'master' - } + config = {"kind": "git", "url": "file://" + self.repo, "track": "master"} if ref is not None: - config['ref'] = ref + config["ref"] = ref if checkout_submodules is not None: - config['checkout-submodules'] = checkout_submodules + config["checkout-submodules"] = checkout_submodules if self.submodules: - config['submodules'] = dict(self.submodules) + config["submodules"] = dict(self.submodules) return config def latest_commit(self): - return self._run_git( - 'rev-parse', 'HEAD', - stdout=subprocess.PIPE, - universal_newlines=True, - ).stdout.strip() + return self._run_git("rev-parse", "HEAD", stdout=subprocess.PIPE, universal_newlines=True,).stdout.strip() def branch(self, branch_name): - self._run_git('checkout', '-b', branch_name) + self._run_git("checkout", "-b", branch_name) def delete_tag(self, tag_name): - self._run_git('tag', '-d', tag_name) + self._run_git("tag", "-d", tag_name) def checkout(self, commit): - self._run_git('checkout', commit) + self._run_git("checkout", commit) def merge(self, commit): - self._run_git('merge', '-m', 'Merge', commit) + self._run_git("merge", "-m", "Merge", commit) return self.latest_commit() def rev_parse(self, rev): - return self._run_git( - 'rev-parse', rev, - stdout=subprocess.PIPE, - universal_newlines=True, - ).stdout.strip() + return self._run_git("rev-parse", rev, stdout=subprocess.PIPE, universal_newlines=True,).stdout.strip() diff --git a/tests/testutils/repo/tar.py b/tests/testutils/repo/tar.py index 63231fa4b..3eb9d896b 100644 --- a/tests/testutils/repo/tar.py +++ b/tests/testutils/repo/tar.py @@ -7,9 +7,8 @@ from buildstream.testing import Repo class Tar(Repo): - def create(self, directory): - tarball = os.path.join(self.repo, 'file.tar.gz') + tarball = os.path.join(self.repo, "file.tar.gz") old_dir = os.getcwd() os.chdir(directory) @@ -20,14 +19,9 @@ class Tar(Repo): return sha256sum(tarball) def source_config(self, ref=None): - tarball = os.path.join(self.repo, 'file.tar.gz') - config = { - 'kind': 'tar', - 'url': 'file://' + tarball, - 'directory': '', - 'base-dir': '' - } + tarball = os.path.join(self.repo, "file.tar.gz") + config = {"kind": "tar", "url": "file://" + tarball, "directory": "", "base-dir": ""} if ref is not None: - config['ref'] = ref + config["ref"] = ref return config diff --git a/tests/testutils/repo/zip.py b/tests/testutils/repo/zip.py index df3f834a9..38419e642 100644 --- a/tests/testutils/repo/zip.py +++ b/tests/testutils/repo/zip.py @@ -7,14 +7,13 @@ from buildstream.testing import Repo class Zip(Repo): - def create(self, directory): - archive = os.path.join(self.repo, 'file.zip') + archive = os.path.join(self.repo, "file.zip") old_dir = os.getcwd() os.chdir(directory) with zipfile.ZipFile(archive, "w") as zipfp: - for root, dirs, files in os.walk('.'): + for root, dirs, files in os.walk("."): names = dirs + files names = [os.path.join(root, name) for name in names] @@ -26,14 +25,9 @@ class Zip(Repo): return sha256sum(archive) def source_config(self, ref=None): - archive = os.path.join(self.repo, 'file.zip') - config = { - 'kind': 'zip', - 'url': 'file://' + archive, - 'directory': '', - 'base-dir': '' - } + archive = os.path.join(self.repo, "file.zip") + config = {"kind": "zip", "url": "file://" + archive, "directory": "", "base-dir": ""} if ref is not None: - config['ref'] = ref + config["ref"] = ref return config diff --git a/tests/testutils/setuptools.py b/tests/testutils/setuptools.py index cb61e1976..0f7f30f91 100644 --- a/tests/testutils/setuptools.py +++ b/tests/testutils/setuptools.py @@ -4,19 +4,17 @@ import pkg_resources # A mock setuptools dist object. -class MockDist(): +class MockDist: def __init__(self, datafiles, module_name): self.datafiles = datafiles self.module_name = module_name def get_resource_filename(self, *_args, **_kwargs): - return os.path.join(self.datafiles.dirname, - self.datafiles.basename, - self.module_name) + return os.path.join(self.datafiles.dirname, self.datafiles.basename, self.module_name) # A mock setuptools entry object. -class MockEntry(): +class MockEntry: def __init__(self, datafiles, module_name): self.dist = MockDist(datafiles, module_name) self.module_name = module_name @@ -30,7 +28,7 @@ class MockEntry(): @pytest.fixture() def entry_fixture(monkeypatch): def patch(datafiles, entry_point, lookup_string): - dist, package = lookup_string.split(':') + dist, package = lookup_string.split(":") def mock_entry(pdist, pentry_point, ppackage): assert pdist == dist @@ -38,6 +36,7 @@ def entry_fixture(monkeypatch): assert ppackage == package return MockEntry(datafiles, package) - monkeypatch.setattr(pkg_resources, 'get_entry_info', mock_entry) + + monkeypatch.setattr(pkg_resources, "get_entry_info", mock_entry) return patch |