diff options
Diffstat (limited to 'src/buildstream/storage')
| -rw-r--r-- | src/buildstream/storage/_casbaseddirectory.py | 134 | ||||
| -rw-r--r-- | src/buildstream/storage/_filebaseddirectory.py | 59 |
2 files changed, 43 insertions, 150 deletions
diff --git a/src/buildstream/storage/_casbaseddirectory.py b/src/buildstream/storage/_casbaseddirectory.py index 3b248f3ae..9c5c179b0 100644 --- a/src/buildstream/storage/_casbaseddirectory.py +++ b/src/buildstream/storage/_casbaseddirectory.py @@ -99,15 +99,7 @@ class CasBasedDirectory(Directory): _pb2_path_sep = "/" _pb2_absolute_path_prefix = "/" - def __init__( - self, - cas_cache, - *, - digest=None, - parent=None, - common_name="untitled", - filename=None - ): + def __init__(self, cas_cache, *, digest=None, parent=None, common_name="untitled", filename=None): self.filename = filename self.common_name = common_name self.cas_cache = cas_cache @@ -123,25 +115,16 @@ class CasBasedDirectory(Directory): with open(self.cas_cache.objpath(digest), "rb") as f: pb2_directory.ParseFromString(f.read()) except FileNotFoundError as e: - raise VirtualDirectoryError( - "Directory not found in local cache: {}".format(e) - ) from e + raise VirtualDirectoryError("Directory not found in local cache: {}".format(e)) from e for entry in pb2_directory.directories: - self.index[entry.name] = IndexEntry( - entry.name, _FileType.DIRECTORY, digest=entry.digest - ) + self.index[entry.name] = IndexEntry(entry.name, _FileType.DIRECTORY, digest=entry.digest) for entry in pb2_directory.files: self.index[entry.name] = IndexEntry( - entry.name, - _FileType.REGULAR_FILE, - digest=entry.digest, - is_executable=entry.is_executable, + entry.name, _FileType.REGULAR_FILE, digest=entry.digest, is_executable=entry.is_executable, ) for entry in pb2_directory.symlinks: - self.index[entry.name] = IndexEntry( - entry.name, _FileType.SYMLINK, target=entry.target - ) + self.index[entry.name] = IndexEntry(entry.name, _FileType.SYMLINK, target=entry.target) def _find_self_in_parent(self): assert self.parent is not None @@ -156,20 +139,14 @@ class CasBasedDirectory(Directory): newdir = CasBasedDirectory(self.cas_cache, parent=self, filename=name) - self.index[name] = IndexEntry( - name, _FileType.DIRECTORY, buildstream_object=newdir - ) + self.index[name] = IndexEntry(name, _FileType.DIRECTORY, buildstream_object=newdir) self.__invalidate_digest() return newdir def _add_file(self, basename, filename, modified=False, can_link=False): - entry = IndexEntry( - filename, - _FileType.REGULAR_FILE, - modified=modified or filename in self.index, - ) + entry = IndexEntry(filename, _FileType.REGULAR_FILE, modified=modified or filename in self.index,) path = os.path.join(basename, filename) entry.digest = self.cas_cache.add_object(path=path, link_directly=can_link) entry.is_executable = os.access(path, os.X_OK) @@ -178,14 +155,10 @@ class CasBasedDirectory(Directory): self.__invalidate_digest() def _copy_link_from_filesystem(self, basename, filename): - self._add_new_link_direct( - filename, os.readlink(os.path.join(basename, filename)) - ) + self._add_new_link_direct(filename, os.readlink(os.path.join(basename, filename))) def _add_new_link_direct(self, name, target): - self.index[name] = IndexEntry( - name, _FileType.SYMLINK, target=target, modified=name in self.index - ) + self.index[name] = IndexEntry(name, _FileType.SYMLINK, target=target, modified=name in self.index) self.__invalidate_digest() @@ -237,20 +210,13 @@ class CasBasedDirectory(Directory): linklocation = entry.target newpaths = linklocation.split(os.path.sep) if os.path.isabs(linklocation): - current_dir = current_dir.find_root().descend( - *newpaths, follow_symlinks=True - ) + current_dir = current_dir.find_root().descend(*newpaths, follow_symlinks=True) else: - current_dir = current_dir.descend( - *newpaths, follow_symlinks=True - ) + current_dir = current_dir.descend(*newpaths, follow_symlinks=True) else: - error = ( - "Cannot descend into {}, which is a '{}' in the directory {}" - ) + error = "Cannot descend into {}, which is a '{}' in the directory {}" raise VirtualDirectoryError( - error.format(path, current_dir.index[path].type, current_dir), - reason="not-a-directory", + error.format(path, current_dir.index[path].type, current_dir), reason="not-a-directory", ) else: if path == ".": @@ -265,8 +231,7 @@ class CasBasedDirectory(Directory): else: error = "'{}' not found in {}" raise VirtualDirectoryError( - error.format(path, str(current_dir)), - reason="directory-not-found", + error.format(path, str(current_dir)), reason="directory-not-found", ) return current_dir @@ -297,9 +262,7 @@ class CasBasedDirectory(Directory): fileListResult.overwritten.append(relative_pathname) return True - def _partial_import_cas_into_cas( - self, source_directory, filter_callback, *, path_prefix="", origin=None, result - ): + def _partial_import_cas_into_cas(self, source_directory, filter_callback, *, path_prefix="", origin=None, result): """ Import files from a CAS-based directory. """ if origin is None: origin = self @@ -318,9 +281,7 @@ class CasBasedDirectory(Directory): # we can import the whole source directory by digest instead # of importing each directory entry individually. subdir_digest = entry.get_digest() - dest_entry = IndexEntry( - name, _FileType.DIRECTORY, digest=subdir_digest - ) + dest_entry = IndexEntry(name, _FileType.DIRECTORY, digest=subdir_digest) self.index[name] = dest_entry self.__invalidate_digest() @@ -337,9 +298,7 @@ class CasBasedDirectory(Directory): else: subdir = dest_entry.get_directory(self) - subdir.__add_files_to_result( - path_prefix=relative_pathname, result=result - ) + subdir.__add_files_to_result(path_prefix=relative_pathname, result=result) else: src_subdir = source_directory.descend(name) if src_subdir == origin: @@ -350,17 +309,11 @@ class CasBasedDirectory(Directory): except VirtualDirectoryError: filetype = self.index[name].type raise VirtualDirectoryError( - "Destination is a {}, not a directory: /{}".format( - filetype, relative_pathname - ) + "Destination is a {}, not a directory: /{}".format(filetype, relative_pathname) ) dest_subdir._partial_import_cas_into_cas( - src_subdir, - filter_callback, - path_prefix=relative_pathname, - origin=origin, - result=result, + src_subdir, filter_callback, path_prefix=relative_pathname, origin=origin, result=result, ) if filter_callback and not filter_callback(relative_pathname): @@ -388,13 +341,7 @@ class CasBasedDirectory(Directory): result.files_written.append(relative_pathname) def import_files( - self, - external_pathspec, - *, - filter_callback=None, - report_written=True, - update_mtime=False, - can_link=False + self, external_pathspec, *, filter_callback=None, report_written=True, update_mtime=False, can_link=False ): """ See superclass Directory for arguments """ @@ -413,9 +360,7 @@ class CasBasedDirectory(Directory): external_pathspec = CasBasedDirectory(self.cas_cache, digest=digest) assert isinstance(external_pathspec, CasBasedDirectory) - self._partial_import_cas_into_cas( - external_pathspec, filter_callback, result=result - ) + self._partial_import_cas_into_cas(external_pathspec, filter_callback, result=result) # TODO: No notice is taken of report_written or update_mtime. # Current behaviour is to fully populate the report, which is inefficient, @@ -425,11 +370,7 @@ class CasBasedDirectory(Directory): def import_single_file(self, external_pathspec): result = FileListResult() - if self._check_replacement( - os.path.basename(external_pathspec), - os.path.dirname(external_pathspec), - result, - ): + if self._check_replacement(os.path.basename(external_pathspec), os.path.dirname(external_pathspec), result,): self._add_file( os.path.dirname(external_pathspec), os.path.basename(external_pathspec), @@ -495,9 +436,7 @@ class CasBasedDirectory(Directory): f = StringIO(entry.target) tarfile.addfile(tarinfo, f) else: - raise VirtualDirectoryError( - "can not export file type {} to tar".format(entry.type) - ) + raise VirtualDirectoryError("can not export file type {} to tar".format(entry.type)) def _mark_changed(self): """ It should not be possible to externally modify a CAS-based @@ -588,12 +527,8 @@ class CasBasedDirectory(Directory): """ - file_list = list( - filter(lambda i: i[1].type != _FileType.DIRECTORY, self.index.items()) - ) - directory_list = filter( - lambda i: i[1].type == _FileType.DIRECTORY, self.index.items() - ) + file_list = list(filter(lambda i: i[1].type != _FileType.DIRECTORY, self.index.items())) + directory_list = filter(lambda i: i[1].type == _FileType.DIRECTORY, self.index.items()) if prefix != "": yield prefix @@ -603,9 +538,7 @@ class CasBasedDirectory(Directory): for (k, v) in sorted(directory_list): subdir = v.get_directory(self) - yield from subdir._list_prefixed_relative_paths( - prefix=os.path.join(prefix, k) - ) + yield from subdir._list_prefixed_relative_paths(prefix=os.path.join(prefix, k)) def walk(self): """Provide a list of dictionaries containing information about the files. @@ -673,8 +606,7 @@ class CasBasedDirectory(Directory): """ There is no underlying directory for a CAS-backed directory, so throw an exception. """ raise VirtualDirectoryError( - "_get_underlying_directory was called on a CAS-backed directory," - + " which has no underlying directory." + "_get_underlying_directory was called on a CAS-backed directory," + " which has no underlying directory." ) # _get_digest(): @@ -712,9 +644,7 @@ class CasBasedDirectory(Directory): symlinknode.name = name symlinknode.target = entry.target - self.__digest = self.cas_cache.add_object( - buffer=pb2_directory.SerializeToString() - ) + self.__digest = self.cas_cache.add_object(buffer=pb2_directory.SerializeToString()) return self.__digest @@ -729,9 +659,7 @@ class CasBasedDirectory(Directory): linklocation = target.target newpath = linklocation.split(os.path.sep) if os.path.isabs(linklocation): - return subdir.find_root()._exists( - *newpath, follow_symlinks=True - ) + return subdir.find_root()._exists(*newpath, follow_symlinks=True) return subdir._exists(*newpath, follow_symlinks=True) return False except VirtualDirectoryError: @@ -750,8 +678,6 @@ class CasBasedDirectory(Directory): if entry.type == _FileType.DIRECTORY: subdir = self.descend(name) - subdir.__add_files_to_result( - path_prefix=relative_pathname, result=result - ) + subdir.__add_files_to_result(path_prefix=relative_pathname, result=result) else: result.files_written.append(relative_pathname) diff --git a/src/buildstream/storage/_filebaseddirectory.py b/src/buildstream/storage/_filebaseddirectory.py index 21515649d..0926c1b3a 100644 --- a/src/buildstream/storage/_filebaseddirectory.py +++ b/src/buildstream/storage/_filebaseddirectory.py @@ -58,9 +58,7 @@ class FileBasedDirectory(Directory): """ See superclass Directory for arguments """ if follow_symlinks: - ImplError( - "FileBasedDirectory.Decend dose not implement follow_symlinks=True" - ) + ImplError("FileBasedDirectory.Decend dose not implement follow_symlinks=True") current_dir = self @@ -74,38 +72,24 @@ class FileBasedDirectory(Directory): st = os.lstat(new_path) if not stat.S_ISDIR(st.st_mode): raise VirtualDirectoryError( - "Cannot descend into '{}': '{}' is not a directory".format( - path, new_path - ) + "Cannot descend into '{}': '{}' is not a directory".format(path, new_path) ) except FileNotFoundError: if create: os.mkdir(new_path) else: - raise VirtualDirectoryError( - "Cannot descend into '{}': '{}' does not exist".format( - path, new_path - ) - ) + raise VirtualDirectoryError("Cannot descend into '{}': '{}' does not exist".format(path, new_path)) current_dir = FileBasedDirectory(new_path) return current_dir def import_files( - self, - external_pathspec, - *, - filter_callback=None, - report_written=True, - update_mtime=False, - can_link=False + self, external_pathspec, *, filter_callback=None, report_written=True, update_mtime=False, can_link=False ): """ See superclass Directory for arguments """ - from ._casbaseddirectory import ( - CasBasedDirectory, - ) # pylint: disable=cyclic-import + from ._casbaseddirectory import CasBasedDirectory # pylint: disable=cyclic-import if isinstance(external_pathspec, CasBasedDirectory): if can_link and not update_mtime: @@ -114,9 +98,7 @@ class FileBasedDirectory(Directory): actionfunc = utils.safe_copy import_result = FileListResult() - self._import_files_from_cas( - external_pathspec, actionfunc, filter_callback, result=import_result - ) + self._import_files_from_cas(external_pathspec, actionfunc, filter_callback, result=import_result) else: if isinstance(external_pathspec, Directory): source_directory = external_pathspec.external_directory @@ -144,15 +126,11 @@ class FileBasedDirectory(Directory): cur_time = time.time() for f in import_result.files_written: - os.utime( - os.path.join(self.external_directory, f), times=(cur_time, cur_time) - ) + os.utime(os.path.join(self.external_directory, f), times=(cur_time, cur_time)) return import_result def import_single_file(self, external_pathspec): - dstpath = os.path.join( - self.external_directory, os.path.basename(external_pathspec) - ) + dstpath = os.path.join(self.external_directory, os.path.basename(external_pathspec)) result = FileListResult() if os.path.exists(dstpath): result.ignored.append(dstpath) @@ -206,9 +184,7 @@ class FileBasedDirectory(Directory): tarfile.addfile(tarinfo, f) elif tarinfo.isdir(): tarfile.addfile(tarinfo) - self.descend(*filename.split(os.path.sep)).export_to_tar( - tarfile, arcname, mtime - ) + self.descend(*filename.split(os.path.sep)).export_to_tar(tarfile, arcname, mtime) else: tarfile.addfile(tarinfo) @@ -230,8 +206,7 @@ class FileBasedDirectory(Directory): return [ f for f in list_relative_paths(self.external_directory) - if _get_link_mtime(os.path.join(self.external_directory, f)) - != BST_ARBITRARY_TIMESTAMP + if _get_link_mtime(os.path.join(self.external_directory, f)) != BST_ARBITRARY_TIMESTAMP ] def list_relative_paths(self): @@ -272,9 +247,7 @@ class FileBasedDirectory(Directory): else: return _FileType.SPECIAL_FILE - def _import_files_from_cas( - self, source_directory, actionfunc, filter_callback, *, path_prefix="", result - ): + def _import_files_from_cas(self, source_directory, actionfunc, filter_callback, *, path_prefix="", result): """ Import files from a CAS-based directory. """ for name, entry in source_directory.index.items(): @@ -295,17 +268,11 @@ class FileBasedDirectory(Directory): except VirtualDirectoryError: filetype = self._get_filetype(name) raise VirtualDirectoryError( - "Destination is a {}, not a directory: /{}".format( - filetype, relative_pathname - ) + "Destination is a {}, not a directory: /{}".format(filetype, relative_pathname) ) dest_subdir._import_files_from_cas( - src_subdir, - actionfunc, - filter_callback, - path_prefix=relative_pathname, - result=result, + src_subdir, actionfunc, filter_callback, path_prefix=relative_pathname, result=result, ) if filter_callback and not filter_callback(relative_pathname): |