diff options
| author | Jim MacArthur <jim.macarthur@codethink.co.uk> | 2018-07-13 17:44:52 +0100 |
|---|---|---|
| committer | Jim MacArthur <jim.macarthur@codethink.co.uk> | 2018-07-13 17:44:52 +0100 |
| commit | 25065284c2a42626bdd36b6c200153ca6a0c6740 (patch) | |
| tree | 3aa27de2e7a91300c773e5585bd1ce4104b916b9 | |
| parent | 48cca2b555036c39371d9bce379e1d62d5728c72 (diff) | |
| download | buildstream-25065284c2a42626bdd36b6c200153ca6a0c6740.tar.gz | |
Cleanup of previous code
| -rw-r--r-- | buildstream/storage/_casbaseddirectory.py | 106 |
1 files changed, 47 insertions, 59 deletions
diff --git a/buildstream/storage/_casbaseddirectory.py b/buildstream/storage/_casbaseddirectory.py index 61ee976d2..edde893b5 100644 --- a/buildstream/storage/_casbaseddirectory.py +++ b/buildstream/storage/_casbaseddirectory.py @@ -398,126 +398,114 @@ class CasBasedDirectory(Directory): result.file_written.apppend(os.path.join(prefix, entry.name)) return result - def filelist_iterator(sorted_files): - """ An interator which just returns the top level directories from sorted_files. It will return any directories listed as a single return value. The return value is a tuple consisting - of a path component and a list of sub-components. If the list is empty, this indicates that the return value is a file. """ - last_directory = None - for f in sorted_files: - components = f.split(os.path.sep) - if len(components)>1: - if components[0] != last_directory: - if last_directory is not None: - yield (components[0], subcomponents) - subcomponents = [] - subcomponents.append(os.path.sep.join(components[1:])) - else: - if last_directory is not None: - yield (last_directory, subcomponents) - yield(f, []) - if last_directory is not None: - yield (last_directory, subcomponents) - def files_in_subdir(sorted_files, dirname): - """ Filters sorted_files and returns only the ones which have 'dirname' as a prefix, with that prefix removed. """ + """Filters sorted_files and returns only the ones which have + 'dirname' as a prefix, with that prefix removed. + + """ if not dirname.endswith(os.path.sep): dirname += os.path.sep - return [f.lstrip(dirname) for f in sorted_files if f.startswith(dirname) ] + return [f.lstrip(dirname) for f in sorted_files if f.startswith(dirname)] def _partial_import_cas_into_cas(self, source_directory, files, path_prefix="", file_list_required=True): + """ Import only the files and symlinks listed in 'files' from source_directory to this one. + Args: + source_directory (:class:`.CasBasedDirectory`): The directory to import from + files ([str]): List of pathnames to import. + path_prefix (str): Prefix used to add entries to the file list result. + file_list_required: Whether to update the file list while processing. + """ + result = FileListResult() sorted_files = sorted(files) # Check if this is necessary - processed_entries = set() + processed_directories = set() for f in sorted_files: if f == ".": continue + fullname = os.path.join(path_prefix, f) components = f.split(os.path.sep) if len(components)>1 or isinstance(source_directory.index[components[0]].buildstream_object, CasBasedDirectory): # Then we are importing a directory dirname = components[0] - if dirname not in processed_entries: + if dirname not in processed_directories: subcomponents = CasBasedDirectory.files_in_subdir(sorted_files, dirname) dest_subdir = self.descend(dirname, create=True) src_subdir = source_directory.descend(dirname) - import_result = dest_subdir._partial_import_cas_into_cas(src_subdir, subcomponents, path_prefix=os.path.join(path_prefix,f), file_list_required=file_list_required) + import_result = dest_subdir._partial_import_cas_into_cas(src_subdir, subcomponents, + path_prefix=fullname, file_list_required=file_list_required) result.combine(import_result) - processed_entries.add(dirname) + processed_directories.add(dirname) else: self._check_replacement(f, path_prefix, result) - fullname = os.path.join(path_prefix, f) - item = source_directory.index[f] - if isinstance(item.pb2_object, remote_execution_pb2.FileNode): - filenode = self.pb2_directory.files.add(digest=item.pb2_object.digest) - filenode.name = f - is_executable = item.pb2_object.is_executable - filenode.is_executable = is_executable + item = source_directory.index[f].pb2_object + if isinstance(item, remote_execution_pb2.FileNode): + filenode = self.pb2_directory.files.add(digest=item.digest, name=f, + is_executable=item.is_executable) self.index[f] = IndexEntry(filenode, modified=(fullname in result.overwritten)) - else: # Must be a symlink - symlinknode = self.pb2_directory.symlinks.add() - symlinknode.name = fullname + else: + assert(isinstance(item.pb2_object, remote_execution_pb2.SymlinkNode)) + symlinknode = self.pb2_directory.symlinks.add(name=f, target=item.pb2_object.target) # A symlink node has no digest. - symlinknode.target = item.pb2_object.target self.index[filename] = IndexEntry(symlinknode, modified=(fullname in result.overwritten)) return result def _full_import_cas_into_cas(self, source_directory, path_prefix="", file_list_required=True): result = FileListResult() for entry in source_directory.pb2_directory.directories: + existing_item = self.find_pb2_entry(entry.name) # Create a cloned CasBasedDirectory, since we may import more files # into a subdirectory of it and we don't want to affect the original. - existing_dir = self.find_pb2_entry(entry.name) - if existing_dir: - existing_dir.digest = entry.digest + if existing_item: + existing_item.digest = entry.digest else: new_pb2_dirnode = self.pb2_directory.directories.add(digest=entry.digest, name=entry.name) - buildStreamDirectory = CasBasedDirectory(self.context, ref=entry.digest, + buildStreamDirectory = CasBasedDirectory(self.context, ref=entry.digest, parent=self, filename=entry.name) self.index[entry.name] = IndexEntry(entry, buildstream_object=buildStreamDirectory) if file_list_required: - if existing_dir: - updated_files = existing_dir.find_updated_files(source_directory.descend(entry.name), entry.name) + if existing_item: + updated_files = existing_item.find_updated_files(source_directory.descend(entry.name), entry.name) result.combine(updated_files) else: for i in source_directory.descend(entry.name).list_relative_paths(): result.files_written.append(i) for entry in source_directory.pb2_directory.files: - existing_file = self.find_pb2_entry(entry.name) - if existing_file: - result.files_overwritten.append(relative_pathname) - filenode = existing_file + existing_item = self.find_pb2_entry(entry.name) + if existing_item: + filenode = existing_item filenode.digest = entry.digest + filenode.size_bytes = entry.size_bytes + result.files_overwritten.append(relative_pathname) else: filenode = self.pb2_directory.files.add(name=entry.name, digest=entry.digest) - is_executable = entry.is_executable - filenode.is_executable = is_executable - self.index[entry.name] = IndexEntry(filenode, modified=(existing_file is not None)) + filenode.is_executable = entry.is_executable + self.index[entry.name] = IndexEntry(filenode, modified=(existing_item is not None)) relative_pathname = os.path.join(path_prefix, entry.name) result.files_written.append(relative_pathname) for entry in source_directory.pb2_directory.symlinks: - print(" Importing symlink {} into this directory.".format(entry.name)) - existing_link = self.find_pb2_entry(entry.name) + existing_item = self.find_pb2_entry(entry.name) relative_pathname = os.path.join(path_prefix, entry.name) - if existing_link: - symlinknode = existing_link + if existing_item: + symlinknode = existing_item result.files_overwritten.append(relative_pathname) else: symlinknode = self.pb2_directory.symlinks.add() symlinknode.name = entry.name - # A symlink node has no digest. symlinknode.target = entry.target - self.index[entry.name] = IndexEntry(symlinknode, modified=(existing_link is not None)) + # A symlink node has no digest. + self.index[entry.name] = IndexEntry(symlinknode, modified=(existing_item is not None)) result.files_written.append(relative_pathname) return result def _import_cas_into_cas(self, source_directory, files=None): - """ A full import is significantly easier than a partial import """ + """ A full import is significantly quicker than a partial import, because we can just + replace one directory with another's hash, without doing any recursion. + """ if files is None: - print("Performing FULL import of {} into {}".format(source_directory, self)) return self._full_import_cas_into_cas(source_directory) else: - print("Performing PARTIAL import of {} into {}".format(source_directory, self)) - print("File list is: {}".format(files)) return self._partial_import_cas_into_cas(source_directory, files) def import_files(self, external_pathspec: any, files: List[str] = None, |
