summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJim MacArthur <jim.macarthur@codethink.co.uk>2018-07-13 17:44:52 +0100
committerJim MacArthur <jim.macarthur@codethink.co.uk>2018-07-13 17:44:52 +0100
commit25065284c2a42626bdd36b6c200153ca6a0c6740 (patch)
tree3aa27de2e7a91300c773e5585bd1ce4104b916b9
parent48cca2b555036c39371d9bce379e1d62d5728c72 (diff)
downloadbuildstream-25065284c2a42626bdd36b6c200153ca6a0c6740.tar.gz
Cleanup of previous code
-rw-r--r--buildstream/storage/_casbaseddirectory.py106
1 files changed, 47 insertions, 59 deletions
diff --git a/buildstream/storage/_casbaseddirectory.py b/buildstream/storage/_casbaseddirectory.py
index 61ee976d2..edde893b5 100644
--- a/buildstream/storage/_casbaseddirectory.py
+++ b/buildstream/storage/_casbaseddirectory.py
@@ -398,126 +398,114 @@ class CasBasedDirectory(Directory):
result.file_written.apppend(os.path.join(prefix, entry.name))
return result
- def filelist_iterator(sorted_files):
- """ An interator which just returns the top level directories from sorted_files. It will return any directories listed as a single return value. The return value is a tuple consisting
- of a path component and a list of sub-components. If the list is empty, this indicates that the return value is a file. """
- last_directory = None
- for f in sorted_files:
- components = f.split(os.path.sep)
- if len(components)>1:
- if components[0] != last_directory:
- if last_directory is not None:
- yield (components[0], subcomponents)
- subcomponents = []
- subcomponents.append(os.path.sep.join(components[1:]))
- else:
- if last_directory is not None:
- yield (last_directory, subcomponents)
- yield(f, [])
- if last_directory is not None:
- yield (last_directory, subcomponents)
-
def files_in_subdir(sorted_files, dirname):
- """ Filters sorted_files and returns only the ones which have 'dirname' as a prefix, with that prefix removed. """
+ """Filters sorted_files and returns only the ones which have
+ 'dirname' as a prefix, with that prefix removed.
+
+ """
if not dirname.endswith(os.path.sep):
dirname += os.path.sep
- return [f.lstrip(dirname) for f in sorted_files if f.startswith(dirname) ]
+ return [f.lstrip(dirname) for f in sorted_files if f.startswith(dirname)]
def _partial_import_cas_into_cas(self, source_directory, files, path_prefix="", file_list_required=True):
+ """ Import only the files and symlinks listed in 'files' from source_directory to this one.
+ Args:
+ source_directory (:class:`.CasBasedDirectory`): The directory to import from
+ files ([str]): List of pathnames to import.
+ path_prefix (str): Prefix used to add entries to the file list result.
+ file_list_required: Whether to update the file list while processing.
+ """
+
result = FileListResult()
sorted_files = sorted(files) # Check if this is necessary
- processed_entries = set()
+ processed_directories = set()
for f in sorted_files:
if f == ".": continue
+ fullname = os.path.join(path_prefix, f)
components = f.split(os.path.sep)
if len(components)>1 or isinstance(source_directory.index[components[0]].buildstream_object, CasBasedDirectory):
# Then we are importing a directory
dirname = components[0]
- if dirname not in processed_entries:
+ if dirname not in processed_directories:
subcomponents = CasBasedDirectory.files_in_subdir(sorted_files, dirname)
dest_subdir = self.descend(dirname, create=True)
src_subdir = source_directory.descend(dirname)
- import_result = dest_subdir._partial_import_cas_into_cas(src_subdir, subcomponents, path_prefix=os.path.join(path_prefix,f), file_list_required=file_list_required)
+ import_result = dest_subdir._partial_import_cas_into_cas(src_subdir, subcomponents,
+ path_prefix=fullname, file_list_required=file_list_required)
result.combine(import_result)
- processed_entries.add(dirname)
+ processed_directories.add(dirname)
else:
self._check_replacement(f, path_prefix, result)
- fullname = os.path.join(path_prefix, f)
- item = source_directory.index[f]
- if isinstance(item.pb2_object, remote_execution_pb2.FileNode):
- filenode = self.pb2_directory.files.add(digest=item.pb2_object.digest)
- filenode.name = f
- is_executable = item.pb2_object.is_executable
- filenode.is_executable = is_executable
+ item = source_directory.index[f].pb2_object
+ if isinstance(item, remote_execution_pb2.FileNode):
+ filenode = self.pb2_directory.files.add(digest=item.digest, name=f,
+ is_executable=item.is_executable)
self.index[f] = IndexEntry(filenode, modified=(fullname in result.overwritten))
- else: # Must be a symlink
- symlinknode = self.pb2_directory.symlinks.add()
- symlinknode.name = fullname
+ else:
+ assert(isinstance(item.pb2_object, remote_execution_pb2.SymlinkNode))
+ symlinknode = self.pb2_directory.symlinks.add(name=f, target=item.pb2_object.target)
# A symlink node has no digest.
- symlinknode.target = item.pb2_object.target
self.index[filename] = IndexEntry(symlinknode, modified=(fullname in result.overwritten))
return result
def _full_import_cas_into_cas(self, source_directory, path_prefix="", file_list_required=True):
result = FileListResult()
for entry in source_directory.pb2_directory.directories:
+ existing_item = self.find_pb2_entry(entry.name)
# Create a cloned CasBasedDirectory, since we may import more files
# into a subdirectory of it and we don't want to affect the original.
- existing_dir = self.find_pb2_entry(entry.name)
- if existing_dir:
- existing_dir.digest = entry.digest
+ if existing_item:
+ existing_item.digest = entry.digest
else:
new_pb2_dirnode = self.pb2_directory.directories.add(digest=entry.digest, name=entry.name)
- buildStreamDirectory = CasBasedDirectory(self.context, ref=entry.digest,
+ buildStreamDirectory = CasBasedDirectory(self.context, ref=entry.digest,
parent=self, filename=entry.name)
self.index[entry.name] = IndexEntry(entry, buildstream_object=buildStreamDirectory)
if file_list_required:
- if existing_dir:
- updated_files = existing_dir.find_updated_files(source_directory.descend(entry.name), entry.name)
+ if existing_item:
+ updated_files = existing_item.find_updated_files(source_directory.descend(entry.name), entry.name)
result.combine(updated_files)
else:
for i in source_directory.descend(entry.name).list_relative_paths():
result.files_written.append(i)
for entry in source_directory.pb2_directory.files:
- existing_file = self.find_pb2_entry(entry.name)
- if existing_file:
- result.files_overwritten.append(relative_pathname)
- filenode = existing_file
+ existing_item = self.find_pb2_entry(entry.name)
+ if existing_item:
+ filenode = existing_item
filenode.digest = entry.digest
+ filenode.size_bytes = entry.size_bytes
+ result.files_overwritten.append(relative_pathname)
else:
filenode = self.pb2_directory.files.add(name=entry.name, digest=entry.digest)
- is_executable = entry.is_executable
- filenode.is_executable = is_executable
- self.index[entry.name] = IndexEntry(filenode, modified=(existing_file is not None))
+ filenode.is_executable = entry.is_executable
+ self.index[entry.name] = IndexEntry(filenode, modified=(existing_item is not None))
relative_pathname = os.path.join(path_prefix, entry.name)
result.files_written.append(relative_pathname)
for entry in source_directory.pb2_directory.symlinks:
- print(" Importing symlink {} into this directory.".format(entry.name))
- existing_link = self.find_pb2_entry(entry.name)
+ existing_item = self.find_pb2_entry(entry.name)
relative_pathname = os.path.join(path_prefix, entry.name)
- if existing_link:
- symlinknode = existing_link
+ if existing_item:
+ symlinknode = existing_item
result.files_overwritten.append(relative_pathname)
else:
symlinknode = self.pb2_directory.symlinks.add()
symlinknode.name = entry.name
- # A symlink node has no digest.
symlinknode.target = entry.target
- self.index[entry.name] = IndexEntry(symlinknode, modified=(existing_link is not None))
+ # A symlink node has no digest.
+ self.index[entry.name] = IndexEntry(symlinknode, modified=(existing_item is not None))
result.files_written.append(relative_pathname)
return result
def _import_cas_into_cas(self, source_directory, files=None):
- """ A full import is significantly easier than a partial import """
+ """ A full import is significantly quicker than a partial import, because we can just
+ replace one directory with another's hash, without doing any recursion.
+ """
if files is None:
- print("Performing FULL import of {} into {}".format(source_directory, self))
return self._full_import_cas_into_cas(source_directory)
else:
- print("Performing PARTIAL import of {} into {}".format(source_directory, self))
- print("File list is: {}".format(files))
return self._partial_import_cas_into_cas(source_directory, files)
def import_files(self, external_pathspec: any, files: List[str] = None,