#
# Copyright (C) 2018 Bloomberg LP
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see .
#
# Authors:
# Jim MacArthur
"""
CasBasedDirectory
=========
Implementation of the Directory class which backs onto a Merkle-tree based content
addressable storage system.
See also: :ref:`sandboxing`.
"""
from collections import OrderedDict
import os
import stat
from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
from .._exceptions import BstError
from .directory import Directory, VirtualDirectoryError
from ._filebaseddirectory import FileBasedDirectory
from ..utils import FileListResult, safe_copy, list_relative_paths
class IndexEntry():
""" Used in our index of names to objects to store the 'modified' flag
for directory entries. Because we need both the remote_execution_pb2 object
and our own Directory object for directory entries, we store both. For files
and symlinks, only pb_object is used. """
def __init__(self, pb_object, buildstream_object=None, modified=False):
self.pb_object = pb_object # Short for 'protocol buffer object')
self.buildstream_object = buildstream_object
self.modified = modified
class ResolutionException(VirtualDirectoryError):
""" Superclass of all exceptions that can be raised by
CasBasedDirectory._resolve. Should not be used outside this module. """
class InfiniteSymlinkException(ResolutionException):
""" Raised when an infinite symlink loop is found. """
class AbsoluteSymlinkException(ResolutionException):
"""Raised if we try to follow an absolute symlink (i.e. one whose
target starts with the path separator) and we have disallowed
following such symlinks.
"""
class UnexpectedFileException(ResolutionException):
"""Raised if we were found a file where a directory or symlink was
expected, for example we try to resolve a symlink pointing to
/a/b/c but /a/b is a file.
"""
def __init__(self, message=""):
"""Allow constructor with no arguments, since this can be raised in
places where there isn't sufficient information to write the
message.
"""
super().__init__(message)
class _Resolver():
"""A class for resolving symlinks inside CAS-based directories. As
well as providing a namespace for some functions, this also
contains two flags which are constant throughout one resolution
operation and the 'seen_objects' list used to detect infinite
symlink loops.
"""
def __init__(self, absolute_symlinks_resolve=True, force_create=False):
self.absolute_symlinks_resolve = absolute_symlinks_resolve
self.force_create = force_create
self.seen_objects = []
def resolve(self, name, directory):
"""Resolves any name to an object. If the name points to a symlink in
the directory, it returns the thing it points to,
recursively.
Returns a CasBasedDirectory, FileNode or None. None indicates
either that 'target' does not exist in this directory, or is a
symlink chain which points to a nonexistent name (broken
symlink).
Raises:
- InfiniteSymlinkException if 'name' points to an infinite
symlink loop.
- AbsoluteSymlinkException if 'name' points to an absolute
symlink and absolute_symlinks_resolve is False.
- UnexpectedFileException if at any point during resolution we
find a file which we expected to be a directory or symlink.
If force_create is set, this will attempt to create
directories to make symlinks and directories resolve. Files
present in symlink target paths will also be removed and
replaced with directories. If force_create is off, this will
never alter 'directory'.
"""
# First check for nonexistent things or 'normal' objects and return them
if name not in directory.index:
return None
index_entry = directory.index[name]
if isinstance(index_entry.buildstream_object, Directory):
return index_entry.buildstream_object
elif isinstance(index_entry.pb_object, remote_execution_pb2.FileNode):
return index_entry.pb_object
# Now we must be dealing with a symlink.
assert isinstance(index_entry.pb_object, remote_execution_pb2.SymlinkNode)
symlink_object = index_entry.pb_object
if symlink_object in self.seen_objects:
# Infinite symlink loop detected
message = ("Infinite symlink loop found during resolution. " +
"First repeated element is {}".format(name))
raise InfiniteSymlinkException(message=message)
self.seen_objects.append(symlink_object)
components = symlink_object.target.split(CasBasedDirectory._pb2_path_sep)
absolute = symlink_object.target.startswith(CasBasedDirectory._pb2_absolute_path_prefix)
if absolute:
if self.absolute_symlinks_resolve:
directory = directory.find_root()
# Discard the first empty element
components.pop(0)
else:
# Unresolvable absolute symlink
message = "{} is an absolute symlink, which was disallowed during resolution".format(name)
raise AbsoluteSymlinkException(message=message)
resolution = directory
while components and isinstance(resolution, CasBasedDirectory):
c = components.pop(0)
directory = resolution
try:
resolution = self._resolve_path_component(c, directory, components)
except UnexpectedFileException as original:
errormsg = ("Reached a file called {} while trying to resolve a symlink; " +
"cannot proceed. The remaining path components are {}.")
raise UnexpectedFileException(errormsg.format(c, components)) from original
return resolution
def _resolve_path_component(self, c, directory, components_remaining):
if c == ".":
resolution = directory
elif c == "..":
if directory.parent is not None:
resolution = directory.parent
else:
# If directory.parent *is* None, this is an attempt to
# access '..' from the root, which is valid under
# POSIX; it just returns the root.
resolution = directory
elif c in directory.index:
try:
resolution = self._resolve_through_files(c, directory, components_remaining)
except UnexpectedFileException as original:
errormsg = ("Reached a file called {} while trying to resolve a symlink; " +
"cannot proceed. The remaining path components are {}.")
raise UnexpectedFileException(errormsg.format(c, components_remaining)) from original
else:
# c is not in our index
if self.force_create:
resolution = directory.descend(c, create=True)
else:
resolution = None
return resolution
def _resolve_through_files(self, c, directory, require_traversable):
"""A wrapper to resolve() which deals with files being found
in the middle of paths, for example trying to resolve a symlink
which points to /usr/lib64/libfoo when 'lib64' is a file.
require_traversable: If this is True, never return a file
node. Instead, if force_create is set, destroy the file node,
then create and return a normal directory in its place. If
force_create is off, throws ResolutionException.
"""
resolved_thing = self.resolve(c, directory)
if isinstance(resolved_thing, remote_execution_pb2.FileNode):
if require_traversable:
# We have components still to resolve, but one of the path components
# is a file.
if self.force_create:
directory.delete_entry(c)
resolved_thing = directory.descend(c, create=True)
else:
# This is a signal that we hit a file, but don't
# have the data to give a proper message, so the
# caller should reraise this with a proper
# description.
raise UnexpectedFileException()
return resolved_thing
# CasBasedDirectory intentionally doesn't call its superclass constuctor,
# which is meant to be unimplemented.
# pylint: disable=super-init-not-called
class CasBasedDirectory(Directory):
"""
CAS-based directories can have two names; one is a 'common name' which has no effect
on functionality, and the 'filename'. If a CasBasedDirectory has a parent, then 'filename'
must be the name of an entry in the parent directory's index which points to this object.
This is used to inform a parent directory that it must update the given hash for this
object when this object changes.
Typically a top-level CasBasedDirectory will have a common_name and no filename, and
subdirectories wil have a filename and no common_name. common_name can used to identify
CasBasedDirectory objects in a log file, since they have no unique position in a file
system.
"""
# Two constants which define the separators used by the remote execution API.
_pb2_path_sep = "/"
_pb2_absolute_path_prefix = "/"
def __init__(self, cas_cache, ref=None, parent=None, common_name="untitled", filename=None):
self.filename = filename
self.common_name = common_name
self.pb2_directory = remote_execution_pb2.Directory()
self.cas_cache = cas_cache
if ref:
with open(self.cas_cache.objpath(ref), 'rb') as f:
self.pb2_directory.ParseFromString(f.read())
self.ref = ref
self.index = OrderedDict()
self.parent = parent
self._directory_read = False
self._populate_index()
def _populate_index(self):
if self._directory_read:
return
for entry in self.pb2_directory.directories:
buildStreamDirectory = CasBasedDirectory(self.cas_cache, ref=entry.digest,
parent=self, filename=entry.name)
self.index[entry.name] = IndexEntry(entry, buildstream_object=buildStreamDirectory)
for entry in self.pb2_directory.files:
self.index[entry.name] = IndexEntry(entry)
for entry in self.pb2_directory.symlinks:
self.index[entry.name] = IndexEntry(entry)
self._directory_read = True
def _recalculate_recursing_up(self, caller=None):
"""Recalcuate the hash for this directory and store the results in
the cache. If this directory has a parent, tell it to
recalculate (since changing this directory changes an entry in
the parent).
"""
self.ref = self.cas_cache.add_object(buffer=self.pb2_directory.SerializeToString())
if caller:
old_dir = self._find_pb2_entry(caller.filename)
self.cas_cache.add_object(digest=old_dir.digest, buffer=caller.pb2_directory.SerializeToString())
if self.parent:
self.parent._recalculate_recursing_up(self)
def _recalculate_recursing_down(self, parent=None):
"""Recalcuate the hash for this directory and any
subdirectories. Hashes for subdirectories should be calculated
and stored after a significant operation (e.g. an
import_files() call) but not after adding each file, as that
is extremely wasteful.
"""
for entry in self.pb2_directory.directories:
self.index[entry.name].buildstream_object._recalculate_recursing_down(entry)
if parent:
self.ref = self.cas_cache.add_object(digest=parent.digest, buffer=self.pb2_directory.SerializeToString())
else:
self.ref = self.cas_cache.add_object(buffer=self.pb2_directory.SerializeToString())
# We don't need to do anything more than that; files were already added ealier, and symlinks are
# part of the directory structure.
def _find_pb2_entry(self, name):
if name in self.index:
return self.index[name].pb_object
return None
def _find_self_in_parent(self):
assert self.parent is not None
parent = self.parent
for (k, v) in parent.index.items():
if v.buildstream_object == self:
return k
return None
def _add_directory(self, name):
if name in self.index:
newdir = self.index[name].buildstream_object
if not isinstance(newdir, CasBasedDirectory):
# TODO: This may not be an actual error; it may actually overwrite it
raise VirtualDirectoryError("New directory {} in {} would overwrite existing non-directory of type {}"
.format(name, str(self), type(newdir)))
dirnode = self._find_pb2_entry(name)
else:
newdir = CasBasedDirectory(self.cas_cache, parent=self, filename=name)
dirnode = self.pb2_directory.directories.add()
dirnode.name = name
# Calculate the hash for an empty directory
new_directory = remote_execution_pb2.Directory()
self.cas_cache.add_object(digest=dirnode.digest, buffer=new_directory.SerializeToString())
self.index[name] = IndexEntry(dirnode, buildstream_object=newdir)
return newdir
def _add_file(self, basename, filename, modified=False):
filenode = self.pb2_directory.files.add()
filenode.name = filename
self.cas_cache.add_object(digest=filenode.digest, path=os.path.join(basename, filename))
is_executable = os.access(os.path.join(basename, filename), os.X_OK)
filenode.is_executable = is_executable
self.index[filename] = IndexEntry(filenode, modified=modified or filename in self.index)
def _copy_link_from_filesystem(self, basename, filename):
self._add_new_link_direct(filename, os.readlink(os.path.join(basename, filename)))
def _add_new_link_direct(self, name, target):
existing_link = self._find_pb2_entry(name)
if existing_link:
symlinknode = existing_link
else:
symlinknode = self.pb2_directory.symlinks.add()
assert isinstance(symlinknode, remote_execution_pb2.SymlinkNode)
symlinknode.name = name
# A symlink node has no digest.
symlinknode.target = target
self.index[name] = IndexEntry(symlinknode, modified=(existing_link is not None))
def delete_entry(self, name):
for collection in [self.pb2_directory.files, self.pb2_directory.symlinks, self.pb2_directory.directories]:
for thing in collection:
if thing.name == name:
collection.remove(thing)
if name in self.index:
del self.index[name]
def descend(self, subdirectory_spec, create=False):
"""Descend one or more levels of directory hierarchy and return a new
Directory object for that directory.
Arguments:
* subdirectory_spec (list of strings): A list of strings which are all directory
names.
* create (boolean): If this is true, the directories will be created if
they don't already exist.
Note: At the moment, creating a directory by descending does
not update this object in the CAS cache. However, performing
an import_files() into a subdirectory of any depth obtained by
descending from this object *will* cause this directory to be
updated and stored.
"""
# It's very common to send a directory name instead of a list and this causes
# bizarre errors, so check for it here
if not isinstance(subdirectory_spec, list):
subdirectory_spec = [subdirectory_spec]
# Because of the way split works, it's common to get a list which begins with
# an empty string. Detect these and remove them.
while subdirectory_spec and subdirectory_spec[0] == "":
subdirectory_spec.pop(0)
# Descending into [] returns the same directory.
if not subdirectory_spec:
return self
if subdirectory_spec[0] in self.index:
entry = self.index[subdirectory_spec[0]].buildstream_object
if isinstance(entry, CasBasedDirectory):
return entry.descend(subdirectory_spec[1:], create)
else:
# May be a symlink
target = self._resolve(subdirectory_spec[0], force_create=create)
if isinstance(target, CasBasedDirectory):
return target
error = "Cannot descend into {}, which is a '{}' in the directory {}"
raise VirtualDirectoryError(error.format(subdirectory_spec[0],
type(self.index[subdirectory_spec[0]].pb_object).__name__,
self))
else:
if create:
newdir = self._add_directory(subdirectory_spec[0])
return newdir.descend(subdirectory_spec[1:], create)
else:
error = "No entry called '{}' found in {}. There are directories called {}."
directory_list = ",".join([entry.name for entry in self.pb2_directory.directories])
raise VirtualDirectoryError(error.format(subdirectory_spec[0], str(self),
directory_list))
return None
def find_root(self):
""" Finds the root of this directory tree by following 'parent' until there is
no parent. """
if self.parent:
return self.parent.find_root()
else:
return self
def _resolve(self, name, absolute_symlinks_resolve=True, force_create=False):
resolver = _Resolver(absolute_symlinks_resolve, force_create)
return resolver.resolve(name, self)
def _check_replacement(self, name, path_prefix, fileListResult):
""" Checks whether 'name' exists, and if so, whether we can overwrite it.
If we can, add the name to 'overwritten_files' and delete the existing entry.
Returns 'True' if the import should go ahead.
fileListResult.overwritten and fileListResult.ignore are updated depending
on the result. """
existing_entry = self._find_pb2_entry(name)
relative_pathname = os.path.join(path_prefix, name)
if existing_entry is None:
return True
if (isinstance(existing_entry,
(remote_execution_pb2.FileNode, remote_execution_pb2.SymlinkNode))):
self.delete_entry(name)
fileListResult.overwritten.append(relative_pathname)
return True
elif isinstance(existing_entry, remote_execution_pb2.DirectoryNode):
# If 'name' maps to a DirectoryNode, then there must be an entry in index
# pointing to another Directory.
if self.index[name].buildstream_object.is_empty():
self.delete_entry(name)
fileListResult.overwritten.append(relative_pathname)
return True
else:
# We can't overwrite a non-empty directory, so we just ignore it.
fileListResult.ignored.append(relative_pathname)
return False
assert False, ("Entry '{}' is not a recognised file/link/directory and not None; it is {}"
.format(name, type(existing_entry)))
return False # In case asserts are disabled
def _replace_anything_with_dir(self, name, path_prefix, overwritten_files_list):
self.delete_entry(name)
subdir = self._add_directory(name)
overwritten_files_list.append(os.path.join(path_prefix, name))
return subdir
def _import_files_from_directory(self, source_directory, files, path_prefix=""):
""" Imports files from a traditional directory. """
def _ensure_followable(name, path_prefix):
""" Makes sure 'name' is a directory or symlink to a directory which can be descended into. """
if isinstance(self.index[name].buildstream_object, Directory):
return self.descend(name)
try:
target = self._resolve(name, force_create=True)
except InfiniteSymlinkException:
return self._replace_anything_with_dir(name, path_prefix, result.overwritten)
if isinstance(target, CasBasedDirectory):
return target
elif isinstance(target, remote_execution_pb2.FileNode):
return self._replace_anything_with_dir(name, path_prefix, result.overwritten)
return target
def _import_directory_recursively(directory_name, source_directory, remaining_path, path_prefix):
""" _import_directory_recursively and _import_files_from_directory will be called alternately
as a directory tree is descended. """
if directory_name in self.index:
subdir = _ensure_followable(directory_name, path_prefix)
else:
subdir = self._add_directory(directory_name)
new_path_prefix = os.path.join(path_prefix, directory_name)
subdir_result = subdir._import_files_from_directory(os.path.join(source_directory, directory_name),
[os.path.sep.join(remaining_path)],
path_prefix=new_path_prefix)
return subdir_result
result = FileListResult()
for entry in files:
split_path = entry.split(os.path.sep)
# The actual file on the FS we're importing
import_file = os.path.join(source_directory, entry)
# The destination filename, relative to the root where the import started
relative_pathname = os.path.join(path_prefix, entry)
if len(split_path) > 1:
directory_name = split_path[0]
# Hand this off to the importer for that subdir.
# It would be advantageous to batch these together by
# directory_name. However, we can't do it out of
# order, since importing symlinks affects the results
# of other imports.
subdir_result = _import_directory_recursively(directory_name, source_directory,
split_path[1:], path_prefix)
result.combine(subdir_result)
elif os.path.islink(import_file):
if self._check_replacement(entry, path_prefix, result):
self._copy_link_from_filesystem(source_directory, entry)
result.files_written.append(relative_pathname)
elif os.path.isdir(import_file):
# A plain directory which already exists isn't a problem; just ignore it.
if entry not in self.index:
self._add_directory(entry)
elif os.path.isfile(import_file):
if self._check_replacement(entry, path_prefix, result):
self._add_file(source_directory, entry, modified=relative_pathname in result.overwritten)
result.files_written.append(relative_pathname)
return result
@staticmethod
def _files_in_subdir(sorted_files, dirname):
"""Filters sorted_files and returns only the ones which have
'dirname' as a prefix, with that prefix removed.
"""
if not dirname.endswith(os.path.sep):
dirname += os.path.sep
return [f[len(dirname):] for f in sorted_files if f.startswith(dirname)]
def _partial_import_cas_into_cas(self, source_directory, files, path_prefix="", file_list_required=True):
""" Import only the files and symlinks listed in 'files' from source_directory to this one.
Args:
source_directory (:class:`.CasBasedDirectory`): The directory to import from
files ([str]): List of pathnames to import. Must be a list, not a generator.
path_prefix (str): Prefix used to add entries to the file list result.
file_list_required: Whether to update the file list while processing.
"""
result = FileListResult()
processed_directories = set()
for f in files:
fullname = os.path.join(path_prefix, f)
components = f.split(os.path.sep)
if len(components) > 1:
# We are importing a thing which is in a subdirectory. We may have already seen this dirname
# for a previous file.
dirname = components[0]
if dirname not in processed_directories:
# Now strip off the first directory name and import files recursively.
subcomponents = CasBasedDirectory._files_in_subdir(files, dirname)
# We will fail at this point if there is a file or symlink to file called 'dirname'.
if dirname in self.index:
resolved_component = self._resolve(dirname, force_create=True)
if isinstance(resolved_component, remote_execution_pb2.FileNode):
dest_subdir = self._replace_anything_with_dir(dirname, path_prefix, result.overwritten)
else:
dest_subdir = resolved_component
else:
dest_subdir = self.descend(dirname, create=True)
src_subdir = source_directory.descend(dirname)
import_result = dest_subdir._partial_import_cas_into_cas(src_subdir, subcomponents,
path_prefix=fullname,
file_list_required=file_list_required)
result.combine(import_result)
processed_directories.add(dirname)
elif isinstance(source_directory.index[f].buildstream_object, CasBasedDirectory):
# The thing in the input file list is a directory on
# its own. We don't need to do anything other than create it if it doesn't exist.
# If we already have an entry with the same name that isn't a directory, that
# will be dealt with when importing files in this directory.
if f not in self.index:
self.descend(f, create=True)
else:
# We're importing a file or symlink - replace anything with the same name.
importable = self._check_replacement(f, path_prefix, result)
if importable:
item = source_directory.index[f].pb_object
if isinstance(item, remote_execution_pb2.FileNode):
filenode = self.pb2_directory.files.add(digest=item.digest, name=f,
is_executable=item.is_executable)
self.index[f] = IndexEntry(filenode, modified=True)
else:
assert isinstance(item, remote_execution_pb2.SymlinkNode)
self._add_new_link_direct(name=f, target=item.target)
else:
result.ignored.append(os.path.join(path_prefix, f))
return result
def import_files(self, external_pathspec, *, files=None,
report_written=True, update_utimes=False,
can_link=False):
"""Imports some or all files from external_path into this directory.
Keyword arguments: external_pathspec: Either a string
containing a pathname, or a Directory object, to use as the
source.
files (list of strings): A list of all the files relative to
the external_pathspec to copy. If 'None' is supplied, all
files are copied.
report_written (bool): Return the full list of files
written. Defaults to true. If false, only a list of
overwritten files is returned.
update_utimes (bool): Currently ignored, since CAS does not store utimes.
can_link (bool): Ignored, since hard links do not have any meaning within CAS.
"""
if files is None:
if isinstance(external_pathspec, str):
files = list_relative_paths(external_pathspec)
else:
assert isinstance(external_pathspec, Directory)
files = external_pathspec.list_relative_paths()
if isinstance(external_pathspec, FileBasedDirectory):
source_directory = external_pathspec.get_underlying_directory()
result = self._import_files_from_directory(source_directory, files=files)
elif isinstance(external_pathspec, str):
source_directory = external_pathspec
result = self._import_files_from_directory(source_directory, files=files)
else:
assert isinstance(external_pathspec, CasBasedDirectory)
result = self._partial_import_cas_into_cas(external_pathspec, files=list(files))
# TODO: No notice is taken of report_written, update_utimes or can_link.
# Current behaviour is to fully populate the report, which is inefficient,
# but still correct.
# We need to recalculate and store the hashes of all directories both
# up and down the tree; we have changed our directory by importing files
# which changes our hash and all our parents' hashes of us. The trees
# lower down need to be stored in the CAS as they are not automatically
# added during construction.
self._recalculate_recursing_down()
if self.parent:
self.parent._recalculate_recursing_up(self)
return result
def set_deterministic_mtime(self):
""" Sets a static modification time for all regular files in this directory.
Since we don't store any modification time, we don't need to do anything.
"""
def set_deterministic_user(self):
""" Sets all files in this directory to the current user's euid/egid.
We also don't store user data, so this can be ignored.
"""
def export_files(self, to_directory, *, can_link=False, can_destroy=False):
"""Copies everything from this into to_directory, which must be the name
of a traditional filesystem directory.
Arguments:
to_directory (string): a path outside this directory object
where the contents will be copied to.
can_link (bool): Whether we can create hard links in to_directory
instead of copying.
can_destroy (bool): Whether we can destroy elements in this
directory to export them (e.g. by renaming them as the
target).
"""
if not os.path.exists(to_directory):
os.mkdir(to_directory)
for entry in self.pb2_directory.directories:
if entry.name not in self.index:
raise VirtualDirectoryError("CasDir {} contained {} in directories but not in the index"
.format(str(self), entry.name))
if not self._directory_read:
raise VirtualDirectoryError("CasDir {} has not been indexed yet".format(str(self)))
dest_dir = os.path.join(to_directory, entry.name)
if not os.path.exists(dest_dir):
os.mkdir(dest_dir)
target = self.descend([entry.name])
target.export_files(dest_dir)
for entry in self.pb2_directory.files:
# Extract the entry to a single file
dest_name = os.path.join(to_directory, entry.name)
src_name = self.cas_cache.objpath(entry.digest)
safe_copy(src_name, dest_name)
if entry.is_executable:
os.chmod(dest_name, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR |
stat.S_IRGRP | stat.S_IXGRP |
stat.S_IROTH | stat.S_IXOTH)
for entry in self.pb2_directory.symlinks:
src_name = os.path.join(to_directory, entry.name)
target_name = entry.target
try:
os.symlink(target_name, src_name)
except FileExistsError as e:
raise BstError(("Cannot create a symlink named {} pointing to {}." +
" The original error was: {}").
format(src_name, entry.target, e))
def export_to_tar(self, tarfile, destination_dir, mtime=0):
raise NotImplementedError()
def mark_changed(self):
""" It should not be possible to externally modify a CAS-based
directory at the moment."""
raise NotImplementedError()
def is_empty(self):
""" Return true if this directory has no files, subdirectories or links in it.
"""
return len(self.index) == 0
def _mark_directory_unmodified(self):
# Marks all entries in this directory and all child directories as unmodified.
for i in self.index.values():
i.modified = False
if isinstance(i.buildstream_object, CasBasedDirectory):
i.buildstream_object._mark_directory_unmodified()
def _mark_entry_unmodified(self, name):
# Marks an entry as unmodified. If the entry is a directory, it will
# recursively mark all its tree as unmodified.
self.index[name].modified = False
if self.index[name].buildstream_object:
self.index[name].buildstream_object._mark_directory_unmodified()
def mark_unmodified(self):
""" Marks all files in this directory (recursively) as unmodified.
If we have a parent, we mark our own entry as unmodified in that parent's
index.
"""
if self.parent:
self.parent._mark_entry_unmodified(self._find_self_in_parent())
else:
self._mark_directory_unmodified()
def _lightweight_resolve_to_index(self, path):
"""A lightweight function for transforming paths into IndexEntry
objects. This does not follow symlinks.
path: The string to resolve. This should be a series of path
components separated by the protocol buffer path separator
_pb2_path_sep.
Returns: the IndexEntry found, or None if any of the path components were not present.
"""
directory = self
path_components = path.split(CasBasedDirectory._pb2_path_sep)
for component in path_components[:-1]:
if component not in directory.index:
return None
if isinstance(directory.index[component].buildstream_object, CasBasedDirectory):
directory = directory.index[component].buildstream_object
else:
return None
return directory.index.get(path_components[-1], None)
def list_modified_paths(self):
"""Provide a list of relative paths which have been modified since the
last call to mark_unmodified.
Return value: List(str) - list of modified paths
"""
for p in self.list_relative_paths():
i = self._lightweight_resolve_to_index(p)
if i and i.modified:
yield p
def list_relative_paths(self, relpath=""):
"""Provide a list of all relative paths.
Return value: List(str) - list of all paths
"""
symlink_list = filter(lambda i: isinstance(i[1].pb_object, remote_execution_pb2.SymlinkNode),
self.index.items())
file_list = list(filter(lambda i: isinstance(i[1].pb_object, remote_execution_pb2.FileNode),
self.index.items()))
directory_list = filter(lambda i: isinstance(i[1].buildstream_object, CasBasedDirectory),
self.index.items())
# We need to mimic the behaviour of os.walk, in which symlinks
# to directories count as directories and symlinks to file or
# broken symlinks count as files. os.walk doesn't follow
# symlinks, so we don't recurse.
for (k, v) in sorted(symlink_list):
target = self._resolve(k, absolute_symlinks_resolve=True)
if isinstance(target, CasBasedDirectory):
yield os.path.join(relpath, k)
else:
file_list.append((k, v))
if file_list == [] and relpath != "":
yield relpath
else:
for (k, v) in sorted(file_list):
yield os.path.join(relpath, k)
for (k, v) in sorted(directory_list):
yield from v.buildstream_object.list_relative_paths(relpath=os.path.join(relpath, k))
def recalculate_hash(self):
""" Recalcuates the hash for this directory and store the results in
the cache. If this directory has a parent, tell it to
recalculate (since changing this directory changes an entry in
the parent). Hashes for subdirectories also get recalculated.
"""
self._recalculate_recursing_up()
self._recalculate_recursing_down()
def _get_identifier(self):
path = ""
if self.parent:
path = self.parent._get_identifier()
if self.filename:
path += "/" + self.filename
else:
path += "/" + self.common_name
return path
def __str__(self):
return "[CAS:{}]".format(self._get_identifier())
def _get_underlying_directory(self):
""" There is no underlying directory for a CAS-backed directory, so
throw an exception. """
raise VirtualDirectoryError("_get_underlying_directory was called on a CAS-backed directory," +
" which has no underlying directory.")