diff options
-rw-r--r-- | buildstream/sandbox/_mount.py | 2 | ||||
-rw-r--r-- | buildstream/sandbox/_sandboxbwrap.py | 2 | ||||
-rw-r--r-- | buildstream/storage/_casbaseddirectory.py | 213 | ||||
-rw-r--r-- | buildstream/storage/_filebaseddirectory.py | 5 | ||||
-rw-r--r-- | buildstream/storage/directory.py | 6 |
5 files changed, 226 insertions, 2 deletions
diff --git a/buildstream/sandbox/_mount.py b/buildstream/sandbox/_mount.py index 225236d48..07e588388 100644 --- a/buildstream/sandbox/_mount.py +++ b/buildstream/sandbox/_mount.py @@ -34,7 +34,7 @@ class Mount(): def __init__(self, sandbox, mount_point, safe_hardlinks): scratch_directory = sandbox._get_scratch_directory() # Getting external_directory here is acceptable as we're part of the sandbox code. - root_directory = sandbox.get_virtual_directory().external_directory + root_directory = sandbox.get_virtual_directory().get_underlying_directory() self.mount_point = mount_point self.safe_hardlinks = safe_hardlinks diff --git a/buildstream/sandbox/_sandboxbwrap.py b/buildstream/sandbox/_sandboxbwrap.py index dc1b47d74..ea8b6682b 100644 --- a/buildstream/sandbox/_sandboxbwrap.py +++ b/buildstream/sandbox/_sandboxbwrap.py @@ -58,7 +58,7 @@ class SandboxBwrap(Sandbox): stdout, stderr = self._get_output() # Allowable access to underlying storage as we're part of the sandbox - root_directory = self.get_virtual_directory().external_directory + root_directory = self.get_virtual_directory().get_underlying_directory() # Fallback to the sandbox default settings for # the cwd and env. diff --git a/buildstream/storage/_casbaseddirectory.py b/buildstream/storage/_casbaseddirectory.py new file mode 100644 index 000000000..354449778 --- /dev/null +++ b/buildstream/storage/_casbaseddirectory.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2018 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Jim MacArthur <jim.macarthur@codethink.co.uk> + +""" +CasBasedDirectory +========= + +Implementation of the Directory class which backs onto a Merkle-tree based content +addressable storage system. + +See also: :ref:`sandboxing`. +""" + +from typing import List +from collections import OrderedDict + +import calendar +import os +import time +from .._exceptions import BstError, ErrorDomain +from .directory import Directory +from ._filebaseddirectory import VirtualDirectoryError # TODO: That shouldn't be in _filebaseddirectory +from ..utils import FileListResult + +# CasBasedDirectory intentionally doesn't call its superclass constuctor, +# which is mean to be unimplemented. +# pylint: disable=super-init-not-called + +class _FileObject(): + """A description of a file in a virtual directory. The contents of + this class are never used, but there needs to be something present + for files so is_empty() works correctly. + + """ + def __init__(self, virtual_directory: Directory, filename: str): + self.directory = virtual_directory + self.filename = filename + + +class CasBasedDirectory(Directory): + def __init__(self, ref, context): + # I need a context... + self.cas_directory = os.path.join(context.artifactdir, 'googlecas') + self.ref = ref + self.index = OrderedDict() + self._directory_read = False + + def _populate_index(self) -> None: + if self._directory_read: + return + # Replace with tree read + + for entry in self.cas.read_tree_object(self.ref): +# if entry is a directory: +# self.index[entry] = CasBasedDirectory(entry.ref) +# elif entry is a file: +# self.index[entry] = _FileObject(self, entry) +# elif entry is a symlink: +# # Dunno + raise NotImplementedError() + self._directory_read = True + + def descend(self, subdirectory_spec: List[str], create: bool = False) -> Directory: + """ Descend one or more levels of directory hierarchy and return a new + Directory object for that directory. + + Arguments: + * subdirectory_spec (list of strings): A list of strings which are all directory + names. + * create (boolean): If this is true, the directories will be created if + they don't already exist. + """ + + # It's very common to send a directory name instead of a list and this causes + # bizarre errors, so check for it here + if not isinstance(subdirectory_spec, list): + subdirectory_spec = [subdirectory_spec] + if not subdirectory_spec: + return self + + # Because of the way split works, it's common to get a list which begins with + # an empty string. Detect these and remove them, then start again. + if subdirectory_spec[0] == "": + return self.descend(subdirectory_spec[1:], create) + + self._populate_index() + if subdirectory_spec[0] in self.index: + entry = self.index[subdirectory_spec[0]] + if isinstance(entry, CasBasedDirectory): + new_path = os.path.join(self.external_directory, subdirectory_spec[0]) + return entry.descend(subdirectory_spec[1:], create) + else: + error = "Cannot descend into {}, which is a '{}' in the directory {}" + raise VirtualDirectoryError(error.format(subdirectory_spec[0], + type(entry).__name__, + self.external_directory)) + else: + if create: + # Adding an entry to this node makes it a new node. + raise NotImplementedError() + else: + error = "No entry called '{}' found in the directory rooted at {}" + raise VirtualDirectoryError(error.format(subdirectory_spec[0], self.external_directory)) + return None + + def import_files(self, external_pathspec: any, files: List[str] = None, + report_written: bool = True, update_utimes: bool = False, + can_link: bool = False) -> FileListResult: + """Imports some or all files from external_path into this directory. + + Keyword arguments: external_pathspec: Either a string + containing a pathname, or a Directory object, to use as the + source. + + files (list of strings): A list of all the files relative to + the external_pathspec to copy. If 'None' is supplied, all + files are copied. + + report_written (bool): Return the full list of files + written. Defaults to true. If false, only a list of + overwritten files is returned. + + update_utimes (bool): Update the access and modification time + of each file copied to the current time. + + can_link (bool): Whether it's OK to create a hard link to the + original content, meaning the stored copy will change when the + original files change. Setting this doesn't guarantee hard + links will be made. can_link will never be used if + update_utimes is set. + """ + + if isinstance(external_pathspec, Directory): + source_directory = external_pathspec.external_directory + else: + source_directory = external_pathspec + + raise NotImplementedError() + + def set_deterministic_mtime(self) -> None: + """ Sets a static modification time for all regular files in this directory. + But we don't have metadata in the CAS. + """ + raise NotImplementedError() + + + def set_deterministic_user(self) -> None: + """ Sets all files in this directory to the current user's euid/egid. + But we don't have metadata in the CAS. + """ + raise NotImplementedError() + + def export_files(self, to_directory: str, can_link: bool = False, can_destroy: bool = False) -> None: + """Copies everything from this into to_directory. + + Arguments: + + to_directory (string): a path outside this directory object + where the contents will be copied to. + + can_link (bool): Whether we can create hard links in to_directory + instead of copying. + + """ + raise NotImplementedError() + + def is_empty(self) -> bool: + """ Return true if this directory has no files, subdirectories or links in it. + """ + self._populate_index() + return len(self.index) == 0 + + def mark_unmodified(self) -> None: + """ Marks all files in this directory (recursively) as unmodified. + """ + raise NotImplementedError() + + def list_modified_paths(self) -> List[str]: + """Provide a list of relative paths which have been modified since the + last call to mark_unmodified. + + Return value: List(str) - list of modified paths + """ + raise NotImplementedError() + + def list_relative_paths(self) -> List[str]: + """Provide a list of all relative paths. + + Return value: List(str) - list of all paths + """ + + # Should be easy - just iterate all files in this listing recursively + raise NotImplementedError() + + def __str__(self) -> str: + return "[CAS:{}]".format(self.ref) diff --git a/buildstream/storage/_filebaseddirectory.py b/buildstream/storage/_filebaseddirectory.py index 60379eaed..ae26fd4f2 100644 --- a/buildstream/storage/_filebaseddirectory.py +++ b/buildstream/storage/_filebaseddirectory.py @@ -251,3 +251,8 @@ class FileBasedDirectory(Directory): # which exposes the sandbox directory; we will have to assume for the time being # that people will not abuse __str__. return self.external_directory + + def get_underlying_directory(self) -> str: + """ Returns the underlying (real) file system directory this + object refers to. """ + return self.external_directory diff --git a/buildstream/storage/directory.py b/buildstream/storage/directory.py index f37fb98ad..b2233c6b0 100644 --- a/buildstream/storage/directory.py +++ b/buildstream/storage/directory.py @@ -132,3 +132,9 @@ class Directory(): Return value: List(str) - dictionary with all paths """ raise NotImplementedError() + + def get_underlying_directory(self) -> str: + """ Returns the underlying (real) file system directory this + object refers to. This will throw an exception if there isn't + a real directory behind the object. """ + raise NotImplementedError() |