diff options
Diffstat (limited to 'src/buildstream/_fuse')
| -rw-r--r-- | src/buildstream/_fuse/__init__.py | 20 | ||||
| -rw-r--r-- | src/buildstream/_fuse/fuse.py | 1006 | ||||
| -rw-r--r-- | src/buildstream/_fuse/hardlinks.py | 218 | ||||
| -rw-r--r-- | src/buildstream/_fuse/mount.py | 196 |
4 files changed, 1440 insertions, 0 deletions
diff --git a/src/buildstream/_fuse/__init__.py b/src/buildstream/_fuse/__init__.py new file mode 100644 index 000000000..a5e882634 --- /dev/null +++ b/src/buildstream/_fuse/__init__.py @@ -0,0 +1,20 @@ +# +# Copyright (C) 2017 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> + +from .hardlinks import SafeHardlinks diff --git a/src/buildstream/_fuse/fuse.py b/src/buildstream/_fuse/fuse.py new file mode 100644 index 000000000..4ff6b9903 --- /dev/null +++ b/src/buildstream/_fuse/fuse.py @@ -0,0 +1,1006 @@ +# This is an embedded copy of fuse.py taken from the following upstream commit: +# +# https://github.com/terencehonles/fusepy/commit/0eafeb557e0e70926ed9450008ef17057d302391 +# +# Our local modifications are recorded in the Git history of this repo. + +# Copyright (c) 2012 Terence Honles <terence@honles.com> (maintainer) +# Copyright (c) 2008 Giorgos Verigakis <verigak@gmail.com> (author) +# +# Permission to use, copy, modify, and distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +# pylint: skip-file + +from __future__ import print_function, absolute_import, division + +from ctypes import * +from ctypes.util import find_library +from errno import * +from os import strerror +from platform import machine, system +from signal import signal, SIGINT, SIG_DFL +from stat import S_IFDIR +from traceback import print_exc + +import logging + +try: + from functools import partial +except ImportError: + # http://docs.python.org/library/functools.html#functools.partial + def partial(func, *args, **keywords): + def newfunc(*fargs, **fkeywords): + newkeywords = keywords.copy() + newkeywords.update(fkeywords) + return func(*(args + fargs), **newkeywords) + + newfunc.func = func + newfunc.args = args + newfunc.keywords = keywords + return newfunc + +try: + basestring +except NameError: + basestring = str + +class c_timespec(Structure): + _fields_ = [('tv_sec', c_long), ('tv_nsec', c_long)] + +class c_utimbuf(Structure): + _fields_ = [('actime', c_timespec), ('modtime', c_timespec)] + +class c_stat(Structure): + pass # Platform dependent + +_system = system() +_machine = machine() + +if _system == 'Darwin': + _libiconv = CDLL(find_library('iconv'), RTLD_GLOBAL) # libfuse dependency + _libfuse_path = (find_library('fuse4x') or find_library('osxfuse') or + find_library('fuse')) +else: + _libfuse_path = find_library('fuse') + +if not _libfuse_path: + raise EnvironmentError('Unable to find libfuse') +else: + _libfuse = CDLL(_libfuse_path) + +if _system == 'Darwin' and hasattr(_libfuse, 'macfuse_version'): + _system = 'Darwin-MacFuse' + + +if _system in ('Darwin', 'Darwin-MacFuse', 'FreeBSD'): + ENOTSUP = 45 + c_dev_t = c_int32 + c_fsblkcnt_t = c_ulong + c_fsfilcnt_t = c_ulong + c_gid_t = c_uint32 + c_mode_t = c_uint16 + c_off_t = c_int64 + c_pid_t = c_int32 + c_uid_t = c_uint32 + setxattr_t = CFUNCTYPE(c_int, c_char_p, c_char_p, POINTER(c_byte), + c_size_t, c_int, c_uint32) + getxattr_t = CFUNCTYPE(c_int, c_char_p, c_char_p, POINTER(c_byte), + c_size_t, c_uint32) + if _system == 'Darwin': + c_stat._fields_ = [ + ('st_dev', c_dev_t), + ('st_mode', c_mode_t), + ('st_nlink', c_uint16), + ('st_ino', c_uint64), + ('st_uid', c_uid_t), + ('st_gid', c_gid_t), + ('st_rdev', c_dev_t), + ('st_atimespec', c_timespec), + ('st_mtimespec', c_timespec), + ('st_ctimespec', c_timespec), + ('st_birthtimespec', c_timespec), + ('st_size', c_off_t), + ('st_blocks', c_int64), + ('st_blksize', c_int32), + ('st_flags', c_int32), + ('st_gen', c_int32), + ('st_lspare', c_int32), + ('st_qspare', c_int64)] + else: + c_stat._fields_ = [ + ('st_dev', c_dev_t), + ('st_ino', c_uint32), + ('st_mode', c_mode_t), + ('st_nlink', c_uint16), + ('st_uid', c_uid_t), + ('st_gid', c_gid_t), + ('st_rdev', c_dev_t), + ('st_atimespec', c_timespec), + ('st_mtimespec', c_timespec), + ('st_ctimespec', c_timespec), + ('st_size', c_off_t), + ('st_blocks', c_int64), + ('st_blksize', c_int32)] +elif _system == 'Linux': + ENOTSUP = 95 + c_dev_t = c_ulonglong + c_fsblkcnt_t = c_ulonglong + c_fsfilcnt_t = c_ulonglong + c_gid_t = c_uint + c_mode_t = c_uint + c_off_t = c_longlong + c_pid_t = c_int + c_uid_t = c_uint + setxattr_t = CFUNCTYPE(c_int, c_char_p, c_char_p, POINTER(c_byte), + c_size_t, c_int) + + getxattr_t = CFUNCTYPE(c_int, c_char_p, c_char_p, POINTER(c_byte), + c_size_t) + + if _machine == 'x86_64': + c_stat._fields_ = [ + ('st_dev', c_dev_t), + ('st_ino', c_ulong), + ('st_nlink', c_ulong), + ('st_mode', c_mode_t), + ('st_uid', c_uid_t), + ('st_gid', c_gid_t), + ('__pad0', c_int), + ('st_rdev', c_dev_t), + ('st_size', c_off_t), + ('st_blksize', c_long), + ('st_blocks', c_long), + ('st_atimespec', c_timespec), + ('st_mtimespec', c_timespec), + ('st_ctimespec', c_timespec)] + elif _machine == 'mips': + c_stat._fields_ = [ + ('st_dev', c_dev_t), + ('__pad1_1', c_ulong), + ('__pad1_2', c_ulong), + ('__pad1_3', c_ulong), + ('st_ino', c_ulong), + ('st_mode', c_mode_t), + ('st_nlink', c_ulong), + ('st_uid', c_uid_t), + ('st_gid', c_gid_t), + ('st_rdev', c_dev_t), + ('__pad2_1', c_ulong), + ('__pad2_2', c_ulong), + ('st_size', c_off_t), + ('__pad3', c_ulong), + ('st_atimespec', c_timespec), + ('__pad4', c_ulong), + ('st_mtimespec', c_timespec), + ('__pad5', c_ulong), + ('st_ctimespec', c_timespec), + ('__pad6', c_ulong), + ('st_blksize', c_long), + ('st_blocks', c_long), + ('__pad7_1', c_ulong), + ('__pad7_2', c_ulong), + ('__pad7_3', c_ulong), + ('__pad7_4', c_ulong), + ('__pad7_5', c_ulong), + ('__pad7_6', c_ulong), + ('__pad7_7', c_ulong), + ('__pad7_8', c_ulong), + ('__pad7_9', c_ulong), + ('__pad7_10', c_ulong), + ('__pad7_11', c_ulong), + ('__pad7_12', c_ulong), + ('__pad7_13', c_ulong), + ('__pad7_14', c_ulong)] + elif _machine == 'ppc': + c_stat._fields_ = [ + ('st_dev', c_dev_t), + ('st_ino', c_ulonglong), + ('st_mode', c_mode_t), + ('st_nlink', c_uint), + ('st_uid', c_uid_t), + ('st_gid', c_gid_t), + ('st_rdev', c_dev_t), + ('__pad2', c_ushort), + ('st_size', c_off_t), + ('st_blksize', c_long), + ('st_blocks', c_longlong), + ('st_atimespec', c_timespec), + ('st_mtimespec', c_timespec), + ('st_ctimespec', c_timespec)] + elif _machine == 'ppc64' or _machine == 'ppc64le': + c_stat._fields_ = [ + ('st_dev', c_dev_t), + ('st_ino', c_ulong), + ('st_nlink', c_ulong), + ('st_mode', c_mode_t), + ('st_uid', c_uid_t), + ('st_gid', c_gid_t), + ('__pad', c_uint), + ('st_rdev', c_dev_t), + ('st_size', c_off_t), + ('st_blksize', c_long), + ('st_blocks', c_long), + ('st_atimespec', c_timespec), + ('st_mtimespec', c_timespec), + ('st_ctimespec', c_timespec)] + elif _machine == 'aarch64': + c_stat._fields_ = [ + ('st_dev', c_dev_t), + ('st_ino', c_ulong), + ('st_mode', c_mode_t), + ('st_nlink', c_uint), + ('st_uid', c_uid_t), + ('st_gid', c_gid_t), + ('st_rdev', c_dev_t), + ('__pad1', c_ulong), + ('st_size', c_off_t), + ('st_blksize', c_int), + ('__pad2', c_int), + ('st_blocks', c_long), + ('st_atimespec', c_timespec), + ('st_mtimespec', c_timespec), + ('st_ctimespec', c_timespec)] + else: + # i686, use as fallback for everything else + c_stat._fields_ = [ + ('st_dev', c_dev_t), + ('__pad1', c_ushort), + ('__st_ino', c_ulong), + ('st_mode', c_mode_t), + ('st_nlink', c_uint), + ('st_uid', c_uid_t), + ('st_gid', c_gid_t), + ('st_rdev', c_dev_t), + ('__pad2', c_ushort), + ('st_size', c_off_t), + ('st_blksize', c_long), + ('st_blocks', c_longlong), + ('st_atimespec', c_timespec), + ('st_mtimespec', c_timespec), + ('st_ctimespec', c_timespec), + ('st_ino', c_ulonglong)] +else: + raise NotImplementedError('{} is not supported.'.format(_system)) + + +class c_statvfs(Structure): + _fields_ = [ + ('f_bsize', c_ulong), + ('f_frsize', c_ulong), + ('f_blocks', c_fsblkcnt_t), + ('f_bfree', c_fsblkcnt_t), + ('f_bavail', c_fsblkcnt_t), + ('f_files', c_fsfilcnt_t), + ('f_ffree', c_fsfilcnt_t), + ('f_favail', c_fsfilcnt_t), + ('f_fsid', c_ulong), + #('unused', c_int), + ('f_flag', c_ulong), + ('f_namemax', c_ulong)] + +if _system == 'FreeBSD': + c_fsblkcnt_t = c_uint64 + c_fsfilcnt_t = c_uint64 + setxattr_t = CFUNCTYPE(c_int, c_char_p, c_char_p, POINTER(c_byte), + c_size_t, c_int) + + getxattr_t = CFUNCTYPE(c_int, c_char_p, c_char_p, POINTER(c_byte), + c_size_t) + + class c_statvfs(Structure): + _fields_ = [ + ('f_bavail', c_fsblkcnt_t), + ('f_bfree', c_fsblkcnt_t), + ('f_blocks', c_fsblkcnt_t), + ('f_favail', c_fsfilcnt_t), + ('f_ffree', c_fsfilcnt_t), + ('f_files', c_fsfilcnt_t), + ('f_bsize', c_ulong), + ('f_flag', c_ulong), + ('f_frsize', c_ulong)] + +class fuse_file_info(Structure): + _fields_ = [ + ('flags', c_int), + ('fh_old', c_ulong), + ('writepage', c_int), + ('direct_io', c_uint, 1), + ('keep_cache', c_uint, 1), + ('flush', c_uint, 1), + ('padding', c_uint, 29), + ('fh', c_uint64), + ('lock_owner', c_uint64)] + +class fuse_context(Structure): + _fields_ = [ + ('fuse', c_voidp), + ('uid', c_uid_t), + ('gid', c_gid_t), + ('pid', c_pid_t), + ('private_data', c_voidp)] + +_libfuse.fuse_get_context.restype = POINTER(fuse_context) + + +class fuse_operations(Structure): + _fields_ = [ + ('getattr', CFUNCTYPE(c_int, c_char_p, POINTER(c_stat))), + ('readlink', CFUNCTYPE(c_int, c_char_p, POINTER(c_byte), c_size_t)), + ('getdir', c_voidp), # Deprecated, use readdir + ('mknod', CFUNCTYPE(c_int, c_char_p, c_mode_t, c_dev_t)), + ('mkdir', CFUNCTYPE(c_int, c_char_p, c_mode_t)), + ('unlink', CFUNCTYPE(c_int, c_char_p)), + ('rmdir', CFUNCTYPE(c_int, c_char_p)), + ('symlink', CFUNCTYPE(c_int, c_char_p, c_char_p)), + ('rename', CFUNCTYPE(c_int, c_char_p, c_char_p)), + ('link', CFUNCTYPE(c_int, c_char_p, c_char_p)), + ('chmod', CFUNCTYPE(c_int, c_char_p, c_mode_t)), + ('chown', CFUNCTYPE(c_int, c_char_p, c_uid_t, c_gid_t)), + ('truncate', CFUNCTYPE(c_int, c_char_p, c_off_t)), + ('utime', c_voidp), # Deprecated, use utimens + ('open', CFUNCTYPE(c_int, c_char_p, POINTER(fuse_file_info))), + + ('read', CFUNCTYPE(c_int, c_char_p, POINTER(c_byte), c_size_t, + c_off_t, POINTER(fuse_file_info))), + + ('write', CFUNCTYPE(c_int, c_char_p, POINTER(c_byte), c_size_t, + c_off_t, POINTER(fuse_file_info))), + + ('statfs', CFUNCTYPE(c_int, c_char_p, POINTER(c_statvfs))), + ('flush', CFUNCTYPE(c_int, c_char_p, POINTER(fuse_file_info))), + ('release', CFUNCTYPE(c_int, c_char_p, POINTER(fuse_file_info))), + ('fsync', CFUNCTYPE(c_int, c_char_p, c_int, POINTER(fuse_file_info))), + ('setxattr', setxattr_t), + ('getxattr', getxattr_t), + ('listxattr', CFUNCTYPE(c_int, c_char_p, POINTER(c_byte), c_size_t)), + ('removexattr', CFUNCTYPE(c_int, c_char_p, c_char_p)), + ('opendir', CFUNCTYPE(c_int, c_char_p, POINTER(fuse_file_info))), + + ('readdir', CFUNCTYPE(c_int, c_char_p, c_voidp, + CFUNCTYPE(c_int, c_voidp, c_char_p, + POINTER(c_stat), c_off_t), + c_off_t, POINTER(fuse_file_info))), + + ('releasedir', CFUNCTYPE(c_int, c_char_p, POINTER(fuse_file_info))), + + ('fsyncdir', CFUNCTYPE(c_int, c_char_p, c_int, + POINTER(fuse_file_info))), + + ('init', CFUNCTYPE(c_voidp, c_voidp)), + ('destroy', CFUNCTYPE(c_voidp, c_voidp)), + ('access', CFUNCTYPE(c_int, c_char_p, c_int)), + + ('create', CFUNCTYPE(c_int, c_char_p, c_mode_t, + POINTER(fuse_file_info))), + + ('ftruncate', CFUNCTYPE(c_int, c_char_p, c_off_t, + POINTER(fuse_file_info))), + + ('fgetattr', CFUNCTYPE(c_int, c_char_p, POINTER(c_stat), + POINTER(fuse_file_info))), + + ('lock', CFUNCTYPE(c_int, c_char_p, POINTER(fuse_file_info), + c_int, c_voidp)), + + ('utimens', CFUNCTYPE(c_int, c_char_p, POINTER(c_utimbuf))), + ('bmap', CFUNCTYPE(c_int, c_char_p, c_size_t, POINTER(c_ulonglong))), + ('flag_nullpath_ok', c_uint, 1), + ('flag_nopath', c_uint, 1), + ('flag_utime_omit_ok', c_uint, 1), + ('flag_reserved', c_uint, 29), + ] + + +def time_of_timespec(ts): + return ts.tv_sec + ts.tv_nsec / 10 ** 9 + +def set_st_attrs(st, attrs): + for key, val in attrs.items(): + if key in ('st_atime', 'st_mtime', 'st_ctime', 'st_birthtime'): + timespec = getattr(st, key + 'spec', None) + if timespec is None: + continue + timespec.tv_sec = int(val) + timespec.tv_nsec = int((val - timespec.tv_sec) * 10 ** 9) + elif hasattr(st, key): + setattr(st, key, val) + + +def fuse_get_context(): + 'Returns a (uid, gid, pid) tuple' + + ctxp = _libfuse.fuse_get_context() + ctx = ctxp.contents + return ctx.uid, ctx.gid, ctx.pid + + +class FuseOSError(OSError): + def __init__(self, errno): + super(FuseOSError, self).__init__(errno, strerror(errno)) + + +class FUSE(object): + ''' + This class is the lower level interface and should not be subclassed under + normal use. Its methods are called by fuse. + + Assumes API version 2.6 or later. + ''' + + OPTIONS = ( + ('foreground', '-f'), + ('debug', '-d'), + ('nothreads', '-s'), + ) + + def __init__(self, operations, mountpoint, raw_fi=False, encoding='utf-8', + **kwargs): + + ''' + Setting raw_fi to True will cause FUSE to pass the fuse_file_info + class as is to Operations, instead of just the fh field. + + This gives you access to direct_io, keep_cache, etc. + ''' + + self.operations = operations + self.raw_fi = raw_fi + self.encoding = encoding + + args = ['fuse'] + + args.extend(flag for arg, flag in self.OPTIONS + if kwargs.pop(arg, False)) + + kwargs.setdefault('fsname', operations.__class__.__name__) + args.append('-o') + args.append(','.join(self._normalize_fuse_options(**kwargs))) + args.append(mountpoint) + + args = [arg.encode(encoding) for arg in args] + argv = (c_char_p * len(args))(*args) + + fuse_ops = fuse_operations() + for ent in fuse_operations._fields_: + name, prototype = ent[:2] + + val = getattr(operations, name, None) + if val is None: + continue + + # Function pointer members are tested for using the + # getattr(operations, name) above but are dynamically + # invoked using self.operations(name) + if hasattr(prototype, 'argtypes'): + val = prototype(partial(self._wrapper, getattr(self, name))) + + setattr(fuse_ops, name, val) + + try: + old_handler = signal(SIGINT, SIG_DFL) + except ValueError: + old_handler = SIG_DFL + + err = _libfuse.fuse_main_real(len(args), argv, pointer(fuse_ops), + sizeof(fuse_ops), None) + + try: + signal(SIGINT, old_handler) + except ValueError: + pass + + del self.operations # Invoke the destructor + if err: + raise RuntimeError(err) + + @staticmethod + def _normalize_fuse_options(**kargs): + for key, value in kargs.items(): + if isinstance(value, bool): + if value is True: yield key + else: + yield '{}={}'.format(key, value) + + @staticmethod + def _wrapper(func, *args, **kwargs): + 'Decorator for the methods that follow' + + try: + return func(*args, **kwargs) or 0 + except OSError as e: + return -(e.errno or EFAULT) + except: + print_exc() + return -EFAULT + + def _decode_optional_path(self, path): + # NB: this method is intended for fuse operations that + # allow the path argument to be NULL, + # *not* as a generic path decoding method + if path is None: + return None + return path.decode(self.encoding) + + def getattr(self, path, buf): + return self.fgetattr(path, buf, None) + + def readlink(self, path, buf, bufsize): + ret = self.operations('readlink', path.decode(self.encoding)) \ + .encode(self.encoding) + + # copies a string into the given buffer + # (null terminated and truncated if necessary) + data = create_string_buffer(ret[:bufsize - 1]) + memmove(buf, data, len(data)) + return 0 + + def mknod(self, path, mode, dev): + return self.operations('mknod', path.decode(self.encoding), mode, dev) + + def mkdir(self, path, mode): + return self.operations('mkdir', path.decode(self.encoding), mode) + + def unlink(self, path): + return self.operations('unlink', path.decode(self.encoding)) + + def rmdir(self, path): + return self.operations('rmdir', path.decode(self.encoding)) + + def symlink(self, source, target): + 'creates a symlink `target -> source` (e.g. ln -s source target)' + + return self.operations('symlink', target.decode(self.encoding), + source.decode(self.encoding)) + + def rename(self, old, new): + return self.operations('rename', old.decode(self.encoding), + new.decode(self.encoding)) + + def link(self, source, target): + 'creates a hard link `target -> source` (e.g. ln source target)' + + return self.operations('link', target.decode(self.encoding), + source.decode(self.encoding)) + + def chmod(self, path, mode): + return self.operations('chmod', path.decode(self.encoding), mode) + + def chown(self, path, uid, gid): + # Check if any of the arguments is a -1 that has overflowed + if c_uid_t(uid + 1).value == 0: + uid = -1 + if c_gid_t(gid + 1).value == 0: + gid = -1 + + return self.operations('chown', path.decode(self.encoding), uid, gid) + + def truncate(self, path, length): + return self.operations('truncate', path.decode(self.encoding), length) + + def open(self, path, fip): + fi = fip.contents + if self.raw_fi: + return self.operations('open', path.decode(self.encoding), fi) + else: + fi.fh = self.operations('open', path.decode(self.encoding), + fi.flags) + + return 0 + + def read(self, path, buf, size, offset, fip): + if self.raw_fi: + fh = fip.contents + else: + fh = fip.contents.fh + + ret = self.operations('read', self._decode_optional_path(path), size, + offset, fh) + + if not ret: return 0 + + retsize = len(ret) + assert retsize <= size, \ + 'actual amount read {:d} greater than expected {:d}'.format(retsize, size) + + data = create_string_buffer(ret, retsize) + memmove(buf, data, retsize) + return retsize + + def write(self, path, buf, size, offset, fip): + data = string_at(buf, size) + + if self.raw_fi: + fh = fip.contents + else: + fh = fip.contents.fh + + return self.operations('write', self._decode_optional_path(path), data, + offset, fh) + + def statfs(self, path, buf): + stv = buf.contents + attrs = self.operations('statfs', path.decode(self.encoding)) + for key, val in attrs.items(): + if hasattr(stv, key): + setattr(stv, key, val) + + return 0 + + def flush(self, path, fip): + if self.raw_fi: + fh = fip.contents + else: + fh = fip.contents.fh + + return self.operations('flush', self._decode_optional_path(path), fh) + + def release(self, path, fip): + if self.raw_fi: + fh = fip.contents + else: + fh = fip.contents.fh + + return self.operations('release', self._decode_optional_path(path), fh) + + def fsync(self, path, datasync, fip): + if self.raw_fi: + fh = fip.contents + else: + fh = fip.contents.fh + + return self.operations('fsync', self._decode_optional_path(path), datasync, + fh) + + def setxattr(self, path, name, value, size, options, *args): + return self.operations('setxattr', path.decode(self.encoding), + name.decode(self.encoding), + string_at(value, size), options, *args) + + def getxattr(self, path, name, value, size, *args): + ret = self.operations('getxattr', path.decode(self.encoding), + name.decode(self.encoding), *args) + + retsize = len(ret) + # allow size queries + if not value: return retsize + + # do not truncate + if retsize > size: return -ERANGE + + buf = create_string_buffer(ret, retsize) # Does not add trailing 0 + memmove(value, buf, retsize) + + return retsize + + def listxattr(self, path, namebuf, size): + attrs = self.operations('listxattr', path.decode(self.encoding)) or '' + ret = '\x00'.join(attrs).encode(self.encoding) + if len(ret) > 0: + ret += '\x00'.encode(self.encoding) + + retsize = len(ret) + # allow size queries + if not namebuf: return retsize + + # do not truncate + if retsize > size: return -ERANGE + + buf = create_string_buffer(ret, retsize) + memmove(namebuf, buf, retsize) + + return retsize + + def removexattr(self, path, name): + return self.operations('removexattr', path.decode(self.encoding), + name.decode(self.encoding)) + + def opendir(self, path, fip): + # Ignore raw_fi + fip.contents.fh = self.operations('opendir', + path.decode(self.encoding)) + + return 0 + + def readdir(self, path, buf, filler, offset, fip): + # Ignore raw_fi + for item in self.operations('readdir', self._decode_optional_path(path), + fip.contents.fh): + + if isinstance(item, basestring): + name, st, offset = item, None, 0 + else: + name, attrs, offset = item + if attrs: + st = c_stat() + set_st_attrs(st, attrs) + else: + st = None + + if filler(buf, name.encode(self.encoding), st, offset) != 0: + break + + return 0 + + def releasedir(self, path, fip): + # Ignore raw_fi + return self.operations('releasedir', self._decode_optional_path(path), + fip.contents.fh) + + def fsyncdir(self, path, datasync, fip): + # Ignore raw_fi + return self.operations('fsyncdir', self._decode_optional_path(path), + datasync, fip.contents.fh) + + def init(self, conn): + return self.operations('init', '/') + + def destroy(self, private_data): + return self.operations('destroy', '/') + + def access(self, path, amode): + return self.operations('access', path.decode(self.encoding), amode) + + def create(self, path, mode, fip): + fi = fip.contents + path = path.decode(self.encoding) + + if self.raw_fi: + return self.operations('create', path, mode, fi) + else: + # This line is different from upstream to fix issues + # reading file opened with O_CREAT|O_RDWR. + # See issue #143. + fi.fh = self.operations('create', path, mode, fi.flags) + # END OF MODIFICATION + return 0 + + def ftruncate(self, path, length, fip): + if self.raw_fi: + fh = fip.contents + else: + fh = fip.contents.fh + + return self.operations('truncate', self._decode_optional_path(path), + length, fh) + + def fgetattr(self, path, buf, fip): + memset(buf, 0, sizeof(c_stat)) + + st = buf.contents + if not fip: + fh = fip + elif self.raw_fi: + fh = fip.contents + else: + fh = fip.contents.fh + + attrs = self.operations('getattr', self._decode_optional_path(path), fh) + set_st_attrs(st, attrs) + return 0 + + def lock(self, path, fip, cmd, lock): + if self.raw_fi: + fh = fip.contents + else: + fh = fip.contents.fh + + return self.operations('lock', self._decode_optional_path(path), fh, cmd, + lock) + + def utimens(self, path, buf): + if buf: + atime = time_of_timespec(buf.contents.actime) + mtime = time_of_timespec(buf.contents.modtime) + times = (atime, mtime) + else: + times = None + + return self.operations('utimens', path.decode(self.encoding), times) + + def bmap(self, path, blocksize, idx): + return self.operations('bmap', path.decode(self.encoding), blocksize, + idx) + + +class Operations(object): + ''' + This class should be subclassed and passed as an argument to FUSE on + initialization. All operations should raise a FuseOSError exception on + error. + + When in doubt of what an operation should do, check the FUSE header file + or the corresponding system call man page. + ''' + + def __call__(self, op, *args): + if not hasattr(self, op): + raise FuseOSError(EFAULT) + return getattr(self, op)(*args) + + def access(self, path, amode): + return 0 + + bmap = None + + def chmod(self, path, mode): + raise FuseOSError(EROFS) + + def chown(self, path, uid, gid): + raise FuseOSError(EROFS) + + def create(self, path, mode, fi=None): + ''' + When raw_fi is False (default case), fi is None and create should + return a numerical file handle. + + When raw_fi is True the file handle should be set directly by create + and return 0. + ''' + + raise FuseOSError(EROFS) + + def destroy(self, path): + 'Called on filesystem destruction. Path is always /' + + pass + + def flush(self, path, fh): + return 0 + + def fsync(self, path, datasync, fh): + return 0 + + def fsyncdir(self, path, datasync, fh): + return 0 + + def getattr(self, path, fh=None): + ''' + Returns a dictionary with keys identical to the stat C structure of + stat(2). + + st_atime, st_mtime and st_ctime should be floats. + + NOTE: There is an incombatibility between Linux and Mac OS X + concerning st_nlink of directories. Mac OS X counts all files inside + the directory, while Linux counts only the subdirectories. + ''' + + if path != '/': + raise FuseOSError(ENOENT) + return dict(st_mode=(S_IFDIR | 0o755), st_nlink=2) + + def getxattr(self, path, name, position=0): + raise FuseOSError(ENOTSUP) + + def init(self, path): + ''' + Called on filesystem initialization. (Path is always /) + + Use it instead of __init__ if you start threads on initialization. + ''' + + pass + + def link(self, target, source): + 'creates a hard link `target -> source` (e.g. ln source target)' + + raise FuseOSError(EROFS) + + def listxattr(self, path): + return [] + + lock = None + + def mkdir(self, path, mode): + raise FuseOSError(EROFS) + + def mknod(self, path, mode, dev): + raise FuseOSError(EROFS) + + def open(self, path, flags): + ''' + When raw_fi is False (default case), open should return a numerical + file handle. + + When raw_fi is True the signature of open becomes: + open(self, path, fi) + + and the file handle should be set directly. + ''' + + return 0 + + def opendir(self, path): + 'Returns a numerical file handle.' + + return 0 + + def read(self, path, size, offset, fh): + 'Returns a string containing the data requested.' + + raise FuseOSError(EIO) + + def readdir(self, path, fh): + ''' + Can return either a list of names, or a list of (name, attrs, offset) + tuples. attrs is a dict as in getattr. + ''' + + return ['.', '..'] + + def readlink(self, path): + raise FuseOSError(ENOENT) + + def release(self, path, fh): + return 0 + + def releasedir(self, path, fh): + return 0 + + def removexattr(self, path, name): + raise FuseOSError(ENOTSUP) + + def rename(self, old, new): + raise FuseOSError(EROFS) + + def rmdir(self, path): + raise FuseOSError(EROFS) + + def setxattr(self, path, name, value, options, position=0): + raise FuseOSError(ENOTSUP) + + def statfs(self, path): + ''' + Returns a dictionary with keys identical to the statvfs C structure of + statvfs(3). + + On Mac OS X f_bsize and f_frsize must be a power of 2 + (minimum 512). + ''' + + return {} + + def symlink(self, target, source): + 'creates a symlink `target -> source` (e.g. ln -s source target)' + + raise FuseOSError(EROFS) + + def truncate(self, path, length, fh=None): + raise FuseOSError(EROFS) + + def unlink(self, path): + raise FuseOSError(EROFS) + + def utimens(self, path, times=None): + 'Times is a (atime, mtime) tuple. If None use current time.' + + return 0 + + def write(self, path, data, offset, fh): + raise FuseOSError(EROFS) + + +class LoggingMixIn: + log = logging.getLogger('fuse.log-mixin') + + def __call__(self, op, path, *args): + self.log.debug('-> %s %s %s', op, path, repr(args)) + ret = '[Unhandled Exception]' + try: + ret = getattr(self, op)(path, *args) + return ret + except OSError as e: + ret = str(e) + raise + finally: + self.log.debug('<- %s %s', op, repr(ret)) diff --git a/src/buildstream/_fuse/hardlinks.py b/src/buildstream/_fuse/hardlinks.py new file mode 100644 index 000000000..ff2e81eea --- /dev/null +++ b/src/buildstream/_fuse/hardlinks.py @@ -0,0 +1,218 @@ +# +# Copyright (C) 2016 Stavros Korokithakis +# Copyright (C) 2017 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> +# +# The filesystem operations implementation here is based +# on some example code written by Stavros Korokithakis. + +import errno +import os +import shutil +import stat +import tempfile + +from .fuse import FuseOSError, Operations + +from .mount import Mount + + +# SafeHardlinks() +# +# A FUSE mount which implements a copy on write hardlink experience. +# +# Args: +# root (str): The underlying filesystem path to mirror +# tmp (str): A directory on the same filesystem for creating temp files +# +class SafeHardlinks(Mount): + + def __init__(self, directory, tempdir, fuse_mount_options=None): + self.directory = directory + self.tempdir = tempdir + if fuse_mount_options is None: + fuse_mount_options = {} + super().__init__(fuse_mount_options=fuse_mount_options) + + def create_operations(self): + return SafeHardlinkOps(self.directory, self.tempdir) + + +# SafeHardlinkOps() +# +# The actual FUSE Operations implementation below. +# +class SafeHardlinkOps(Operations): + + def __init__(self, root, tmp): + self.root = root + self.tmp = tmp + + def _full_path(self, partial): + if partial.startswith("/"): + partial = partial[1:] + path = os.path.join(self.root, partial) + return path + + def _ensure_copy(self, full_path): + try: + # Follow symbolic links manually here + real_path = os.path.realpath(full_path) + file_stat = os.stat(real_path) + + # Dont bother with files that cannot be hardlinked, oddly it + # directories actually usually have st_nlink > 1 so just avoid + # that. + # + # We already wont get symlinks here, and stat will throw + # the FileNotFoundError below if a followed symlink did not exist. + # + if not stat.S_ISDIR(file_stat.st_mode) and file_stat.st_nlink > 1: + with tempfile.TemporaryDirectory(dir=self.tmp) as tempdir: + basename = os.path.basename(real_path) + temp_path = os.path.join(tempdir, basename) + + # First copy, then unlink origin and rename + shutil.copy2(real_path, temp_path) + os.unlink(real_path) + os.rename(temp_path, real_path) + + except FileNotFoundError: + # This doesnt exist yet, assume we're about to create it + # so it's not a problem. + pass + + ########################################################### + # Fuse Methods # + ########################################################### + def access(self, path, mode): + full_path = self._full_path(path) + if not os.access(full_path, mode): + raise FuseOSError(errno.EACCES) + + def chmod(self, path, mode): + full_path = self._full_path(path) + + # Ensure copies on chmod + self._ensure_copy(full_path) + return os.chmod(full_path, mode) + + def chown(self, path, uid, gid): + full_path = self._full_path(path) + + # Ensure copies on chown + self._ensure_copy(full_path) + return os.chown(full_path, uid, gid) + + def getattr(self, path, fh=None): + full_path = self._full_path(path) + st = os.lstat(full_path) + return dict((key, getattr(st, key)) for key in ( + 'st_atime', 'st_ctime', 'st_gid', 'st_mode', + 'st_mtime', 'st_nlink', 'st_size', 'st_uid', 'st_rdev')) + + def readdir(self, path, fh): + full_path = self._full_path(path) + + dirents = ['.', '..'] + if os.path.isdir(full_path): + dirents.extend(os.listdir(full_path)) + for r in dirents: + yield r + + def readlink(self, path): + pathname = os.readlink(self._full_path(path)) + if pathname.startswith("/"): + # Path name is absolute, sanitize it. + return os.path.relpath(pathname, self.root) + else: + return pathname + + def mknod(self, path, mode, dev): + return os.mknod(self._full_path(path), mode, dev) + + def rmdir(self, path): + full_path = self._full_path(path) + return os.rmdir(full_path) + + def mkdir(self, path, mode): + return os.mkdir(self._full_path(path), mode) + + def statfs(self, path): + full_path = self._full_path(path) + stv = os.statvfs(full_path) + return dict((key, getattr(stv, key)) for key in ( + 'f_bavail', 'f_bfree', 'f_blocks', 'f_bsize', 'f_favail', + 'f_ffree', 'f_files', 'f_flag', 'f_frsize', 'f_namemax')) + + def unlink(self, path): + return os.unlink(self._full_path(path)) + + def symlink(self, name, target): + return os.symlink(target, self._full_path(name)) + + def rename(self, old, new): + return os.rename(self._full_path(old), self._full_path(new)) + + def link(self, target, name): + + # When creating a hard link here, should we ensure the original + # file is not a hardlink itself first ? + # + return os.link(self._full_path(name), self._full_path(target)) + + def utimens(self, path, times=None): + return os.utime(self._full_path(path), times) + + def open(self, path, flags): + full_path = self._full_path(path) + + # If we're opening for writing, ensure it's a copy first + if flags & os.O_WRONLY or flags & os.O_RDWR: + self._ensure_copy(full_path) + + return os.open(full_path, flags) + + def create(self, path, mode, flags): + full_path = self._full_path(path) + + # If it already exists, ensure it's a copy first + self._ensure_copy(full_path) + return os.open(full_path, flags, mode) + + def read(self, path, length, offset, fh): + os.lseek(fh, offset, os.SEEK_SET) + return os.read(fh, length) + + def write(self, path, buf, offset, fh): + os.lseek(fh, offset, os.SEEK_SET) + return os.write(fh, buf) + + def truncate(self, path, length, fh=None): + full_path = self._full_path(path) + with open(full_path, 'r+') as f: + f.truncate(length) + + def flush(self, path, fh): + return os.fsync(fh) + + def release(self, path, fh): + return os.close(fh) + + def fsync(self, path, fdatasync, fh): + return self.flush(path, fh) diff --git a/src/buildstream/_fuse/mount.py b/src/buildstream/_fuse/mount.py new file mode 100644 index 000000000..e31684100 --- /dev/null +++ b/src/buildstream/_fuse/mount.py @@ -0,0 +1,196 @@ +# +# Copyright (C) 2017 Codethink Limited +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> + +import os +import signal +import time +import sys + +from contextlib import contextmanager +from multiprocessing import Process +from .fuse import FUSE + +from .._exceptions import ImplError +from .. import _signals + + +# Just a custom exception to raise here, for identifying possible +# bugs with a fuse layer implementation +# +class FuseMountError(Exception): + pass + + +# This is a convenience class which takes care of synchronizing the +# startup of FUSE and shutting it down. +# +# The implementations / subclasses should: +# +# - Overload the instance initializer to add any parameters +# needed for their fuse Operations implementation +# +# - Implement create_operations() to create the Operations +# instance on behalf of the superclass, using any additional +# parameters collected in the initializer. +# +# Mount objects can be treated as contextmanagers, the volume +# will be mounted during the context. +# +# UGLY CODE NOTE: +# +# This is a horrible little piece of code. The problem we face +# here is that the highlevel libfuse API has fuse_main(), which +# will either block in the foreground, or become a full daemon. +# +# With the daemon approach, we know that the fuse is mounted right +# away when fuse_main() returns, then the daemon will go and handle +# requests on its own, but then we have no way to shut down the +# daemon. +# +# With the blocking approach, we still have it as a child process +# so we can tell it to gracefully terminate; but it's impossible +# to know when the mount is done, there is no callback for that +# +# The solution we use here without digging too deep into the +# low level fuse API, is to fork a child process which will +# fun the fuse loop in foreground, and we block the parent +# process until the volume is mounted with a busy loop with timeouts. +# +class Mount(): + + # These are not really class data, they are + # just here for the sake of having None setup instead + # of missing attributes, since we do not provide any + # initializer and leave the initializer to the subclass. + # + __mountpoint = None + __operations = None + __process = None + + ################################################ + # User Facing API # + ################################################ + + def __init__(self, fuse_mount_options=None): + self._fuse_mount_options = {} if fuse_mount_options is None else fuse_mount_options + + # mount(): + # + # User facing API for mounting a fuse subclass implementation + # + # Args: + # (str): Location to mount this fuse fs + # + def mount(self, mountpoint): + + assert self.__process is None + + self.__mountpoint = mountpoint + self.__process = Process(target=self.__run_fuse) + + # Ensure the child fork() does not inherit our signal handlers, if the + # child wants to handle a signal then it will first set its own + # handler, and then unblock it. + with _signals.blocked([signal.SIGTERM, signal.SIGTSTP, signal.SIGINT], ignore=False): + self.__process.start() + + # This is horrible, we're going to wait until mountpoint is mounted and that's it. + while not os.path.ismount(mountpoint): + time.sleep(1 / 100) + + # unmount(): + # + # User facing API for unmounting a fuse subclass implementation + # + def unmount(self): + + # Terminate child process and join + if self.__process is not None: + self.__process.terminate() + self.__process.join() + + # Report an error if ever the underlying operations crashed for some reason. + if self.__process.exitcode != 0: + raise FuseMountError("{} reported exit code {} when unmounting" + .format(type(self).__name__, self.__process.exitcode)) + + self.__mountpoint = None + self.__process = None + + # mounted(): + # + # A context manager to run a code block with this fuse Mount + # mounted, this will take care of automatically unmounting + # in the case that the calling process is terminated. + # + # Args: + # (str): Location to mount this fuse fs + # + @contextmanager + def mounted(self, mountpoint): + + self.mount(mountpoint) + try: + with _signals.terminator(self.unmount): + yield + finally: + self.unmount() + + ################################################ + # Abstract Methods # + ################################################ + + # create_operations(): + # + # Create an Operations class (from fusepy) and return it + # + # Returns: + # (Operations): A FUSE Operations implementation + def create_operations(self): + raise ImplError("Mount subclass '{}' did not implement create_operations()" + .format(type(self).__name__)) + + ################################################ + # Child Process # + ################################################ + def __run_fuse(self): + + # First become session leader while signals are still blocked + # + # Then reset the SIGTERM handler to the default and finally + # unblock SIGTERM. + # + os.setsid() + signal.signal(signal.SIGTERM, signal.SIG_DFL) + signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGTERM]) + + # Ask the subclass to give us an Operations object + # + self.__operations = self.create_operations() # pylint: disable=assignment-from-no-return + + # Run fuse in foreground in this child process, internally libfuse + # will handle SIGTERM and gracefully exit its own little main loop. + # + FUSE(self.__operations, self.__mountpoint, nothreads=True, foreground=True, nonempty=True, + **self._fuse_mount_options) + + # Explicit 0 exit code, if the operations crashed for some reason, the exit + # code will not be 0, and we want to know about it. + # + sys.exit(0) |