From 65ee2f821c8c642aae60f462ff8f209ebdc93e20 Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Thu, 19 Mar 2015 12:38:11 +0000 Subject: Morph build 11ac5b237b3640718da94ad8e252d330 System branch: master --- morphlib/__init__.py | 2 + morphlib/app.py | 7 + morphlib/bins.py | 59 ++-- morphlib/bins_tests.py | 98 +------ morphlib/buildcommand.py | 32 ++- morphlib/builder.py | 118 ++++---- morphlib/builder_tests.py | 18 +- morphlib/fsutils.py | 23 +- morphlib/ostree.py | 139 +++++++++ morphlib/ostreeartifactcache.py | 229 +++++++++++++++ morphlib/plugins/deploy_plugin.py | 76 ++++- morphlib/plugins/distbuild_plugin.py | 30 +- morphlib/plugins/gc_plugin.py | 8 +- morphlib/pylru.py | 532 ----------------------------------- morphlib/remoteartifactcache.py | 25 ++ morphlib/stagingarea.py | 57 ++-- morphlib/stagingarea_tests.py | 39 ++- morphlib/util.py | 6 +- 18 files changed, 694 insertions(+), 804 deletions(-) create mode 100644 morphlib/ostree.py create mode 100644 morphlib/ostreeartifactcache.py delete mode 100644 morphlib/pylru.py (limited to 'morphlib') diff --git a/morphlib/__init__.py b/morphlib/__init__.py index 7c462aad..695241cc 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -71,6 +71,8 @@ import morphologyfinder import morphology import morphloader import morphset +import ostree +import ostreeartifactcache import remoteartifactcache import remoterepocache import repoaliasresolver diff --git a/morphlib/app.py b/morphlib/app.py index c8fe397d..f7c07726 100644 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -120,6 +120,13 @@ class Morph(cliapp.Application): metavar='URL', default=None, group=group_advanced) + self.settings.string(['union-filesystem'], + 'filesystem used to provide "union filesystem" ' + 'functionality when building and deploying. ' + 'Only "overlayfs" and "unionfs-fuse" are ' + 'supported at this time.', + default='overlayfs', + group=group_advanced) group_build = 'Build Options' self.settings.integer(['max-jobs'], diff --git a/morphlib/bins.py b/morphlib/bins.py index 2e8ba0b3..c5bacc26 100644 --- a/morphlib/bins.py +++ b/morphlib/bins.py @@ -78,12 +78,8 @@ if sys.version_info < (2, 7, 3): # pragma: no cover raise ExtractError("could not change owner") tarfile.TarFile.chown = fixed_chown -def create_chunk(rootdir, f, include, dump_memory_profile=None): - '''Create a chunk from the contents of a directory. - - ``f`` is an open file handle, to which the tar file is written. - - ''' +def create_chunk(rootdir, chunkdir, include, dump_memory_profile=None): + '''Create a chunk from the contents of a directory.''' dump_memory_profile = dump_memory_profile or (lambda msg: None) @@ -91,31 +87,42 @@ def create_chunk(rootdir, f, include, dump_memory_profile=None): # chunk artifact. This is useful to avoid problems from smallish # clock skew. It needs to be recent enough, however, that GNU tar # does not complain about an implausibly old timestamp. - normalized_timestamp = 683074800 + normalized_timestamp = (683074800, 683074800) dump_memory_profile('at beginning of create_chunk') - - path_pairs = [(relname, os.path.join(rootdir, relname)) - for relname in include] - tar = tarfile.open(fileobj=f, mode='w') - for relname, filename in path_pairs: + + def check_parent(name, paths): + parent = os.path.dirname(name) + if parent: + path = os.path.join(rootdir, parent) + if parent != rootdir and path not in paths: + paths.append(path) + check_parent(parent, paths) + + def filter_contents(dirname, filenames): + paths = [os.path.join(rootdir, relname) for relname in include] + for name in include: + check_parent(name, paths) + + return [f for f in filenames if os.path.join(dirname, f) not in paths] + + logging.debug('Copying artifact into %s.' % chunkdir) + shutil.copytree(rootdir, chunkdir, + symlinks=True, ignore=filter_contents) + + path_triplets = [(relname, os.path.join(chunkdir, relname), + os.path.join(rootdir, relname)) + for relname in include] + for relname, filename, orig in path_triplets: # Normalize mtime for everything. - tarinfo = tar.gettarinfo(filename, - arcname=relname) - tarinfo.ctime = normalized_timestamp - tarinfo.mtime = normalized_timestamp - if tarinfo.isreg(): - with open(filename, 'rb') as f: - tar.addfile(tarinfo, fileobj=f) - else: - tar.addfile(tarinfo) - tar.close() + if not os.path.islink(filename): + os.utime(filename, normalized_timestamp) - for relname, filename in reversed(path_pairs): - if os.path.isdir(filename) and not os.path.islink(filename): + for relname, filename, orig in reversed(path_triplets): + if os.path.isdir(orig) and not os.path.islink(orig): continue else: - os.remove(filename) + os.remove(orig) dump_memory_profile('after removing in create_chunks') @@ -209,7 +216,7 @@ def unpack_binary_from_file(f, dirname): # pragma: no cover tf.close() -def unpack_binary(filename, dirname): +def unpack_binary(filename, dirname): # pragma: no cover with open(filename, "rb") as f: unpack_binary_from_file(f, dirname) diff --git a/morphlib/bins_tests.py b/morphlib/bins_tests.py index 3895680f..879aada4 100644 --- a/morphlib/bins_tests.py +++ b/morphlib/bins_tests.py @@ -78,11 +78,9 @@ class ChunkTests(BinsTest): self.tempdir = tempfile.mkdtemp() self.instdir = os.path.join(self.tempdir, 'inst') self.chunk_file = os.path.join(self.tempdir, 'chunk') - self.chunk_f = open(self.chunk_file, 'wb') self.unpacked = os.path.join(self.tempdir, 'unpacked') def tearDown(self): - self.chunk_f.close() shutil.rmtree(self.tempdir) def populate_instdir(self): @@ -108,109 +106,21 @@ class ChunkTests(BinsTest): def create_chunk(self, includes): self.populate_instdir() - morphlib.bins.create_chunk(self.instdir, self.chunk_f, includes) - self.chunk_f.flush() - - def unpack_chunk(self): - os.mkdir(self.unpacked) - morphlib.bins.unpack_binary(self.chunk_file, self.unpacked) + morphlib.bins.create_chunk(self.instdir, self.chunk_file, includes) def test_empties_files(self): self.create_chunk(['bin/foo', 'lib/libfoo.so']) self.assertEqual([x for x, y in self.recursive_lstat(self.instdir)], ['.', 'bin', 'lib']) - def test_creates_and_unpacks_chunk_exactly(self): + def test_creates_chunk_exactly(self): self.create_chunk(['bin', 'bin/foo', 'lib', 'lib/libfoo.so']) - self.unpack_chunk() self.assertEqual(self.instdir_orig_files, - self.recursive_lstat(self.unpacked)) + self.recursive_lstat(self.chunk_file)) def test_uses_only_matching_names(self): self.create_chunk(['bin/foo']) - self.unpack_chunk() - self.assertEqual([x for x, y in self.recursive_lstat(self.unpacked)], + self.assertEqual([x for x, y in self.recursive_lstat(self.chunk_file)], ['.', 'bin', 'bin/foo']) self.assertEqual([x for x, y in self.recursive_lstat(self.instdir)], ['.', 'bin', 'lib', 'lib/libfoo.so']) - - def test_does_not_compress_artifact(self): - self.create_chunk(['bin']) - f = gzip.open(self.chunk_file) - self.assertRaises(IOError, f.read) - f.close() - - -class ExtractTests(unittest.TestCase): - - def setUp(self): - self.tempdir = tempfile.mkdtemp() - self.instdir = os.path.join(self.tempdir, 'inst') - self.unpacked = os.path.join(self.tempdir, 'unpacked') - - def tearDown(self): - shutil.rmtree(self.tempdir) - - def create_chunk(self, callback): - fh = StringIO.StringIO() - os.mkdir(self.instdir) - patterns = callback(self.instdir) - morphlib.bins.create_chunk(self.instdir, fh, patterns) - shutil.rmtree(self.instdir) - fh.flush() - fh.seek(0) - return fh - - def test_extracted_files_replace_links(self): - def make_linkfile(basedir): - with open(os.path.join(basedir, 'babar'), 'w') as f: - pass - os.symlink('babar', os.path.join(basedir, 'bar')) - return ['babar'] - linktar = self.create_chunk(make_linkfile) - - def make_file(basedir): - with open(os.path.join(basedir, 'bar'), 'w') as f: - pass - return ['bar'] - filetar = self.create_chunk(make_file) - - os.mkdir(self.unpacked) - morphlib.bins.unpack_binary_from_file(linktar, self.unpacked) - morphlib.bins.unpack_binary_from_file(filetar, self.unpacked) - mode = os.lstat(os.path.join(self.unpacked, 'bar')).st_mode - self.assertTrue(stat.S_ISREG(mode)) - - def test_extracted_dirs_keep_links(self): - def make_usrlink(basedir): - os.symlink('.', os.path.join(basedir, 'usr')) - return ['usr'] - linktar = self.create_chunk(make_usrlink) - - def make_usrdir(basedir): - os.mkdir(os.path.join(basedir, 'usr')) - return ['usr'] - dirtar = self.create_chunk(make_usrdir) - - morphlib.bins.unpack_binary_from_file(linktar, self.unpacked) - morphlib.bins.unpack_binary_from_file(dirtar, self.unpacked) - mode = os.lstat(os.path.join(self.unpacked, 'usr')).st_mode - self.assertTrue(stat.S_ISLNK(mode)) - - def test_extracted_files_follow_links(self): - def make_usrlink(basedir): - os.symlink('.', os.path.join(basedir, 'usr')) - return ['usr'] - linktar = self.create_chunk(make_usrlink) - - def make_usrdir(basedir): - os.mkdir(os.path.join(basedir, 'usr')) - with open(os.path.join(basedir, 'usr', 'foo'), 'w') as f: - pass - return ['usr', 'usr/foo'] - dirtar = self.create_chunk(make_usrdir) - - morphlib.bins.unpack_binary_from_file(linktar, self.unpacked) - morphlib.bins.unpack_binary_from_file(dirtar, self.unpacked) - mode = os.lstat(os.path.join(self.unpacked, 'foo')).st_mode - self.assertTrue(stat.S_ISREG(mode)) diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index be8a1507..c83abca6 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -418,8 +418,10 @@ class BuildCommand(object): # module into morphlib.remoteartififactcache first. to_fetch = [] if not self.lac.has(artifact): - to_fetch.append((self.rac.get(artifact), - self.lac.put(artifact))) + self.app.status( + msg='Fetching to local cache: artifact %(name)s', + name=artifact.name) + self.lac.copy_from_remote(artifact, self.rac) if artifact.source.morphology.needs_artifact_metadata_cached: if not self.lac.has_artifact_metadata(artifact, 'meta'): @@ -428,9 +430,6 @@ class BuildCommand(object): self.lac.put_artifact_metadata(artifact, 'meta'))) if len(to_fetch) > 0: - self.app.status( - msg='Fetching to local cache: artifact %(name)s', - name=artifact.name) fetch_files(to_fetch) def create_staging_area(self, build_env, use_chroot=True, extra_env={}, @@ -493,8 +492,27 @@ class BuildCommand(object): chunk_name=artifact.name, cache=artifact.source.cache_key[:7], chatty=True) - handle = self.lac.get(artifact) - staging_area.install_artifact(handle) + chunk_cache_dir = os.path.join(self.app.settings['tempdir'], + 'chunks') + artifact_checkout = os.path.join( + chunk_cache_dir, os.path.basename(artifact.basename()) + '.d') + if not os.path.exists(artifact_checkout): + self.app.status( + msg='Checking out %(chunk)s from cache.', + chunk=artifact.name + ) + temp_checkout = os.path.join(self.app.settings['tempdir'], + artifact.basename()) + try: + self.lac.get(artifact, temp_checkout) + except BaseException: + shutil.rmtree(temp_checkout) + raise + # TODO: This rename is not concurrency safe if two builds are + # extracting the same chunk, one build will fail because + # the other renamed its tempdir here first. + os.rename(temp_checkout, artifact_checkout) + staging_area.install_artifact(artifact, artifact_checkout) if target_source.build_mode == 'staging': morphlib.builder.ldconfig(self.app.runcmd, staging_area.dirname) diff --git a/morphlib/builder.py b/morphlib/builder.py index 04ebd149..9b01f983 100644 --- a/morphlib/builder.py +++ b/morphlib/builder.py @@ -125,11 +125,7 @@ def ldconfig(runcmd, rootdir): # pragma: no cover def download_depends(constituents, lac, rac, metadatas=None): for constituent in constituents: if not lac.has(constituent): - source = rac.get(constituent) - target = lac.put(constituent) - shutil.copyfileobj(source, target) - target.close() - source.close() + lac.copy_from_remote(constituent, rac) if metadatas is not None: for metadata in metadatas: if not lac.has_artifact_metadata(constituent, metadata): @@ -246,28 +242,6 @@ class ChunkBuilder(BuilderBase): '''Build chunk artifacts.''' - def create_devices(self, destdir): # pragma: no cover - '''Creates device nodes if the morphology specifies them''' - morphology = self.source.morphology - perms_mask = stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO - if 'devices' in morphology and morphology['devices'] is not None: - for dev in morphology['devices']: - destfile = os.path.join(destdir, './' + dev['filename']) - mode = int(dev['permissions'], 8) & perms_mask - if dev['type'] == 'c': - mode = mode | stat.S_IFCHR - elif dev['type'] == 'b': - mode = mode | stat.S_IFBLK - else: - raise IOError('Cannot create device node %s,' - 'unrecognized device type "%s"' - % (destfile, dev['type'])) - self.app.status(msg="Creating device node %s" - % destfile) - os.mknod(destfile, mode, - os.makedev(dev['major'], dev['minor'])) - os.chown(destfile, dev['uid'], dev['gid']) - def build_and_cache(self): # pragma: no cover with self.build_watch('overall-build'): @@ -286,7 +260,6 @@ class ChunkBuilder(BuilderBase): try: self.get_sources(builddir) self.run_commands(builddir, destdir, temppath, stdout) - self.create_devices(destdir) os.rename(temppath, logpath) except BaseException as e: @@ -459,13 +432,23 @@ class ChunkBuilder(BuilderBase): extra_files += ['baserock/%s.meta' % chunk_artifact_name] parented_paths = parentify(file_paths + extra_files) - with self.local_artifact_cache.put(chunk_artifact) as f: - self.write_metadata(destdir, chunk_artifact_name, - parented_paths) + self.write_metadata(destdir, chunk_artifact_name, + parented_paths) - self.app.status(msg='Creating chunk artifact %(name)s', - name=chunk_artifact_name) - morphlib.bins.create_chunk(destdir, f, parented_paths) + self.app.status(msg='Creating chunk artifact %(name)s', + name=chunk_artifact_name) + # TODO: This is not concurrency safe, bins.create_chunk will + # fail if tempdir already exists (eg if another build + # has created it). + tempdir = os.path.join(self.app.settings['tempdir'], + chunk_artifact.basename()) + try: + morphlib.bins.create_chunk(destdir, tempdir, + parented_paths) + self.local_artifact_cache.put(tempdir, chunk_artifact) + finally: + if os.path.isdir(tempdir): + shutil.rmtree(tempdir) built_artifacts.append(chunk_artifact) for dirname, subdirs, files in os.walk(destdir): @@ -509,8 +492,13 @@ class StratumBuilder(BuilderBase): [x.name for x in constituents]) with lac.put_artifact_metadata(a, 'meta') as f: json.dump(meta, f, indent=4, sort_keys=True) - with self.local_artifact_cache.put(a) as f: + # TODO: This is not concurrency safe, put_stratum_artifact + # deletes temp which could be in use by another + # build. + temp = os.path.join(self.app.settings['tempdir'], a.name) + with open(temp, 'w+') as f: json.dump([c.basename() for c in constituents], f) + self.local_artifact_cache.put_non_ostree_artifact(a, temp) self.save_build_times() return self.source.artifacts.values() @@ -532,33 +520,40 @@ class SystemBuilder(BuilderBase): # pragma: no cover arch = self.source.morphology['arch'] for a_name, artifact in self.source.artifacts.iteritems(): - handle = self.local_artifact_cache.put(artifact) - try: fs_root = self.staging_area.destdir(self.source) self.unpack_strata(fs_root) - self.write_metadata(fs_root, a_name) - self.run_system_integration_commands(fs_root) - unslashy_root = fs_root[1:] - def uproot_info(info): - info.name = relpath(info.name, unslashy_root) - if info.islnk(): - info.linkname = relpath(info.linkname, - unslashy_root) - return info - tar = tarfile.open(fileobj=handle, mode="w", name=a_name) - self.app.status(msg='Constructing tarball of rootfs', - chatty=True) - tar.add(fs_root, recursive=True, filter=uproot_info) - tar.close() + upperdir = self.staging_area.overlay_upperdir( + self.source) + editable_root = self.staging_area.overlaydir(self.source) + workdir = os.path.join(self.staging_area.dirname, + 'overlayfs-workdir') + if not os.path.exists(workdir): + os.makedirs(workdir) + union_filesystem = self.app.settings['union-filesystem'] + morphlib.fsutils.overlay_mount(self.app.runcmd, + 'overlay-%s' % a_name, + editable_root, fs_root, + upperdir, workdir, + union_filesystem) + self.write_metadata(editable_root, a_name) + self.run_system_integration_commands(editable_root) + # Put the contents of upperdir into the local artifact + # cache. Don't use editable root as we only want to + # store the modified files. + self.local_artifact_cache.put(upperdir, artifact) except BaseException as e: logging.error(traceback.format_exc()) self.app.status(msg='Error while building system', error=True) - handle.abort() + if editable_root and os.path.exists(editable_root): + morphlib.fsutils.unmount(self.app.runcmd, + editable_root) raise else: - handle.close() + if editable_root and os.path.exists(editable_root): + morphlib.fsutils.unmount(self.app.runcmd, + editable_root) self.save_build_times() return self.source.artifacts.itervalues() @@ -567,13 +562,12 @@ class SystemBuilder(BuilderBase): # pragma: no cover '''Unpack a single stratum into a target directory''' cache = self.local_artifact_cache - with cache.get(stratum_artifact) as stratum_file: + with open(cache.get(stratum_artifact), 'r') as stratum_file: artifact_list = json.load(stratum_file, encoding='unicode-escape') for chunk in (ArtifactCacheReference(a) for a in artifact_list): - self.app.status(msg='Unpacking chunk %(basename)s', + self.app.status(msg='Checkout chunk %(basename)s', basename=chunk.basename(), chatty=True) - with cache.get(chunk) as chunk_file: - morphlib.bins.unpack_binary_from_file(chunk_file, target) + cache.get(chunk, target) target_metadata = os.path.join( target, 'baserock', '%s.meta' % stratum_artifact.name) @@ -584,7 +578,7 @@ class SystemBuilder(BuilderBase): # pragma: no cover def unpack_strata(self, path): '''Unpack strata into a directory.''' - self.app.status(msg='Unpacking strata to %(path)s', + self.app.status(msg='Checking out strata to %(path)s', path=path, chatty=True) with self.build_watch('unpack-strata'): for a_name, a in self.source.artifacts.iteritems(): @@ -596,12 +590,14 @@ class SystemBuilder(BuilderBase): # pragma: no cover # download the chunk artifacts if necessary for stratum_artifact in self.source.dependencies: - f = self.local_artifact_cache.get(stratum_artifact) - chunks = [ArtifactCacheReference(c) for c in json.load(f)] + stratum_path = self.local_artifact_cache.get( + stratum_artifact) + with open(stratum_path, 'r') as stratum: + chunks = [ArtifactCacheReference(c) + for c in json.load(stratum)] download_depends(chunks, self.local_artifact_cache, self.remote_artifact_cache) - f.close() # unpack it from the local artifact cache for stratum_artifact in self.source.dependencies: diff --git a/morphlib/builder_tests.py b/morphlib/builder_tests.py index a571e3d0..b5e66521 100644 --- a/morphlib/builder_tests.py +++ b/morphlib/builder_tests.py @@ -105,8 +105,8 @@ class FakeArtifactCache(object): def __init__(self): self._cached = {} - def put(self, artifact): - return FakeFileHandle(self, (artifact.cache_key, artifact.name)) + def put(self, artifact, directory): + self._cached[(artifact.cache_key, artifact.name)] = artifact.name def put_artifact_metadata(self, artifact, name): return FakeFileHandle(self, (artifact.cache_key, artifact.name, name)) @@ -114,7 +114,7 @@ class FakeArtifactCache(object): def put_source_metadata(self, source, cachekey, name): return FakeFileHandle(self, (cachekey, name)) - def get(self, artifact): + def get(self, artifact, directory=None): return StringIO.StringIO( self._cached[(artifact.cache_key, artifact.name)]) @@ -134,6 +134,10 @@ class FakeArtifactCache(object): def has_source_metadata(self, source, cachekey, name): return (cachekey, name) in self._cached + def copy_from_remote(self, artifact, remote): + self._cached[(artifact.cache_key, artifact.name)] = \ + remote._cached[(artifact.cache_key, artifact.name)] + class BuilderBaseTests(unittest.TestCase): @@ -191,9 +195,7 @@ class BuilderBaseTests(unittest.TestCase): rac = FakeArtifactCache() afacts = [FakeArtifact(name) for name in ('a', 'b', 'c')] for a in afacts: - fh = rac.put(a) - fh.write(a.name) - fh.close() + rac.put(a, 'not-a-dir') morphlib.builder.download_depends(afacts, lac, rac) self.assertTrue(all(lac.has(a) for a in afacts)) @@ -202,9 +204,7 @@ class BuilderBaseTests(unittest.TestCase): rac = FakeArtifactCache() afacts = [FakeArtifact(name) for name in ('a', 'b', 'c')] for a in afacts: - fh = rac.put(a) - fh.write(a.name) - fh.close() + rac.put(a, 'not-a-dir') fh = rac.put_artifact_metadata(a, 'meta') fh.write('metadata') fh.close() diff --git a/morphlib/fsutils.py b/morphlib/fsutils.py index a3b73bf6..400ff7d8 100644 --- a/morphlib/fsutils.py +++ b/morphlib/fsutils.py @@ -46,14 +46,33 @@ def create_fs(runcmd, partition): # pragma: no cover runcmd(['mkfs.btrfs', '-L', 'baserock', partition]) -def mount(runcmd, partition, mount_point, fstype=None): # pragma: no cover +def mount(runcmd, partition, mount_point, + fstype=None, options=[]): # pragma: no cover if not os.path.exists(mount_point): os.mkdir(mount_point) if not fstype: fstype = [] else: fstype = ['-t', fstype] - runcmd(['mount', partition, mount_point] + fstype) + if not type(options) == list: + options = [options] + runcmd(['mount', partition, mount_point] + fstype + options) + + +def overlay_mount(runcmd, partition, mount_point, + lowerdir, upperdir, workdir, method): # pragma: no cover + if method == 'overlayfs': + options = '-olowerdir=%s,upperdir=%s,workdir=%s' % \ + (lowerdir, upperdir, workdir) + mount(runcmd, partition, mount_point, 'overlay', options) + elif method == 'unionfs': + if not os.path.exists(mount_point): + os.mkdir(mount_point) + dir_string = '%s=RW:%s=RO' % (upperdir, lowerdir) + runcmd(['unionfs', '-o', 'cow', dir_string, mount_point]) + else: + raise cliapp.AppException('Union filesystem %s not supported' % + method) def unmount(runcmd, mount_point): # pragma: no cover diff --git a/morphlib/ostree.py b/morphlib/ostree.py new file mode 100644 index 00000000..a2c133f2 --- /dev/null +++ b/morphlib/ostree.py @@ -0,0 +1,139 @@ +from gi.repository import OSTree +from gi.repository import Gio +from gi.repository import GLib + +import os + + +class OSTreeRepo(object): + + """Class to wrap the OSTree API.""" + + OSTREE_GIO_FAST_QUERYINFO = 'standard::name,standard::type,standard::' \ + 'size,standard::is-symlink,standard::syml' \ + 'ink-target,unix::device,unix::inode,unix' \ + '::mode,unix::uid,unix::gid,unix::rdev' + G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS = Gio.FileQueryInfoFlags(1) + cancellable = Gio.Cancellable.new() + + def __init__(self, path, disable_fsync=True): + self.path = path + self.repo = self._open_repo(path, disable_fsync) + + def _open_repo(self, path, disable_fsync=True): + """Create and open and OSTree.Repo, and return it.""" + repo_dir = Gio.file_new_for_path(path) + repo = OSTree.Repo.new(repo_dir) + try: + repo.open(self.cancellable) + except GLib.GError: + if not os.path.exists(path): + os.makedirs(path) + repo.create(OSTree.RepoMode.ARCHIVE_Z2, self.cancellable) + repo.set_disable_fsync(disable_fsync) + return repo + + def refsdir(self): + """Return the abspath to the refs/heads directory in the repo.""" + return os.path.join(os.path.abspath(self.path), 'refs/heads') + + def touch_ref(self, ref): + """Update the mtime of a ref file in repo/refs/heads.""" + os.utime(os.path.join(self.refsdir(), ref), None) + + def resolve_rev(self, branch, allow_noent=True): + """Return the SHA256 corresponding to 'branch'.""" + return self.repo.resolve_rev(branch, allow_noent)[1] + + def read_commit(self, branch): + """Return an OSTree.RepoFile representing a committed tree.""" + return self.repo.read_commit(branch, self.cancellable)[1] + + def query_info(self, file_object): + """Quickly return a Gio.FileInfo for file_object.""" + return file_object.query_info(self.OSTREE_GIO_FAST_QUERYINFO, + self.G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, + self.cancellable) + + def checkout(self, branch, destdir): + """Checkout branch into destdir.""" + checkout_path = destdir + if not os.path.exists(checkout_path): + os.makedirs(checkout_path) + checkout = Gio.file_new_for_path(checkout_path) + + commit = self.read_commit(branch) + commit_info = self.query_info(commit) + self.repo.checkout_tree(0, 1, checkout, commit, + commit_info, self.cancellable) + + def commit(self, subject, srcdir, branch): + """Commit the contents of 'srcdir' to 'branch'.""" + self.repo.prepare_transaction(self.cancellable) + parent = self.resolve_rev(branch) + if parent: + parent_root = self.read_commit(parent) + + mtree = OSTree.MutableTree() + src = Gio.file_new_for_path(srcdir) + self.repo.write_directory_to_mtree(src, mtree, None, self.cancellable) + root = self.repo.write_mtree(mtree, self.cancellable)[1] + if parent and root.equal(parent_root): + return + checksum = self.repo.write_commit(parent, subject, '', None, + root, self.cancellable)[1] + self.repo.transaction_set_ref(None, branch, checksum) + stats = self.repo.commit_transaction(self.cancellable) + + def cat_file(self, ref, path): + """Return the file descriptor of path at ref.""" + commit = self.read_commit(ref) + relative = commit.resolve_relative_path(path) + ret, content, etag = relative.load_contents() + return content + + def list_refs(self, resolved=False): + """Return a list of all refs in the repo.""" + refs = self.repo.list_refs()[1] + if not resolved: + return refs.keys() + return refs + + def delete_ref(self, ref): + """Remove refspec from the repo.""" + if not self.list_refs(ref): + raise Exception("Failed to delete ref, it doesn't exist") + self.repo.set_ref_immediate(None, ref, None, self.cancellable) + + def prune(self): + """Remove unreachable objects from the repo.""" + return self.repo.prune(OSTree.RepoPruneFlags.REFS_ONLY, + -1, self.cancellable) + + def add_remote(self, name, url): + """Add a remote with a given name and url.""" + options_type = GLib.VariantType.new('a{sv}') + options_builder = GLib.VariantBuilder.new(options_type) + options = options_builder.end() + self.repo.remote_add(name, url, options, self.cancellable) + + def remove_remote(self, name): + """Remove a remote with a given name.""" + self.repo.remote_delete(name, self.cancellable) + + def get_remote_url(self, name): + """Return the URL for a remote.""" + return self.repo.remote_get_url(name)[1] + + def list_remotes(self): + """Return a list of all remotes for this repo.""" + return self.repo.remote_list() + + def has_remote(self, name): + """Return True if name is a remote for the repo.""" + return name in self.list_remotes() + + def pull(self, refs, remote): + """Pull ref from remote into the local repo.""" + flags = OSTree.RepoPullFlags.NONE + self.repo.pull(remote, refs, flags, None, self.cancellable) diff --git a/morphlib/ostreeartifactcache.py b/morphlib/ostreeartifactcache.py new file mode 100644 index 00000000..fdb7cb5d --- /dev/null +++ b/morphlib/ostreeartifactcache.py @@ -0,0 +1,229 @@ +# Copyright (C) 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import collections +import logging +import os +import shutil +import tarfile +import tempfile + +import cliapp +from gi.repository import GLib + +import morphlib +from morphlib.artifactcachereference import ArtifactCacheReference + +class OSTreeArtifactCache(object): + """Class to provide the artifact cache API using an OSTree repo.""" + + def __init__(self, cachedir): + repo_dir = os.path.join(cachedir, 'repo') + self.repo = morphlib.ostree.OSTreeRepo(repo_dir) + self.cachedir = cachedir + + def _get_file_from_remote(self, artifact, remote, metadata_name=None): + if metadata_name: + handle = remote.get_artifact_metadata(artifact, metadata_name) + else: + handle = remote.get(artifact) + fd, path = tempfile.mkstemp() + with open(path, 'w+') as temp: + shutil.copyfileobj(handle, temp) + return path + + def _get_artifact_cache_name(self, artifact): + logging.debug('LAC: %s' % artifact.basename()) + cache_key, kind, name = artifact.basename().split('.', 2) + suffix = name.split('-')[-1] + return '%s-%s' % (cache_key, suffix) + + def put(self, directory, artifact): + """Commit the contents of 'directory' to the repo. + + This uses the artifact name and cache key to create the ref, so the + contents of directory should be the contents of the artifact. + + """ + ref = self._get_artifact_cache_name(artifact) + subject = artifact.name + try: + logging.debug('Committing %s to artifact cache at %s.' % + (subject, ref)) + self.repo.commit(subject, directory, ref) + except GLib.GError as e: + logging.debug('OSTree raised an exception: %s' % e) + raise cliapp.AppException('Failed to commit %s to artifact ' + 'cache.' % ref) + + def put_non_ostree_artifact(self, artifact, location, metadata_name=None): + """Store a single file in the artifact cachedir.""" + if metadata_name: + filename = self._artifact_metadata_filename(artifact, + metadata_name) + else: + filename = self.artifact_filename(artifact) + shutil.copy(location, filename) + os.remove(location) + + def copy_from_remote(self, artifact, remote): + """Get 'artifact' from remote artifact cache and store it locally.""" + if remote.method == 'tarball': + logging.debug('Downloading artifact tarball for %s.' % + artifact.name) + location = self._get_file_from_remote(artifact, remote) + try: + tempdir = tempfile.mkdtemp() + with tarfile.open(name=location) as tf: + tf.extractall(path=tempdir) + try: + self.put(tempdir, artifact) + finally: + os.remove(location) + shutil.rmtree(tempdir) + except tarfile.ReadError: + # Reading the artifact as a tarball failed, so it must be a + # single file (for example a stratum artifact). + self.put_non_ostree_artifact(artifact, location) + + elif remote.method == 'ostree': + logging.debug('Pulling artifact for %s from remote.' % + artifact.basename()) + try: + ref = self._get_artifact_cache_name(artifact) + except Exception: + # if we can't split the name properly, we must want metadata + a, name = artifact.basename().split('.', 1) + location = self._get_file_from_remote( + ArtifactCacheReference(a), remote, name) + self.put_non_ostree_artifact(artifact, location, name) + return + + if artifact.basename().split('.', 2)[1] == 'stratum': + location = self._get_file_from_remote(artifact, remote) + self.put_non_ostree_artifact(artifact, location) + return + + try: + if not self.repo.has_remote(remote.name): + self.repo.add_remote(remote.name, remote.ostree_url) + self.repo.pull([ref], remote.name) + except GLib.GError as e: + logging.debug('OSTree raised an exception: %s' % e) + raise cliapp.AppException('Failed to pull %s from remote ' + 'cache.' % ref) + + def get(self, artifact, directory=None): + """Checkout an artifact from the repo and return its location.""" + cache_key, kind, name = artifact.basename().split('.', 2) + if kind == 'stratum': + return self.artifact_filename(artifact) + if directory is None: + directory = tempfile.mkdtemp() + ref = self._get_artifact_cache_name(artifact) + try: + self.repo.checkout(ref, directory) + self.repo.touch_ref(ref) + except GLib.GError as e: + logging.debug('OSTree raised an exception: %s' % e) + raise cliapp.AppException('Failed to checkout %s from artifact ' + 'cache.' % ref) + return directory + + def list_contents(self): + """Return the set of sources cached and related information. + + returns a [(cache_key, set(artifacts), last_used)] + + """ + CacheInfo = collections.namedtuple('CacheInfo', ('artifacts', 'mtime')) + contents = collections.defaultdict(lambda: CacheInfo(set(), 0)) + for ref in self.repo.list_refs(): + cachekey = ref[:63] + artifact = ref[65:] + artifacts, max_mtime = contents[cachekey] + artifacts.add(artifact) + ref_filename = os.path.join(self.repo.refsdir(), ref) + mtime = os.path.getmtime(ref_filename) + contents[cachekey] = CacheInfo(artifacts, max(max_mtime, mtime)) + return ((cache_key, info.artifacts, info.mtime) + for cache_key, info in contents.iteritems()) + + def remove(self, cachekey): + """Remove all artifacts associated with the given cachekey.""" + for ref in (r for r in self.repo.list_refs() + if r.startswith(cachekey)): + self.repo.delete_ref(ref) + + def prune(self): + """Delete orphaned objects in the repo.""" + self.repo.prune() + + def has(self, artifact): + cachekey, kind, name = artifact.basename().split('.', 2) + logging.debug('OSTreeArtifactCache: got %s, %s, %s' % + (cachekey, kind, name)) + if self._get_artifact_cache_name(artifact) in self.repo.list_refs(): + self.repo.touch_ref(self._get_artifact_cache_name(artifact)) + return True + if kind == 'stratum' and \ + self._has_file(self.artifact_filename(artifact)): + return True + return False + + def get_artifact_metadata(self, artifact, name): + filename = self._artifact_metadata_filename(artifact, name) + os.utime(filename, None) + return open(filename) + + def get_source_metadata_filename(self, source, cachekey, name): + return self._source_metadata_filename(source, cachekey, name) + + def get_source_metadata(self, source, cachekey, name): + filename = self._source_metadata_filename(source, cachekey, name) + os.utime(filename, None) + return open(filename) + + def artifact_filename(self, artifact): + return os.path.join(self.cachedir, artifact.basename()) + + def _artifact_metadata_filename(self, artifact, name): + return os.path.join(self.cachedir, artifact.metadata_basename(name)) + + def _source_metadata_filename(self, source, cachekey, name): + return os.path.join(self.cachedir, '%s.%s' % (cachekey, name)) + + def put_artifact_metadata(self, artifact, name): + filename = self._artifact_metadata_filename(artifact, name) + return morphlib.savefile.SaveFile(filename, mode='w') + + def put_source_metadata(self, source, cachekey, name): + filename = self._source_metadata_filename(source, cachekey, name) + return morphlib.savefile.SaveFile(filename, mode='w') + + def _has_file(self, filename): + if os.path.exists(filename): + os.utime(filename, None) + return True + return False + + def has_artifact_metadata(self, artifact, name): + filename = self._artifact_metadata_filename(artifact, name) + return self._has_file(filename) + + def has_source_metadata(self, source, cachekey, name): + filename = self._source_metadata_filename(source, cachekey, name) + return self._has_file(filename) diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index 7635a7b4..c9890b13 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -24,6 +24,7 @@ import uuid import cliapp import morphlib +from morphlib.artifactcachereference import ArtifactCacheReference class DeployPlugin(cliapp.Plugin): @@ -439,6 +440,8 @@ class DeployPlugin(cliapp.Plugin): system_status_prefix = '%s[%s]' % (old_status_prefix, system['morph']) self.app.status_prefix = system_status_prefix try: + system_tree = None + # Find the artifact to build morph = morphlib.util.sanitise_morphology_path(system['morph']) srcpool = build_command.create_source_pool(build_repo, ref, morph) @@ -502,6 +505,9 @@ class DeployPlugin(cliapp.Plugin): system_tree, deploy_location) finally: self.app.status_prefix = system_status_prefix + if system_tree and os.path.exists(system_tree): + morphlib.fsutils.unmount(self.app.runcmd, system_tree) + shutil.rmtree(system_tree) finally: self.app.status_prefix = old_status_prefix @@ -535,46 +541,94 @@ class DeployPlugin(cliapp.Plugin): except morphlib.extensions.ExtensionNotFoundError: pass + def checkout_stratum(self, path, artifact, lac, rac): + with open(lac.get(artifact), 'r') as stratum: + chunks = [ArtifactCacheReference(c) for c in json.load(stratum)] + morphlib.builder.download_depends(chunks, lac, rac) + for chunk in chunks: + self.app.status(msg='Checkout chunk %(name)s.', + name=chunk.basename(), chatty=True) + lac.get(chunk, path) + + metadata = os.path.join(path, 'baserock', '%s.meta' % artifact.name) + with lac.get_artifact_metadata(artifact, 'meta') as meta_src: + with morphlib.savefile.SaveFile(metadata, 'w') as meta_dst: + shutil.copyfileobj(meta_src, meta_dst) + + def checkout_strata(self, path, artifact, lac, rac): + deps = artifact.source.dependencies + morphlib.builder.download_depends(deps, lac, rac) + for stratum in deps: + self.checkout_stratum(path, stratum, lac, rac) + morphlib.builder.ldconfig(self.app.runcmd, path) + def setup_deploy(self, build_command, deploy_tempdir, root_repo_dir, ref, artifact, deployment_type, location, env): # deployment_type, location and env are only used for saving metadata - # Create a tempdir to extract the rootfs in - system_tree = tempfile.mkdtemp(dir=deploy_tempdir) + deployment_dir = tempfile.mkdtemp(dir=deploy_tempdir) + # Create a tempdir to extract the rootfs in + system_tree = tempfile.mkdtemp(dir=deployment_dir) + + # Create temporary directory for overlayfs + overlay_dir = os.path.join(deployment_dir, + '%s-upperdir' % artifact.name) + if not os.path.exists(overlay_dir): + os.makedirs(overlay_dir) + work_dir = os.path.join(deployment_dir, '%s-workdir' % artifact.name) + if not os.path.exists(work_dir): + os.makedirs(work_dir) + + deploy_tree = os.path.join(deployment_dir, + 'overlay-deploy-%s' % artifact.name) try: - # Unpack the artifact (tarball) to a temporary directory. - self.app.status(msg='Unpacking system for configuration') + # Checkout the strata involved in the artifact into a tempdir + self.app.status(msg='Checking out strata in system') + self.checkout_strata(system_tree, artifact, + build_command.lac, build_command.rac) + self.app.status(msg='Checking out system for configuration') if build_command.lac.has(artifact): - f = build_command.lac.get(artifact) + build_command.lac.get(artifact, system_tree) elif build_command.rac.has(artifact): build_command.cache_artifacts_locally([artifact]) - f = build_command.lac.get(artifact) + build_command.lac.get(artifact, system_tree) else: raise cliapp.AppException('Deployment failed as system is' ' not yet built.\nPlease ensure' ' the system is built before' ' deployment.') - tf = tarfile.open(fileobj=f) - tf.extractall(path=system_tree) self.app.status( - msg='System unpacked at %(system_tree)s', + msg='System checked out at %(system_tree)s', system_tree=system_tree) + union_filesystem = self.app.settings['union-filesystem'] + morphlib.fsutils.overlay_mount(self.app.runcmd, + 'overlay-deploy-%s' % + artifact.name, + deploy_tree, system_tree, + overlay_dir, work_dir, + union_filesystem) + self.app.status( msg='Writing deployment metadata file') metadata = self.create_metadata( artifact, root_repo_dir, deployment_type, location, env) metadata_path = os.path.join( - system_tree, 'baserock', 'deployment.meta') + deploy_tree, 'baserock', 'deployment.meta') with morphlib.savefile.SaveFile(metadata_path, 'w') as f: json.dump(metadata, f, indent=4, sort_keys=True, encoding='unicode-escape') - return system_tree + return deploy_tree except Exception: + if deploy_tree and os.path.exists(deploy_tree): + morphlib.fsutils.unmount(self.app.runcmd, deploy_tree) + shutil.rmtree(deploy_tree) shutil.rmtree(system_tree) + shutil.rmtree(overlay_dir) + shutil.rmtree(work_dir) raise def run_deploy_commands(self, deploy_tempdir, env, artifact, root_repo_dir, diff --git a/morphlib/plugins/distbuild_plugin.py b/morphlib/plugins/distbuild_plugin.py index 1900b1bd..ac3957f3 100644 --- a/morphlib/plugins/distbuild_plugin.py +++ b/morphlib/plugins/distbuild_plugin.py @@ -71,9 +71,7 @@ class SerialiseArtifactPlugin(cliapp.Plugin): srcpool = build_command.create_source_pool( repo_name, ref, filename, original_ref=original_ref) artifact = build_command.resolve_artifacts(srcpool) - self.app.output.write(distbuild.serialise_artifact(artifact, - repo_name, - ref)) + self.app.output.write(distbuild.serialise_artifact(artifact)) self.app.output.write('\n') @@ -97,14 +95,9 @@ class WorkerBuild(cliapp.Plugin): distbuild.add_crash_conditions(self.app.settings['crash-condition']) serialized = sys.stdin.readline() - artifact_reference = distbuild.deserialise_artifact(serialized) - + artifact = distbuild.deserialise_artifact(serialized) + bc = morphlib.buildcommand.BuildCommand(self.app) - source_pool = bc.create_source_pool(artifact_reference.repo, - artifact_reference.ref, - artifact_reference.root_filename) - - root = bc.resolve_artifacts(source_pool) # Now, before we start the build, we garbage collect the caches # to ensure we have room. First we remove all system artifacts @@ -117,21 +110,8 @@ class WorkerBuild(cliapp.Plugin): self.app.subcommands['gc']([]) - source = self.find_source(source_pool, artifact_reference) - build_env = bc.new_build_env(artifact_reference.arch) - bc.build_source(source, build_env) - - def find_source(self, source_pool, artifact_reference): - for s in source_pool.lookup(artifact_reference.source_repo, - artifact_reference.source_ref, - artifact_reference.filename): - if s.cache_key == artifact_reference.cache_key: - return s - for s in source_pool.lookup(artifact_reference.source_repo, - artifact_reference.source_sha1, - artifact_reference.filename): - if s.cache_key == artifact_reference.cache_key: - return s + arch = artifact.arch + bc.build_source(artifact.source, bc.new_build_env(arch)) def is_system_artifact(self, filename): return re.match(r'^[0-9a-fA-F]{64}\.system\.', filename) diff --git a/morphlib/plugins/gc_plugin.py b/morphlib/plugins/gc_plugin.py index 71522b04..8b5dc4c2 100644 --- a/morphlib/plugins/gc_plugin.py +++ b/morphlib/plugins/gc_plugin.py @@ -125,8 +125,8 @@ class GCPlugin(cliapp.Plugin): 'sufficient space already cleared', chatty=True) return - lac = morphlib.localartifactcache.LocalArtifactCache( - fs.osfs.OSFS(os.path.join(cache_path, 'artifacts'))) + lac = morphlib.ostreeartifactcache.OSTreeArtifactCache( + os.path.join(cache_path, 'artifacts')) max_age, min_age = self.calculate_delete_range() logging.debug('Must remove artifacts older than timestamp %d' % max_age) @@ -144,6 +144,8 @@ class GCPlugin(cliapp.Plugin): lac.remove(cachekey) removed += 1 + lac.prune() + # Maybe remove remaining middle-aged artifacts for cachekey in may_delete: if sufficient_free(): @@ -157,6 +159,8 @@ class GCPlugin(cliapp.Plugin): lac.remove(cachekey) removed += 1 + lac.prune() + if sufficient_free(): self.app.status(msg='Made sufficient space in %(cache_path)s ' 'after removing %(removed)d sources', diff --git a/morphlib/pylru.py b/morphlib/pylru.py deleted file mode 100644 index 28d55d50..00000000 --- a/morphlib/pylru.py +++ /dev/null @@ -1,532 +0,0 @@ - -# Cache implementaion with a Least Recently Used (LRU) replacement policy and -# a basic dictionary interface. - -# Copyright (C) 2006, 2009, 2010, 2011 Jay Hutchinson - -# This program is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the Free -# Software Foundation; either version 2 of the License, or (at your option) -# any later version. - -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. - -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., 51 -# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - - -# The cache is implemented using a combination of a python dictionary (hash -# table) and a circular doubly linked list. Items in the cache are stored in -# nodes. These nodes make up the linked list. The list is used to efficiently -# maintain the order that the items have been used in. The front or head of -# the list contains the most recently used item, the tail of the list -# contains the least recently used item. When an item is used it can easily -# (in a constant amount of time) be moved to the front of the list, thus -# updating its position in the ordering. These nodes are also placed in the -# hash table under their associated key. The hash table allows efficient -# lookup of values by key. - -# Class for the node objects. -class _dlnode(object): - def __init__(self): - self.empty = True - - -class lrucache(object): - - def __init__(self, size, callback=None): - - self.callback = callback - - # Create an empty hash table. - self.table = {} - - # Initialize the doubly linked list with one empty node. This is an - # invariant. The cache size must always be greater than zero. Each - # node has a 'prev' and 'next' variable to hold the node that comes - # before it and after it respectively. Initially the two variables - # each point to the head node itself, creating a circular doubly - # linked list of size one. Then the size() method is used to adjust - # the list to the desired size. - - self.head = _dlnode() - self.head.next = self.head - self.head.prev = self.head - - self.listSize = 1 - - # Adjust the size - self.size(size) - - - def __len__(self): - return len(self.table) - - def clear(self): - for node in self.dli(): - node.empty = True - node.key = None - node.value = None - - self.table.clear() - - - def __contains__(self, key): - return key in self.table - - # Looks up a value in the cache without affecting cache order. - def peek(self, key): - # Look up the node - node = self.table[key] - return node.value - - - def __getitem__(self, key): - # Look up the node - node = self.table[key] - - # Update the list ordering. Move this node so that is directly - # proceeds the head node. Then set the 'head' variable to it. This - # makes it the new head of the list. - self.mtf(node) - self.head = node - - # Return the value. - return node.value - - def get(self, key, default=None): - """Get an item - return default (None) if not present""" - try: - return self[key] - except KeyError: - return default - - def __setitem__(self, key, value): - # First, see if any value is stored under 'key' in the cache already. - # If so we are going to replace that value with the new one. - if key in self.table: - - # Lookup the node - node = self.table[key] - - # Replace the value. - node.value = value - - # Update the list ordering. - self.mtf(node) - self.head = node - - return - - # Ok, no value is currently stored under 'key' in the cache. We need - # to choose a node to place the new item in. There are two cases. If - # the cache is full some item will have to be pushed out of the - # cache. We want to choose the node with the least recently used - # item. This is the node at the tail of the list. If the cache is not - # full we want to choose a node that is empty. Because of the way the - # list is managed, the empty nodes are always together at the tail - # end of the list. Thus, in either case, by chooseing the node at the - # tail of the list our conditions are satisfied. - - # Since the list is circular, the tail node directly preceeds the - # 'head' node. - node = self.head.prev - - # If the node already contains something we need to remove the old - # key from the dictionary. - if not node.empty: - if self.callback is not None: - self.callback(node.key, node.value) - del self.table[node.key] - - # Place the new key and value in the node - node.empty = False - node.key = key - node.value = value - - # Add the node to the dictionary under the new key. - self.table[key] = node - - # We need to move the node to the head of the list. The node is the - # tail node, so it directly preceeds the head node due to the list - # being circular. Therefore, the ordering is already correct, we just - # need to adjust the 'head' variable. - self.head = node - - - def __delitem__(self, key): - - # Lookup the node, then remove it from the hash table. - node = self.table[key] - del self.table[key] - - node.empty = True - - # Not strictly necessary. - node.key = None - node.value = None - - # Because this node is now empty we want to reuse it before any - # non-empty node. To do that we want to move it to the tail of the - # list. We move it so that it directly preceeds the 'head' node. This - # makes it the tail node. The 'head' is then adjusted. This - # adjustment ensures correctness even for the case where the 'node' - # is the 'head' node. - self.mtf(node) - self.head = node.next - - def __iter__(self): - - # Return an iterator that returns the keys in the cache in order from - # the most recently to least recently used. Does not modify the cache - # order. - for node in self.dli(): - yield node.key - - def items(self): - - # Return an iterator that returns the (key, value) pairs in the cache - # in order from the most recently to least recently used. Does not - # modify the cache order. - for node in self.dli(): - yield (node.key, node.value) - - def keys(self): - - # Return an iterator that returns the keys in the cache in order from - # the most recently to least recently used. Does not modify the cache - # order. - for node in self.dli(): - yield node.key - - def values(self): - - # Return an iterator that returns the values in the cache in order - # from the most recently to least recently used. Does not modify the - # cache order. - for node in self.dli(): - yield node.value - - def size(self, size=None): - - if size is not None: - assert size > 0 - if size > self.listSize: - self.addTailNode(size - self.listSize) - elif size < self.listSize: - self.removeTailNode(self.listSize - size) - - return self.listSize - - # Increases the size of the cache by inserting n empty nodes at the tail - # of the list. - def addTailNode(self, n): - for i in range(n): - node = _dlnode() - node.next = self.head - node.prev = self.head.prev - - self.head.prev.next = node - self.head.prev = node - - self.listSize += n - - # Decreases the size of the list by removing n nodes from the tail of the - # list. - def removeTailNode(self, n): - assert self.listSize > n - for i in range(n): - node = self.head.prev - if not node.empty: - if self.callback is not None: - self.callback(node.key, node.value) - del self.table[node.key] - - # Splice the tail node out of the list - self.head.prev = node.prev - node.prev.next = self.head - - # The next four lines are not strictly necessary. - node.prev = None - node.next = None - - node.key = None - node.value = None - - self.listSize -= n - - - # This method adjusts the ordering of the doubly linked list so that - # 'node' directly precedes the 'head' node. Because of the order of - # operations, if 'node' already directly precedes the 'head' node or if - # 'node' is the 'head' node the order of the list will be unchanged. - def mtf(self, node): - node.prev.next = node.next - node.next.prev = node.prev - - node.prev = self.head.prev - node.next = self.head.prev.next - - node.next.prev = node - node.prev.next = node - - # This method returns an iterator that iterates over the non-empty nodes - # in the doubly linked list in order from the most recently to the least - # recently used. - def dli(self): - node = self.head - for i in range(len(self.table)): - yield node - node = node.next - - - - -class WriteThroughCacheManager(object): - def __init__(self, store, size): - self.store = store - self.cache = lrucache(size) - - def __len__(self): - return len(self.store) - - # Returns/sets the size of the managed cache. - def size(self, size=None): - return self.cache.size(size) - - def clear(self): - self.cache.clear() - self.store.clear() - - def __contains__(self, key): - # Check the cache first. If it is there we can return quickly. - if key in self.cache: - return True - - # Not in the cache. Might be in the underlying store. - if key in self.store: - return True - - return False - - def __getitem__(self, key): - # First we try the cache. If successful we just return the value. If - # not we catch KeyError and ignore it since that just means the key - # was not in the cache. - try: - return self.cache[key] - except KeyError: - pass - - # It wasn't in the cache. Look it up in the store, add the entry to - # the cache, and return the value. - value = self.store[key] - self.cache[key] = value - return value - - def get(self, key, default=None): - """Get an item - return default (None) if not present""" - try: - return self[key] - except KeyError: - return default - - def __setitem__(self, key, value): - # Add the key/value pair to the cache and store. - self.cache[key] = value - self.store[key] = value - - def __delitem__(self, key): - # Write-through behavior cache and store should be consistent. Delete - # it from the store. - del self.store[key] - try: - # Ok, delete from the store was successful. It might also be in - # the cache, try and delete it. If not we catch the KeyError and - # ignore it. - del self.cache[key] - except KeyError: - pass - - def __iter__(self): - return self.keys() - - def keys(self): - return self.store.keys() - - def values(self): - return self.store.values() - - def items(self): - return self.store.items() - - - -class WriteBackCacheManager(object): - def __init__(self, store, size): - self.store = store - - # Create a set to hold the dirty keys. - self.dirty = set() - - # Define a callback function to be called by the cache when a - # key/value pair is about to be ejected. This callback will check to - # see if the key is in the dirty set. If so, then it will update the - # store object and remove the key from the dirty set. - def callback(key, value): - if key in self.dirty: - self.store[key] = value - self.dirty.remove(key) - - # Create a cache and give it the callback function. - self.cache = lrucache(size, callback) - - # Returns/sets the size of the managed cache. - def size(self, size=None): - return self.cache.size(size) - - def clear(self): - self.cache.clear() - self.dirty.clear() - self.store.clear() - - def __contains__(self, key): - # Check the cache first, since if it is there we can return quickly. - if key in self.cache: - return True - - # Not in the cache. Might be in the underlying store. - if key in self.store: - return True - - return False - - def __getitem__(self, key): - # First we try the cache. If successful we just return the value. If - # not we catch KeyError and ignore it since that just means the key - # was not in the cache. - try: - return self.cache[key] - except KeyError: - pass - - # It wasn't in the cache. Look it up in the store, add the entry to - # the cache, and return the value. - value = self.store[key] - self.cache[key] = value - return value - - def get(self, key, default=None): - """Get an item - return default (None) if not present""" - try: - return self[key] - except KeyError: - return default - - def __setitem__(self, key, value): - # Add the key/value pair to the cache. - self.cache[key] = value - self.dirty.add(key) - - def __delitem__(self, key): - - found = False - try: - del self.cache[key] - found = True - self.dirty.remove(key) - except KeyError: - pass - - try: - del self.store[key] - found = True - except KeyError: - pass - - if not found: # If not found in cache or store, raise error. - raise KeyError - - - def __iter__(self): - return self.keys() - - def keys(self): - for key in self.store.keys(): - if key not in self.dirty: - yield key - - for key in self.dirty: - yield key - - - def values(self): - for key, value in self.items(): - yield value - - - def items(self): - for key, value in self.store.items(): - if key not in self.dirty: - yield (key, value) - - for key in self.dirty: - value = self.cache.peek(key) - yield (key, value) - - - - def sync(self): - # For each dirty key, peek at its value in the cache and update the - # store. Doesn't change the cache's order. - for key in self.dirty: - self.store[key] = self.cache.peek(key) - # There are no dirty keys now. - self.dirty.clear() - - def flush(self): - self.sync() - self.cache.clear() - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.sync() - return False - - - - - -def lruwrap(store, size, writeback=False): - if writeback: - return WriteBackCacheManager(store, size) - else: - return WriteThroughCacheManager(store, size) - - - - -class lrudecorator(object): - def __init__(self, size): - self.cache = lrucache(size) - - def __call__(self, func): - def wrapped(*args, **kwargs): - kwtuple = tuple((key, kwargs[key]) for key in sorted(kwargs.keys())) - key = (args, kwtuple) - try: - return self.cache[key] - except KeyError: - pass - - value = func(*args, **kwargs) - self.cache[key] = value - return value - return wrapped diff --git a/morphlib/remoteartifactcache.py b/morphlib/remoteartifactcache.py index 427e4cbb..f5115cd6 100644 --- a/morphlib/remoteartifactcache.py +++ b/morphlib/remoteartifactcache.py @@ -57,6 +57,18 @@ class RemoteArtifactCache(object): def __init__(self, server_url): self.server_url = server_url + self.name = urlparse.urlparse(server_url).hostname + try: + self.method = self._get_method() + except urllib2.URLError: + self.method = 'tarball' + except Exception as e: # pragma: no cover + logging.debug('Failed to determine cache method: %s' % e) + raise cliapp.AppException('Failed to determine method used by ' + 'remote cache.') + if self.method == 'ostree': # pragma: no cover + self.ostree_url = 'http://%s:%s/' % (self.name, + self._get_ostree_info()) def has(self, artifact): return self._has_file(artifact.basename()) @@ -112,5 +124,18 @@ class RemoteArtifactCache(object): server_url, '/1.0/artifacts?filename=%s' % urllib.quote(filename)) + def _get_method(self): # pragma: no cover + logging.debug('Getting cache method of %s' % self.server_url) + request_url = urlparse.urljoin(self.server_url, '/1.0/method') + req = urllib2.urlopen(request_url) + return req.read() + + def _get_ostree_info(self): # pragma: no cover + logging.debug('Getting OSTree repo info.') + request_url = urlparse.urljoin(self.server_url, '/1.0/ostreeinfo') + logging.debug('sending %s' % request_url) + req = urllib2.urlopen(request_url) + return req.read() + def __str__(self): # pragma: no cover return self.server_url diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py index 8c2781aa..768ec643 100644 --- a/morphlib/stagingarea.py +++ b/morphlib/stagingarea.py @@ -87,6 +87,14 @@ class StagingArea(object): return self._dir_for_source(source, 'inst') + def overlay_upperdir(self, source): + '''Create a directory to be upperdir for overlayfs, and return it.''' + return self._dir_for_source(source, 'overlay_upper') + + def overlaydir(self, source): + '''Create a directory to be a mount point for overlayfs, return it''' + return self._dir_for_source(source, 'overlay') + def relative(self, filename): '''Return a filename relative to the staging area.''' @@ -146,37 +154,42 @@ class StagingArea(object): raise IOError('Cannot extract %s into staging-area. Unsupported' ' type.' % srcpath) - def install_artifact(self, handle): + def create_devices(self, morphology): # pragma: no cover + '''Creates device nodes if the morphology specifies them''' + perms_mask = stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO + if 'devices' in morphology and morphology['devices'] is not None: + for dev in morphology['devices']: + destfile = os.path.join(self.dirname, './' + dev['filename']) + mode = int(dev['permissions'], 8) & perms_mask + if dev['type'] == 'c': + mode = mode | stat.S_IFCHR + elif dev['type'] == 'b': + mode = mode | stat.S_IFBLK + else: + raise IOError('Cannot create device node %s,' + 'unrecognized device type "%s"' + % (destfile, dev['type'])) + parent = os.path.dirname(destfile) + if not os.path.exists(parent): + os.makedirs(parent) + if not os.path.exists(destfile): + logging.debug("Creating device node %s" % destfile) + os.mknod(destfile, mode, + os.makedev(dev['major'], dev['minor'])) + os.chown(destfile, dev['uid'], dev['gid']) + + def install_artifact(self, artifact, artifact_checkout): '''Install a build artifact into the staging area. We access the artifact via an open file handle. For now, we assume the artifact is a tarball. ''' - - chunk_cache_dir = os.path.join(self._app.settings['tempdir'], 'chunks') - unpacked_artifact = os.path.join( - chunk_cache_dir, os.path.basename(handle.name) + '.d') - if not os.path.exists(unpacked_artifact): - self._app.status( - msg='Unpacking chunk from cache %(filename)s', - filename=os.path.basename(handle.name)) - savedir = tempfile.mkdtemp(dir=chunk_cache_dir) - try: - morphlib.bins.unpack_binary_from_file( - handle, savedir + '/') - except BaseException as e: # pragma: no cover - shutil.rmtree(savedir) - raise - # TODO: This rename is not concurrency safe if two builds are - # extracting the same chunk, one build will fail because - # the other renamed its tempdir here first. - os.rename(savedir, unpacked_artifact) - if not os.path.exists(self.dirname): self._mkdir(self.dirname) - self.hardlink_all_files(unpacked_artifact, self.dirname) + self.hardlink_all_files(artifact_checkout, self.dirname) + self.create_devices(artifact.source.morphology) def remove(self): '''Remove the entire staging area. diff --git a/morphlib/stagingarea_tests.py b/morphlib/stagingarea_tests.py index 97d78236..ffdf5eaa 100644 --- a/morphlib/stagingarea_tests.py +++ b/morphlib/stagingarea_tests.py @@ -30,6 +30,7 @@ class FakeBuildEnvironment(object): } self.extra_path = ['/extra-path'] + class FakeSource(object): def __init__(self): @@ -39,6 +40,12 @@ class FakeSource(object): self.name = 'le-name' +class FakeArtifact(object): + + def __init__(self): + self.source = FakeSource() + + class FakeApplication(object): def __init__(self, cachedir, tempdir): @@ -83,12 +90,8 @@ class StagingAreaTests(unittest.TestCase): os.mkdir(chunkdir) with open(os.path.join(chunkdir, 'file.txt'), 'w'): pass - chunk_tar = os.path.join(self.tempdir, 'chunk.tar') - tf = tarfile.TarFile(name=chunk_tar, mode='w') - tf.add(chunkdir, arcname='.') - tf.close() - return chunk_tar + return chunkdir def list_tree(self, root): files = [] @@ -118,20 +121,34 @@ class StagingAreaTests(unittest.TestCase): self.assertEqual(self.created_dirs, [dirname]) self.assertTrue(dirname.startswith(self.staging)) + def test_creates_overlay_upper_directory(self): + source = FakeSource() + self.sa._mkdir = self.fake_mkdir + dirname = self.sa.overlay_upperdir(source) + self.assertEqual(self.created_dirs, [dirname]) + self.assertTrue(dirname.startswith(self.staging)) + + def test_creates_overlay_directory(self): + source = FakeSource() + self.sa._mkdir = self.fake_mkdir + dirname = self.sa.overlaydir(source) + self.assertEqual(self.created_dirs, [dirname]) + self.assertTrue(dirname.startswith(self.staging)) + def test_makes_relative_name(self): filename = os.path.join(self.staging, 'foobar') self.assertEqual(self.sa.relative(filename), '/foobar') def test_installs_artifact(self): - chunk_tar = self.create_chunk() - with open(chunk_tar, 'rb') as f: - self.sa.install_artifact(f) + artifact = FakeArtifact() + chunkdir = self.create_chunk() + self.sa.install_artifact(artifact, chunkdir) self.assertEqual(self.list_tree(self.staging), ['/', '/file.txt']) def test_removes_everything(self): - chunk_tar = self.create_chunk() - with open(chunk_tar, 'rb') as f: - self.sa.install_artifact(f) + artifact = FakeArtifact() + chunkdir = self.create_chunk() + self.sa.install_artifact(artifact, chunkdir) self.sa.remove() self.assertFalse(os.path.exists(self.staging)) diff --git a/morphlib/util.py b/morphlib/util.py index a3a07cce..00111ff7 100644 --- a/morphlib/util.py +++ b/morphlib/util.py @@ -131,8 +131,10 @@ def new_artifact_caches(settings): # pragma: no cover if not os.path.exists(artifact_cachedir): os.mkdir(artifact_cachedir) - lac = morphlib.localartifactcache.LocalArtifactCache( - fs.osfs.OSFS(artifact_cachedir)) + #lac = morphlib.localartifactcache.LocalArtifactCache( + # fs.osfs.OSFS(artifact_cachedir)) + + lac = morphlib.ostreeartifactcache.OSTreeArtifactCache(artifact_cachedir) rac_url = get_artifact_cache_server(settings) rac = None -- cgit v1.2.1