diff options
Diffstat (limited to 'morphlib')
-rw-r--r-- | morphlib/__init__.py | 2 | ||||
-rw-r--r-- | morphlib/app.py | 7 | ||||
-rw-r--r-- | morphlib/bins.py | 59 | ||||
-rw-r--r-- | morphlib/bins_tests.py | 98 | ||||
-rw-r--r-- | morphlib/buildcommand.py | 32 | ||||
-rw-r--r-- | morphlib/builder.py | 118 | ||||
-rw-r--r-- | morphlib/builder_tests.py | 18 | ||||
-rwxr-xr-x | morphlib/exts/fstab.configure | 25 | ||||
-rwxr-xr-x | morphlib/exts/hosts.configure | 48 | ||||
-rw-r--r-- | morphlib/fsutils.py | 23 | ||||
-rw-r--r-- | morphlib/ostree.py | 139 | ||||
-rw-r--r-- | morphlib/ostreeartifactcache.py | 229 | ||||
-rw-r--r-- | morphlib/plugins/deploy_plugin.py | 79 | ||||
-rw-r--r-- | morphlib/plugins/gc_plugin.py | 8 | ||||
-rw-r--r-- | morphlib/remoteartifactcache.py | 25 | ||||
-rw-r--r-- | morphlib/sourceresolver.py | 36 | ||||
-rw-r--r-- | morphlib/stagingarea.py | 57 | ||||
-rw-r--r-- | morphlib/stagingarea_tests.py | 39 | ||||
-rw-r--r-- | morphlib/util.py | 38 | ||||
-rw-r--r-- | morphlib/writeexts.py | 6 |
20 files changed, 719 insertions, 367 deletions
diff --git a/morphlib/__init__.py b/morphlib/__init__.py index 7c462aad..695241cc 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -71,6 +71,8 @@ import morphologyfinder import morphology import morphloader import morphset +import ostree +import ostreeartifactcache import remoteartifactcache import remoterepocache import repoaliasresolver diff --git a/morphlib/app.py b/morphlib/app.py index c8fe397d..f7c07726 100644 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -120,6 +120,13 @@ class Morph(cliapp.Application): metavar='URL', default=None, group=group_advanced) + self.settings.string(['union-filesystem'], + 'filesystem used to provide "union filesystem" ' + 'functionality when building and deploying. ' + 'Only "overlayfs" and "unionfs-fuse" are ' + 'supported at this time.', + default='overlayfs', + group=group_advanced) group_build = 'Build Options' self.settings.integer(['max-jobs'], diff --git a/morphlib/bins.py b/morphlib/bins.py index 2e8ba0b3..c5bacc26 100644 --- a/morphlib/bins.py +++ b/morphlib/bins.py @@ -78,12 +78,8 @@ if sys.version_info < (2, 7, 3): # pragma: no cover raise ExtractError("could not change owner") tarfile.TarFile.chown = fixed_chown -def create_chunk(rootdir, f, include, dump_memory_profile=None): - '''Create a chunk from the contents of a directory. - - ``f`` is an open file handle, to which the tar file is written. - - ''' +def create_chunk(rootdir, chunkdir, include, dump_memory_profile=None): + '''Create a chunk from the contents of a directory.''' dump_memory_profile = dump_memory_profile or (lambda msg: None) @@ -91,31 +87,42 @@ def create_chunk(rootdir, f, include, dump_memory_profile=None): # chunk artifact. This is useful to avoid problems from smallish # clock skew. It needs to be recent enough, however, that GNU tar # does not complain about an implausibly old timestamp. - normalized_timestamp = 683074800 + normalized_timestamp = (683074800, 683074800) dump_memory_profile('at beginning of create_chunk') - - path_pairs = [(relname, os.path.join(rootdir, relname)) - for relname in include] - tar = tarfile.open(fileobj=f, mode='w') - for relname, filename in path_pairs: + + def check_parent(name, paths): + parent = os.path.dirname(name) + if parent: + path = os.path.join(rootdir, parent) + if parent != rootdir and path not in paths: + paths.append(path) + check_parent(parent, paths) + + def filter_contents(dirname, filenames): + paths = [os.path.join(rootdir, relname) for relname in include] + for name in include: + check_parent(name, paths) + + return [f for f in filenames if os.path.join(dirname, f) not in paths] + + logging.debug('Copying artifact into %s.' % chunkdir) + shutil.copytree(rootdir, chunkdir, + symlinks=True, ignore=filter_contents) + + path_triplets = [(relname, os.path.join(chunkdir, relname), + os.path.join(rootdir, relname)) + for relname in include] + for relname, filename, orig in path_triplets: # Normalize mtime for everything. - tarinfo = tar.gettarinfo(filename, - arcname=relname) - tarinfo.ctime = normalized_timestamp - tarinfo.mtime = normalized_timestamp - if tarinfo.isreg(): - with open(filename, 'rb') as f: - tar.addfile(tarinfo, fileobj=f) - else: - tar.addfile(tarinfo) - tar.close() + if not os.path.islink(filename): + os.utime(filename, normalized_timestamp) - for relname, filename in reversed(path_pairs): - if os.path.isdir(filename) and not os.path.islink(filename): + for relname, filename, orig in reversed(path_triplets): + if os.path.isdir(orig) and not os.path.islink(orig): continue else: - os.remove(filename) + os.remove(orig) dump_memory_profile('after removing in create_chunks') @@ -209,7 +216,7 @@ def unpack_binary_from_file(f, dirname): # pragma: no cover tf.close() -def unpack_binary(filename, dirname): +def unpack_binary(filename, dirname): # pragma: no cover with open(filename, "rb") as f: unpack_binary_from_file(f, dirname) diff --git a/morphlib/bins_tests.py b/morphlib/bins_tests.py index 3895680f..879aada4 100644 --- a/morphlib/bins_tests.py +++ b/morphlib/bins_tests.py @@ -78,11 +78,9 @@ class ChunkTests(BinsTest): self.tempdir = tempfile.mkdtemp() self.instdir = os.path.join(self.tempdir, 'inst') self.chunk_file = os.path.join(self.tempdir, 'chunk') - self.chunk_f = open(self.chunk_file, 'wb') self.unpacked = os.path.join(self.tempdir, 'unpacked') def tearDown(self): - self.chunk_f.close() shutil.rmtree(self.tempdir) def populate_instdir(self): @@ -108,109 +106,21 @@ class ChunkTests(BinsTest): def create_chunk(self, includes): self.populate_instdir() - morphlib.bins.create_chunk(self.instdir, self.chunk_f, includes) - self.chunk_f.flush() - - def unpack_chunk(self): - os.mkdir(self.unpacked) - morphlib.bins.unpack_binary(self.chunk_file, self.unpacked) + morphlib.bins.create_chunk(self.instdir, self.chunk_file, includes) def test_empties_files(self): self.create_chunk(['bin/foo', 'lib/libfoo.so']) self.assertEqual([x for x, y in self.recursive_lstat(self.instdir)], ['.', 'bin', 'lib']) - def test_creates_and_unpacks_chunk_exactly(self): + def test_creates_chunk_exactly(self): self.create_chunk(['bin', 'bin/foo', 'lib', 'lib/libfoo.so']) - self.unpack_chunk() self.assertEqual(self.instdir_orig_files, - self.recursive_lstat(self.unpacked)) + self.recursive_lstat(self.chunk_file)) def test_uses_only_matching_names(self): self.create_chunk(['bin/foo']) - self.unpack_chunk() - self.assertEqual([x for x, y in self.recursive_lstat(self.unpacked)], + self.assertEqual([x for x, y in self.recursive_lstat(self.chunk_file)], ['.', 'bin', 'bin/foo']) self.assertEqual([x for x, y in self.recursive_lstat(self.instdir)], ['.', 'bin', 'lib', 'lib/libfoo.so']) - - def test_does_not_compress_artifact(self): - self.create_chunk(['bin']) - f = gzip.open(self.chunk_file) - self.assertRaises(IOError, f.read) - f.close() - - -class ExtractTests(unittest.TestCase): - - def setUp(self): - self.tempdir = tempfile.mkdtemp() - self.instdir = os.path.join(self.tempdir, 'inst') - self.unpacked = os.path.join(self.tempdir, 'unpacked') - - def tearDown(self): - shutil.rmtree(self.tempdir) - - def create_chunk(self, callback): - fh = StringIO.StringIO() - os.mkdir(self.instdir) - patterns = callback(self.instdir) - morphlib.bins.create_chunk(self.instdir, fh, patterns) - shutil.rmtree(self.instdir) - fh.flush() - fh.seek(0) - return fh - - def test_extracted_files_replace_links(self): - def make_linkfile(basedir): - with open(os.path.join(basedir, 'babar'), 'w') as f: - pass - os.symlink('babar', os.path.join(basedir, 'bar')) - return ['babar'] - linktar = self.create_chunk(make_linkfile) - - def make_file(basedir): - with open(os.path.join(basedir, 'bar'), 'w') as f: - pass - return ['bar'] - filetar = self.create_chunk(make_file) - - os.mkdir(self.unpacked) - morphlib.bins.unpack_binary_from_file(linktar, self.unpacked) - morphlib.bins.unpack_binary_from_file(filetar, self.unpacked) - mode = os.lstat(os.path.join(self.unpacked, 'bar')).st_mode - self.assertTrue(stat.S_ISREG(mode)) - - def test_extracted_dirs_keep_links(self): - def make_usrlink(basedir): - os.symlink('.', os.path.join(basedir, 'usr')) - return ['usr'] - linktar = self.create_chunk(make_usrlink) - - def make_usrdir(basedir): - os.mkdir(os.path.join(basedir, 'usr')) - return ['usr'] - dirtar = self.create_chunk(make_usrdir) - - morphlib.bins.unpack_binary_from_file(linktar, self.unpacked) - morphlib.bins.unpack_binary_from_file(dirtar, self.unpacked) - mode = os.lstat(os.path.join(self.unpacked, 'usr')).st_mode - self.assertTrue(stat.S_ISLNK(mode)) - - def test_extracted_files_follow_links(self): - def make_usrlink(basedir): - os.symlink('.', os.path.join(basedir, 'usr')) - return ['usr'] - linktar = self.create_chunk(make_usrlink) - - def make_usrdir(basedir): - os.mkdir(os.path.join(basedir, 'usr')) - with open(os.path.join(basedir, 'usr', 'foo'), 'w') as f: - pass - return ['usr', 'usr/foo'] - dirtar = self.create_chunk(make_usrdir) - - morphlib.bins.unpack_binary_from_file(linktar, self.unpacked) - morphlib.bins.unpack_binary_from_file(dirtar, self.unpacked) - mode = os.lstat(os.path.join(self.unpacked, 'foo')).st_mode - self.assertTrue(stat.S_ISREG(mode)) diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index be8a1507..c83abca6 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -418,8 +418,10 @@ class BuildCommand(object): # module into morphlib.remoteartififactcache first. to_fetch = [] if not self.lac.has(artifact): - to_fetch.append((self.rac.get(artifact), - self.lac.put(artifact))) + self.app.status( + msg='Fetching to local cache: artifact %(name)s', + name=artifact.name) + self.lac.copy_from_remote(artifact, self.rac) if artifact.source.morphology.needs_artifact_metadata_cached: if not self.lac.has_artifact_metadata(artifact, 'meta'): @@ -428,9 +430,6 @@ class BuildCommand(object): self.lac.put_artifact_metadata(artifact, 'meta'))) if len(to_fetch) > 0: - self.app.status( - msg='Fetching to local cache: artifact %(name)s', - name=artifact.name) fetch_files(to_fetch) def create_staging_area(self, build_env, use_chroot=True, extra_env={}, @@ -493,8 +492,27 @@ class BuildCommand(object): chunk_name=artifact.name, cache=artifact.source.cache_key[:7], chatty=True) - handle = self.lac.get(artifact) - staging_area.install_artifact(handle) + chunk_cache_dir = os.path.join(self.app.settings['tempdir'], + 'chunks') + artifact_checkout = os.path.join( + chunk_cache_dir, os.path.basename(artifact.basename()) + '.d') + if not os.path.exists(artifact_checkout): + self.app.status( + msg='Checking out %(chunk)s from cache.', + chunk=artifact.name + ) + temp_checkout = os.path.join(self.app.settings['tempdir'], + artifact.basename()) + try: + self.lac.get(artifact, temp_checkout) + except BaseException: + shutil.rmtree(temp_checkout) + raise + # TODO: This rename is not concurrency safe if two builds are + # extracting the same chunk, one build will fail because + # the other renamed its tempdir here first. + os.rename(temp_checkout, artifact_checkout) + staging_area.install_artifact(artifact, artifact_checkout) if target_source.build_mode == 'staging': morphlib.builder.ldconfig(self.app.runcmd, staging_area.dirname) diff --git a/morphlib/builder.py b/morphlib/builder.py index 04ebd149..9b01f983 100644 --- a/morphlib/builder.py +++ b/morphlib/builder.py @@ -125,11 +125,7 @@ def ldconfig(runcmd, rootdir): # pragma: no cover def download_depends(constituents, lac, rac, metadatas=None): for constituent in constituents: if not lac.has(constituent): - source = rac.get(constituent) - target = lac.put(constituent) - shutil.copyfileobj(source, target) - target.close() - source.close() + lac.copy_from_remote(constituent, rac) if metadatas is not None: for metadata in metadatas: if not lac.has_artifact_metadata(constituent, metadata): @@ -246,28 +242,6 @@ class ChunkBuilder(BuilderBase): '''Build chunk artifacts.''' - def create_devices(self, destdir): # pragma: no cover - '''Creates device nodes if the morphology specifies them''' - morphology = self.source.morphology - perms_mask = stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO - if 'devices' in morphology and morphology['devices'] is not None: - for dev in morphology['devices']: - destfile = os.path.join(destdir, './' + dev['filename']) - mode = int(dev['permissions'], 8) & perms_mask - if dev['type'] == 'c': - mode = mode | stat.S_IFCHR - elif dev['type'] == 'b': - mode = mode | stat.S_IFBLK - else: - raise IOError('Cannot create device node %s,' - 'unrecognized device type "%s"' - % (destfile, dev['type'])) - self.app.status(msg="Creating device node %s" - % destfile) - os.mknod(destfile, mode, - os.makedev(dev['major'], dev['minor'])) - os.chown(destfile, dev['uid'], dev['gid']) - def build_and_cache(self): # pragma: no cover with self.build_watch('overall-build'): @@ -286,7 +260,6 @@ class ChunkBuilder(BuilderBase): try: self.get_sources(builddir) self.run_commands(builddir, destdir, temppath, stdout) - self.create_devices(destdir) os.rename(temppath, logpath) except BaseException as e: @@ -459,13 +432,23 @@ class ChunkBuilder(BuilderBase): extra_files += ['baserock/%s.meta' % chunk_artifact_name] parented_paths = parentify(file_paths + extra_files) - with self.local_artifact_cache.put(chunk_artifact) as f: - self.write_metadata(destdir, chunk_artifact_name, - parented_paths) + self.write_metadata(destdir, chunk_artifact_name, + parented_paths) - self.app.status(msg='Creating chunk artifact %(name)s', - name=chunk_artifact_name) - morphlib.bins.create_chunk(destdir, f, parented_paths) + self.app.status(msg='Creating chunk artifact %(name)s', + name=chunk_artifact_name) + # TODO: This is not concurrency safe, bins.create_chunk will + # fail if tempdir already exists (eg if another build + # has created it). + tempdir = os.path.join(self.app.settings['tempdir'], + chunk_artifact.basename()) + try: + morphlib.bins.create_chunk(destdir, tempdir, + parented_paths) + self.local_artifact_cache.put(tempdir, chunk_artifact) + finally: + if os.path.isdir(tempdir): + shutil.rmtree(tempdir) built_artifacts.append(chunk_artifact) for dirname, subdirs, files in os.walk(destdir): @@ -509,8 +492,13 @@ class StratumBuilder(BuilderBase): [x.name for x in constituents]) with lac.put_artifact_metadata(a, 'meta') as f: json.dump(meta, f, indent=4, sort_keys=True) - with self.local_artifact_cache.put(a) as f: + # TODO: This is not concurrency safe, put_stratum_artifact + # deletes temp which could be in use by another + # build. + temp = os.path.join(self.app.settings['tempdir'], a.name) + with open(temp, 'w+') as f: json.dump([c.basename() for c in constituents], f) + self.local_artifact_cache.put_non_ostree_artifact(a, temp) self.save_build_times() return self.source.artifacts.values() @@ -532,33 +520,40 @@ class SystemBuilder(BuilderBase): # pragma: no cover arch = self.source.morphology['arch'] for a_name, artifact in self.source.artifacts.iteritems(): - handle = self.local_artifact_cache.put(artifact) - try: fs_root = self.staging_area.destdir(self.source) self.unpack_strata(fs_root) - self.write_metadata(fs_root, a_name) - self.run_system_integration_commands(fs_root) - unslashy_root = fs_root[1:] - def uproot_info(info): - info.name = relpath(info.name, unslashy_root) - if info.islnk(): - info.linkname = relpath(info.linkname, - unslashy_root) - return info - tar = tarfile.open(fileobj=handle, mode="w", name=a_name) - self.app.status(msg='Constructing tarball of rootfs', - chatty=True) - tar.add(fs_root, recursive=True, filter=uproot_info) - tar.close() + upperdir = self.staging_area.overlay_upperdir( + self.source) + editable_root = self.staging_area.overlaydir(self.source) + workdir = os.path.join(self.staging_area.dirname, + 'overlayfs-workdir') + if not os.path.exists(workdir): + os.makedirs(workdir) + union_filesystem = self.app.settings['union-filesystem'] + morphlib.fsutils.overlay_mount(self.app.runcmd, + 'overlay-%s' % a_name, + editable_root, fs_root, + upperdir, workdir, + union_filesystem) + self.write_metadata(editable_root, a_name) + self.run_system_integration_commands(editable_root) + # Put the contents of upperdir into the local artifact + # cache. Don't use editable root as we only want to + # store the modified files. + self.local_artifact_cache.put(upperdir, artifact) except BaseException as e: logging.error(traceback.format_exc()) self.app.status(msg='Error while building system', error=True) - handle.abort() + if editable_root and os.path.exists(editable_root): + morphlib.fsutils.unmount(self.app.runcmd, + editable_root) raise else: - handle.close() + if editable_root and os.path.exists(editable_root): + morphlib.fsutils.unmount(self.app.runcmd, + editable_root) self.save_build_times() return self.source.artifacts.itervalues() @@ -567,13 +562,12 @@ class SystemBuilder(BuilderBase): # pragma: no cover '''Unpack a single stratum into a target directory''' cache = self.local_artifact_cache - with cache.get(stratum_artifact) as stratum_file: + with open(cache.get(stratum_artifact), 'r') as stratum_file: artifact_list = json.load(stratum_file, encoding='unicode-escape') for chunk in (ArtifactCacheReference(a) for a in artifact_list): - self.app.status(msg='Unpacking chunk %(basename)s', + self.app.status(msg='Checkout chunk %(basename)s', basename=chunk.basename(), chatty=True) - with cache.get(chunk) as chunk_file: - morphlib.bins.unpack_binary_from_file(chunk_file, target) + cache.get(chunk, target) target_metadata = os.path.join( target, 'baserock', '%s.meta' % stratum_artifact.name) @@ -584,7 +578,7 @@ class SystemBuilder(BuilderBase): # pragma: no cover def unpack_strata(self, path): '''Unpack strata into a directory.''' - self.app.status(msg='Unpacking strata to %(path)s', + self.app.status(msg='Checking out strata to %(path)s', path=path, chatty=True) with self.build_watch('unpack-strata'): for a_name, a in self.source.artifacts.iteritems(): @@ -596,12 +590,14 @@ class SystemBuilder(BuilderBase): # pragma: no cover # download the chunk artifacts if necessary for stratum_artifact in self.source.dependencies: - f = self.local_artifact_cache.get(stratum_artifact) - chunks = [ArtifactCacheReference(c) for c in json.load(f)] + stratum_path = self.local_artifact_cache.get( + stratum_artifact) + with open(stratum_path, 'r') as stratum: + chunks = [ArtifactCacheReference(c) + for c in json.load(stratum)] download_depends(chunks, self.local_artifact_cache, self.remote_artifact_cache) - f.close() # unpack it from the local artifact cache for stratum_artifact in self.source.dependencies: diff --git a/morphlib/builder_tests.py b/morphlib/builder_tests.py index a571e3d0..b5e66521 100644 --- a/morphlib/builder_tests.py +++ b/morphlib/builder_tests.py @@ -105,8 +105,8 @@ class FakeArtifactCache(object): def __init__(self): self._cached = {} - def put(self, artifact): - return FakeFileHandle(self, (artifact.cache_key, artifact.name)) + def put(self, artifact, directory): + self._cached[(artifact.cache_key, artifact.name)] = artifact.name def put_artifact_metadata(self, artifact, name): return FakeFileHandle(self, (artifact.cache_key, artifact.name, name)) @@ -114,7 +114,7 @@ class FakeArtifactCache(object): def put_source_metadata(self, source, cachekey, name): return FakeFileHandle(self, (cachekey, name)) - def get(self, artifact): + def get(self, artifact, directory=None): return StringIO.StringIO( self._cached[(artifact.cache_key, artifact.name)]) @@ -134,6 +134,10 @@ class FakeArtifactCache(object): def has_source_metadata(self, source, cachekey, name): return (cachekey, name) in self._cached + def copy_from_remote(self, artifact, remote): + self._cached[(artifact.cache_key, artifact.name)] = \ + remote._cached[(artifact.cache_key, artifact.name)] + class BuilderBaseTests(unittest.TestCase): @@ -191,9 +195,7 @@ class BuilderBaseTests(unittest.TestCase): rac = FakeArtifactCache() afacts = [FakeArtifact(name) for name in ('a', 'b', 'c')] for a in afacts: - fh = rac.put(a) - fh.write(a.name) - fh.close() + rac.put(a, 'not-a-dir') morphlib.builder.download_depends(afacts, lac, rac) self.assertTrue(all(lac.has(a) for a in afacts)) @@ -202,9 +204,7 @@ class BuilderBaseTests(unittest.TestCase): rac = FakeArtifactCache() afacts = [FakeArtifact(name) for name in ('a', 'b', 'c')] for a in afacts: - fh = rac.put(a) - fh.write(a.name) - fh.close() + rac.put(a, 'not-a-dir') fh = rac.put_artifact_metadata(a, 'meta') fh.write('metadata') fh.close() diff --git a/morphlib/exts/fstab.configure b/morphlib/exts/fstab.configure index b9154eee..3bbc9102 100755 --- a/morphlib/exts/fstab.configure +++ b/morphlib/exts/fstab.configure @@ -1,6 +1,5 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright © 2013-2015 Codethink Limited +#!/usr/bin/python +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -20,9 +19,21 @@ import os import sys -import morphlib -envvars = {k: v for (k, v) in os.environ.iteritems() if k.startswith('FSTAB_')} +def asciibetical(strings): -conf_file = os.path.join(sys.argv[1], 'etc/fstab') -morphlib.util.write_from_dict(conf_file, envvars) + def key(s): + return [ord(c) for c in s] + + return sorted(strings, key=key) + + +fstab_filename = os.path.join(sys.argv[1], 'etc', 'fstab') + +fstab_vars = asciibetical(x for x in os.environ if x.startswith('FSTAB_')) +with open(fstab_filename, 'a') as f: + for var in fstab_vars: + f.write('%s\n' % os.environ[var]) + +os.chown(fstab_filename, 0, 0) +os.chmod(fstab_filename, 0644) diff --git a/morphlib/exts/hosts.configure b/morphlib/exts/hosts.configure deleted file mode 100755 index 6b068d04..00000000 --- a/morphlib/exts/hosts.configure +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright © 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -# =*= License: GPL-2 =*= - - -import os -import sys -import socket - -import morphlib - -def validate(var, line): - xs = line.split() - if len(xs) == 0: - raise morphlib.Error("`%s: %s': line is empty" % (var, line)) - - ip = xs[0] - hostnames = xs[1:] - - if len(hostnames) == 0: - raise morphlib.Error("`%s: %s': missing hostname" % (var, line)) - - family = socket.AF_INET6 if ':' in ip else socket.AF_INET - - try: - socket.inet_pton(family, ip) - except socket.error: - raise morphlib.Error("`%s: %s' invalid ip" % (var, ip)) - -envvars = {k: v for (k, v) in os.environ.iteritems() if k.startswith('HOSTS_')} - -conf_file = os.path.join(sys.argv[1], 'etc/hosts') -morphlib.util.write_from_dict(conf_file, envvars, validate) diff --git a/morphlib/fsutils.py b/morphlib/fsutils.py index a3b73bf6..400ff7d8 100644 --- a/morphlib/fsutils.py +++ b/morphlib/fsutils.py @@ -46,14 +46,33 @@ def create_fs(runcmd, partition): # pragma: no cover runcmd(['mkfs.btrfs', '-L', 'baserock', partition]) -def mount(runcmd, partition, mount_point, fstype=None): # pragma: no cover +def mount(runcmd, partition, mount_point, + fstype=None, options=[]): # pragma: no cover if not os.path.exists(mount_point): os.mkdir(mount_point) if not fstype: fstype = [] else: fstype = ['-t', fstype] - runcmd(['mount', partition, mount_point] + fstype) + if not type(options) == list: + options = [options] + runcmd(['mount', partition, mount_point] + fstype + options) + + +def overlay_mount(runcmd, partition, mount_point, + lowerdir, upperdir, workdir, method): # pragma: no cover + if method == 'overlayfs': + options = '-olowerdir=%s,upperdir=%s,workdir=%s' % \ + (lowerdir, upperdir, workdir) + mount(runcmd, partition, mount_point, 'overlay', options) + elif method == 'unionfs': + if not os.path.exists(mount_point): + os.mkdir(mount_point) + dir_string = '%s=RW:%s=RO' % (upperdir, lowerdir) + runcmd(['unionfs', '-o', 'cow', dir_string, mount_point]) + else: + raise cliapp.AppException('Union filesystem %s not supported' % + method) def unmount(runcmd, mount_point): # pragma: no cover diff --git a/morphlib/ostree.py b/morphlib/ostree.py new file mode 100644 index 00000000..a2c133f2 --- /dev/null +++ b/morphlib/ostree.py @@ -0,0 +1,139 @@ +from gi.repository import OSTree +from gi.repository import Gio +from gi.repository import GLib + +import os + + +class OSTreeRepo(object): + + """Class to wrap the OSTree API.""" + + OSTREE_GIO_FAST_QUERYINFO = 'standard::name,standard::type,standard::' \ + 'size,standard::is-symlink,standard::syml' \ + 'ink-target,unix::device,unix::inode,unix' \ + '::mode,unix::uid,unix::gid,unix::rdev' + G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS = Gio.FileQueryInfoFlags(1) + cancellable = Gio.Cancellable.new() + + def __init__(self, path, disable_fsync=True): + self.path = path + self.repo = self._open_repo(path, disable_fsync) + + def _open_repo(self, path, disable_fsync=True): + """Create and open and OSTree.Repo, and return it.""" + repo_dir = Gio.file_new_for_path(path) + repo = OSTree.Repo.new(repo_dir) + try: + repo.open(self.cancellable) + except GLib.GError: + if not os.path.exists(path): + os.makedirs(path) + repo.create(OSTree.RepoMode.ARCHIVE_Z2, self.cancellable) + repo.set_disable_fsync(disable_fsync) + return repo + + def refsdir(self): + """Return the abspath to the refs/heads directory in the repo.""" + return os.path.join(os.path.abspath(self.path), 'refs/heads') + + def touch_ref(self, ref): + """Update the mtime of a ref file in repo/refs/heads.""" + os.utime(os.path.join(self.refsdir(), ref), None) + + def resolve_rev(self, branch, allow_noent=True): + """Return the SHA256 corresponding to 'branch'.""" + return self.repo.resolve_rev(branch, allow_noent)[1] + + def read_commit(self, branch): + """Return an OSTree.RepoFile representing a committed tree.""" + return self.repo.read_commit(branch, self.cancellable)[1] + + def query_info(self, file_object): + """Quickly return a Gio.FileInfo for file_object.""" + return file_object.query_info(self.OSTREE_GIO_FAST_QUERYINFO, + self.G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, + self.cancellable) + + def checkout(self, branch, destdir): + """Checkout branch into destdir.""" + checkout_path = destdir + if not os.path.exists(checkout_path): + os.makedirs(checkout_path) + checkout = Gio.file_new_for_path(checkout_path) + + commit = self.read_commit(branch) + commit_info = self.query_info(commit) + self.repo.checkout_tree(0, 1, checkout, commit, + commit_info, self.cancellable) + + def commit(self, subject, srcdir, branch): + """Commit the contents of 'srcdir' to 'branch'.""" + self.repo.prepare_transaction(self.cancellable) + parent = self.resolve_rev(branch) + if parent: + parent_root = self.read_commit(parent) + + mtree = OSTree.MutableTree() + src = Gio.file_new_for_path(srcdir) + self.repo.write_directory_to_mtree(src, mtree, None, self.cancellable) + root = self.repo.write_mtree(mtree, self.cancellable)[1] + if parent and root.equal(parent_root): + return + checksum = self.repo.write_commit(parent, subject, '', None, + root, self.cancellable)[1] + self.repo.transaction_set_ref(None, branch, checksum) + stats = self.repo.commit_transaction(self.cancellable) + + def cat_file(self, ref, path): + """Return the file descriptor of path at ref.""" + commit = self.read_commit(ref) + relative = commit.resolve_relative_path(path) + ret, content, etag = relative.load_contents() + return content + + def list_refs(self, resolved=False): + """Return a list of all refs in the repo.""" + refs = self.repo.list_refs()[1] + if not resolved: + return refs.keys() + return refs + + def delete_ref(self, ref): + """Remove refspec from the repo.""" + if not self.list_refs(ref): + raise Exception("Failed to delete ref, it doesn't exist") + self.repo.set_ref_immediate(None, ref, None, self.cancellable) + + def prune(self): + """Remove unreachable objects from the repo.""" + return self.repo.prune(OSTree.RepoPruneFlags.REFS_ONLY, + -1, self.cancellable) + + def add_remote(self, name, url): + """Add a remote with a given name and url.""" + options_type = GLib.VariantType.new('a{sv}') + options_builder = GLib.VariantBuilder.new(options_type) + options = options_builder.end() + self.repo.remote_add(name, url, options, self.cancellable) + + def remove_remote(self, name): + """Remove a remote with a given name.""" + self.repo.remote_delete(name, self.cancellable) + + def get_remote_url(self, name): + """Return the URL for a remote.""" + return self.repo.remote_get_url(name)[1] + + def list_remotes(self): + """Return a list of all remotes for this repo.""" + return self.repo.remote_list() + + def has_remote(self, name): + """Return True if name is a remote for the repo.""" + return name in self.list_remotes() + + def pull(self, refs, remote): + """Pull ref from remote into the local repo.""" + flags = OSTree.RepoPullFlags.NONE + self.repo.pull(remote, refs, flags, None, self.cancellable) diff --git a/morphlib/ostreeartifactcache.py b/morphlib/ostreeartifactcache.py new file mode 100644 index 00000000..fdb7cb5d --- /dev/null +++ b/morphlib/ostreeartifactcache.py @@ -0,0 +1,229 @@ +# Copyright (C) 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import collections +import logging +import os +import shutil +import tarfile +import tempfile + +import cliapp +from gi.repository import GLib + +import morphlib +from morphlib.artifactcachereference import ArtifactCacheReference + +class OSTreeArtifactCache(object): + """Class to provide the artifact cache API using an OSTree repo.""" + + def __init__(self, cachedir): + repo_dir = os.path.join(cachedir, 'repo') + self.repo = morphlib.ostree.OSTreeRepo(repo_dir) + self.cachedir = cachedir + + def _get_file_from_remote(self, artifact, remote, metadata_name=None): + if metadata_name: + handle = remote.get_artifact_metadata(artifact, metadata_name) + else: + handle = remote.get(artifact) + fd, path = tempfile.mkstemp() + with open(path, 'w+') as temp: + shutil.copyfileobj(handle, temp) + return path + + def _get_artifact_cache_name(self, artifact): + logging.debug('LAC: %s' % artifact.basename()) + cache_key, kind, name = artifact.basename().split('.', 2) + suffix = name.split('-')[-1] + return '%s-%s' % (cache_key, suffix) + + def put(self, directory, artifact): + """Commit the contents of 'directory' to the repo. + + This uses the artifact name and cache key to create the ref, so the + contents of directory should be the contents of the artifact. + + """ + ref = self._get_artifact_cache_name(artifact) + subject = artifact.name + try: + logging.debug('Committing %s to artifact cache at %s.' % + (subject, ref)) + self.repo.commit(subject, directory, ref) + except GLib.GError as e: + logging.debug('OSTree raised an exception: %s' % e) + raise cliapp.AppException('Failed to commit %s to artifact ' + 'cache.' % ref) + + def put_non_ostree_artifact(self, artifact, location, metadata_name=None): + """Store a single file in the artifact cachedir.""" + if metadata_name: + filename = self._artifact_metadata_filename(artifact, + metadata_name) + else: + filename = self.artifact_filename(artifact) + shutil.copy(location, filename) + os.remove(location) + + def copy_from_remote(self, artifact, remote): + """Get 'artifact' from remote artifact cache and store it locally.""" + if remote.method == 'tarball': + logging.debug('Downloading artifact tarball for %s.' % + artifact.name) + location = self._get_file_from_remote(artifact, remote) + try: + tempdir = tempfile.mkdtemp() + with tarfile.open(name=location) as tf: + tf.extractall(path=tempdir) + try: + self.put(tempdir, artifact) + finally: + os.remove(location) + shutil.rmtree(tempdir) + except tarfile.ReadError: + # Reading the artifact as a tarball failed, so it must be a + # single file (for example a stratum artifact). + self.put_non_ostree_artifact(artifact, location) + + elif remote.method == 'ostree': + logging.debug('Pulling artifact for %s from remote.' % + artifact.basename()) + try: + ref = self._get_artifact_cache_name(artifact) + except Exception: + # if we can't split the name properly, we must want metadata + a, name = artifact.basename().split('.', 1) + location = self._get_file_from_remote( + ArtifactCacheReference(a), remote, name) + self.put_non_ostree_artifact(artifact, location, name) + return + + if artifact.basename().split('.', 2)[1] == 'stratum': + location = self._get_file_from_remote(artifact, remote) + self.put_non_ostree_artifact(artifact, location) + return + + try: + if not self.repo.has_remote(remote.name): + self.repo.add_remote(remote.name, remote.ostree_url) + self.repo.pull([ref], remote.name) + except GLib.GError as e: + logging.debug('OSTree raised an exception: %s' % e) + raise cliapp.AppException('Failed to pull %s from remote ' + 'cache.' % ref) + + def get(self, artifact, directory=None): + """Checkout an artifact from the repo and return its location.""" + cache_key, kind, name = artifact.basename().split('.', 2) + if kind == 'stratum': + return self.artifact_filename(artifact) + if directory is None: + directory = tempfile.mkdtemp() + ref = self._get_artifact_cache_name(artifact) + try: + self.repo.checkout(ref, directory) + self.repo.touch_ref(ref) + except GLib.GError as e: + logging.debug('OSTree raised an exception: %s' % e) + raise cliapp.AppException('Failed to checkout %s from artifact ' + 'cache.' % ref) + return directory + + def list_contents(self): + """Return the set of sources cached and related information. + + returns a [(cache_key, set(artifacts), last_used)] + + """ + CacheInfo = collections.namedtuple('CacheInfo', ('artifacts', 'mtime')) + contents = collections.defaultdict(lambda: CacheInfo(set(), 0)) + for ref in self.repo.list_refs(): + cachekey = ref[:63] + artifact = ref[65:] + artifacts, max_mtime = contents[cachekey] + artifacts.add(artifact) + ref_filename = os.path.join(self.repo.refsdir(), ref) + mtime = os.path.getmtime(ref_filename) + contents[cachekey] = CacheInfo(artifacts, max(max_mtime, mtime)) + return ((cache_key, info.artifacts, info.mtime) + for cache_key, info in contents.iteritems()) + + def remove(self, cachekey): + """Remove all artifacts associated with the given cachekey.""" + for ref in (r for r in self.repo.list_refs() + if r.startswith(cachekey)): + self.repo.delete_ref(ref) + + def prune(self): + """Delete orphaned objects in the repo.""" + self.repo.prune() + + def has(self, artifact): + cachekey, kind, name = artifact.basename().split('.', 2) + logging.debug('OSTreeArtifactCache: got %s, %s, %s' % + (cachekey, kind, name)) + if self._get_artifact_cache_name(artifact) in self.repo.list_refs(): + self.repo.touch_ref(self._get_artifact_cache_name(artifact)) + return True + if kind == 'stratum' and \ + self._has_file(self.artifact_filename(artifact)): + return True + return False + + def get_artifact_metadata(self, artifact, name): + filename = self._artifact_metadata_filename(artifact, name) + os.utime(filename, None) + return open(filename) + + def get_source_metadata_filename(self, source, cachekey, name): + return self._source_metadata_filename(source, cachekey, name) + + def get_source_metadata(self, source, cachekey, name): + filename = self._source_metadata_filename(source, cachekey, name) + os.utime(filename, None) + return open(filename) + + def artifact_filename(self, artifact): + return os.path.join(self.cachedir, artifact.basename()) + + def _artifact_metadata_filename(self, artifact, name): + return os.path.join(self.cachedir, artifact.metadata_basename(name)) + + def _source_metadata_filename(self, source, cachekey, name): + return os.path.join(self.cachedir, '%s.%s' % (cachekey, name)) + + def put_artifact_metadata(self, artifact, name): + filename = self._artifact_metadata_filename(artifact, name) + return morphlib.savefile.SaveFile(filename, mode='w') + + def put_source_metadata(self, source, cachekey, name): + filename = self._source_metadata_filename(source, cachekey, name) + return morphlib.savefile.SaveFile(filename, mode='w') + + def _has_file(self, filename): + if os.path.exists(filename): + os.utime(filename, None) + return True + return False + + def has_artifact_metadata(self, artifact, name): + filename = self._artifact_metadata_filename(artifact, name) + return self._has_file(filename) + + def has_source_metadata(self, source, cachekey, name): + filename = self._source_metadata_filename(source, cachekey, name) + return self._has_file(filename) diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index 7635a7b4..3c74a13f 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -24,6 +24,7 @@ import uuid import cliapp import morphlib +from morphlib.artifactcachereference import ArtifactCacheReference class DeployPlugin(cliapp.Plugin): @@ -439,6 +440,8 @@ class DeployPlugin(cliapp.Plugin): system_status_prefix = '%s[%s]' % (old_status_prefix, system['morph']) self.app.status_prefix = system_status_prefix try: + system_tree = None + # Find the artifact to build morph = morphlib.util.sanitise_morphology_path(system['morph']) srcpool = build_command.create_source_pool(build_repo, ref, morph) @@ -502,6 +505,9 @@ class DeployPlugin(cliapp.Plugin): system_tree, deploy_location) finally: self.app.status_prefix = system_status_prefix + if system_tree and os.path.exists(system_tree): + morphlib.fsutils.unmount(self.app.runcmd, system_tree) + shutil.rmtree(system_tree) finally: self.app.status_prefix = old_status_prefix @@ -535,46 +541,97 @@ class DeployPlugin(cliapp.Plugin): except morphlib.extensions.ExtensionNotFoundError: pass + def checkout_stratum(self, path, artifact, lac, rac): + with open(lac.get(artifact), 'r') as stratum: + chunks = [ArtifactCacheReference(c) for c in json.load(stratum)] + #morphlib.builder.download_depends(chunks, lac, rac) + for chunk in chunks: + self.app.status(msg='Checkout chunk %(name)s.', + name=chunk.basename(), chatty=True) + try: + lac.get(chunk, path) + except: + raise Exception('%s not cached' % chunk.basename()) + + metadata = os.path.join(path, 'baserock', '%s.meta' % artifact.name) + with lac.get_artifact_metadata(artifact, 'meta') as meta_src: + with morphlib.savefile.SaveFile(metadata, 'w') as meta_dst: + shutil.copyfileobj(meta_src, meta_dst) + + def checkout_strata(self, path, artifact, lac, rac): + deps = artifact.source.dependencies + #morphlib.builder.download_depends(deps, lac, rac) + for stratum in deps: + self.checkout_stratum(path, stratum, lac, rac) + morphlib.builder.ldconfig(self.app.runcmd, path) + def setup_deploy(self, build_command, deploy_tempdir, root_repo_dir, ref, artifact, deployment_type, location, env): # deployment_type, location and env are only used for saving metadata - # Create a tempdir to extract the rootfs in - system_tree = tempfile.mkdtemp(dir=deploy_tempdir) + deployment_dir = tempfile.mkdtemp(dir=deploy_tempdir) + # Create a tempdir to extract the rootfs in + system_tree = tempfile.mkdtemp(dir=deployment_dir) + + # Create temporary directory for overlayfs + overlay_dir = os.path.join(deployment_dir, + '%s-upperdir' % artifact.name) + if not os.path.exists(overlay_dir): + os.makedirs(overlay_dir) + work_dir = os.path.join(deployment_dir, '%s-workdir' % artifact.name) + if not os.path.exists(work_dir): + os.makedirs(work_dir) + + deploy_tree = os.path.join(deployment_dir, + 'overlay-deploy-%s' % artifact.name) try: - # Unpack the artifact (tarball) to a temporary directory. - self.app.status(msg='Unpacking system for configuration') + # Checkout the strata involved in the artifact into a tempdir + self.app.status(msg='Checking out strata in system') + self.checkout_strata(system_tree, artifact, + build_command.lac, build_command.rac) + self.app.status(msg='Checking out system for configuration') if build_command.lac.has(artifact): - f = build_command.lac.get(artifact) + build_command.lac.get(artifact, system_tree) elif build_command.rac.has(artifact): build_command.cache_artifacts_locally([artifact]) - f = build_command.lac.get(artifact) + build_command.lac.get(artifact, system_tree) else: raise cliapp.AppException('Deployment failed as system is' ' not yet built.\nPlease ensure' ' the system is built before' ' deployment.') - tf = tarfile.open(fileobj=f) - tf.extractall(path=system_tree) self.app.status( - msg='System unpacked at %(system_tree)s', + msg='System checked out at %(system_tree)s', system_tree=system_tree) + union_filesystem = self.app.settings['union-filesystem'] + morphlib.fsutils.overlay_mount(self.app.runcmd, + 'overlay-deploy-%s' % + artifact.name, + deploy_tree, system_tree, + overlay_dir, work_dir, + union_filesystem) + self.app.status( msg='Writing deployment metadata file') metadata = self.create_metadata( artifact, root_repo_dir, deployment_type, location, env) metadata_path = os.path.join( - system_tree, 'baserock', 'deployment.meta') + deploy_tree, 'baserock', 'deployment.meta') with morphlib.savefile.SaveFile(metadata_path, 'w') as f: json.dump(metadata, f, indent=4, sort_keys=True, encoding='unicode-escape') - return system_tree + return deploy_tree except Exception: + if deploy_tree and os.path.exists(deploy_tree): + morphlib.fsutils.unmount(self.app.runcmd, deploy_tree) + shutil.rmtree(deploy_tree) shutil.rmtree(system_tree) + shutil.rmtree(overlay_dir) + shutil.rmtree(work_dir) raise def run_deploy_commands(self, deploy_tempdir, env, artifact, root_repo_dir, diff --git a/morphlib/plugins/gc_plugin.py b/morphlib/plugins/gc_plugin.py index 71522b04..8b5dc4c2 100644 --- a/morphlib/plugins/gc_plugin.py +++ b/morphlib/plugins/gc_plugin.py @@ -125,8 +125,8 @@ class GCPlugin(cliapp.Plugin): 'sufficient space already cleared', chatty=True) return - lac = morphlib.localartifactcache.LocalArtifactCache( - fs.osfs.OSFS(os.path.join(cache_path, 'artifacts'))) + lac = morphlib.ostreeartifactcache.OSTreeArtifactCache( + os.path.join(cache_path, 'artifacts')) max_age, min_age = self.calculate_delete_range() logging.debug('Must remove artifacts older than timestamp %d' % max_age) @@ -144,6 +144,8 @@ class GCPlugin(cliapp.Plugin): lac.remove(cachekey) removed += 1 + lac.prune() + # Maybe remove remaining middle-aged artifacts for cachekey in may_delete: if sufficient_free(): @@ -157,6 +159,8 @@ class GCPlugin(cliapp.Plugin): lac.remove(cachekey) removed += 1 + lac.prune() + if sufficient_free(): self.app.status(msg='Made sufficient space in %(cache_path)s ' 'after removing %(removed)d sources', diff --git a/morphlib/remoteartifactcache.py b/morphlib/remoteartifactcache.py index 427e4cbb..f5115cd6 100644 --- a/morphlib/remoteartifactcache.py +++ b/morphlib/remoteartifactcache.py @@ -57,6 +57,18 @@ class RemoteArtifactCache(object): def __init__(self, server_url): self.server_url = server_url + self.name = urlparse.urlparse(server_url).hostname + try: + self.method = self._get_method() + except urllib2.URLError: + self.method = 'tarball' + except Exception as e: # pragma: no cover + logging.debug('Failed to determine cache method: %s' % e) + raise cliapp.AppException('Failed to determine method used by ' + 'remote cache.') + if self.method == 'ostree': # pragma: no cover + self.ostree_url = 'http://%s:%s/' % (self.name, + self._get_ostree_info()) def has(self, artifact): return self._has_file(artifact.basename()) @@ -112,5 +124,18 @@ class RemoteArtifactCache(object): server_url, '/1.0/artifacts?filename=%s' % urllib.quote(filename)) + def _get_method(self): # pragma: no cover + logging.debug('Getting cache method of %s' % self.server_url) + request_url = urlparse.urljoin(self.server_url, '/1.0/method') + req = urllib2.urlopen(request_url) + return req.read() + + def _get_ostree_info(self): # pragma: no cover + logging.debug('Getting OSTree repo info.') + request_url = urlparse.urljoin(self.server_url, '/1.0/ostreeinfo') + logging.debug('sending %s' % request_url) + req = urllib2.urlopen(request_url) + return req.read() + def __str__(self): # pragma: no cover return self.server_url diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py index 1e64c23a..d2b47d35 100644 --- a/morphlib/sourceresolver.py +++ b/morphlib/sourceresolver.py @@ -31,7 +31,7 @@ tree_cache_filename = 'trees.cache.pickle' buildsystem_cache_size = 10000 buildsystem_cache_filename = 'detected-chunk-buildsystems.cache.pickle' -supported_versions = [0, 1] +not_supported_versions = [] class PickleCacheManager(object): # pragma: no cover '''Cache manager for PyLRU that reads and writes to Pickle files. @@ -346,29 +346,6 @@ class SourceResolver(object): loader.set_defaults(morph) return morph - def _parse_version_file(self, version_file): # pragma : no cover - '''Parse VERSION file and return the version of the format if: - - VERSION is a YAML file - and it's a dict - and has the key 'version' - and the type stored in the 'version' key is an int - and that int is not in the supported format - - otherwise returns None - - ''' - version = None - - yaml_obj = yaml.safe_load(version_file) - if yaml_obj is not None: - if type(yaml_obj) is dict: - if 'version' in yaml_obj.keys(): - if type(yaml_obj['version']) is int: - version = yaml_obj['version'] - - return version - def _check_version_file(self,definitions_repo, definitions_absref): # pragma: no cover version_file = self._get_file_contents( @@ -377,10 +354,13 @@ class SourceResolver(object): if version_file is None: return - version = self._parse_version_file(version_file) - if version is not None: - if version not in supported_versions: - raise UnknownVersionError(version) + try: + version = yaml.safe_load(version_file)['version'] + except (yaml.error.YAMLError, KeyError, TypeError): + version = 0 + + if version in not_supported_versions: + raise UnknownVersionError(version) def _process_definitions_with_children(self, system_filenames, definitions_repo, diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py index 8c2781aa..768ec643 100644 --- a/morphlib/stagingarea.py +++ b/morphlib/stagingarea.py @@ -87,6 +87,14 @@ class StagingArea(object): return self._dir_for_source(source, 'inst') + def overlay_upperdir(self, source): + '''Create a directory to be upperdir for overlayfs, and return it.''' + return self._dir_for_source(source, 'overlay_upper') + + def overlaydir(self, source): + '''Create a directory to be a mount point for overlayfs, return it''' + return self._dir_for_source(source, 'overlay') + def relative(self, filename): '''Return a filename relative to the staging area.''' @@ -146,37 +154,42 @@ class StagingArea(object): raise IOError('Cannot extract %s into staging-area. Unsupported' ' type.' % srcpath) - def install_artifact(self, handle): + def create_devices(self, morphology): # pragma: no cover + '''Creates device nodes if the morphology specifies them''' + perms_mask = stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO + if 'devices' in morphology and morphology['devices'] is not None: + for dev in morphology['devices']: + destfile = os.path.join(self.dirname, './' + dev['filename']) + mode = int(dev['permissions'], 8) & perms_mask + if dev['type'] == 'c': + mode = mode | stat.S_IFCHR + elif dev['type'] == 'b': + mode = mode | stat.S_IFBLK + else: + raise IOError('Cannot create device node %s,' + 'unrecognized device type "%s"' + % (destfile, dev['type'])) + parent = os.path.dirname(destfile) + if not os.path.exists(parent): + os.makedirs(parent) + if not os.path.exists(destfile): + logging.debug("Creating device node %s" % destfile) + os.mknod(destfile, mode, + os.makedev(dev['major'], dev['minor'])) + os.chown(destfile, dev['uid'], dev['gid']) + + def install_artifact(self, artifact, artifact_checkout): '''Install a build artifact into the staging area. We access the artifact via an open file handle. For now, we assume the artifact is a tarball. ''' - - chunk_cache_dir = os.path.join(self._app.settings['tempdir'], 'chunks') - unpacked_artifact = os.path.join( - chunk_cache_dir, os.path.basename(handle.name) + '.d') - if not os.path.exists(unpacked_artifact): - self._app.status( - msg='Unpacking chunk from cache %(filename)s', - filename=os.path.basename(handle.name)) - savedir = tempfile.mkdtemp(dir=chunk_cache_dir) - try: - morphlib.bins.unpack_binary_from_file( - handle, savedir + '/') - except BaseException as e: # pragma: no cover - shutil.rmtree(savedir) - raise - # TODO: This rename is not concurrency safe if two builds are - # extracting the same chunk, one build will fail because - # the other renamed its tempdir here first. - os.rename(savedir, unpacked_artifact) - if not os.path.exists(self.dirname): self._mkdir(self.dirname) - self.hardlink_all_files(unpacked_artifact, self.dirname) + self.hardlink_all_files(artifact_checkout, self.dirname) + self.create_devices(artifact.source.morphology) def remove(self): '''Remove the entire staging area. diff --git a/morphlib/stagingarea_tests.py b/morphlib/stagingarea_tests.py index 97d78236..ffdf5eaa 100644 --- a/morphlib/stagingarea_tests.py +++ b/morphlib/stagingarea_tests.py @@ -30,6 +30,7 @@ class FakeBuildEnvironment(object): } self.extra_path = ['/extra-path'] + class FakeSource(object): def __init__(self): @@ -39,6 +40,12 @@ class FakeSource(object): self.name = 'le-name' +class FakeArtifact(object): + + def __init__(self): + self.source = FakeSource() + + class FakeApplication(object): def __init__(self, cachedir, tempdir): @@ -83,12 +90,8 @@ class StagingAreaTests(unittest.TestCase): os.mkdir(chunkdir) with open(os.path.join(chunkdir, 'file.txt'), 'w'): pass - chunk_tar = os.path.join(self.tempdir, 'chunk.tar') - tf = tarfile.TarFile(name=chunk_tar, mode='w') - tf.add(chunkdir, arcname='.') - tf.close() - return chunk_tar + return chunkdir def list_tree(self, root): files = [] @@ -118,20 +121,34 @@ class StagingAreaTests(unittest.TestCase): self.assertEqual(self.created_dirs, [dirname]) self.assertTrue(dirname.startswith(self.staging)) + def test_creates_overlay_upper_directory(self): + source = FakeSource() + self.sa._mkdir = self.fake_mkdir + dirname = self.sa.overlay_upperdir(source) + self.assertEqual(self.created_dirs, [dirname]) + self.assertTrue(dirname.startswith(self.staging)) + + def test_creates_overlay_directory(self): + source = FakeSource() + self.sa._mkdir = self.fake_mkdir + dirname = self.sa.overlaydir(source) + self.assertEqual(self.created_dirs, [dirname]) + self.assertTrue(dirname.startswith(self.staging)) + def test_makes_relative_name(self): filename = os.path.join(self.staging, 'foobar') self.assertEqual(self.sa.relative(filename), '/foobar') def test_installs_artifact(self): - chunk_tar = self.create_chunk() - with open(chunk_tar, 'rb') as f: - self.sa.install_artifact(f) + artifact = FakeArtifact() + chunkdir = self.create_chunk() + self.sa.install_artifact(artifact, chunkdir) self.assertEqual(self.list_tree(self.staging), ['/', '/file.txt']) def test_removes_everything(self): - chunk_tar = self.create_chunk() - with open(chunk_tar, 'rb') as f: - self.sa.install_artifact(f) + artifact = FakeArtifact() + chunkdir = self.create_chunk() + self.sa.install_artifact(artifact, chunkdir) self.sa.remove() self.assertFalse(os.path.exists(self.staging)) diff --git a/morphlib/util.py b/morphlib/util.py index e733af9d..00111ff7 100644 --- a/morphlib/util.py +++ b/morphlib/util.py @@ -131,8 +131,10 @@ def new_artifact_caches(settings): # pragma: no cover if not os.path.exists(artifact_cachedir): os.mkdir(artifact_cachedir) - lac = morphlib.localartifactcache.LocalArtifactCache( - fs.osfs.OSFS(artifact_cachedir)) + #lac = morphlib.localartifactcache.LocalArtifactCache( + # fs.osfs.OSFS(artifact_cachedir)) + + lac = morphlib.ostreeartifactcache.OSTreeArtifactCache(artifact_cachedir) rac_url = get_artifact_cache_server(settings) rac = None @@ -644,35 +646,3 @@ def error_message_for_containerised_commandline( 'Containerisation settings: %s\n' \ 'Error output:\n%s' \ % (argv_string, container_kwargs, err) - - -def write_from_dict(filepath, d, validate=lambda x, y: True): #pragma: no cover - '''Takes a dictionary and appends the contents to a file - - An optional validation callback can be passed to perform validation on - each value in the dictionary. - - e.g. - - def validation_callback(dictionary_key, dictionary_value): - if not dictionary_value.isdigit(): - raise Exception('value contains non-digit character(s)') - - Any callback supplied to this function should raise an exception - if validation fails. - ''' - - # Sort items asciibetically - # the output of the deployment should not depend - # on the locale of the machine running the deployment - items = sorted(d.iteritems(), key=lambda (k, v): [ord(c) for c in v]) - - for (k, v) in items: - validate(k, v) - - with open(filepath, 'a') as f: - for (_, v) in items: - f.write('%s\n' % v) - - os.fchown(f.fileno(), 0, 0) - os.fchmod(f.fileno(), 0644) diff --git a/morphlib/writeexts.py b/morphlib/writeexts.py index aa185a2b..129b2bc4 100644 --- a/morphlib/writeexts.py +++ b/morphlib/writeexts.py @@ -604,16 +604,12 @@ class WriteExtension(cliapp.Application): def check_ssh_connectivity(self, ssh_host): try: - output = cliapp.ssh_runcmd(ssh_host, ['echo', 'test']) + cliapp.ssh_runcmd(ssh_host, ['true']) except cliapp.AppException as e: logging.error("Error checking SSH connectivity: %s", str(e)) raise cliapp.AppException( 'Unable to SSH to %s: %s' % (ssh_host, e)) - if output.strip() != 'test': - raise cliapp.AppException( - 'Unexpected output from remote machine: %s' % output.strip()) - def is_device(self, location): try: st = os.stat(location) |