diff options
Diffstat (limited to 'morphlib')
29 files changed, 1240 insertions, 291 deletions
diff --git a/morphlib/__init__.py b/morphlib/__init__.py index 695241cc..79e829a4 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -37,8 +37,9 @@ __version__ = gitversion.version # List of architectures that Morph supports -valid_archs = ['armv7l', 'armv7lhf', 'armv7b', 'testarch', - 'x86_32', 'x86_64', 'ppc64', 'armv8l64', 'armv8b64'] +valid_archs = ['armv7l', 'armv7lhf', 'armv7b', 'testarch', 'x86_32', + 'x86_64', 'ppc64', 'armv8l64', 'armv8b64', 'mips32l', + 'mips32b', 'mips64l', 'mips64b'] class Error(cliapp.AppException): diff --git a/morphlib/app.py b/morphlib/app.py index c367cafb..f5823dd3 100644 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -20,6 +20,7 @@ import pipes import sys import time import urlparse +import warnings import extensions import morphlib @@ -59,7 +60,7 @@ class Morph(cliapp.Application): 'show no output unless there is an error') self.settings.boolean(['help', 'h'], - 'show this help message and exit') + 'show this help message and exit') self.settings.boolean(['help-all'], 'show help message including hidden subcommands') @@ -154,6 +155,10 @@ class Morph(cliapp.Application): 'always push temporary build branches to the ' 'remote repository', group=group_build) + self.settings.boolean(['partial'], + 'only build up to a given chunk', + default=False, + group=group_build) self.settings.choice (['local-changes'], ['include', 'ignore'], 'the `build` and `deploy` commands detect ' @@ -229,6 +234,12 @@ class Morph(cliapp.Application): with morphlib.util.hide_password_environment_variables(os.environ): cliapp.Application.log_config(self) + def pretty_warnings(message, category, filename, lineno, + file=None, line=None): + return 'WARNING: %s' % (message) + + warnings.formatwarning = pretty_warnings + def process_args(self, args): self.check_time() @@ -278,8 +289,7 @@ class Morph(cliapp.Application): sys.exit(0) tmpdir = self.settings['tempdir'] - for required_dir in (os.path.join(tmpdir, 'chunks'), - os.path.join(tmpdir, 'staging'), + for required_dir in (os.path.join(tmpdir, 'staging'), os.path.join(tmpdir, 'failed'), os.path.join(tmpdir, 'deployments'), self.settings['cachedir']): @@ -291,11 +301,13 @@ class Morph(cliapp.Application): def setup_plugin_manager(self): cliapp.Application.setup_plugin_manager(self) - self.pluginmgr.locations += os.path.join( - os.path.dirname(morphlib.__file__), 'plugins') + s = os.path.join(os.path.dirname(morphlib.__file__), 'plugins') + if not s in self.pluginmgr.locations: + self.pluginmgr.locations.append(s) - s = os.environ.get('MORPH_PLUGIN_PATH', '') - self.pluginmgr.locations += s.split(':') + s = os.environ.get('MORPH_PLUGIN_PATH', '').split(':') + for path in s: + self.pluginmgr.locations.append(path) self.hookmgr = cliapp.HookManager() self.hookmgr.new('new-build-command', cliapp.FilterHook()) @@ -330,7 +342,7 @@ class Morph(cliapp.Application): * ``error`` should be true when it is an error message All other keywords are ignored unless embedded in ``msg``. - + The ``self.status_prefix`` string is prepended to the output. It is set to the empty string by default. @@ -385,7 +397,7 @@ class Morph(cliapp.Application): self._write_status(self._commandline_as_message(argv, args)) # Log the environment. - prev = getattr(self, 'prev_env', {}) + prev = getattr(self, 'prev_env', os.environ) morphlib.util.log_environment_changes(self, kwargs['env'], prev) self.prev_env = kwargs['env'] diff --git a/morphlib/buildbranch.py b/morphlib/buildbranch.py index 80cecd75..2a2530b0 100644 --- a/morphlib/buildbranch.py +++ b/morphlib/buildbranch.py @@ -103,7 +103,7 @@ class BuildBranch(object): in index.get_uncommitted_changes()] if not changed: continue - add_cb(gd=gd, build_ref=gd, changed=changed) + add_cb(gd=gd, build_ref=build_ref, changed=changed) changes_made = True index.add_files_from_working_tree(changed) return changes_made @@ -303,9 +303,8 @@ def pushed_build_branch(bb, loader, changes_need_pushing, name, email, build_uuid, status): with contextlib.closing(bb) as bb: def report_add(gd, build_ref, changed): - status(msg='Adding uncommitted changes '\ - 'in %(dirname)s to %(ref)s', - dirname=gd.dirname, ref=build_ref, chatty=True) + status(msg='Creating temporary branch in %(dirname)s '\ + 'named %(ref)s', dirname=gd.dirname, ref=build_ref) changes_made = bb.add_uncommitted_changes(add_cb=report_add) unpushed = any(bb.get_unpushed_branches()) diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index c83abca6..cab38395 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -74,7 +74,8 @@ class BuildCommand(object): This includes creating the directories on disk if they are missing. ''' - return morphlib.util.new_artifact_caches(self.app.settings) + return morphlib.util.new_artifact_caches( + self.app.settings, status_cb=self.app.status) def new_repo_caches(self): return morphlib.util.new_repo_caches(self.app) @@ -119,7 +120,10 @@ class BuildCommand(object): root_kind = root_artifact.source.morphology['kind'] if root_kind != 'system': raise morphlib.Error( - 'Building a %s directly is not supported' % root_kind) + 'In order to build this %s directly, please give the filename ' + 'of the system which contains it, and the name of the %s. ' + 'See `morph build --help` for more information.' + % (root_kind, root_kind)) def _validate_architecture(self, root_artifact): '''Perform the validation between root and target architectures.''' @@ -271,7 +275,8 @@ class BuildCommand(object): def build_in_order(self, root_artifact): '''Build everything specified in a build order.''' - self.app.status(msg='Building a set of sources') + self.app.status(msg='Starting build of %(name)s', + name=root_artifact.source.name) build_env = root_artifact.build_env ordered_sources = list(self.get_ordered_sources(root_artifact.walk())) old_prefix = self.app.status_prefix @@ -487,32 +492,13 @@ class BuildCommand(object): if artifact.source.build_mode == 'bootstrap': if not self.in_same_stratum(artifact.source, target_source): continue + self.app.status( msg='Installing chunk %(chunk_name)s from cache %(cache)s', chunk_name=artifact.name, cache=artifact.source.cache_key[:7], chatty=True) - chunk_cache_dir = os.path.join(self.app.settings['tempdir'], - 'chunks') - artifact_checkout = os.path.join( - chunk_cache_dir, os.path.basename(artifact.basename()) + '.d') - if not os.path.exists(artifact_checkout): - self.app.status( - msg='Checking out %(chunk)s from cache.', - chunk=artifact.name - ) - temp_checkout = os.path.join(self.app.settings['tempdir'], - artifact.basename()) - try: - self.lac.get(artifact, temp_checkout) - except BaseException: - shutil.rmtree(temp_checkout) - raise - # TODO: This rename is not concurrency safe if two builds are - # extracting the same chunk, one build will fail because - # the other renamed its tempdir here first. - os.rename(temp_checkout, artifact_checkout) - staging_area.install_artifact(artifact, artifact_checkout) + staging_area.install_artifact(self.lac, artifact) if target_source.build_mode == 'staging': morphlib.builder.ldconfig(self.app.runcmd, staging_area.dirname) @@ -540,7 +526,8 @@ class InitiatorBuildCommand(BuildCommand): self.app.settings['push-build-branches'] = True super(InitiatorBuildCommand, self).__init__(app) - def build(self, repo_name, ref, filename, original_ref=None): + def build(self, repo_name, ref, filename, original_ref=None, + component_names=[]): '''Initiate a distributed build on a controller''' distbuild.add_crash_conditions(self.app.settings['crash-condition']) @@ -551,7 +538,8 @@ class InitiatorBuildCommand(BuildCommand): self.app.status(msg='Starting distributed build') loop = distbuild.MainLoop() - args = [repo_name, ref, filename, original_ref or ref] + args = [repo_name, ref, filename, original_ref or ref, + component_names] cm = distbuild.InitiatorConnectionMachine(self.app, self.addr, self.port, diff --git a/morphlib/buildenvironment.py b/morphlib/buildenvironment.py index 6ec82d45..266510f6 100644 --- a/morphlib/buildenvironment.py +++ b/morphlib/buildenvironment.py @@ -114,16 +114,30 @@ class BuildEnvironment(): # than leaving it up to individual morphologies. if arch == 'x86_32': cpu = 'i686' + abi = '' + elif arch.startswith('armv7'): + cpu = arch + abi = 'eabi' elif arch == 'armv8l64': # pragma: no cover cpu = 'aarch64' + abi = '' elif arch == 'armv8b64': # pragma: no cover cpu = 'aarch64_be' + abi = '' + elif arch == 'mips64b': # pragma: no cover + cpu = 'mips64' + abi = 'abi64' + elif arch == 'mips64l': # pragma: no cover + cpu = 'mips64el' + abi = 'abi64' + elif arch == 'mips32b': # pragma: no cover + cpu = 'mips' + abi = '' + elif arch == 'mips32l': # pragma: no cover + cpu = 'mipsel' + abi = '' else: cpu = arch - - if arch.startswith('armv7'): - abi = 'eabi' - else: abi = '' env['TARGET'] = cpu + '-baserock-linux-gnu' + abi diff --git a/morphlib/builder.py b/morphlib/builder.py index e5b891b2..b0c95bb3 100644 --- a/morphlib/builder.py +++ b/morphlib/builder.py @@ -558,16 +558,32 @@ class SystemBuilder(BuilderBase): # pragma: no cover self.save_build_times() return self.source.artifacts.itervalues() + def load_stratum(self, stratum_artifact): + '''Load a stratum from the local artifact cache. + + Returns a list of ArtifactCacheReference instances for the chunks + contained in the stratum. + + ''' + cache = self.local_artifact_cache + with open(cache.get(stratum_artifact), 'r') as stratum_file: + try: + artifact_list = json.load(stratum_file, + encoding='unicode-escape') + except ValueError as e: + raise cliapp.AppException( + 'Corruption detected: %s while loading %s' % + (e, cache.artifact_filename(stratum_artifact))) + return [ArtifactCacheReference(a) for a in artifact_list] + def unpack_one_stratum(self, stratum_artifact, target): '''Unpack a single stratum into a target directory''' cache = self.local_artifact_cache - with open(cache.get(stratum_artifact), 'r') as stratum_file: - artifact_list = json.load(stratum_file, encoding='unicode-escape') - for chunk in (ArtifactCacheReference(a) for a in artifact_list): - self.app.status(msg='Checkout chunk %(basename)s', - basename=chunk.basename(), chatty=True) - cache.get(chunk, target) + for chunk in self.load_stratum(stratum_artifact): + self.app.status(msg='Checkout chunk %(basename)s', + basename=chunk.basename(), chatty=True) + cache.get(chunk, target) target_metadata = os.path.join( target, 'baserock', '%s.meta' % stratum_artifact.name) @@ -590,11 +606,7 @@ class SystemBuilder(BuilderBase): # pragma: no cover # download the chunk artifacts if necessary for stratum_artifact in self.source.dependencies: - stratum_path = self.local_artifact_cache.get( - stratum_artifact) - with open(stratum_path, 'r') as stratum: - chunks = [ArtifactCacheReference(c) - for c in json.load(stratum)] + chunks = self.load_stratum(stratum_artifact) download_depends(chunks, self.local_artifact_cache, self.remote_artifact_cache) diff --git a/morphlib/cachedrepo.py b/morphlib/cachedrepo.py index 23639043..b41ba86f 100644 --- a/morphlib/cachedrepo.py +++ b/morphlib/cachedrepo.py @@ -123,6 +123,26 @@ class CachedRepo(object): ''' return self._gitdir.read_file(filename, ref) + def tags_containing_sha1(self, ref): # pragma: no cover + '''Check whether given sha1 is contained in any tags + + Raises a gitdir.InvalidRefError if the ref is not found in the + repository. Raises gitdir.ExpectedSha1Error if the ref is not + a sha1. + + ''' + return self._gitdir.tags_containing_sha1(ref) + + def branches_containing_sha1(self, ref): # pragma: no cover + '''Check whether given sha1 is contained in any branches + + Raises a gitdir.InvalidRefError if the ref is not found in the + repository. Raises gitdir.ExpectedSha1Error if the ref is not + a sha1. + + ''' + return self._gitdir.branches_containing_sha1(ref) + def list_files(self, ref, recurse=True): # pragma: no cover '''Return filenames found in the tree pointed to by the given ref. diff --git a/morphlib/exts/fstab.configure b/morphlib/exts/fstab.configure index 3bbc9102..b9154eee 100755 --- a/morphlib/exts/fstab.configure +++ b/morphlib/exts/fstab.configure @@ -1,5 +1,6 @@ -#!/usr/bin/python -# Copyright (C) 2013,2015 Codethink Limited +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright © 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,21 +20,9 @@ import os import sys +import morphlib -def asciibetical(strings): +envvars = {k: v for (k, v) in os.environ.iteritems() if k.startswith('FSTAB_')} - def key(s): - return [ord(c) for c in s] - - return sorted(strings, key=key) - - -fstab_filename = os.path.join(sys.argv[1], 'etc', 'fstab') - -fstab_vars = asciibetical(x for x in os.environ if x.startswith('FSTAB_')) -with open(fstab_filename, 'a') as f: - for var in fstab_vars: - f.write('%s\n' % os.environ[var]) - -os.chown(fstab_filename, 0, 0) -os.chmod(fstab_filename, 0644) +conf_file = os.path.join(sys.argv[1], 'etc/fstab') +morphlib.util.write_from_dict(conf_file, envvars) diff --git a/morphlib/exts/hosts.configure b/morphlib/exts/hosts.configure new file mode 100755 index 00000000..6b068d04 --- /dev/null +++ b/morphlib/exts/hosts.configure @@ -0,0 +1,48 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright © 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# =*= License: GPL-2 =*= + + +import os +import sys +import socket + +import morphlib + +def validate(var, line): + xs = line.split() + if len(xs) == 0: + raise morphlib.Error("`%s: %s': line is empty" % (var, line)) + + ip = xs[0] + hostnames = xs[1:] + + if len(hostnames) == 0: + raise morphlib.Error("`%s: %s': missing hostname" % (var, line)) + + family = socket.AF_INET6 if ':' in ip else socket.AF_INET + + try: + socket.inet_pton(family, ip) + except socket.error: + raise morphlib.Error("`%s: %s' invalid ip" % (var, ip)) + +envvars = {k: v for (k, v) in os.environ.iteritems() if k.startswith('HOSTS_')} + +conf_file = os.path.join(sys.argv[1], 'etc/hosts') +morphlib.util.write_from_dict(conf_file, envvars, validate) diff --git a/morphlib/exts/install-files.configure b/morphlib/exts/install-files.configure index 58cf373a..c2970243 100755 --- a/morphlib/exts/install-files.configure +++ b/morphlib/exts/install-files.configure @@ -30,6 +30,12 @@ import shlex import shutil import stat +try: + import jinja2 + jinja_available = True +except ImportError: + jinja_available = False + class InstallFilesConfigureExtension(cliapp.Application): def process_args(self, args): @@ -48,18 +54,20 @@ class InstallFilesConfigureExtension(cliapp.Application): self.install_entry(entry, manifest_dir, target_root) def install_entry(self, entry, manifest_root, target_root): - m = re.match('(overwrite )?([0-7]+) ([0-9]+) ([0-9]+) (\S+)', entry) + m = re.match('(template )?(overwrite )?' + '([0-7]+) ([0-9]+) ([0-9]+) (\S+)', entry) if m: - overwrite = m.group(1) - mode = int(m.group(2), 8) # mode is octal - uid = int(m.group(3)) - gid = int(m.group(4)) - path = m.group(5) + template = m.group(1) + overwrite = m.group(2) + mode = int(m.group(3), 8) # mode is octal + uid = int(m.group(4)) + gid = int(m.group(5)) + path = m.group(6) else: raise cliapp.AppException('Invalid manifest entry, ' - 'format: [overwrite] <octal mode> <uid decimal> <gid decimal> ' - '<filename>') + 'format: [template] [overwrite] ' + '<octal mode> <uid decimal> <gid decimal> <filename>') dest_path = os.path.join(target_root, './' + path) if stat.S_ISDIR(mode): @@ -91,8 +99,22 @@ class InstallFilesConfigureExtension(cliapp.Application): raise cliapp.AppException('File already exists at %s' % dest_path) else: - shutil.copyfile(os.path.join(manifest_root, './' + path), - dest_path) + if template: + if not jinja_available: + raise cliapp.AppException( + "Failed to install template file `%s': " + 'install-files templates require jinja2' + % path) + + loader = jinja2.FileSystemLoader(manifest_root) + env = jinja2.Environment(loader=loader, + keep_trailing_newline=True) + + env.get_template(path).stream(os.environ).dump(dest_path) + else: + shutil.copyfile(os.path.join(manifest_root, './' + path), + dest_path) + os.chown(dest_path, uid, gid) os.chmod(dest_path, mode) diff --git a/morphlib/exts/kvm.check b/morphlib/exts/kvm.check index 62d76453..67cb3d38 100755 --- a/morphlib/exts/kvm.check +++ b/morphlib/exts/kvm.check @@ -47,6 +47,7 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): self.check_no_existing_libvirt_vm(ssh_host, vm_name) self.check_extra_disks_exist(ssh_host, self.parse_attach_disks()) self.check_virtual_networks_are_started(ssh_host) + self.check_host_has_virtinstall(ssh_host) def check_and_parse_location(self, location): '''Check and parse the location argument to get relevant data.''' @@ -129,14 +130,22 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): def name(nic_entry): if ',' in nic_entry: - # NETWORK_NAME,mac=12:34,model=e1000... - return nic_entry[:nic_entry.find(',')] + # network=NETWORK_NAME,mac=12:34,model=e1000... + return nic_entry[:nic_entry.find(',')].lstrip('network=') else: - return nic_entry # NETWORK_NAME + return nic_entry.lstrip('network=') # NETWORK_NAME if 'NIC_CONFIG' in os.environ: nics = os.environ['NIC_CONFIG'].split() + for n in nics: + if not (n.startswith('network=') + or n.startswith('bridge=') + or n == 'user'): + raise cliapp.AppException('malformed NIC_CONFIG: %s\n' + " (expected 'bridge=BRIDGE' 'network=NAME'" + " or 'user')" % n) + # --network bridge= is used to specify a bridge # --network user is used to specify a form of NAT # (see the virt-install(1) man page) @@ -148,5 +157,13 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): for network in networks: check_virtual_network_is_started(network) + def check_host_has_virtinstall(self, ssh_host): + try: + cliapp.ssh_runcmd(ssh_host, ['which', 'virt-install']) + except cliapp.AppException: + raise cliapp.AppException( + 'virt-install does not seem to be installed on host %s' + % ssh_host) + KvmPlusSshCheckExtension().run() diff --git a/morphlib/exts/simple-network.configure b/morphlib/exts/simple-network.configure index 61113325..1ba94e86 100755 --- a/morphlib/exts/simple-network.configure +++ b/morphlib/exts/simple-network.configure @@ -27,6 +27,7 @@ for DHCP import os import sys +import errno import cliapp import morphlib @@ -80,12 +81,14 @@ class SimpleNetworkConfigurationExtension(cliapp.Application): """ file_path = os.path.join(args[0], "etc", "systemd", "network", "10-dhcp.network") - try: - os.rename(file_path, file_path + ".morph") - self.status(msg="Renaming networkd file from systemd chunk: %(f)s \ - to %(f)s.morph", f=file_path) - except OSError: - pass + + if os.path.isfile(file_path): + try: + os.rename(file_path, file_path + ".morph") + self.status(msg="Renaming networkd file from systemd chunk: \ + %(f)s to %(f)s.morph", f=file_path) + except OSError: + pass def generate_default_network_config(self, args): """Generate default network config: DHCP in all the interfaces""" @@ -106,7 +109,11 @@ class SimpleNetworkConfigurationExtension(cliapp.Application): """Generate /etc/network/interfaces file""" iface_file = self.generate_iface_file(stanzas) - with open(os.path.join(args[0], "etc/network/interfaces"), "w") as f: + + directory_path = os.path.join(args[0], "etc", "network") + self.make_sure_path_exists(directory_path) + file_path = os.path.join(directory_path, "interfaces") + with open(file_path, "w") as f: f.write(iface_file) def generate_iface_file(self, stanzas): @@ -147,10 +154,12 @@ class SimpleNetworkConfigurationExtension(cliapp.Application): if iface_file is None: continue - path = os.path.join(args[0], "etc", "systemd", "network", - "%s-%s.network" % (i, stanza['name'])) + directory_path = os.path.join(args[0], "etc", "systemd", "network") + self.make_sure_path_exists(directory_path) + file_path = os.path.join(directory_path, + "%s-%s.network" % (i, stanza['name'])) - with open(path, "w") as f: + with open(file_path, "w") as f: f.write(iface_file) def generate_networkd_file(self, stanza): @@ -252,6 +261,16 @@ class SimpleNetworkConfigurationExtension(cliapp.Application): return output_stanza + def make_sure_path_exists(self, path): + try: + os.makedirs(path) + except OSError as e: + if e.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise SimpleNetworkError("Unable to create directory '%s'" + % path) + def status(self, **kwargs): '''Provide status output. diff --git a/morphlib/gitdir.py b/morphlib/gitdir.py index 03640a22..1c286720 100644 --- a/morphlib/gitdir.py +++ b/morphlib/gitdir.py @@ -681,6 +681,32 @@ class GitDirectory(object): if not morphlib.git.is_valid_sha1(string): raise ExpectedSha1Error(string) + def _check_ref_exists(self, ref): + self._rev_parse('%s^{commit}' % ref) + + def _gitcmd_output_list(self, *args): + output = morphlib.git.gitcmd(self._runcmd, *args) + separated = [l.strip() for l in output.splitlines()] + prefix = '* ' + for i, l in enumerate(separated): + if l.startswith(prefix): + separated[i] = l[len(prefix):] + return separated + + def tags_containing_sha1(self, ref): # pragma: no cover + self._check_is_sha1(ref) + self._check_ref_exists(ref) + + args = ['tag', '--contains', ref] + return self._gitcmd_output_list(*args) + + def branches_containing_sha1(self, ref): + self._check_is_sha1(ref) + self._check_ref_exists(ref) + + args = ['branch', '--contains', ref] + return self._gitcmd_output_list(*args) + def _update_ref(self, ref_args, message): args = ['update-ref'] # No test coverage, since while this functionality is useful, diff --git a/morphlib/gitdir_tests.py b/morphlib/gitdir_tests.py index a6e1921d..f606dfe7 100644 --- a/morphlib/gitdir_tests.py +++ b/morphlib/gitdir_tests.py @@ -95,6 +95,38 @@ class GitDirectoryTests(unittest.TestCase): self.assertIsInstance(gitdir.get_index(), morphlib.gitindex.GitIndex) +class GitDirectoryAnchoredRefTests(unittest.TestCase): + + def setUp(self): + self.tempdir = tempfile.mkdtemp() + self.dirname = os.path.join(self.tempdir, 'foo') + os.mkdir(self.dirname) + gd = morphlib.gitdir.init(self.dirname) + with open(os.path.join(self.dirname, 'test_file.morph'), "w") as f: + f.write('dummy morphology text') + morphlib.git.gitcmd(gd._runcmd, 'add', '.') + morphlib.git.gitcmd(gd._runcmd, 'commit', '-m', 'Initial commit') + + def tearDown(self): + shutil.rmtree(self.tempdir) + + def test_ref_anchored_in_branch(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + output = morphlib.git.gitcmd(gd._runcmd, 'rev-parse', 'HEAD') + ref = output.strip() + + self.assertEqual(len(gd.branches_containing_sha1(ref)), 1) + self.assertEqual(gd.branches_containing_sha1(ref)[0], 'master') + + def test_ref_not_anchored_in_branch(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + output = morphlib.git.gitcmd(gd._runcmd, 'rev-parse', 'HEAD') + ref = output.strip() + + morphlib.git.gitcmd(gd._runcmd, 'commit', '--amend', '-m', + 'New commit message') + self.assertEqual(len(gd.branches_containing_sha1(ref)), 0) + class GitDirectoryContentsTests(unittest.TestCase): def setUp(self): diff --git a/morphlib/localartifactcache.py b/morphlib/localartifactcache.py index e6695c4e..9e3c9e70 100644 --- a/morphlib/localartifactcache.py +++ b/morphlib/localartifactcache.py @@ -132,8 +132,9 @@ class LocalArtifactCache(object): CacheInfo = collections.namedtuple('CacheInfo', ('artifacts', 'mtime')) contents = collections.defaultdict(lambda: CacheInfo(set(), 0)) for filename in self.cachefs.walkfiles(): - cachekey = filename[:63] - artifact = filename[65:] + if filename.startswith('/repo'): # pragma: no cover + continue + cachekey, artifact = filename.split('.', 1) artifacts, max_mtime = contents[cachekey] artifacts.add(artifact) art_info = self.cachefs.getinfo(filename) diff --git a/morphlib/ostreeartifactcache.py b/morphlib/ostreeartifactcache.py index ea659c8f..230460f8 100644 --- a/morphlib/ostreeartifactcache.py +++ b/morphlib/ostreeartifactcache.py @@ -15,8 +15,10 @@ import collections +import contextlib import logging import os +import stat import shutil import tarfile import tempfile @@ -27,26 +29,48 @@ from gi.repository import GLib import morphlib from morphlib.artifactcachereference import ArtifactCacheReference + +class NotCachedError(morphlib.Error): + + def __init__(self, ref): + self.msg = 'Failed to checkout %s from the artifact cache.' % ref + + class OSTreeArtifactCache(object): """Class to provide the artifact cache API using an OSTree repo.""" - def __init__(self, cachedir, mode): + def __init__(self, cachedir, mode='bare', status_cb=None): repo_dir = os.path.join(cachedir, 'repo') self.repo = morphlib.ostree.OSTreeRepo(repo_dir, mode=mode) self.cachedir = cachedir + self.status_cb = status_cb + + def status(self, *args, **kwargs): + if self.status_cb is not None: + self.status_cb(*args, **kwargs) + @contextlib.contextmanager def _get_file_from_remote(self, artifact, remote, metadata_name=None): if metadata_name: handle = remote.get_artifact_metadata(artifact, metadata_name) + self.status( + msg='Downloading %(name)s %(metadata_name)s as a file.', + chatty=True, name=artifact.basename(), + metadata_name=metadata_name) else: handle = remote.get(artifact) - fd, path = tempfile.mkstemp() - with open(path, 'w+') as temp: - shutil.copyfileobj(handle, temp) - return path + self.status( + msg='Downloading %(name)s as a tarball.', chatty=True, + name=artifact.basename()) + + try: + temporary_download = tempfile.NamedTemporaryFile(dir=self.cachedir) + shutil.copyfileobj(handle, temporary_download) + yield temporary_download.name + finally: + temporary_download.close() def _get_artifact_cache_name(self, artifact): - logging.debug('LAC: %s' % artifact.basename()) cache_key, kind, name = artifact.basename().split('.', 2) suffix = name.split('-')[-1] return '%s-%s' % (cache_key, suffix) @@ -58,11 +82,13 @@ class OSTreeArtifactCache(object): contents of directory should be the contents of the artifact. """ + cache_key, kind, name = artifact.basename().split('.', 2) ref = self._get_artifact_cache_name(artifact) - subject = artifact.name + subject = name try: - logging.debug('Committing %s to artifact cache at %s.' % - (subject, ref)) + self.status( + msg='Committing %(subject)s to artifact cache at %(ref)s.', + chatty=True, subject=subject, ref=ref) self.repo.commit(subject, directory, ref) except GLib.GError as e: logging.debug('OSTree raised an exception: %s' % e) @@ -77,44 +103,71 @@ class OSTreeArtifactCache(object): else: filename = self.artifact_filename(artifact) shutil.copy(location, filename) - os.remove(location) + + def _remove_device_nodes(self, path): + for dirpath, dirnames, filenames in os.walk(path): + for f in filenames: + filepath = os.path.join(dirpath, f) + mode = os.lstat(filepath).st_mode + if stat.S_ISBLK(mode) or stat.S_ISCHR(mode): + logging.debug('Removing device node %s from artifact' % + filepath) + os.remove(filepath) + + def _copy_metadata_from_remote(self, artifact, remote): + """Copy a metadata file from a remote cache.""" + a, name = artifact.basename().split('.', 1) + with self._get_file_from_remote(ArtifactCacheReference(a), + remote, name) as location: + self.put_non_ostree_artifact(ArtifactCacheReference(a), + location, name) def copy_from_remote(self, artifact, remote): - """Get 'artifact' from remote artifact cache and store it locally.""" + """Get 'artifact' from remote artifact cache and store it locally. + + This takes an Artifact object and a RemoteArtifactCache. Note that + `remote` here is not the same as a `remote` for and OSTree repo. + + """ if remote.method == 'tarball': - logging.debug('Downloading artifact tarball for %s.' % - artifact.name) - location = self._get_file_from_remote(artifact, remote) - try: - tempdir = tempfile.mkdtemp() - with tarfile.open(name=location) as tf: - tf.extractall(path=tempdir) + with self._get_file_from_remote(artifact, remote) as location: try: + cache_key, kind, name = artifact.basename().split('.', 2) + except ValueError: + # We can't split the name properly, it must be metadata! + self._copy_metadata_from_remote(artifact, remote) + return + + if kind == 'stratum': + self.put_non_ostree_artifact(artifact, location) + return + try: + tempdir = tempfile.mkdtemp(dir=self.cachedir) + with tarfile.open(name=location) as tf: + tf.extractall(path=tempdir) + self._remove_device_nodes(tempdir) self.put(tempdir, artifact) + except tarfile.ReadError: + # Reading the tarball failed, and we expected a + # tarball artifact. Something must have gone + # wrong. + raise finally: - os.remove(location) shutil.rmtree(tempdir) - except tarfile.ReadError: - # Reading the artifact as a tarball failed, so it must be a - # single file (for example a stratum artifact). - self.put_non_ostree_artifact(artifact, location) elif remote.method == 'ostree': - logging.debug('Pulling artifact for %s from remote.' % - artifact.basename()) + self.status(msg='Pulling artifact for %(name)s from remote.', + chatty=True, name=artifact.basename()) try: ref = self._get_artifact_cache_name(artifact) - except Exception: + except ValueError: # if we can't split the name properly, we must want metadata - a, name = artifact.basename().split('.', 1) - location = self._get_file_from_remote( - ArtifactCacheReference(a), remote, name) - self.put_non_ostree_artifact(artifact, location, name) + self._copy_metadata_from_remote(artifact, remote) return if artifact.basename().split('.', 2)[1] == 'stratum': - location = self._get_file_from_remote(artifact, remote) - self.put_non_ostree_artifact(artifact, location) + with self._get_file_from_remote(artifact, remote) as location: + self.put_non_ostree_artifact(artifact, location) return try: @@ -126,7 +179,7 @@ class OSTreeArtifactCache(object): raise cliapp.AppException('Failed to pull %s from remote ' 'cache.' % ref) - def get(self, artifact, directory=None, status=lambda a: a): + def get(self, artifact, directory=None): """Checkout an artifact from the repo and return its location.""" cache_key, kind, name = artifact.basename().split('.', 2) if kind == 'stratum': @@ -136,11 +189,13 @@ class OSTreeArtifactCache(object): ref = self._get_artifact_cache_name(artifact) try: self.repo.checkout(ref, directory) + # We need to update the mtime and atime of the ref file in the + # repository so that we can decide which refs were least recently + # accessed when doing `morph gc`. self.repo.touch_ref(ref) except GLib.GError as e: logging.debug('OSTree raised an exception: %s' % e) - raise cliapp.AppException('Failed to checkout %s from artifact ' - 'cache.' % ref) + raise NotCachedError(ref) return directory def list_contents(self): @@ -173,16 +228,26 @@ class OSTreeArtifactCache(object): self.repo.prune() def has(self, artifact): - cachekey, kind, name = artifact.basename().split('.', 2) - logging.debug('OSTreeArtifactCache: got %s, %s, %s' % - (cachekey, kind, name)) + try: + cachekey, kind, name = artifact.basename().split('.', 2) + except ValueError: + # We couldn't split the basename properly, we must want metadata + cachekey, name = artifact.basename().split('.', 1) + if self.has_artifact_metadata(artifact, name): + return True + else: + return False + + if kind == 'stratum': + if self._has_file(self.artifact_filename(artifact)): + return True + else: + return False + sha = self.repo.resolve_rev(self._get_artifact_cache_name(artifact)) if sha: self.repo.touch_ref(self._get_artifact_cache_name(artifact)) return True - if kind == 'stratum' and \ - self._has_file(self.artifact_filename(artifact)): - return True return False def get_artifact_metadata(self, artifact, name): diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py index cdbb303f..08589ea6 100644 --- a/morphlib/plugins/branch_and_merge_plugin.py +++ b/morphlib/plugins/branch_and_merge_plugin.py @@ -623,7 +623,6 @@ class BranchAndMergePlugin(cliapp.Plugin): smd = morphlib.systemmetadatadir.SystemMetadataDir(path) metadata = smd.values() - logging.debug(metadata) systems = [md for md in metadata if 'kind' in md and md['kind'] == 'system'] diff --git a/morphlib/plugins/build_plugin.py b/morphlib/plugins/build_plugin.py index 2cc395fc..e5b35853 100644 --- a/morphlib/plugins/build_plugin.py +++ b/morphlib/plugins/build_plugin.py @@ -13,26 +13,39 @@ # with this program. If not, see <http://www.gnu.org/licenses/>. -import cliapp +import collections import contextlib import uuid import logging +import cliapp + import morphlib +class ComponentNotInSystemError(morphlib.Error): + + def __init__(self, components, system): + components = ', '.join(components) + self.msg = ('Components %s are not in %s. Ensure you provided ' + 'component names rather than filenames.' + % (components, system)) + + class BuildPlugin(cliapp.Plugin): def enable(self): self.app.add_subcommand('build-morphology', self.build_morphology, - arg_synopsis='(REPO REF FILENAME)...') + arg_synopsis='REPO REF FILENAME ' + '[COMPONENT...]') self.app.add_subcommand('build', self.build, - arg_synopsis='SYSTEM') + arg_synopsis='SYSTEM [COMPONENT...]') self.app.add_subcommand('distbuild-morphology', self.distbuild_morphology, - arg_synopsis='SYSTEM') + arg_synopsis='REPO REF FILENAME ' + '[COMPONENT...]') self.app.add_subcommand('distbuild', self.distbuild, - arg_synopsis='SYSTEM') + arg_synopsis='SYSTEM [COMPONENT...]') self.use_distbuild = False def disable(self): @@ -46,6 +59,8 @@ class BuildPlugin(cliapp.Plugin): * `REPO` is a git repository URL. * `REF` is a branch or other commit reference in that repository. * `FILENAME` is a morphology filename at that ref. + * `COMPONENT...` is the names of one or more chunks or strata to + build. If none are given the the system at FILENAME is built. See 'help distbuild' and 'help build-morphology' for more information. @@ -54,10 +69,15 @@ class BuildPlugin(cliapp.Plugin): addr = self.app.settings['controller-initiator-address'] port = self.app.settings['controller-initiator-port'] + self.use_distbuild = True build_command = morphlib.buildcommand.InitiatorBuildCommand( self.app, addr, port) - for repo_name, ref, filename in self.app.itertriplets(args): - build_command.build(repo_name, ref, filename) + repo, ref, filename = args[0:3] + filename = morphlib.util.sanitise_morphology_path(filename) + component_names = [morphlib.util.sanitise_morphology_path(name) + for name in args[3:]] + self.start_build(repo, ref, build_command, filename, + component_names) def distbuild(self, args): '''Distbuild a system image in the current system branch @@ -65,6 +85,8 @@ class BuildPlugin(cliapp.Plugin): Command line arguments: * `SYSTEM` is the name of the system to build. + * `COMPONENT...` is the names of one or more chunks or strata to + build. If none are given then SYSTEM is built. This command launches a distributed build, to use this command you must first set up a distbuild cluster. @@ -92,6 +114,8 @@ class BuildPlugin(cliapp.Plugin): * `REPO` is a git repository URL. * `REF` is a branch or other commit reference in that repository. * `FILENAME` is a morphology filename at that ref. + * `COMPONENT...` is the names of one or more chunks or strata to + build. If none are given then the system at FILENAME is built. You probably want `morph build` instead. However, in some cases it is more convenient to not have to create a Morph @@ -104,8 +128,14 @@ class BuildPlugin(cliapp.Plugin): Example: - morph build-morphology baserock:baserock/definitions \ - master devel-system-x86_64-generic.morph + morph build-morphology baserock:baserock/definitions \\ + master systems/devel-system-x86_64-generic.morph + + Partial build example: + + morph build-morphology baserock:baserock/definitions \\ + master systems/devel-system-x86_64-generic.morph \\ + build-essential ''' @@ -117,15 +147,21 @@ class BuildPlugin(cliapp.Plugin): self.app.settings['cachedir-min-space']) build_command = morphlib.buildcommand.BuildCommand(self.app) - for repo_name, ref, filename in self.app.itertriplets(args): - build_command.build(repo_name, ref, filename) + repo, ref, filename = args[0:3] + filename = morphlib.util.sanitise_morphology_path(filename) + component_names = [morphlib.util.sanitise_morphology_path(name) + for name in args[3:]] + self.start_build(repo, ref, build_command, filename, + component_names) def build(self, args): '''Build a system image in the current system branch Command line arguments: - * `SYSTEM` is the name of the system to build. + * `SYSTEM` is the filename of the system to build. + * `COMPONENT...` is the names of one or more chunks or strata to + build. If this is not given then the SYSTEM is built. This builds a system image, and any of its components that need building. The system name is the basename of the system @@ -145,14 +181,14 @@ class BuildPlugin(cliapp.Plugin): Example: - morph build devel-system-x86_64-generic.morph + morph build systems/devel-system-x86_64-generic.morph - ''' + Partial build example: - if len(args) != 1: - raise cliapp.AppException('morph build expects exactly one ' - 'parameter: the system to build') + morph build systems/devel-system-x86_64-generic.morph \\ + build-essential + ''' # Raise an exception if there is not enough space morphlib.util.check_disk_available( self.app.settings['tempdir'], @@ -165,6 +201,7 @@ class BuildPlugin(cliapp.Plugin): system_filename = morphlib.util.sanitise_morphology_path(args[0]) system_filename = sb.relative_to_root_repo(system_filename) + component_names = args[1:] logging.debug('System branch is %s' % sb.root_directory) @@ -178,11 +215,14 @@ class BuildPlugin(cliapp.Plugin): build_command = morphlib.buildcommand.BuildCommand(self.app) if self.app.settings['local-changes'] == 'include': - self._build_with_local_changes(build_command, sb, system_filename) + self._build_with_local_changes(build_command, sb, system_filename, + component_names) else: - self._build_local_commit(build_command, sb, system_filename) + self._build_local_commit(build_command, sb, system_filename, + component_names) - def _build_with_local_changes(self, build_command, sb, system_filename): + def _build_with_local_changes(self, build_command, sb, system_filename, + component_names): '''Construct a branch including user's local changes, and build that. It is often a slow process to check all repos in the system branch for @@ -199,9 +239,12 @@ class BuildPlugin(cliapp.Plugin): email = morphlib.git.get_user_email(self.app.runcmd) build_ref_prefix = self.app.settings['build-ref-prefix'] - self.app.status(msg='Starting build %(uuid)s', uuid=build_uuid) + self.app.status(msg='Looking for uncommitted changes (pass ' + '--local-changes=ignore to skip)') + self.app.status(msg='Collecting morphologies involved in ' 'building %(system)s from %(branch)s', + chatty=True, system=system_filename, branch=sb.system_branch_name) @@ -211,10 +254,11 @@ class BuildPlugin(cliapp.Plugin): name=name, email=email, build_uuid=build_uuid, status=self.app.status) with pbb as (repo, commit, original_ref): - build_command.build(repo, commit, system_filename, - original_ref=original_ref) + self.start_build(repo, commit, build_command, system_filename, + component_names, original_ref=original_ref) - def _build_local_commit(self, build_command, sb, system_filename): + def _build_local_commit(self, build_command, sb, system_filename, + component_names): '''Build whatever commit the user has checked-out locally. This ignores any uncommitted changes. Also, if the user has a commit @@ -242,4 +286,47 @@ class BuildPlugin(cliapp.Plugin): definitions_repo = morphlib.gitdir.GitDirectory(definitions_repo_path) commit = definitions_repo.resolve_ref_to_commit(ref) - build_command.build(root_repo_url, commit, system_filename) + self.start_build(root_repo_url, commit, build_command, + system_filename, component_names) + + def _find_artifacts(self, names, root_artifact): + found = collections.OrderedDict() + not_found = names + for a in root_artifact.walk(): + name = a.source.morphology['name'] + if name in names and name not in found: + found[name] = a + not_found.remove(name) + return found, not_found + + def start_build(self, repo, commit, bc, system_filename, + component_names, original_ref=None): + '''Actually run the build. + + If a set of components was given, only build those. Otherwise, + build the whole system. + + ''' + if self.use_distbuild: + bc.build(repo, commit, system_filename, + original_ref=original_ref, + component_names=component_names) + return + + self.app.status(msg='Deciding on task order') + srcpool = bc.create_source_pool(repo, commit, system_filename) + bc.validate_sources(srcpool) + root = bc.resolve_artifacts(srcpool) + if not component_names: + component_names = [root.source.name] + components, not_found = self._find_artifacts(component_names, root) + if not_found: + raise ComponentNotInSystemError(not_found, system_filename) + + for name, component in components.iteritems(): + component.build_env = root.build_env + bc.build_in_order(component) + self.app.status(msg='%(kind)s %(name)s is cached at %(path)s', + kind=component.source.morphology['kind'], + name=name, + path=bc.lac.artifact_filename(component)) diff --git a/morphlib/plugins/certify_plugin.py b/morphlib/plugins/certify_plugin.py new file mode 100644 index 00000000..10fc19ad --- /dev/null +++ b/morphlib/plugins/certify_plugin.py @@ -0,0 +1,140 @@ +# Copyright (C) 2014-2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# This plugin is used as part of the Baserock automated release process. +# +# See: <http://wiki.baserock.org/guides/release-process> for more information. + +import warnings + +import cliapp +import morphlib + +class CertifyPlugin(cliapp.Plugin): + + def enable(self): + self.app.add_subcommand( + 'certify', self.certify, + arg_synopsis='REPO REF MORPH [MORPH]...') + + def disable(self): + pass + + def certify(self, args): + '''Certify that any given system definition is reproducable. + + Command line arguments: + + * `REPO` is a git repository URL. + * `REF` is a branch or other commit reference in that repository. + * `MORPH` is a system morphology name at that ref. + + ''' + + if len(args) < 3: + raise cliapp.AppException( + 'Wrong number of arguments to certify command ' + '(see help)') + + repo, ref = args[0], args[1] + system_filenames = map(morphlib.util.sanitise_morphology_path, + args[2:]) + + self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app) + self.resolver = morphlib.artifactresolver.ArtifactResolver() + + for system_filename in system_filenames: + self.certify_system(repo, ref, system_filename) + + def certify_system(self, repo, ref, system_filename): + '''Certify reproducibility of system.''' + + self.app.status( + msg='Creating source pool for %s' % system_filename, chatty=True) + source_pool = morphlib.sourceresolver.create_source_pool( + self.lrc, self.rrc, repo, ref, system_filename, + cachedir=self.app.settings['cachedir'], + update_repos = not self.app.settings['no-git-update'], + status_cb=self.app.status) + + self.app.status( + msg='Resolving artifacts for %s' % system_filename, chatty=True) + root_artifacts = self.resolver.resolve_root_artifacts(source_pool) + + def find_artifact_by_name(artifacts_list, filename): + for a in artifacts_list: + if a.source.filename == filename: + return a + raise ValueError + + system_artifact = find_artifact_by_name(root_artifacts, + system_filename) + + self.app.status( + msg='Computing cache keys for %s' % system_filename, chatty=True) + build_env = morphlib.buildenvironment.BuildEnvironment( + self.app.settings, system_artifact.source.morphology['arch']) + ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env) + + aliases = self.app.settings['repo-alias'] + resolver = morphlib.repoaliasresolver.RepoAliasResolver(aliases) + + certified = True + + for source in set(a.source for a in system_artifact.walk()): + source.cache_key = ckc.compute_key(source) + source.cache_id = ckc.get_cache_id(source) + + if source.morphology['kind'] != 'chunk': + continue + + name = source.morphology['name'] + ref = source.original_ref + + # Test that chunk has a sha1 ref + # TODO: Could allow either sha1 or existent tag. + if not morphlib.git.is_valid_sha1(ref): + warnings.warn('Chunk "{}" has non-sha1 ref: "{}"\n' + .format(name, ref)) + certified = False + + # Ensure we have a cache of the repo + if not self.lrc.has_repo(source.repo_name): + self.lrc.cache_repo(source.repo_name) + + cached = self.lrc.get_repo(source.repo_name) + + # Test that sha1 ref is anchored in a tag or branch, + # and thus not a candidate for removal on `git gc`. + if (morphlib.git.is_valid_sha1(ref) and + not len(cached.tags_containing_sha1(ref)) and + not len(cached.branches_containing_sha1(ref))): + warnings.warn('Chunk "{}" has unanchored ref: "{}"\n' + .format(name, ref)) + certified = False + + # Test that chunk repo is on trove-host + pull_url = resolver.pull_url(source.repo_name) + if self.app.settings['trove-host'] not in pull_url: + warnings.warn('Chunk "{}" has repo not on trove-host: "{}"\n' + .format(name, pull_url)) + certified = False + + if certified: + print('=> Reproducibility certification PASSED for\n {}' + .format(system_filename)) + else: + print('=> Reproducibility certification FAILED for\n {}' + .format(system_filename)) diff --git a/morphlib/plugins/cross-bootstrap_plugin.py b/morphlib/plugins/cross-bootstrap_plugin.py index 79609cb5..9bec5646 100644 --- a/morphlib/plugins/cross-bootstrap_plugin.py +++ b/morphlib/plugins/cross-bootstrap_plugin.py @@ -27,7 +27,7 @@ echo "Generated by Morph version %s\n" set -eu -export PATH=$PATH:/tools/bin:/tools/sbin +export PATH=/usr/bin:/bin:/usr/sbin:/sbin:/tools/bin:/tools/sbin export SRCDIR=/src ''' % morphlib.__version__ diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index efe6735b..3c19553e 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -13,6 +13,7 @@ # with this program. If not, see <http://www.gnu.org/licenses/>. +import collections import json import logging import os @@ -27,6 +28,14 @@ import morphlib from morphlib.artifactcachereference import ArtifactCacheReference +class NotYetBuiltError(morphlib.Error): + + def __init__(self, name): + self.msg = ('Deployment failed as %s is not yet built.\n' + 'Please ensure the system is built before deployment.' + % name) + + class DeployPlugin(cliapp.Plugin): def enable(self): @@ -385,7 +394,7 @@ class DeployPlugin(cliapp.Plugin): name=name, email=email, build_uuid=build_uuid, status=self.app.status) with pbb as (repo, commit, original_ref): - self.deploy_cluster(build_command, cluster_morphology, + self.deploy_cluster(sb, build_command, cluster_morphology, root_repo_dir, repo, commit, env_vars, deployments) else: @@ -398,6 +407,11 @@ class DeployPlugin(cliapp.Plugin): deployments) self.app.status(msg='Finished deployment') + if self.app.settings['partial']: + self.app.status(msg='WARNING: This was a partial deployment. ' + 'Configuration extensions have not been ' + 'run. Applying the result to an existing ' + 'system may not have reproducible results.') def validate_deployment_options( self, env_vars, all_deployments, all_subsystems): @@ -415,21 +429,54 @@ class DeployPlugin(cliapp.Plugin): 'Variable referenced a non-existent deployment ' 'name: %s' % var) - def deploy_cluster(self, build_command, cluster_morphology, root_repo_dir, - repo, commit, env_vars, deployments): + def deploy_cluster(self, sb, build_command, cluster_morphology, + root_repo_dir, repo, commit, env_vars, deployments): # Create a tempdir for this deployment to work in deploy_tempdir = tempfile.mkdtemp( dir=os.path.join(self.app.settings['tempdir'], 'deployments')) try: for system in cluster_morphology['systems']: - self.deploy_system(build_command, deploy_tempdir, + self.deploy_system(sb, build_command, deploy_tempdir, root_repo_dir, repo, commit, system, env_vars, deployments, parent_location='') finally: shutil.rmtree(deploy_tempdir) - def deploy_system(self, build_command, deploy_tempdir, + def _sanitise_morphology_paths(self, paths, sb): + sanitised_paths = [] + for path in paths: + path = morphlib.util.sanitise_morphology_path(path) + sanitised_paths.append(sb.relative_to_root_repo(path)) + return sanitised_paths + + def _find_artifacts(self, filenames, root_artifact): + found = collections.OrderedDict() + not_found = filenames + for a in root_artifact.walk(): + if a.source.filename in filenames and a.source.name not in found: + found[a.source.name] = a + not_found.remove(a.source.filename) + return found, not_found + + def _validate_partial_deployment(self, deployment_type, + artifact, component_names): + supported_types = ('tar', 'sysroot') + if deployment_type not in supported_types: + raise cliapp.AppException('Not deploying %s, --partial was ' + 'set and partial deployment only ' + 'supports %s deployments.' % + (artifact.source.name, + ', '.join(supported_types))) + components, not_found = self._find_artifacts(component_names, + artifact) + if not_found: + raise cliapp.AppException('Components %s not found in system %s.' % + (', '.join(not_found), + artifact.source.name)) + return components + + def deploy_system(self, sb, build_command, deploy_tempdir, root_repo_dir, build_repo, ref, system, env_vars, deployment_filter, parent_location): sys_ids = set(system['deploy'].iterkeys()) @@ -475,6 +522,12 @@ class DeployPlugin(cliapp.Plugin): raise morphlib.Error('"type" is undefined ' 'for system "%s"' % system_id) + components = self._sanitise_morphology_paths( + deploy_params.get('partial-deploy-components', []), sb) + if self.app.settings['partial']: + components = self._validate_partial_deployment( + deployment_type, artifact, components) + location = final_env.pop('location', None) if not location: raise morphlib.Error('"location" is undefined ' @@ -488,9 +541,10 @@ class DeployPlugin(cliapp.Plugin): root_repo_dir, ref, artifact, deployment_type, - location, final_env) + location, final_env, + components=components) for subsystem in system.get('subsystems', []): - self.deploy_system(build_command, deploy_tempdir, + self.deploy_system(sb, build_command, deploy_tempdir, root_repo_dir, build_repo, ref, subsystem, env_vars, [], parent_location=system_tree) @@ -542,13 +596,27 @@ class DeployPlugin(cliapp.Plugin): pass def checkout_stratum(self, path, artifact, lac, rac): + """Pull the chunks in a stratum, and checkout them into `path`. + + This reads a stratum artifact and pulls the chunks it contains from + the remote into the local artifact cache if they are not already + cached locally. Each of these chunks is then checked out into `path`. + + Also download the stratum metadata into the local cache, then place + it in the /baserock directory of the system checkout indicated by + `path`. + + If any of the chunks have not been cached either locally or remotely, + a morphlib.remoteartifactcache.GetError is raised. + + """ with open(lac.get(artifact), 'r') as stratum: chunks = [ArtifactCacheReference(c) for c in json.load(stratum)] morphlib.builder.download_depends(chunks, lac, rac) for chunk in chunks: self.app.status(msg='Checkout chunk %(name)s.', name=chunk.basename(), chatty=True) - lac.get(chunk, path, self.app.status) + lac.get(chunk, path) metadata = os.path.join(path, 'baserock', '%s.meta' % artifact.name) with lac.get_artifact_metadata(artifact, 'meta') as meta_src: @@ -556,14 +624,94 @@ class DeployPlugin(cliapp.Plugin): shutil.copyfileobj(meta_src, meta_dst) def checkout_strata(self, path, artifact, lac, rac): + """Pull the dependencies of `artifact` and checkout them into `path`. + + This assumes that `artifact` is a system artifact. If any of the + dependencies aren't cached remotely or locally, this raises a + morphlib.remoteartifactcache.GetError. + + """ deps = artifact.source.dependencies morphlib.builder.download_depends(deps, lac, rac) for stratum in deps: self.checkout_stratum(path, stratum, lac, rac) morphlib.builder.ldconfig(self.app.runcmd, path) + def checkout_system(self, build_command, artifact, path): + """Checkout a system into `path`. + + This checks out each of the strata into the directory given by `path`, + then checks out the system artifact into the same directory. This uses + OSTree's `union` checkout mode to overwrite duplicate files but not + need an empty directory. Artifacts which aren't cached locally are + fetched from the remote cache. + + Raises a NotYetBuiltError if either the system artifact or any of the + chunk artifacts in the strata which make up the system aren't cached + either locally or remotely. + + """ + # Check if the system artifact is in the local or remote cache. + # If it isn't, we don't need to bother checking out strata before + # we fail. + if not (build_command.lac.has(artifact) + or build_command.rac.has(artifact)): + raise NotYetBuiltError(artifact.name) + + # Checkout the strata involved in the artifact into a tempdir + self.app.status(msg='Checking out strata in system') + try: + self.checkout_strata(path, artifact, + build_command.lac, build_command.rac) + + self.app.status(msg='Checking out system for configuration') + build_command.cache_artifacts_locally([artifact]) + build_command.lac.get(artifact, path) + except (morphlib.ostreeartifactcache.NotCachedError, + morphlib.remoteartifactcache.GetError): + raise NotYetBuiltError(artifact.name) + + self.app.status( + msg='System checked out at %(system_tree)s', + system_tree=path) + + def checkout_components(self, bc, components, path): + if not components: + raise cliapp.AppException('Deployment failed as no components ' + 'were specified for deployment and ' + '--partial was set.') + for name, artifact in components.iteritems(): + deps = artifact.source.dependencies + morphlib.builder.download_depends(deps, bc.lac, bc.rac) + for dep in deps: + if dep.source.morphology['kind'] == 'stratum': + self.checkout_stratum(path, dep, bc.lac, bc.rac) + elif dep.source.morphology['kind'] == 'chunk': + self.app.status(msg='Checkout chunk %(name)s.', + name=dep.basename(), chatty=True) + bc.lac.get(dep, path) + if artifact.source.morphology['kind'] == 'stratum': + self.checkout_stratum(path, artifact, bc.lac, bc.rac) + elif artifact.source.morphology['kind'] == 'chunk': + self.app.status(msg='Checkout chunk %(name)s.', + name=name, chatty=True) + bc.lac.get(artifact, path) + self.app.status( + msg='Components %(components)s checkout out at %(path)s', + components=', '.join(components), path=path) + def setup_deploy(self, build_command, deploy_tempdir, root_repo_dir, ref, - artifact, deployment_type, location, env): + artifact, deployment_type, location, env, components=[]): + """Checkout the artifact, create metadata and return the location. + + This checks out the system into a temporary directory, and then mounts + this temporary directory alongside a different temporary directory + using a union filesystem. This allows changes to be made without + touching the checked out artifacts. The deployment metadata file is + created and then the directory at which the two temporary directories + are mounted is returned. + + """ # deployment_type, location and env are only used for saving metadata deployment_dir = tempfile.mkdtemp(dir=deploy_tempdir) @@ -583,26 +731,11 @@ class DeployPlugin(cliapp.Plugin): deploy_tree = os.path.join(deployment_dir, 'overlay-deploy-%s' % artifact.name) try: - # Checkout the strata involved in the artifact into a tempdir - self.app.status(msg='Checking out strata in system') - self.checkout_strata(system_tree, artifact, - build_command.lac, build_command.rac) - - self.app.status(msg='Checking out system for configuration') - if build_command.lac.has(artifact): - build_command.lac.get(artifact, system_tree) - elif build_command.rac.has(artifact): - build_command.cache_artifacts_locally([artifact]) - build_command.lac.get(artifact, system_tree) + if self.app.settings['partial']: + self.checkout_components(build_command, components, + system_tree) else: - raise cliapp.AppException('Deployment failed as system is' - ' not yet built.\nPlease ensure' - ' the system is built before' - ' deployment.') - - self.app.status( - msg='System checked out at %(system_tree)s', - system_tree=system_tree) + self.checkout_system(build_command, artifact, system_tree) union_filesystem = self.app.settings['union-filesystem'] morphlib.fsutils.overlay_mount(self.app.runcmd, @@ -625,10 +758,7 @@ class DeployPlugin(cliapp.Plugin): except Exception: if deploy_tree and os.path.exists(deploy_tree): morphlib.fsutils.unmount(self.app.runcmd, deploy_tree) - shutil.rmtree(deploy_tree) - shutil.rmtree(system_tree) - shutil.rmtree(overlay_dir) - shutil.rmtree(work_dir) + shutil.rmtree(deployment_dir) raise def run_deploy_commands(self, deploy_tempdir, env, artifact, root_repo_dir, @@ -640,15 +770,19 @@ class DeployPlugin(cliapp.Plugin): try: # Run configuration extensions. - self.app.status(msg='Configure system') - names = artifact.source.morphology['configuration-extensions'] - for name in names: - self._run_extension( - root_repo_dir, - name, - '.configure', - [system_tree], - env) + if not self.app.settings['partial']: + self.app.status(msg='Configure system') + names = artifact.source.morphology['configuration-extensions'] + for name in names: + self._run_extension( + root_repo_dir, + name, + '.configure', + [system_tree], + env) + else: + self.app.status(msg='WARNING: Not running configuration ' + 'extensions as --partial is set!') # Run write extension. self.app.status(msg='Writing to device') @@ -665,7 +799,7 @@ class DeployPlugin(cliapp.Plugin): shutil.rmtree(deploy_private_tempdir) def _report_extension_stdout(self, line): - self.app.status(msg=line.replace('%s', '%%')) + self.app.status(msg=line.replace('%', '%%')) def _report_extension_stderr(self, error_list): def cb(line): error_list.append(line) @@ -699,7 +833,7 @@ class DeployPlugin(cliapp.Plugin): raise cliapp.AppException(message) def create_metadata(self, system_artifact, root_repo_dir, deployment_type, - location, env): + location, env, components=[]): '''Deployment-specific metadata. The `build` and `deploy` operations must be from the same ref, so full @@ -731,6 +865,9 @@ class DeployPlugin(cliapp.Plugin): 'commit': morphlib.gitversion.commit, 'version': morphlib.gitversion.version, }, + 'partial': self.app.settings['partial'], } + if self.app.settings['partial']: + meta['partial-components'] = components return meta diff --git a/morphlib/plugins/gc_plugin.py b/morphlib/plugins/gc_plugin.py index 8b5dc4c2..54c1b43e 100644 --- a/morphlib/plugins/gc_plugin.py +++ b/morphlib/plugins/gc_plugin.py @@ -157,10 +157,9 @@ class GCPlugin(cliapp.Plugin): self.app.status(msg='Removing source %(cachekey)s', cachekey=cachekey, chatty=True) lac.remove(cachekey) + lac.prune() removed += 1 - lac.prune() - if sufficient_free(): self.app.status(msg='Made sufficient space in %(cache_path)s ' 'after removing %(removed)d sources', diff --git a/morphlib/plugins/get_chunk_details_plugin.py b/morphlib/plugins/get_chunk_details_plugin.py new file mode 100644 index 00000000..842b4afe --- /dev/null +++ b/morphlib/plugins/get_chunk_details_plugin.py @@ -0,0 +1,79 @@ +# Copyright (C) 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + +import cliapp +import morphlib + +class GetChunkDetailsPlugin(cliapp.Plugin): + + def enable(self): + self.app.add_subcommand( + 'get-chunk-details', self.get_chunk_details, + arg_synopsis='[STRATUM] CHUNK') + + def disable(self): + pass + + def get_chunk_details(self, args): + '''Print out details for the given chunk + + Command line arguments: + + * `STRATUM` is the stratum to search for chunk (optional). + * `CHUNK` is the component to obtain a URL for. + + ''' + + stratum_name = None + + if len(args) == 1: + chunk_name = args[0] + elif len(args) == 2: + stratum_name = args[0] + chunk_name = args[1] + else: + raise cliapp.AppException( + 'Wrong number of arguments to get-chunk-details command ' + '(see help)') + + sb = morphlib.sysbranchdir.open_from_within('.') + loader = morphlib.morphloader.MorphologyLoader() + + aliases = self.app.settings['repo-alias'] + self.resolver = morphlib.repoaliasresolver.RepoAliasResolver(aliases) + + found = 0 + for morph in sb.load_all_morphologies(loader): + if morph['kind'] == 'stratum': + if (stratum_name == None or + morph['name'] == stratum_name): + for chunk in morph['chunks']: + if chunk['name'] == chunk_name: + found = found + 1 + self._print_chunk_details(chunk, morph) + + if found == 0: + if stratum_name == None: + print('Chunk `{}` not found' + .format(chunk_name)) + else: + print('Chunk `{}` not found in stratum `{}`' + .format(chunk_name, stratum_name)) + + def _print_chunk_details(self, chunk, morph): + repo = self.resolver.pull_url(chunk['repo']) + print('In stratum {}:'.format(morph['name'])) + print(' Chunk: {}'.format(chunk['name'])) + print(' Repo: {}'.format(repo)) + print(' Ref: {}'.format(chunk['ref'])) diff --git a/morphlib/plugins/ostree_artifacts_plugin.py b/morphlib/plugins/ostree_artifacts_plugin.py new file mode 100644 index 00000000..eedcd1e7 --- /dev/null +++ b/morphlib/plugins/ostree_artifacts_plugin.py @@ -0,0 +1,169 @@ +# Copyright (C) 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + + +import collections +import fs +import os + +import cliapp + +import morphlib +from morphlib.artifactcachereference import ArtifactCacheReference + + +class NoCacheError(morphlib.Error): + + def __init__(self, cachedir): + self.msg = ("Expected artifact cache directory %s doesn't exist.\n" + "No existing cache to convert!" % cachedir) + + +class ComponentNotInSystemError(morphlib.Error): + + def __init__(self, components, system): + components = ', '.join(components) + self.msg = ('Components %s are not in %s. Ensure you provided ' + 'component names rather than filenames.' + % (components, system)) + + +class OSTreeArtifactsPlugin(cliapp.Plugin): + + def enable(self): + self.app.add_subcommand('convert-local-cache', self.convert_cache, + arg_synopsis='[DELETE]') + self.app.add_subcommand('query-cache', self.query_cache, + arg_synopsis='SYSTEM NAME...') + + def disable(self): + pass + + def convert_cache(self, args): + """Convert a local tarball cache into an OSTree cache. + + Command line arguments: + + * DELETE: This is an optional argument, which if given as "delete" + will cause tarball artifacts to be removed once they are converted. + + This command will extract all the tarball artifacts in your local + artifact cache and store them in an OSTree repository in that + artifact cache. This will be quicker than redownloading all that + content from a remote cache server, but may still be time consuming + if your cache is large. + + """ + delete = False + if args: + if args[0] == 'delete': + delete = True + + artifact_cachedir = os.path.join(self.app.settings['cachedir'], + 'artifacts') + if not os.path.exists(artifact_cachedir): + raise NoCacheError(artifact_cachedir) + + tarball_cache = morphlib.localartifactcache.LocalArtifactCache( + fs.osfs.OSFS(artifact_cachedir)) + ostree_cache = morphlib.ostreeartifactcache.OSTreeArtifactCache( + artifact_cachedir, mode=self.app.settings['ostree-repo-mode'], + status_cb=self.app.status) + + cached_artifacts = [] + for cachekey, artifacts, last_used in tarball_cache.list_contents(): + for artifact in artifacts: + basename = '.'.join((cachekey.lstrip('/'), artifact)) + cached_artifacts.append(ArtifactCacheReference(basename)) + + # Set the method property of the tarball cache to allow us to + # treat it like a RemoteArtifactCache. + tarball_cache.method = 'tarball' + + for artifact in cached_artifacts: + if not ostree_cache.has(artifact): + try: + cache_key, kind, name = artifact.basename().split('.', 2) + if kind in ('system', 'stratum'): + # System artifacts are quick to recreate now, and + # stratum artifacts are still stored in the same way. + continue + except ValueError: + # We must have metadata, which doesn't need converting + continue + self.app.status(msg='Converting %(name)s', + name=artifact.basename()) + ostree_cache.copy_from_remote(artifact, tarball_cache) + if delete: + os.remove(tarball_cache.artifact_filename(artifact)) + + def _find_artifacts(self, names, root_artifact): + found = collections.OrderedDict() + not_found = list(names) + for a in root_artifact.walk(): + name = a.source.morphology['name'] + if name in names and name not in found: + found[name] = [a] + if name in not_found: + not_found.remove(name) + elif name in names: + found[name].append(a) + if name in not_found: + not_found.remove(name) + return found, not_found + + def query_cache(self, args): + """Check if the cache contains an artifact. + + Command line arguments: + + * `SYSTEM` is the filename of the system containing the components + to be looked for. + * `NAME...` is the name of one or more components to look for. + + """ + if not args: + raise cliapp.AppException('You must provide at least a system ' + 'filename.\nUsage: `morph query-cache ' + 'SYSTEM [NAME...]`') + ws = morphlib.workspace.open('.') + sb = morphlib.sysbranchdir.open_from_within('.') + + system_filename = morphlib.util.sanitise_morphology_path(args[0]) + system_filename = sb.relative_to_root_repo(system_filename) + component_names = args[1:] + + bc = morphlib.buildcommand.BuildCommand(self.app) + repo = sb.get_config('branch.root') + ref = sb.get_config('branch.name') + + definitions_repo_path = sb.get_git_directory_name(repo) + definitions_repo = morphlib.gitdir.GitDirectory(definitions_repo_path) + commit = definitions_repo.resolve_ref_to_commit(ref) + + srcpool = bc.create_source_pool(repo, commit, system_filename) + bc.validate_sources(srcpool) + root = bc.resolve_artifacts(srcpool) + if not component_names: + component_names = [root.source.name] + components, not_found = self._find_artifacts(component_names, root) + if not_found: + raise ComponentNotInSystemError(not_found, system_filename) + + for name, artifacts in components.iteritems(): + for component in artifacts: + if bc.lac.has(component): + print bc.lac._get_artifact_cache_name(component) + else: + print '%s is not cached' % name diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py index d2b47d35..771e81e3 100644 --- a/morphlib/sourceresolver.py +++ b/morphlib/sourceresolver.py @@ -31,7 +31,7 @@ tree_cache_filename = 'trees.cache.pickle' buildsystem_cache_size = 10000 buildsystem_cache_filename = 'detected-chunk-buildsystems.cache.pickle' -not_supported_versions = [] +supported_versions = [0, 1, 2] class PickleCacheManager(object): # pragma: no cover '''Cache manager for PyLRU that reads and writes to Pickle files. @@ -90,12 +90,26 @@ class MorphologyNotFoundError(SourceResolverError): # pragma: no cover SourceResolverError.__init__( self, "Couldn't find morphology: %s" % filename) + +class MorphologyReferenceNotFoundError(SourceResolverError): # pragma: no cover + def __init__(self, filename, reference_file): + SourceResolverError.__init__(self, + "Couldn't find morphology: %s " + "referenced in %s" + % (filename, reference_file)) + + class UnknownVersionError(SourceResolverError): # pragma: no cover def __init__(self, version): SourceResolverError.__init__( self, "Definitions format version %s is not supported" % version) +class InvalidVersionFileError(SourceResolverError): #pragma: no cover + def __init__(self): + SourceResolverError.__init__(self, "invalid VERSION file") + + class SourceResolver(object): '''Provides a way of resolving the set of sources for a given system. @@ -286,7 +300,7 @@ class SourceResolver(object): loader = morphlib.morphloader.MorphologyLoader() text = self._get_file_contents(reponame, sha1, filename) - morph = loader.load_from_string(text) + morph = loader.load_from_string(text, filename) if morph is not None: self._resolved_morphologies[key] = morph @@ -346,22 +360,45 @@ class SourceResolver(object): loader.set_defaults(morph) return morph - def _check_version_file(self,definitions_repo, + def _parse_version_file(self, version_file): # pragma : no cover + '''Parse VERSION file and return the version of the format if: + + VERSION is a YAML file + and it's a dict + and has the key 'version' + and the type stored in the 'version' key is an int + + otherwise returns None + + ''' + + yaml_obj = yaml.safe_load(version_file) + + return (yaml_obj['version'] if yaml_obj is not None + and isinstance(yaml_obj, dict) + and 'version' in yaml_obj + and isinstance(yaml_obj['version'], int) + + else None) + + def _check_version_file(self, definitions_repo, definitions_absref): # pragma: no cover - version_file = self._get_file_contents( - definitions_repo, definitions_absref, 'VERSION') + version_file = self._get_file_contents(definitions_repo, + definitions_absref, 'VERSION') - if version_file is None: - return + if version_file == None: + return 0 # Assume version 0 if no version file - try: - version = yaml.safe_load(version_file)['version'] - except (yaml.error.YAMLError, KeyError, TypeError): - version = 0 + version = self._parse_version_file(version_file) - if version in not_supported_versions: + if version == None: + raise InvalidVersionFileError() + + if version not in supported_versions: raise UnknownVersionError(version) + return version + def _process_definitions_with_children(self, system_filenames, definitions_repo, definitions_ref, @@ -371,7 +408,8 @@ class SourceResolver(object): definitions_queue = collections.deque(system_filenames) chunk_queue = set() - self._check_version_file(definitions_repo, definitions_absref) + definitions_version = self._check_version_file(definitions_repo, + definitions_absref) while definitions_queue: filename = definitions_queue.popleft() @@ -410,9 +448,27 @@ class SourceResolver(object): # code path should be removed. path = morphlib.util.sanitise_morphology_path( c.get('morph', c['name'])) + chunk_queue.add((c['repo'], c['ref'], path)) else: - chunk_queue.add((c['repo'], c['ref'], c['morph'])) + # Now, does this path actually exist? + path = c['morph'] + + morphology = self._get_morphology(definitions_repo, + definitions_absref, + path) + if morphology is None: + if definitions_version > 1: + raise MorphologyReferenceNotFoundError( + path, filename) + else: + self.status( + msg="Warning! `%(path)s' referenced in " + "`%(stratum)s' does not exist", + path=path, + stratum=filename) + + chunk_queue.add((c['repo'], c['ref'], path)) return chunk_queue diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py index 768ec643..df38a2e8 100644 --- a/morphlib/stagingarea.py +++ b/morphlib/stagingarea.py @@ -108,52 +108,6 @@ class StagingArea(object): assert filename.startswith(dirname) return filename[len(dirname) - 1:] # include leading slash - def hardlink_all_files(self, srcpath, destpath): # pragma: no cover - '''Hardlink every file in the path to the staging-area - - If an exception is raised, the staging-area is indeterminate. - - ''' - - file_stat = os.lstat(srcpath) - mode = file_stat.st_mode - - if stat.S_ISDIR(mode): - # Ensure directory exists in destination, then recurse. - if not os.path.lexists(destpath): - os.makedirs(destpath) - dest_stat = os.stat(os.path.realpath(destpath)) - if not stat.S_ISDIR(dest_stat.st_mode): - raise IOError('Destination not a directory. source has %s' - ' destination has %s' % (srcpath, destpath)) - - for entry in os.listdir(srcpath): - self.hardlink_all_files(os.path.join(srcpath, entry), - os.path.join(destpath, entry)) - elif stat.S_ISLNK(mode): - # Copy the symlink. - if os.path.lexists(destpath): - os.remove(destpath) - os.symlink(os.readlink(srcpath), destpath) - - elif stat.S_ISREG(mode): - # Hardlink the file. - if os.path.lexists(destpath): - os.remove(destpath) - os.link(srcpath, destpath) - - elif stat.S_ISCHR(mode) or stat.S_ISBLK(mode): - # Block or character device. Put contents of st_dev in a mknod. - if os.path.lexists(destpath): - os.remove(destpath) - os.mknod(destpath, file_stat.st_mode, file_stat.st_rdev) - os.chmod(destpath, file_stat.st_mode) - - else: - # Unsupported type. - raise IOError('Cannot extract %s into staging-area. Unsupported' - ' type.' % srcpath) - def create_devices(self, morphology): # pragma: no cover '''Creates device nodes if the morphology specifies them''' perms_mask = stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO @@ -178,17 +132,13 @@ class StagingArea(object): os.makedev(dev['major'], dev['minor'])) os.chown(destfile, dev['uid'], dev['gid']) - def install_artifact(self, artifact, artifact_checkout): - '''Install a build artifact into the staging area. - - We access the artifact via an open file handle. For now, we assume - the artifact is a tarball. - - ''' + def install_artifact(self, artifact_cache, artifact): + '''Install a build artifact into the staging area.''' if not os.path.exists(self.dirname): self._mkdir(self.dirname) - self.hardlink_all_files(artifact_checkout, self.dirname) + artifact_cache.get(artifact, directory=self.dirname) + self.create_devices(artifact.source.morphology) def remove(self): diff --git a/morphlib/stagingarea_tests.py b/morphlib/stagingarea_tests.py index ffdf5eaa..3d378573 100644 --- a/morphlib/stagingarea_tests.py +++ b/morphlib/stagingarea_tests.py @@ -46,6 +46,25 @@ class FakeArtifact(object): self.source = FakeSource() +class FakeArtifactCache(object): + + def __init__(self, tempdir): + self.tempdir = tempdir + + def create_chunk(self, chunkdir): + if not chunkdir: + chunkdir = os.path.join(self.tempdir, 'chunk') + if not os.path.exists(chunkdir): + os.mkdir(chunkdir) + with open(os.path.join(chunkdir, 'file.txt'), 'w'): + pass + + return chunkdir + + def get(self, artifact, directory=None): + return self.create_chunk(directory) + + class FakeApplication(object): def __init__(self, cachedir, tempdir): @@ -141,14 +160,14 @@ class StagingAreaTests(unittest.TestCase): def test_installs_artifact(self): artifact = FakeArtifact() - chunkdir = self.create_chunk() - self.sa.install_artifact(artifact, chunkdir) + artifact_cache = FakeArtifactCache(self.tempdir) + self.sa.install_artifact(artifact_cache, artifact) self.assertEqual(self.list_tree(self.staging), ['/', '/file.txt']) def test_removes_everything(self): artifact = FakeArtifact() - chunkdir = self.create_chunk() - self.sa.install_artifact(artifact, chunkdir) + artifact_cache = FakeArtifactCache(self.tempdir) + self.sa.install_artifact(artifact_cache, artifact) self.sa.remove() self.assertFalse(os.path.exists(self.staging)) diff --git a/morphlib/util.py b/morphlib/util.py index 8566345d..91880988 100644 --- a/morphlib/util.py +++ b/morphlib/util.py @@ -19,6 +19,7 @@ import pipes import re import subprocess import textwrap +import sys import fs.osfs @@ -119,7 +120,7 @@ def get_git_resolve_cache_server(settings): # pragma: no cover return None -def new_artifact_caches(settings): # pragma: no cover +def new_artifact_caches(settings, status_cb=None): # pragma: no cover '''Create new objects for local and remote artifact caches. This includes creating the directories on disk, if missing. @@ -132,10 +133,8 @@ def new_artifact_caches(settings): # pragma: no cover os.mkdir(artifact_cachedir) mode = settings['ostree-repo-mode'] - import logging - logging.debug(mode) - lac = morphlib.ostreeartifactcache.OSTreeArtifactCache(artifact_cachedir, - mode=mode) + lac = morphlib.ostreeartifactcache.OSTreeArtifactCache( + artifact_cachedir, mode=mode, status_cb=status_cb) rac_url = get_artifact_cache_server(settings) rac = None @@ -453,6 +452,13 @@ def has_hardware_fp(): # pragma: no cover output = subprocess.check_output(['readelf', '-A', '/proc/self/exe']) return 'Tag_ABI_VFP_args: VFP registers' in output +def determine_endianness(): # pragma: no cover + ''' + This function returns whether the host is running + in big or little endian. This is needed for MIPS. + ''' + + return sys.byteorder def get_host_architecture(): # pragma: no cover '''Get the canonical Morph name for the host's architecture.''' @@ -470,7 +476,9 @@ def get_host_architecture(): # pragma: no cover 'armv8l': 'armv8l', 'armv8b': 'armv8b', 'aarch64': 'armv8l64', - 'aarch64b': 'armv8b64', + 'aarch64_be': 'armv8b64', + 'mips': 'mips32', + 'mips64': 'mips64', 'ppc64': 'ppc64' } @@ -479,6 +487,11 @@ def get_host_architecture(): # pragma: no cover if machine == 'armv7l' and has_hardware_fp(): return 'armv7lhf' + elif machine in ('mips', 'mips64'): + if determine_endianness() == 'big': + return table[machine]+'b' + else: + return table[machine]+'l' return table[machine] @@ -647,3 +660,35 @@ def error_message_for_containerised_commandline( 'Containerisation settings: %s\n' \ 'Error output:\n%s' \ % (argv_string, container_kwargs, err) + + +def write_from_dict(filepath, d, validate=lambda x, y: True): #pragma: no cover + '''Takes a dictionary and appends the contents to a file + + An optional validation callback can be passed to perform validation on + each value in the dictionary. + + e.g. + + def validation_callback(dictionary_key, dictionary_value): + if not dictionary_value.isdigit(): + raise Exception('value contains non-digit character(s)') + + Any callback supplied to this function should raise an exception + if validation fails. + ''' + + # Sort items asciibetically + # the output of the deployment should not depend + # on the locale of the machine running the deployment + items = sorted(d.iteritems(), key=lambda (k, v): [ord(c) for c in v]) + + for (k, v) in items: + validate(k, v) + + with open(filepath, 'a') as f: + for (_, v) in items: + f.write('%s\n' % v) + + os.fchown(f.fileno(), 0, 0) + os.fchmod(f.fileno(), 0644) diff --git a/morphlib/writeexts.py b/morphlib/writeexts.py index 129b2bc4..aa185a2b 100644 --- a/morphlib/writeexts.py +++ b/morphlib/writeexts.py @@ -604,12 +604,16 @@ class WriteExtension(cliapp.Application): def check_ssh_connectivity(self, ssh_host): try: - cliapp.ssh_runcmd(ssh_host, ['true']) + output = cliapp.ssh_runcmd(ssh_host, ['echo', 'test']) except cliapp.AppException as e: logging.error("Error checking SSH connectivity: %s", str(e)) raise cliapp.AppException( 'Unable to SSH to %s: %s' % (ssh_host, e)) + if output.strip() != 'test': + raise cliapp.AppException( + 'Unexpected output from remote machine: %s' % output.strip()) + def is_device(self, location): try: st = os.stat(location) |