diff options
Diffstat (limited to 'morphlib/plugins')
| -rw-r--r-- | morphlib/plugins/branch_and_merge_plugin.py | 1 | ||||
| -rw-r--r-- | morphlib/plugins/build_plugin.py | 137 | ||||
| -rw-r--r-- | morphlib/plugins/certify_plugin.py | 140 | ||||
| -rw-r--r-- | morphlib/plugins/cross-bootstrap_plugin.py | 2 | ||||
| -rw-r--r-- | morphlib/plugins/deploy_plugin.py | 223 | ||||
| -rw-r--r-- | morphlib/plugins/gc_plugin.py | 3 | ||||
| -rw-r--r-- | morphlib/plugins/get_chunk_details_plugin.py | 79 | ||||
| -rw-r--r-- | morphlib/plugins/ostree_artifacts_plugin.py | 169 |
8 files changed, 682 insertions, 72 deletions
diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py index cdbb303f..08589ea6 100644 --- a/morphlib/plugins/branch_and_merge_plugin.py +++ b/morphlib/plugins/branch_and_merge_plugin.py @@ -623,7 +623,6 @@ class BranchAndMergePlugin(cliapp.Plugin): smd = morphlib.systemmetadatadir.SystemMetadataDir(path) metadata = smd.values() - logging.debug(metadata) systems = [md for md in metadata if 'kind' in md and md['kind'] == 'system'] diff --git a/morphlib/plugins/build_plugin.py b/morphlib/plugins/build_plugin.py index 2cc395fc..e5b35853 100644 --- a/morphlib/plugins/build_plugin.py +++ b/morphlib/plugins/build_plugin.py @@ -13,26 +13,39 @@ # with this program. If not, see <http://www.gnu.org/licenses/>. -import cliapp +import collections import contextlib import uuid import logging +import cliapp + import morphlib +class ComponentNotInSystemError(morphlib.Error): + + def __init__(self, components, system): + components = ', '.join(components) + self.msg = ('Components %s are not in %s. Ensure you provided ' + 'component names rather than filenames.' + % (components, system)) + + class BuildPlugin(cliapp.Plugin): def enable(self): self.app.add_subcommand('build-morphology', self.build_morphology, - arg_synopsis='(REPO REF FILENAME)...') + arg_synopsis='REPO REF FILENAME ' + '[COMPONENT...]') self.app.add_subcommand('build', self.build, - arg_synopsis='SYSTEM') + arg_synopsis='SYSTEM [COMPONENT...]') self.app.add_subcommand('distbuild-morphology', self.distbuild_morphology, - arg_synopsis='SYSTEM') + arg_synopsis='REPO REF FILENAME ' + '[COMPONENT...]') self.app.add_subcommand('distbuild', self.distbuild, - arg_synopsis='SYSTEM') + arg_synopsis='SYSTEM [COMPONENT...]') self.use_distbuild = False def disable(self): @@ -46,6 +59,8 @@ class BuildPlugin(cliapp.Plugin): * `REPO` is a git repository URL. * `REF` is a branch or other commit reference in that repository. * `FILENAME` is a morphology filename at that ref. + * `COMPONENT...` is the names of one or more chunks or strata to + build. If none are given the the system at FILENAME is built. See 'help distbuild' and 'help build-morphology' for more information. @@ -54,10 +69,15 @@ class BuildPlugin(cliapp.Plugin): addr = self.app.settings['controller-initiator-address'] port = self.app.settings['controller-initiator-port'] + self.use_distbuild = True build_command = morphlib.buildcommand.InitiatorBuildCommand( self.app, addr, port) - for repo_name, ref, filename in self.app.itertriplets(args): - build_command.build(repo_name, ref, filename) + repo, ref, filename = args[0:3] + filename = morphlib.util.sanitise_morphology_path(filename) + component_names = [morphlib.util.sanitise_morphology_path(name) + for name in args[3:]] + self.start_build(repo, ref, build_command, filename, + component_names) def distbuild(self, args): '''Distbuild a system image in the current system branch @@ -65,6 +85,8 @@ class BuildPlugin(cliapp.Plugin): Command line arguments: * `SYSTEM` is the name of the system to build. + * `COMPONENT...` is the names of one or more chunks or strata to + build. If none are given then SYSTEM is built. This command launches a distributed build, to use this command you must first set up a distbuild cluster. @@ -92,6 +114,8 @@ class BuildPlugin(cliapp.Plugin): * `REPO` is a git repository URL. * `REF` is a branch or other commit reference in that repository. * `FILENAME` is a morphology filename at that ref. + * `COMPONENT...` is the names of one or more chunks or strata to + build. If none are given then the system at FILENAME is built. You probably want `morph build` instead. However, in some cases it is more convenient to not have to create a Morph @@ -104,8 +128,14 @@ class BuildPlugin(cliapp.Plugin): Example: - morph build-morphology baserock:baserock/definitions \ - master devel-system-x86_64-generic.morph + morph build-morphology baserock:baserock/definitions \\ + master systems/devel-system-x86_64-generic.morph + + Partial build example: + + morph build-morphology baserock:baserock/definitions \\ + master systems/devel-system-x86_64-generic.morph \\ + build-essential ''' @@ -117,15 +147,21 @@ class BuildPlugin(cliapp.Plugin): self.app.settings['cachedir-min-space']) build_command = morphlib.buildcommand.BuildCommand(self.app) - for repo_name, ref, filename in self.app.itertriplets(args): - build_command.build(repo_name, ref, filename) + repo, ref, filename = args[0:3] + filename = morphlib.util.sanitise_morphology_path(filename) + component_names = [morphlib.util.sanitise_morphology_path(name) + for name in args[3:]] + self.start_build(repo, ref, build_command, filename, + component_names) def build(self, args): '''Build a system image in the current system branch Command line arguments: - * `SYSTEM` is the name of the system to build. + * `SYSTEM` is the filename of the system to build. + * `COMPONENT...` is the names of one or more chunks or strata to + build. If this is not given then the SYSTEM is built. This builds a system image, and any of its components that need building. The system name is the basename of the system @@ -145,14 +181,14 @@ class BuildPlugin(cliapp.Plugin): Example: - morph build devel-system-x86_64-generic.morph + morph build systems/devel-system-x86_64-generic.morph - ''' + Partial build example: - if len(args) != 1: - raise cliapp.AppException('morph build expects exactly one ' - 'parameter: the system to build') + morph build systems/devel-system-x86_64-generic.morph \\ + build-essential + ''' # Raise an exception if there is not enough space morphlib.util.check_disk_available( self.app.settings['tempdir'], @@ -165,6 +201,7 @@ class BuildPlugin(cliapp.Plugin): system_filename = morphlib.util.sanitise_morphology_path(args[0]) system_filename = sb.relative_to_root_repo(system_filename) + component_names = args[1:] logging.debug('System branch is %s' % sb.root_directory) @@ -178,11 +215,14 @@ class BuildPlugin(cliapp.Plugin): build_command = morphlib.buildcommand.BuildCommand(self.app) if self.app.settings['local-changes'] == 'include': - self._build_with_local_changes(build_command, sb, system_filename) + self._build_with_local_changes(build_command, sb, system_filename, + component_names) else: - self._build_local_commit(build_command, sb, system_filename) + self._build_local_commit(build_command, sb, system_filename, + component_names) - def _build_with_local_changes(self, build_command, sb, system_filename): + def _build_with_local_changes(self, build_command, sb, system_filename, + component_names): '''Construct a branch including user's local changes, and build that. It is often a slow process to check all repos in the system branch for @@ -199,9 +239,12 @@ class BuildPlugin(cliapp.Plugin): email = morphlib.git.get_user_email(self.app.runcmd) build_ref_prefix = self.app.settings['build-ref-prefix'] - self.app.status(msg='Starting build %(uuid)s', uuid=build_uuid) + self.app.status(msg='Looking for uncommitted changes (pass ' + '--local-changes=ignore to skip)') + self.app.status(msg='Collecting morphologies involved in ' 'building %(system)s from %(branch)s', + chatty=True, system=system_filename, branch=sb.system_branch_name) @@ -211,10 +254,11 @@ class BuildPlugin(cliapp.Plugin): name=name, email=email, build_uuid=build_uuid, status=self.app.status) with pbb as (repo, commit, original_ref): - build_command.build(repo, commit, system_filename, - original_ref=original_ref) + self.start_build(repo, commit, build_command, system_filename, + component_names, original_ref=original_ref) - def _build_local_commit(self, build_command, sb, system_filename): + def _build_local_commit(self, build_command, sb, system_filename, + component_names): '''Build whatever commit the user has checked-out locally. This ignores any uncommitted changes. Also, if the user has a commit @@ -242,4 +286,47 @@ class BuildPlugin(cliapp.Plugin): definitions_repo = morphlib.gitdir.GitDirectory(definitions_repo_path) commit = definitions_repo.resolve_ref_to_commit(ref) - build_command.build(root_repo_url, commit, system_filename) + self.start_build(root_repo_url, commit, build_command, + system_filename, component_names) + + def _find_artifacts(self, names, root_artifact): + found = collections.OrderedDict() + not_found = names + for a in root_artifact.walk(): + name = a.source.morphology['name'] + if name in names and name not in found: + found[name] = a + not_found.remove(name) + return found, not_found + + def start_build(self, repo, commit, bc, system_filename, + component_names, original_ref=None): + '''Actually run the build. + + If a set of components was given, only build those. Otherwise, + build the whole system. + + ''' + if self.use_distbuild: + bc.build(repo, commit, system_filename, + original_ref=original_ref, + component_names=component_names) + return + + self.app.status(msg='Deciding on task order') + srcpool = bc.create_source_pool(repo, commit, system_filename) + bc.validate_sources(srcpool) + root = bc.resolve_artifacts(srcpool) + if not component_names: + component_names = [root.source.name] + components, not_found = self._find_artifacts(component_names, root) + if not_found: + raise ComponentNotInSystemError(not_found, system_filename) + + for name, component in components.iteritems(): + component.build_env = root.build_env + bc.build_in_order(component) + self.app.status(msg='%(kind)s %(name)s is cached at %(path)s', + kind=component.source.morphology['kind'], + name=name, + path=bc.lac.artifact_filename(component)) diff --git a/morphlib/plugins/certify_plugin.py b/morphlib/plugins/certify_plugin.py new file mode 100644 index 00000000..10fc19ad --- /dev/null +++ b/morphlib/plugins/certify_plugin.py @@ -0,0 +1,140 @@ +# Copyright (C) 2014-2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# This plugin is used as part of the Baserock automated release process. +# +# See: <http://wiki.baserock.org/guides/release-process> for more information. + +import warnings + +import cliapp +import morphlib + +class CertifyPlugin(cliapp.Plugin): + + def enable(self): + self.app.add_subcommand( + 'certify', self.certify, + arg_synopsis='REPO REF MORPH [MORPH]...') + + def disable(self): + pass + + def certify(self, args): + '''Certify that any given system definition is reproducable. + + Command line arguments: + + * `REPO` is a git repository URL. + * `REF` is a branch or other commit reference in that repository. + * `MORPH` is a system morphology name at that ref. + + ''' + + if len(args) < 3: + raise cliapp.AppException( + 'Wrong number of arguments to certify command ' + '(see help)') + + repo, ref = args[0], args[1] + system_filenames = map(morphlib.util.sanitise_morphology_path, + args[2:]) + + self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app) + self.resolver = morphlib.artifactresolver.ArtifactResolver() + + for system_filename in system_filenames: + self.certify_system(repo, ref, system_filename) + + def certify_system(self, repo, ref, system_filename): + '''Certify reproducibility of system.''' + + self.app.status( + msg='Creating source pool for %s' % system_filename, chatty=True) + source_pool = morphlib.sourceresolver.create_source_pool( + self.lrc, self.rrc, repo, ref, system_filename, + cachedir=self.app.settings['cachedir'], + update_repos = not self.app.settings['no-git-update'], + status_cb=self.app.status) + + self.app.status( + msg='Resolving artifacts for %s' % system_filename, chatty=True) + root_artifacts = self.resolver.resolve_root_artifacts(source_pool) + + def find_artifact_by_name(artifacts_list, filename): + for a in artifacts_list: + if a.source.filename == filename: + return a + raise ValueError + + system_artifact = find_artifact_by_name(root_artifacts, + system_filename) + + self.app.status( + msg='Computing cache keys for %s' % system_filename, chatty=True) + build_env = morphlib.buildenvironment.BuildEnvironment( + self.app.settings, system_artifact.source.morphology['arch']) + ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env) + + aliases = self.app.settings['repo-alias'] + resolver = morphlib.repoaliasresolver.RepoAliasResolver(aliases) + + certified = True + + for source in set(a.source for a in system_artifact.walk()): + source.cache_key = ckc.compute_key(source) + source.cache_id = ckc.get_cache_id(source) + + if source.morphology['kind'] != 'chunk': + continue + + name = source.morphology['name'] + ref = source.original_ref + + # Test that chunk has a sha1 ref + # TODO: Could allow either sha1 or existent tag. + if not morphlib.git.is_valid_sha1(ref): + warnings.warn('Chunk "{}" has non-sha1 ref: "{}"\n' + .format(name, ref)) + certified = False + + # Ensure we have a cache of the repo + if not self.lrc.has_repo(source.repo_name): + self.lrc.cache_repo(source.repo_name) + + cached = self.lrc.get_repo(source.repo_name) + + # Test that sha1 ref is anchored in a tag or branch, + # and thus not a candidate for removal on `git gc`. + if (morphlib.git.is_valid_sha1(ref) and + not len(cached.tags_containing_sha1(ref)) and + not len(cached.branches_containing_sha1(ref))): + warnings.warn('Chunk "{}" has unanchored ref: "{}"\n' + .format(name, ref)) + certified = False + + # Test that chunk repo is on trove-host + pull_url = resolver.pull_url(source.repo_name) + if self.app.settings['trove-host'] not in pull_url: + warnings.warn('Chunk "{}" has repo not on trove-host: "{}"\n' + .format(name, pull_url)) + certified = False + + if certified: + print('=> Reproducibility certification PASSED for\n {}' + .format(system_filename)) + else: + print('=> Reproducibility certification FAILED for\n {}' + .format(system_filename)) diff --git a/morphlib/plugins/cross-bootstrap_plugin.py b/morphlib/plugins/cross-bootstrap_plugin.py index 79609cb5..9bec5646 100644 --- a/morphlib/plugins/cross-bootstrap_plugin.py +++ b/morphlib/plugins/cross-bootstrap_plugin.py @@ -27,7 +27,7 @@ echo "Generated by Morph version %s\n" set -eu -export PATH=$PATH:/tools/bin:/tools/sbin +export PATH=/usr/bin:/bin:/usr/sbin:/sbin:/tools/bin:/tools/sbin export SRCDIR=/src ''' % morphlib.__version__ diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index efe6735b..3c19553e 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -13,6 +13,7 @@ # with this program. If not, see <http://www.gnu.org/licenses/>. +import collections import json import logging import os @@ -27,6 +28,14 @@ import morphlib from morphlib.artifactcachereference import ArtifactCacheReference +class NotYetBuiltError(morphlib.Error): + + def __init__(self, name): + self.msg = ('Deployment failed as %s is not yet built.\n' + 'Please ensure the system is built before deployment.' + % name) + + class DeployPlugin(cliapp.Plugin): def enable(self): @@ -385,7 +394,7 @@ class DeployPlugin(cliapp.Plugin): name=name, email=email, build_uuid=build_uuid, status=self.app.status) with pbb as (repo, commit, original_ref): - self.deploy_cluster(build_command, cluster_morphology, + self.deploy_cluster(sb, build_command, cluster_morphology, root_repo_dir, repo, commit, env_vars, deployments) else: @@ -398,6 +407,11 @@ class DeployPlugin(cliapp.Plugin): deployments) self.app.status(msg='Finished deployment') + if self.app.settings['partial']: + self.app.status(msg='WARNING: This was a partial deployment. ' + 'Configuration extensions have not been ' + 'run. Applying the result to an existing ' + 'system may not have reproducible results.') def validate_deployment_options( self, env_vars, all_deployments, all_subsystems): @@ -415,21 +429,54 @@ class DeployPlugin(cliapp.Plugin): 'Variable referenced a non-existent deployment ' 'name: %s' % var) - def deploy_cluster(self, build_command, cluster_morphology, root_repo_dir, - repo, commit, env_vars, deployments): + def deploy_cluster(self, sb, build_command, cluster_morphology, + root_repo_dir, repo, commit, env_vars, deployments): # Create a tempdir for this deployment to work in deploy_tempdir = tempfile.mkdtemp( dir=os.path.join(self.app.settings['tempdir'], 'deployments')) try: for system in cluster_morphology['systems']: - self.deploy_system(build_command, deploy_tempdir, + self.deploy_system(sb, build_command, deploy_tempdir, root_repo_dir, repo, commit, system, env_vars, deployments, parent_location='') finally: shutil.rmtree(deploy_tempdir) - def deploy_system(self, build_command, deploy_tempdir, + def _sanitise_morphology_paths(self, paths, sb): + sanitised_paths = [] + for path in paths: + path = morphlib.util.sanitise_morphology_path(path) + sanitised_paths.append(sb.relative_to_root_repo(path)) + return sanitised_paths + + def _find_artifacts(self, filenames, root_artifact): + found = collections.OrderedDict() + not_found = filenames + for a in root_artifact.walk(): + if a.source.filename in filenames and a.source.name not in found: + found[a.source.name] = a + not_found.remove(a.source.filename) + return found, not_found + + def _validate_partial_deployment(self, deployment_type, + artifact, component_names): + supported_types = ('tar', 'sysroot') + if deployment_type not in supported_types: + raise cliapp.AppException('Not deploying %s, --partial was ' + 'set and partial deployment only ' + 'supports %s deployments.' % + (artifact.source.name, + ', '.join(supported_types))) + components, not_found = self._find_artifacts(component_names, + artifact) + if not_found: + raise cliapp.AppException('Components %s not found in system %s.' % + (', '.join(not_found), + artifact.source.name)) + return components + + def deploy_system(self, sb, build_command, deploy_tempdir, root_repo_dir, build_repo, ref, system, env_vars, deployment_filter, parent_location): sys_ids = set(system['deploy'].iterkeys()) @@ -475,6 +522,12 @@ class DeployPlugin(cliapp.Plugin): raise morphlib.Error('"type" is undefined ' 'for system "%s"' % system_id) + components = self._sanitise_morphology_paths( + deploy_params.get('partial-deploy-components', []), sb) + if self.app.settings['partial']: + components = self._validate_partial_deployment( + deployment_type, artifact, components) + location = final_env.pop('location', None) if not location: raise morphlib.Error('"location" is undefined ' @@ -488,9 +541,10 @@ class DeployPlugin(cliapp.Plugin): root_repo_dir, ref, artifact, deployment_type, - location, final_env) + location, final_env, + components=components) for subsystem in system.get('subsystems', []): - self.deploy_system(build_command, deploy_tempdir, + self.deploy_system(sb, build_command, deploy_tempdir, root_repo_dir, build_repo, ref, subsystem, env_vars, [], parent_location=system_tree) @@ -542,13 +596,27 @@ class DeployPlugin(cliapp.Plugin): pass def checkout_stratum(self, path, artifact, lac, rac): + """Pull the chunks in a stratum, and checkout them into `path`. + + This reads a stratum artifact and pulls the chunks it contains from + the remote into the local artifact cache if they are not already + cached locally. Each of these chunks is then checked out into `path`. + + Also download the stratum metadata into the local cache, then place + it in the /baserock directory of the system checkout indicated by + `path`. + + If any of the chunks have not been cached either locally or remotely, + a morphlib.remoteartifactcache.GetError is raised. + + """ with open(lac.get(artifact), 'r') as stratum: chunks = [ArtifactCacheReference(c) for c in json.load(stratum)] morphlib.builder.download_depends(chunks, lac, rac) for chunk in chunks: self.app.status(msg='Checkout chunk %(name)s.', name=chunk.basename(), chatty=True) - lac.get(chunk, path, self.app.status) + lac.get(chunk, path) metadata = os.path.join(path, 'baserock', '%s.meta' % artifact.name) with lac.get_artifact_metadata(artifact, 'meta') as meta_src: @@ -556,14 +624,94 @@ class DeployPlugin(cliapp.Plugin): shutil.copyfileobj(meta_src, meta_dst) def checkout_strata(self, path, artifact, lac, rac): + """Pull the dependencies of `artifact` and checkout them into `path`. + + This assumes that `artifact` is a system artifact. If any of the + dependencies aren't cached remotely or locally, this raises a + morphlib.remoteartifactcache.GetError. + + """ deps = artifact.source.dependencies morphlib.builder.download_depends(deps, lac, rac) for stratum in deps: self.checkout_stratum(path, stratum, lac, rac) morphlib.builder.ldconfig(self.app.runcmd, path) + def checkout_system(self, build_command, artifact, path): + """Checkout a system into `path`. + + This checks out each of the strata into the directory given by `path`, + then checks out the system artifact into the same directory. This uses + OSTree's `union` checkout mode to overwrite duplicate files but not + need an empty directory. Artifacts which aren't cached locally are + fetched from the remote cache. + + Raises a NotYetBuiltError if either the system artifact or any of the + chunk artifacts in the strata which make up the system aren't cached + either locally or remotely. + + """ + # Check if the system artifact is in the local or remote cache. + # If it isn't, we don't need to bother checking out strata before + # we fail. + if not (build_command.lac.has(artifact) + or build_command.rac.has(artifact)): + raise NotYetBuiltError(artifact.name) + + # Checkout the strata involved in the artifact into a tempdir + self.app.status(msg='Checking out strata in system') + try: + self.checkout_strata(path, artifact, + build_command.lac, build_command.rac) + + self.app.status(msg='Checking out system for configuration') + build_command.cache_artifacts_locally([artifact]) + build_command.lac.get(artifact, path) + except (morphlib.ostreeartifactcache.NotCachedError, + morphlib.remoteartifactcache.GetError): + raise NotYetBuiltError(artifact.name) + + self.app.status( + msg='System checked out at %(system_tree)s', + system_tree=path) + + def checkout_components(self, bc, components, path): + if not components: + raise cliapp.AppException('Deployment failed as no components ' + 'were specified for deployment and ' + '--partial was set.') + for name, artifact in components.iteritems(): + deps = artifact.source.dependencies + morphlib.builder.download_depends(deps, bc.lac, bc.rac) + for dep in deps: + if dep.source.morphology['kind'] == 'stratum': + self.checkout_stratum(path, dep, bc.lac, bc.rac) + elif dep.source.morphology['kind'] == 'chunk': + self.app.status(msg='Checkout chunk %(name)s.', + name=dep.basename(), chatty=True) + bc.lac.get(dep, path) + if artifact.source.morphology['kind'] == 'stratum': + self.checkout_stratum(path, artifact, bc.lac, bc.rac) + elif artifact.source.morphology['kind'] == 'chunk': + self.app.status(msg='Checkout chunk %(name)s.', + name=name, chatty=True) + bc.lac.get(artifact, path) + self.app.status( + msg='Components %(components)s checkout out at %(path)s', + components=', '.join(components), path=path) + def setup_deploy(self, build_command, deploy_tempdir, root_repo_dir, ref, - artifact, deployment_type, location, env): + artifact, deployment_type, location, env, components=[]): + """Checkout the artifact, create metadata and return the location. + + This checks out the system into a temporary directory, and then mounts + this temporary directory alongside a different temporary directory + using a union filesystem. This allows changes to be made without + touching the checked out artifacts. The deployment metadata file is + created and then the directory at which the two temporary directories + are mounted is returned. + + """ # deployment_type, location and env are only used for saving metadata deployment_dir = tempfile.mkdtemp(dir=deploy_tempdir) @@ -583,26 +731,11 @@ class DeployPlugin(cliapp.Plugin): deploy_tree = os.path.join(deployment_dir, 'overlay-deploy-%s' % artifact.name) try: - # Checkout the strata involved in the artifact into a tempdir - self.app.status(msg='Checking out strata in system') - self.checkout_strata(system_tree, artifact, - build_command.lac, build_command.rac) - - self.app.status(msg='Checking out system for configuration') - if build_command.lac.has(artifact): - build_command.lac.get(artifact, system_tree) - elif build_command.rac.has(artifact): - build_command.cache_artifacts_locally([artifact]) - build_command.lac.get(artifact, system_tree) + if self.app.settings['partial']: + self.checkout_components(build_command, components, + system_tree) else: - raise cliapp.AppException('Deployment failed as system is' - ' not yet built.\nPlease ensure' - ' the system is built before' - ' deployment.') - - self.app.status( - msg='System checked out at %(system_tree)s', - system_tree=system_tree) + self.checkout_system(build_command, artifact, system_tree) union_filesystem = self.app.settings['union-filesystem'] morphlib.fsutils.overlay_mount(self.app.runcmd, @@ -625,10 +758,7 @@ class DeployPlugin(cliapp.Plugin): except Exception: if deploy_tree and os.path.exists(deploy_tree): morphlib.fsutils.unmount(self.app.runcmd, deploy_tree) - shutil.rmtree(deploy_tree) - shutil.rmtree(system_tree) - shutil.rmtree(overlay_dir) - shutil.rmtree(work_dir) + shutil.rmtree(deployment_dir) raise def run_deploy_commands(self, deploy_tempdir, env, artifact, root_repo_dir, @@ -640,15 +770,19 @@ class DeployPlugin(cliapp.Plugin): try: # Run configuration extensions. - self.app.status(msg='Configure system') - names = artifact.source.morphology['configuration-extensions'] - for name in names: - self._run_extension( - root_repo_dir, - name, - '.configure', - [system_tree], - env) + if not self.app.settings['partial']: + self.app.status(msg='Configure system') + names = artifact.source.morphology['configuration-extensions'] + for name in names: + self._run_extension( + root_repo_dir, + name, + '.configure', + [system_tree], + env) + else: + self.app.status(msg='WARNING: Not running configuration ' + 'extensions as --partial is set!') # Run write extension. self.app.status(msg='Writing to device') @@ -665,7 +799,7 @@ class DeployPlugin(cliapp.Plugin): shutil.rmtree(deploy_private_tempdir) def _report_extension_stdout(self, line): - self.app.status(msg=line.replace('%s', '%%')) + self.app.status(msg=line.replace('%', '%%')) def _report_extension_stderr(self, error_list): def cb(line): error_list.append(line) @@ -699,7 +833,7 @@ class DeployPlugin(cliapp.Plugin): raise cliapp.AppException(message) def create_metadata(self, system_artifact, root_repo_dir, deployment_type, - location, env): + location, env, components=[]): '''Deployment-specific metadata. The `build` and `deploy` operations must be from the same ref, so full @@ -731,6 +865,9 @@ class DeployPlugin(cliapp.Plugin): 'commit': morphlib.gitversion.commit, 'version': morphlib.gitversion.version, }, + 'partial': self.app.settings['partial'], } + if self.app.settings['partial']: + meta['partial-components'] = components return meta diff --git a/morphlib/plugins/gc_plugin.py b/morphlib/plugins/gc_plugin.py index 8b5dc4c2..54c1b43e 100644 --- a/morphlib/plugins/gc_plugin.py +++ b/morphlib/plugins/gc_plugin.py @@ -157,10 +157,9 @@ class GCPlugin(cliapp.Plugin): self.app.status(msg='Removing source %(cachekey)s', cachekey=cachekey, chatty=True) lac.remove(cachekey) + lac.prune() removed += 1 - lac.prune() - if sufficient_free(): self.app.status(msg='Made sufficient space in %(cache_path)s ' 'after removing %(removed)d sources', diff --git a/morphlib/plugins/get_chunk_details_plugin.py b/morphlib/plugins/get_chunk_details_plugin.py new file mode 100644 index 00000000..842b4afe --- /dev/null +++ b/morphlib/plugins/get_chunk_details_plugin.py @@ -0,0 +1,79 @@ +# Copyright (C) 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + +import cliapp +import morphlib + +class GetChunkDetailsPlugin(cliapp.Plugin): + + def enable(self): + self.app.add_subcommand( + 'get-chunk-details', self.get_chunk_details, + arg_synopsis='[STRATUM] CHUNK') + + def disable(self): + pass + + def get_chunk_details(self, args): + '''Print out details for the given chunk + + Command line arguments: + + * `STRATUM` is the stratum to search for chunk (optional). + * `CHUNK` is the component to obtain a URL for. + + ''' + + stratum_name = None + + if len(args) == 1: + chunk_name = args[0] + elif len(args) == 2: + stratum_name = args[0] + chunk_name = args[1] + else: + raise cliapp.AppException( + 'Wrong number of arguments to get-chunk-details command ' + '(see help)') + + sb = morphlib.sysbranchdir.open_from_within('.') + loader = morphlib.morphloader.MorphologyLoader() + + aliases = self.app.settings['repo-alias'] + self.resolver = morphlib.repoaliasresolver.RepoAliasResolver(aliases) + + found = 0 + for morph in sb.load_all_morphologies(loader): + if morph['kind'] == 'stratum': + if (stratum_name == None or + morph['name'] == stratum_name): + for chunk in morph['chunks']: + if chunk['name'] == chunk_name: + found = found + 1 + self._print_chunk_details(chunk, morph) + + if found == 0: + if stratum_name == None: + print('Chunk `{}` not found' + .format(chunk_name)) + else: + print('Chunk `{}` not found in stratum `{}`' + .format(chunk_name, stratum_name)) + + def _print_chunk_details(self, chunk, morph): + repo = self.resolver.pull_url(chunk['repo']) + print('In stratum {}:'.format(morph['name'])) + print(' Chunk: {}'.format(chunk['name'])) + print(' Repo: {}'.format(repo)) + print(' Ref: {}'.format(chunk['ref'])) diff --git a/morphlib/plugins/ostree_artifacts_plugin.py b/morphlib/plugins/ostree_artifacts_plugin.py new file mode 100644 index 00000000..eedcd1e7 --- /dev/null +++ b/morphlib/plugins/ostree_artifacts_plugin.py @@ -0,0 +1,169 @@ +# Copyright (C) 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + + +import collections +import fs +import os + +import cliapp + +import morphlib +from morphlib.artifactcachereference import ArtifactCacheReference + + +class NoCacheError(morphlib.Error): + + def __init__(self, cachedir): + self.msg = ("Expected artifact cache directory %s doesn't exist.\n" + "No existing cache to convert!" % cachedir) + + +class ComponentNotInSystemError(morphlib.Error): + + def __init__(self, components, system): + components = ', '.join(components) + self.msg = ('Components %s are not in %s. Ensure you provided ' + 'component names rather than filenames.' + % (components, system)) + + +class OSTreeArtifactsPlugin(cliapp.Plugin): + + def enable(self): + self.app.add_subcommand('convert-local-cache', self.convert_cache, + arg_synopsis='[DELETE]') + self.app.add_subcommand('query-cache', self.query_cache, + arg_synopsis='SYSTEM NAME...') + + def disable(self): + pass + + def convert_cache(self, args): + """Convert a local tarball cache into an OSTree cache. + + Command line arguments: + + * DELETE: This is an optional argument, which if given as "delete" + will cause tarball artifacts to be removed once they are converted. + + This command will extract all the tarball artifacts in your local + artifact cache and store them in an OSTree repository in that + artifact cache. This will be quicker than redownloading all that + content from a remote cache server, but may still be time consuming + if your cache is large. + + """ + delete = False + if args: + if args[0] == 'delete': + delete = True + + artifact_cachedir = os.path.join(self.app.settings['cachedir'], + 'artifacts') + if not os.path.exists(artifact_cachedir): + raise NoCacheError(artifact_cachedir) + + tarball_cache = morphlib.localartifactcache.LocalArtifactCache( + fs.osfs.OSFS(artifact_cachedir)) + ostree_cache = morphlib.ostreeartifactcache.OSTreeArtifactCache( + artifact_cachedir, mode=self.app.settings['ostree-repo-mode'], + status_cb=self.app.status) + + cached_artifacts = [] + for cachekey, artifacts, last_used in tarball_cache.list_contents(): + for artifact in artifacts: + basename = '.'.join((cachekey.lstrip('/'), artifact)) + cached_artifacts.append(ArtifactCacheReference(basename)) + + # Set the method property of the tarball cache to allow us to + # treat it like a RemoteArtifactCache. + tarball_cache.method = 'tarball' + + for artifact in cached_artifacts: + if not ostree_cache.has(artifact): + try: + cache_key, kind, name = artifact.basename().split('.', 2) + if kind in ('system', 'stratum'): + # System artifacts are quick to recreate now, and + # stratum artifacts are still stored in the same way. + continue + except ValueError: + # We must have metadata, which doesn't need converting + continue + self.app.status(msg='Converting %(name)s', + name=artifact.basename()) + ostree_cache.copy_from_remote(artifact, tarball_cache) + if delete: + os.remove(tarball_cache.artifact_filename(artifact)) + + def _find_artifacts(self, names, root_artifact): + found = collections.OrderedDict() + not_found = list(names) + for a in root_artifact.walk(): + name = a.source.morphology['name'] + if name in names and name not in found: + found[name] = [a] + if name in not_found: + not_found.remove(name) + elif name in names: + found[name].append(a) + if name in not_found: + not_found.remove(name) + return found, not_found + + def query_cache(self, args): + """Check if the cache contains an artifact. + + Command line arguments: + + * `SYSTEM` is the filename of the system containing the components + to be looked for. + * `NAME...` is the name of one or more components to look for. + + """ + if not args: + raise cliapp.AppException('You must provide at least a system ' + 'filename.\nUsage: `morph query-cache ' + 'SYSTEM [NAME...]`') + ws = morphlib.workspace.open('.') + sb = morphlib.sysbranchdir.open_from_within('.') + + system_filename = morphlib.util.sanitise_morphology_path(args[0]) + system_filename = sb.relative_to_root_repo(system_filename) + component_names = args[1:] + + bc = morphlib.buildcommand.BuildCommand(self.app) + repo = sb.get_config('branch.root') + ref = sb.get_config('branch.name') + + definitions_repo_path = sb.get_git_directory_name(repo) + definitions_repo = morphlib.gitdir.GitDirectory(definitions_repo_path) + commit = definitions_repo.resolve_ref_to_commit(ref) + + srcpool = bc.create_source_pool(repo, commit, system_filename) + bc.validate_sources(srcpool) + root = bc.resolve_artifacts(srcpool) + if not component_names: + component_names = [root.source.name] + components, not_found = self._find_artifacts(component_names, root) + if not_found: + raise ComponentNotInSystemError(not_found, system_filename) + + for name, artifacts in components.iteritems(): + for component in artifacts: + if bc.lac.has(component): + print bc.lac._get_artifact_cache_name(component) + else: + print '%s is not cached' % name |