diff options
author | Sam Thursfield <sam.thursfield@codethink.co.uk> | 2014-06-05 12:00:22 +0100 |
---|---|---|
committer | Sam Thursfield <sam.thursfield@codethink.co.uk> | 2014-06-05 12:00:22 +0100 |
commit | aa177dd08b2695375003b9f188d716c60affa011 (patch) | |
tree | f9b84c902cd96f25c3129e0cb831a6537698de22 | |
parent | 8fe88b78e3d8b3462a3a27a49e4e72456d71f9ad (diff) | |
parent | 179ce80da32a708e8fcb31747d345647f84957aa (diff) | |
download | morph-aa177dd08b2695375003b9f188d716c60affa011.tar.gz |
Merge branch 'sam/list-artifacts'
Reviewed-By: Adam Coldrick <adam.coldrick@codethink.co.uk>
Reviewed-By: Richard Maw <richard.maw@codethink.co.uk>
-rw-r--r-- | morphlib/plugins/copy-artifacts_plugin.py | 142 | ||||
-rw-r--r-- | morphlib/plugins/list_artifacts_plugin.py | 124 | ||||
-rw-r--r-- | without-test-modules | 2 |
3 files changed, 125 insertions, 143 deletions
diff --git a/morphlib/plugins/copy-artifacts_plugin.py b/morphlib/plugins/copy-artifacts_plugin.py deleted file mode 100644 index 577d0ef2..00000000 --- a/morphlib/plugins/copy-artifacts_plugin.py +++ /dev/null @@ -1,142 +0,0 @@ -# Copyright (C) 2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - - -# FIXME: no tests - - - -import os -import glob -import json - - -import cliapp - - - -import morphlib - - - -class CopyArtifactsPlugin(cliapp.Plugin): - - def enable(self): - self.app.add_subcommand( - 'list-artifacts', self.list_artifacts, - arg_synopsis='SYSTEM-ARTIFACT') - self.app.add_subcommand( - 'copy-artifacts', self.copy_artifacts, - arg_synopsis='SYSTEM-ARTIFACT DESTINATION') - - def disable(self): - pass - - def list_artifacts(self, args): - '''List every artifact that makes up a system artifact. - - Command line arguments: - - * `SYSTEM-ARTIFACT` is the filename for a build artifact for - a system (ending in `-rootfs`). - - list-artifacts reads the system artifact and writes out a list - of the filenames, in the local artifact cache, of all the - component artifacts included in the system. It does not include - build-dependencies of the system, unless they're included in - the system. - - ''' - - if len(args) != 1: - raise cliapp.AppException( - 'Wrong number of arguments to list-artifacts command' - '(see help)') - - system = args[0] - artifacts = self.find_artifacts(system) - for artifact in artifacts: - self.app.output.write(artifact + "\n") - - def copy_artifacts(self, args): - '''Copy every artifact that makes up a system to an rsync path. - - Command line arguments: - - * `SYSTEM-ARTIFACT` is the filename for a build artifact for - a system (ending in `-rootfs`). - * `DESTINATION` is a URL for where the artifacts are to be - copied to, in a form suitable for the rsync program. - - This command is useful for copying artifacts from a local - system to a Morph artifact cache server, so they can be - shared by other people. It can also be used to archive - artifacts used for a release. Note, however, that it does - not include artifacts that are build-dependencies of the - strata included in the system, but not actually in cluded - in the system. - - Example: - - morph copy-artifacts /src/cache/artifacts/*.system.* \ - cache.example.com:/srv/cache/ - - ''' - - if len(args) != 2: - raise cliapp.AppException( - 'Wrong number of arguments to copy-artifacts command' - '(see help)') - - system = args[0] - rsync_dest = args[1] - artifacts = self.find_artifacts(system) - cmdline = ['rsync'] - cmdline.extend(artifacts) - cmdline.append(rsync_dest) - cliapp.runcmd(cmdline) - - def find_artifacts(self, system): - artifacts_dir = os.path.join( - self.app.settings['cachedir'], 'artifacts') - artifacts = [] - - def find_in_system(dirname): - metadirs = [ - os.path.join(dirname, 'factory', 'baserock'), - os.path.join(dirname, 'baserock') - ] - existing_metadirs = [x for x in metadirs if os.path.isdir(x)] - if not existing_metadirs: - raise NotASystemArtifactError(system) - metadir = existing_metadirs[0] - for basename in glob.glob(os.path.join(metadir, '*.meta')): - metafile = os.path.join(metadir, basename) - metadata = json.load(open(metafile)) - cache_key = metadata['cache-key'] - artifact_glob = os.path.join(artifacts_dir, cache_key) + '*' - found_artifacts = glob.glob(artifact_glob) - if not found_artifacts: - raise cliapp.AppException('Could not find cache-key ' - + cache_key + 'for artifact ' - + metajson['artifact-name']) - artifacts.extend(found_artifacts) - - morphlib.bins.call_in_artifact_directory( - self.app, system, find_in_system) - - return artifacts - diff --git a/morphlib/plugins/list_artifacts_plugin.py b/morphlib/plugins/list_artifacts_plugin.py new file mode 100644 index 00000000..5e64f708 --- /dev/null +++ b/morphlib/plugins/list_artifacts_plugin.py @@ -0,0 +1,124 @@ +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# This plugin is used as part of the Baserock automated release process. +# +# See: <http://wiki.baserock.org/guides/release-process> for more information. + + +import cliapp +import morphlib + + +class ListArtifactsPlugin(cliapp.Plugin): + + def enable(self): + self.app.add_subcommand( + 'list-artifacts', self.list_artifacts, + arg_synopsis='REPO REF MORPH [MORPH]...') + + def disable(self): + pass + + def list_artifacts(self, args): + '''List every artifact in the build graph of a system. + + Command line arguments: + + * `REPO` is a git repository URL. + * `REF` is a branch or other commit reference in that repository. + * `MORPH` is a system morphology name at that ref. + + You can pass multiple values for `MORPH`, in which case the command + outputs the union of the build graphs of all the systems passed in. + + The output includes any meta-artifacts such as .meta and .build-log + files. + + ''' + + if len(args) < 3: + raise cliapp.AppException( + 'Wrong number of arguments to list-artifacts command ' + '(see help)') + + repo, ref = args[0], args[1] + system_names = map(morphlib.util.strip_morph_extension, args[2:]) + + self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app) + self.resolver = morphlib.artifactresolver.ArtifactResolver() + + artifact_files = set() + for system_name in system_names: + system_artifact_files = self.list_artifacts_for_system( + repo, ref, system_name) + artifact_files.update(system_artifact_files) + + for artifact_file in sorted(artifact_files): + print artifact_file + + def list_artifacts_for_system(self, repo, ref, system_name): + '''List all artifact files in the build graph of a single system.''' + + # Sadly, we must use a fresh source pool and a fresh list of artifacts + # for each system. Creating a source pool is slow (queries every Git + # repo involved in the build) and resolving artifacts isn't so quick + # either. Unfortunately, each Source object can only have one set of + # Artifact objects associated, which means the source pool cannot mix + # sources that are being built for multiple architectures: the build + # graph representation does not distinguish chunks or strata of + # different architectures right now. + + self.app.status( + msg='Creating source pool for %s' % system_name, chatty=True) + source_pool = self.app.create_source_pool( + self.lrc, self.rrc, (repo, ref, system_name + '.morph')) + + self.app.status( + msg='Resolving artifacts for %s' % system_name, chatty=True) + artifacts = self.resolver.resolve_artifacts(source_pool) + + def find_artifact_by_name(artifacts_list, name): + for a in artifacts_list: + if a.source.filename == name + '.morph': + return a + raise ValueError + + system_artifact = find_artifact_by_name(artifacts, system_name) + + self.app.status( + msg='Computing cache keys for %s' % system_name, chatty=True) + build_env = morphlib.buildenvironment.BuildEnvironment( + self.app.settings, system_artifact.source.morphology['arch']) + ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env) + + artifact_files = set() + for artifact in system_artifact.walk(): + artifact.cache_key = ckc.compute_key(artifact) + artifact.cache_id = ckc.get_cache_id(artifact) + + artifact_files.add(artifact.basename()) + + if artifact.source.morphology.needs_artifact_metadata_cached: + artifact_files.add('%s.meta' % artifact.basename()) + + # This is unfortunate hardwiring of behaviour; in future we + # should list all artifacts in the meta-artifact file, so we + # don't have to guess what files there will be. + artifact_files.add('%s.meta' % artifact.cache_key) + if artifact.source.morphology['kind'] == 'chunk': + artifact_files.add('%s.build-log' % artifact.cache_key) + + return artifact_files diff --git a/without-test-modules b/without-test-modules index a42cce97..a847da86 100644 --- a/without-test-modules +++ b/without-test-modules @@ -25,7 +25,7 @@ morphlib/plugins/expand_repo_plugin.py morphlib/plugins/deploy_plugin.py morphlib/plugins/__init__.py morphlib/writeexts.py -morphlib/plugins/copy-artifacts_plugin.py +morphlib/plugins/list_artifacts_plugin.py morphlib/plugins/trovectl_plugin.py morphlib/plugins/gc_plugin.py morphlib/plugins/branch_and_merge_new_plugin.py |