summaryrefslogtreecommitdiff
path: root/morphlib
diff options
context:
space:
mode:
authorSam Thursfield <sam.thursfield@codethink.co.uk>2014-05-15 14:41:19 +0000
committerSam Thursfield <sam.thursfield@codethink.co.uk>2014-06-05 11:58:35 +0100
commit179ce80da32a708e8fcb31747d345647f84957aa (patch)
tree3e3833c4ed03d717cafdc8ffb7d77e9492a280ea /morphlib
parent64263de0123f9e513a300fd179e5b84384fa5343 (diff)
downloadmorph-179ce80da32a708e8fcb31747d345647f84957aa.tar.gz
Rewrite copy_artifacts plugin as list_artifacts
The copy-artifacts and list-artifacts commands are mainly useful for making releases. As part of the release process we copy artifacts for the entire build graph of the release to the artifact cache on trove.baserock.org, to provide Baserock users with ready-built artifacts. This part of the release process is now automated, and the automation require the list-artifacts command to function as a 'plumbing' command. The copy-artifacts command is no longer required. It can be replaced with: morph list-artifacts --quiet REPO REF MORPH | rsync --files-from=- $TARGET The previous version of this plugin looked in the system artifact's metadata for the list of artifacts. This is flawed as the final system does not necessarily contain every build dependency. The new version of the plugin calculates the build graph from source, using the same process as the 'buildcommand' module. It also required looking in Morph's artifact cache for the system artifact file to analyse.
Diffstat (limited to 'morphlib')
-rw-r--r--morphlib/plugins/copy-artifacts_plugin.py142
-rw-r--r--morphlib/plugins/list_artifacts_plugin.py124
2 files changed, 124 insertions, 142 deletions
diff --git a/morphlib/plugins/copy-artifacts_plugin.py b/morphlib/plugins/copy-artifacts_plugin.py
deleted file mode 100644
index 577d0ef2..00000000
--- a/morphlib/plugins/copy-artifacts_plugin.py
+++ /dev/null
@@ -1,142 +0,0 @@
-# Copyright (C) 2013 Codethink Limited
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-
-# FIXME: no tests
-
-
-
-import os
-import glob
-import json
-
-
-import cliapp
-
-
-
-import morphlib
-
-
-
-class CopyArtifactsPlugin(cliapp.Plugin):
-
- def enable(self):
- self.app.add_subcommand(
- 'list-artifacts', self.list_artifacts,
- arg_synopsis='SYSTEM-ARTIFACT')
- self.app.add_subcommand(
- 'copy-artifacts', self.copy_artifacts,
- arg_synopsis='SYSTEM-ARTIFACT DESTINATION')
-
- def disable(self):
- pass
-
- def list_artifacts(self, args):
- '''List every artifact that makes up a system artifact.
-
- Command line arguments:
-
- * `SYSTEM-ARTIFACT` is the filename for a build artifact for
- a system (ending in `-rootfs`).
-
- list-artifacts reads the system artifact and writes out a list
- of the filenames, in the local artifact cache, of all the
- component artifacts included in the system. It does not include
- build-dependencies of the system, unless they're included in
- the system.
-
- '''
-
- if len(args) != 1:
- raise cliapp.AppException(
- 'Wrong number of arguments to list-artifacts command'
- '(see help)')
-
- system = args[0]
- artifacts = self.find_artifacts(system)
- for artifact in artifacts:
- self.app.output.write(artifact + "\n")
-
- def copy_artifacts(self, args):
- '''Copy every artifact that makes up a system to an rsync path.
-
- Command line arguments:
-
- * `SYSTEM-ARTIFACT` is the filename for a build artifact for
- a system (ending in `-rootfs`).
- * `DESTINATION` is a URL for where the artifacts are to be
- copied to, in a form suitable for the rsync program.
-
- This command is useful for copying artifacts from a local
- system to a Morph artifact cache server, so they can be
- shared by other people. It can also be used to archive
- artifacts used for a release. Note, however, that it does
- not include artifacts that are build-dependencies of the
- strata included in the system, but not actually in cluded
- in the system.
-
- Example:
-
- morph copy-artifacts /src/cache/artifacts/*.system.* \
- cache.example.com:/srv/cache/
-
- '''
-
- if len(args) != 2:
- raise cliapp.AppException(
- 'Wrong number of arguments to copy-artifacts command'
- '(see help)')
-
- system = args[0]
- rsync_dest = args[1]
- artifacts = self.find_artifacts(system)
- cmdline = ['rsync']
- cmdline.extend(artifacts)
- cmdline.append(rsync_dest)
- cliapp.runcmd(cmdline)
-
- def find_artifacts(self, system):
- artifacts_dir = os.path.join(
- self.app.settings['cachedir'], 'artifacts')
- artifacts = []
-
- def find_in_system(dirname):
- metadirs = [
- os.path.join(dirname, 'factory', 'baserock'),
- os.path.join(dirname, 'baserock')
- ]
- existing_metadirs = [x for x in metadirs if os.path.isdir(x)]
- if not existing_metadirs:
- raise NotASystemArtifactError(system)
- metadir = existing_metadirs[0]
- for basename in glob.glob(os.path.join(metadir, '*.meta')):
- metafile = os.path.join(metadir, basename)
- metadata = json.load(open(metafile))
- cache_key = metadata['cache-key']
- artifact_glob = os.path.join(artifacts_dir, cache_key) + '*'
- found_artifacts = glob.glob(artifact_glob)
- if not found_artifacts:
- raise cliapp.AppException('Could not find cache-key '
- + cache_key + 'for artifact '
- + metajson['artifact-name'])
- artifacts.extend(found_artifacts)
-
- morphlib.bins.call_in_artifact_directory(
- self.app, system, find_in_system)
-
- return artifacts
-
diff --git a/morphlib/plugins/list_artifacts_plugin.py b/morphlib/plugins/list_artifacts_plugin.py
new file mode 100644
index 00000000..5e64f708
--- /dev/null
+++ b/morphlib/plugins/list_artifacts_plugin.py
@@ -0,0 +1,124 @@
+# Copyright (C) 2014 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# This plugin is used as part of the Baserock automated release process.
+#
+# See: <http://wiki.baserock.org/guides/release-process> for more information.
+
+
+import cliapp
+import morphlib
+
+
+class ListArtifactsPlugin(cliapp.Plugin):
+
+ def enable(self):
+ self.app.add_subcommand(
+ 'list-artifacts', self.list_artifacts,
+ arg_synopsis='REPO REF MORPH [MORPH]...')
+
+ def disable(self):
+ pass
+
+ def list_artifacts(self, args):
+ '''List every artifact in the build graph of a system.
+
+ Command line arguments:
+
+ * `REPO` is a git repository URL.
+ * `REF` is a branch or other commit reference in that repository.
+ * `MORPH` is a system morphology name at that ref.
+
+ You can pass multiple values for `MORPH`, in which case the command
+ outputs the union of the build graphs of all the systems passed in.
+
+ The output includes any meta-artifacts such as .meta and .build-log
+ files.
+
+ '''
+
+ if len(args) < 3:
+ raise cliapp.AppException(
+ 'Wrong number of arguments to list-artifacts command '
+ '(see help)')
+
+ repo, ref = args[0], args[1]
+ system_names = map(morphlib.util.strip_morph_extension, args[2:])
+
+ self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app)
+ self.resolver = morphlib.artifactresolver.ArtifactResolver()
+
+ artifact_files = set()
+ for system_name in system_names:
+ system_artifact_files = self.list_artifacts_for_system(
+ repo, ref, system_name)
+ artifact_files.update(system_artifact_files)
+
+ for artifact_file in sorted(artifact_files):
+ print artifact_file
+
+ def list_artifacts_for_system(self, repo, ref, system_name):
+ '''List all artifact files in the build graph of a single system.'''
+
+ # Sadly, we must use a fresh source pool and a fresh list of artifacts
+ # for each system. Creating a source pool is slow (queries every Git
+ # repo involved in the build) and resolving artifacts isn't so quick
+ # either. Unfortunately, each Source object can only have one set of
+ # Artifact objects associated, which means the source pool cannot mix
+ # sources that are being built for multiple architectures: the build
+ # graph representation does not distinguish chunks or strata of
+ # different architectures right now.
+
+ self.app.status(
+ msg='Creating source pool for %s' % system_name, chatty=True)
+ source_pool = self.app.create_source_pool(
+ self.lrc, self.rrc, (repo, ref, system_name + '.morph'))
+
+ self.app.status(
+ msg='Resolving artifacts for %s' % system_name, chatty=True)
+ artifacts = self.resolver.resolve_artifacts(source_pool)
+
+ def find_artifact_by_name(artifacts_list, name):
+ for a in artifacts_list:
+ if a.source.filename == name + '.morph':
+ return a
+ raise ValueError
+
+ system_artifact = find_artifact_by_name(artifacts, system_name)
+
+ self.app.status(
+ msg='Computing cache keys for %s' % system_name, chatty=True)
+ build_env = morphlib.buildenvironment.BuildEnvironment(
+ self.app.settings, system_artifact.source.morphology['arch'])
+ ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env)
+
+ artifact_files = set()
+ for artifact in system_artifact.walk():
+ artifact.cache_key = ckc.compute_key(artifact)
+ artifact.cache_id = ckc.get_cache_id(artifact)
+
+ artifact_files.add(artifact.basename())
+
+ if artifact.source.morphology.needs_artifact_metadata_cached:
+ artifact_files.add('%s.meta' % artifact.basename())
+
+ # This is unfortunate hardwiring of behaviour; in future we
+ # should list all artifacts in the meta-artifact file, so we
+ # don't have to guess what files there will be.
+ artifact_files.add('%s.meta' % artifact.cache_key)
+ if artifact.source.morphology['kind'] == 'chunk':
+ artifact_files.add('%s.build-log' % artifact.cache_key)
+
+ return artifact_files