Merge branch 'sam/list-artifacts'

Reviewed-By: Adam Coldrick <adam.coldrick@codethink.co.uk>
Reviewed-By: Richard Maw <richard.maw@codethink.co.uk>

3 files changed, 125 insertions, 143 deletions

diff --git a/morphlib/plugins/copy-artifacts_plugin.py b/morphlib/plugins/copy-artifacts_plugin.py
deleted file mode 100644
index 577d0ef2..00000000
--- a/morphlib/plugins/copy-artifacts_plugin.py
+++ /dev/null
@@ -1,142 +0,0 @@
-# Copyright (C) 2013  Codethink Limited
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-
-# FIXME: no tests
-
-
-
-import os
-import glob
-import json
-
-
-import cliapp
-
-
-
-import morphlib
-
-
-
-class CopyArtifactsPlugin(cliapp.Plugin):
-
-    def enable(self):
-        self.app.add_subcommand(
-            'list-artifacts', self.list_artifacts,
-            arg_synopsis='SYSTEM-ARTIFACT')
-        self.app.add_subcommand(
-            'copy-artifacts', self.copy_artifacts,
-            arg_synopsis='SYSTEM-ARTIFACT DESTINATION')
-
-    def disable(self):
-        pass
-
-    def list_artifacts(self, args):
-        '''List every artifact that makes up a system artifact.
-
-        Command line arguments:
-
-        * `SYSTEM-ARTIFACT` is the filename for a build artifact for
-          a system (ending in `-rootfs`).
-
-        list-artifacts reads the system artifact and writes out a list
-        of the filenames, in the local artifact cache, of all the
-        component artifacts included in the system. It does not include
-        build-dependencies of the system, unless they're included in
-        the system.
-
-        '''
-
-        if len(args) != 1:
-            raise cliapp.AppException(
-                'Wrong number of arguments to list-artifacts command'
-                '(see help)')
-
-        system = args[0]
-        artifacts = self.find_artifacts(system)
-        for artifact in artifacts:
-            self.app.output.write(artifact + "\n")
-
-    def copy_artifacts(self, args):
-        '''Copy every artifact that makes up a system to an rsync path.
-
-        Command line arguments:
-
-        * `SYSTEM-ARTIFACT` is the filename for a build artifact for
-          a system (ending in `-rootfs`).
-        * `DESTINATION` is a URL for where the artifacts are to be
-          copied to, in a form suitable for the rsync program.
-
-        This command is useful for copying artifacts from a local
-        system to a Morph artifact cache server, so they can be
-        shared by other people. It can also be used to archive
-        artifacts used for a release. Note, however, that it does
-        not include artifacts that are build-dependencies of the
-        strata included in the system, but not actually in cluded
-        in the system.
-
-        Example:
-
-            morph copy-artifacts /src/cache/artifacts/*.system.* \
-                cache.example.com:/srv/cache/
-
-        '''
-
-        if len(args) != 2:
-            raise cliapp.AppException(
-                'Wrong number of arguments to copy-artifacts command'
-                '(see help)')
-
-        system = args[0]
-        rsync_dest = args[1]
-        artifacts = self.find_artifacts(system)
-        cmdline = ['rsync']
-        cmdline.extend(artifacts)
-        cmdline.append(rsync_dest)
-        cliapp.runcmd(cmdline)
-
-    def find_artifacts(self, system):
-        artifacts_dir = os.path.join(
-            self.app.settings['cachedir'], 'artifacts')
-        artifacts = []
-
-        def find_in_system(dirname):
-            metadirs = [
-                os.path.join(dirname, 'factory', 'baserock'),
-                os.path.join(dirname, 'baserock')
-            ]
-            existing_metadirs = [x for x in metadirs if os.path.isdir(x)]
-            if not existing_metadirs:
-                raise NotASystemArtifactError(system)
-            metadir = existing_metadirs[0]
-            for basename in glob.glob(os.path.join(metadir, '*.meta')):
-                metafile = os.path.join(metadir, basename)
-                metadata = json.load(open(metafile))
-                cache_key = metadata['cache-key']
-                artifact_glob = os.path.join(artifacts_dir, cache_key) + '*'
-                found_artifacts = glob.glob(artifact_glob)
-                if not found_artifacts:
-                    raise cliapp.AppException('Could not find cache-key '
-                                              + cache_key + 'for artifact '
-                                              + metajson['artifact-name'])
-                artifacts.extend(found_artifacts)
-
-        morphlib.bins.call_in_artifact_directory(
-            self.app, system, find_in_system)
-
-        return artifacts
-
diff --git a/morphlib/plugins/list_artifacts_plugin.py b/morphlib/plugins/list_artifacts_plugin.py
new file mode 100644
index 00000000..5e64f708
--- /dev/null
+++ b/morphlib/plugins/list_artifacts_plugin.py
@@ -0,0 +1,124 @@
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# This plugin is used as part of the Baserock automated release process.
+#
+# See: <http://wiki.baserock.org/guides/release-process> for more information.
+
+
+import cliapp
+import morphlib
+
+
+class ListArtifactsPlugin(cliapp.Plugin):
+
+    def enable(self):
+        self.app.add_subcommand(
+            'list-artifacts', self.list_artifacts,
+            arg_synopsis='REPO REF MORPH [MORPH]...')
+
+    def disable(self):
+        pass
+
+    def list_artifacts(self, args):
+        '''List every artifact in the build graph of a system.
+
+        Command line arguments:
+
+        * `REPO` is a git repository URL.
+        * `REF` is a branch or other commit reference in that repository.
+        * `MORPH` is a system morphology name at that ref.
+
+        You can pass multiple values for `MORPH`, in which case the command
+        outputs the union of the build graphs of all the systems passed in.
+
+        The output includes any meta-artifacts such as .meta and .build-log
+        files.
+
+        '''
+
+        if len(args) < 3:
+            raise cliapp.AppException(
+                'Wrong number of arguments to list-artifacts command '
+                '(see help)')
+
+        repo, ref = args[0], args[1]
+        system_names = map(morphlib.util.strip_morph_extension, args[2:])
+
+        self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app)
+        self.resolver = morphlib.artifactresolver.ArtifactResolver()
+
+        artifact_files = set()
+        for system_name in system_names:
+            system_artifact_files = self.list_artifacts_for_system(
+                repo, ref, system_name)
+            artifact_files.update(system_artifact_files)
+
+        for artifact_file in sorted(artifact_files):
+            print artifact_file
+
+    def list_artifacts_for_system(self, repo, ref, system_name):
+        '''List all artifact files in the build graph of a single system.'''
+
+        # Sadly, we must use a fresh source pool and a fresh list of artifacts
+        # for each system. Creating a source pool is slow (queries every Git
+        # repo involved in the build) and resolving artifacts isn't so quick
+        # either. Unfortunately, each Source object can only have one set of
+        # Artifact objects associated, which means the source pool cannot mix
+        # sources that are being built for multiple architectures: the build
+        # graph representation does not distinguish chunks or strata of
+        # different architectures right now.
+
+        self.app.status(
+            msg='Creating source pool for %s' % system_name, chatty=True)
+        source_pool = self.app.create_source_pool(
+            self.lrc, self.rrc, (repo, ref, system_name + '.morph'))
+
+        self.app.status(
+            msg='Resolving artifacts for %s' % system_name, chatty=True)
+        artifacts = self.resolver.resolve_artifacts(source_pool)
+
+        def find_artifact_by_name(artifacts_list, name):
+            for a in artifacts_list:
+                if a.source.filename == name + '.morph':
+                    return a
+            raise ValueError
+
+        system_artifact = find_artifact_by_name(artifacts, system_name)
+
+        self.app.status(
+            msg='Computing cache keys for %s' % system_name, chatty=True)
+        build_env = morphlib.buildenvironment.BuildEnvironment(
+            self.app.settings, system_artifact.source.morphology['arch'])
+        ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env)
+
+        artifact_files = set()
+        for artifact in system_artifact.walk():
+            artifact.cache_key = ckc.compute_key(artifact)
+            artifact.cache_id = ckc.get_cache_id(artifact)
+
+            artifact_files.add(artifact.basename())
+
+            if artifact.source.morphology.needs_artifact_metadata_cached:
+                artifact_files.add('%s.meta' % artifact.basename())
+
+            # This is unfortunate hardwiring of behaviour; in future we
+            # should list all artifacts in the meta-artifact file, so we
+            # don't have to guess what files there will be.
+            artifact_files.add('%s.meta' % artifact.cache_key)
+            if artifact.source.morphology['kind'] == 'chunk':
+                artifact_files.add('%s.build-log' % artifact.cache_key)
+
+        return artifact_files
diff --git a/without-test-modules b/without-test-modules
index a42cce97..a847da86 100644
--- a/without-test-modules
+++ b/without-test-modules
@@ -25,7 +25,7 @@ morphlib/plugins/expand_repo_plugin.py
 morphlib/plugins/deploy_plugin.py
 morphlib/plugins/__init__.py
 morphlib/writeexts.py
-morphlib/plugins/copy-artifacts_plugin.py
+morphlib/plugins/list_artifacts_plugin.py
 morphlib/plugins/trovectl_plugin.py
 morphlib/plugins/gc_plugin.py
 morphlib/plugins/branch_and_merge_new_plugin.py