Add a plugin containing some commands to help with the OSTree artifact cache

Change-Id: I37855e48a8ff099d4ea32d6ec4d44048f9eae077

3 files changed, 173 insertions, 2 deletions

diff --git a/morphlib/localartifactcache.py b/morphlib/localartifactcache.py
index e6695c4e..9e3c9e70 100644
--- a/morphlib/localartifactcache.py
+++ b/morphlib/localartifactcache.py
@@ -132,8 +132,9 @@ class LocalArtifactCache(object):
         CacheInfo = collections.namedtuple('CacheInfo', ('artifacts', 'mtime'))
         contents = collections.defaultdict(lambda: CacheInfo(set(), 0))
         for filename in self.cachefs.walkfiles():
-            cachekey = filename[:63]
-            artifact = filename[65:]
+            if filename.startswith('/repo'): # pragma: no cover
+                continue
+            cachekey, artifact = filename.split('.', 1)
             artifacts, max_mtime = contents[cachekey]
             artifacts.add(artifact)
             art_info = self.cachefs.getinfo(filename)
diff --git a/morphlib/plugins/ostree_artifacts_plugin.py b/morphlib/plugins/ostree_artifacts_plugin.py
new file mode 100644
index 00000000..eedcd1e7
--- /dev/null
+++ b/morphlib/plugins/ostree_artifacts_plugin.py
@@ -0,0 +1,169 @@
+# Copyright (C) 2015  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import collections
+import fs
+import os
+
+import cliapp
+
+import morphlib
+from morphlib.artifactcachereference import ArtifactCacheReference
+
+
+class NoCacheError(morphlib.Error):
+
+    def __init__(self, cachedir):
+        self.msg = ("Expected artifact cache directory %s doesn't exist.\n"
+                    "No existing cache to convert!" % cachedir)
+
+
+class ComponentNotInSystemError(morphlib.Error):
+
+    def __init__(self, components, system):
+        components = ', '.join(components)
+        self.msg = ('Components %s are not in %s. Ensure you provided '
+                    'component names rather than filenames.'
+                    % (components, system))
+
+
+class OSTreeArtifactsPlugin(cliapp.Plugin):
+
+    def enable(self):
+        self.app.add_subcommand('convert-local-cache', self.convert_cache,
+                                arg_synopsis='[DELETE]')
+        self.app.add_subcommand('query-cache', self.query_cache,
+                                arg_synopsis='SYSTEM NAME...')
+
+    def disable(self):
+        pass
+
+    def convert_cache(self, args):
+        """Convert a local tarball cache into an OSTree cache.
+        
+        Command line arguments:
+
+        * DELETE: This is an optional argument, which if given as "delete"
+          will cause tarball artifacts to be removed once they are converted.
+
+        This command will extract all the tarball artifacts in your local
+        artifact cache and store them in an OSTree repository in that
+        artifact cache. This will be quicker than redownloading all that
+        content from a remote cache server, but may still be time consuming
+        if your cache is large.
+
+        """
+        delete = False
+        if args:
+            if args[0] == 'delete':
+                delete = True
+
+        artifact_cachedir = os.path.join(self.app.settings['cachedir'],
+                                         'artifacts')
+        if not os.path.exists(artifact_cachedir):
+            raise NoCacheError(artifact_cachedir)
+
+        tarball_cache = morphlib.localartifactcache.LocalArtifactCache(
+                fs.osfs.OSFS(artifact_cachedir))
+        ostree_cache = morphlib.ostreeartifactcache.OSTreeArtifactCache(
+                artifact_cachedir, mode=self.app.settings['ostree-repo-mode'],
+                status_cb=self.app.status)
+
+        cached_artifacts = []
+        for cachekey, artifacts, last_used in tarball_cache.list_contents():
+            for artifact in artifacts:
+                basename = '.'.join((cachekey.lstrip('/'), artifact))
+                cached_artifacts.append(ArtifactCacheReference(basename))
+
+        # Set the method property of the tarball cache to allow us to
+        # treat it like a RemoteArtifactCache.
+        tarball_cache.method = 'tarball'
+
+        for artifact in cached_artifacts:
+            if not ostree_cache.has(artifact):
+                try:
+                    cache_key, kind, name = artifact.basename().split('.', 2)
+                    if kind in ('system', 'stratum'):
+                        # System artifacts are quick to recreate now, and
+                        # stratum artifacts are still stored in the same way.
+                        continue
+                except ValueError:
+                    # We must have metadata, which doesn't need converting
+                    continue
+                self.app.status(msg='Converting %(name)s',
+                                name=artifact.basename())
+                ostree_cache.copy_from_remote(artifact, tarball_cache)
+                if delete:
+                    os.remove(tarball_cache.artifact_filename(artifact))
+
+    def _find_artifacts(self, names, root_artifact):
+        found = collections.OrderedDict()
+        not_found = list(names)
+        for a in root_artifact.walk():
+            name = a.source.morphology['name']
+            if name in names and name not in found:
+                found[name] = [a]
+                if name in not_found:
+                    not_found.remove(name)
+            elif name in names:
+                found[name].append(a)
+                if name in not_found:
+                    not_found.remove(name)
+        return found, not_found
+
+    def query_cache(self, args):
+        """Check if the cache contains an artifact.
+
+        Command line arguments:
+
+        * `SYSTEM` is the filename of the system containing the components
+          to be looked for.
+        * `NAME...` is the name of one or more components to look for.
+
+        """
+        if not args:
+            raise cliapp.AppException('You must provide at least a system '
+                                      'filename.\nUsage: `morph query-cache '
+                                      'SYSTEM [NAME...]`')
+        ws = morphlib.workspace.open('.')
+        sb = morphlib.sysbranchdir.open_from_within('.')
+
+        system_filename = morphlib.util.sanitise_morphology_path(args[0])
+        system_filename = sb.relative_to_root_repo(system_filename)
+        component_names = args[1:]
+
+        bc = morphlib.buildcommand.BuildCommand(self.app)
+        repo = sb.get_config('branch.root')
+        ref = sb.get_config('branch.name')
+
+        definitions_repo_path = sb.get_git_directory_name(repo)
+        definitions_repo = morphlib.gitdir.GitDirectory(definitions_repo_path)
+        commit = definitions_repo.resolve_ref_to_commit(ref)
+
+        srcpool = bc.create_source_pool(repo, commit, system_filename)
+        bc.validate_sources(srcpool)
+        root = bc.resolve_artifacts(srcpool)
+        if not component_names:
+            component_names = [root.source.name]
+        components, not_found = self._find_artifacts(component_names, root)
+        if not_found:
+            raise ComponentNotInSystemError(not_found, system_filename)
+
+        for name, artifacts in components.iteritems():
+            for component in artifacts:
+                if bc.lac.has(component):
+                    print bc.lac._get_artifact_cache_name(component)
+                else:
+                    print '%s is not cached' % name
diff --git a/without-test-modules b/without-test-modules
index ebbcfb6a..d7173b6b 100644
--- a/without-test-modules
+++ b/without-test-modules
@@ -56,3 +56,4 @@ distbuild/worker_build_scheduler.py
 morphlib/buildbranch.py
 morphlib/ostree.py
 morphlib/ostreeartifactcache.py
+morphlib/plugins/ostree_artifacts_plugin.py