morph-cache-server: Add support for an OSTree artifact cache

Change-Id: I8409eb5816e5fbd6416437f51b97a703e3bddac4

1 files changed, 72 insertions, 52 deletions

diff --git a/morph-cache-server b/morph-cache-server
index 007cfbe8..938ecb1f 100755
--- a/morph-cache-server
+++ b/morph-cache-server
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 #
-# Copyright (C) 2013, 2014-2015 Codethink Limited
+# Copyright (C) 2013-2015 Codethink Limited
 # 
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -27,6 +27,9 @@ import shutil
 from bottle import Bottle, request, response, run, static_file
 from flup.server.fcgi import WSGIServer
 from morphcacheserver.repocache import RepoCache
+from morphlib.artifactcachereference import ArtifactCacheReference
+from morphlib.ostreeartifactcache import OSTreeArtifactCache
+from morphlib.remoteartifactcache import RemoteArtifactCache
 
 
 defaults = {
@@ -34,6 +37,8 @@ defaults = {
     'bundle-dir': '/var/cache/morph-cache-server/bundles',
     'artifact-dir': '/var/cache/morph-cache-server/artifacts',
     'port': 8080,
+    'ostree-port': 12324,
+    'ostree-repo-mode': 'archive_z2',
 }
 
 
@@ -44,6 +49,10 @@ class MorphCacheServer(cliapp.Application):
                               'port to listen on',
                               metavar='PORTNUM',
                               default=defaults['port'])
+        self.settings.integer(['ostree-port'],
+                              'port for accessing the ostree repo for '
+                              'the artifact cache',
+                              default=defaults['ostree-port'])
         self.settings.string(['port-file'],
                               'write port number to FILE',
                               metavar='FILE',
@@ -60,6 +69,11 @@ class MorphCacheServer(cliapp.Application):
                              'path to the artifact cache directory',
                              metavar='PATH',
                              default=defaults['artifact-dir'])
+        self.settings.string(['ostree-repo-mode'],
+                             'mode of the ostree artifact cache - either '
+                             'archive_z2 or bare, for servers and users '
+                             'respectively.',
+                             default=defaults['ostree-repo-mode'])
         self.settings.boolean(['direct-mode'],
                               'cache directories are directly managed')
         self.settings.boolean(['enable-writes'],
@@ -68,50 +82,20 @@ class MorphCacheServer(cliapp.Application):
                               'runs a fcgi-server',
                               default=True)
 
-
-    def _fetch_artifact(self, url, filename):
-        in_fh = None
-        try:
-            in_fh = urllib2.urlopen(url)
-            with open(filename, "w") as localtmp:
-                shutil.copyfileobj(in_fh, localtmp)
-            in_fh.close()
-        except Exception, e:
-            if in_fh is not None:
-                in_fh.close()
-            raise
-        else:
-            if in_fh is not None:
-                in_fh.close()
-        return os.stat(filename)
-
     def _fetch_artifacts(self, server, cacheid, artifacts):
         ret = {}
+        cache = OSTreeArtifactCache(self.settings['artifact-dir'],
+                                    mode=self.settings['ostree-repo-mode'])
+        remote = RemoteArtifactCache('http://%s/' % server)
         try:
             for artifact in artifacts:
-                artifact_name = "%s.%s" % (cacheid, artifact)
-                tmpname = os.path.join(self.settings['artifact-dir'],
-                                       ".dl.%s" % artifact_name)
-                url = "http://%s/1.0/artifacts?filename=%s" % (
-                    server, urllib.quote(artifact_name))
-                stinfo = self._fetch_artifact(url, tmpname)
-                ret[artifact_name] = {
-                    "size": stinfo.st_size,
-                    "used": stinfo.st_blocks * 512,
-                    }
+                logging.debug('%s.%s' % (cacheid, artifact))
+                cache_artifact = ArtifactCacheReference(
+                    '.'.join((cacheid, artifact)))
+                cache.copy_from_remote(cache_artifact, remote)
         except Exception, e:
-            for artifact in ret.iterkeys():
-                os.unlink(os.path.join(self.settings['artifact-dir'],
-                                       ".dl.%s" % artifact))
+            logging.debug('OSTree raised an Exception: %s' % e)
             raise
-
-        for artifact in ret.iterkeys():
-            tmpname = os.path.join(self.settings['artifact-dir'],
-                                   ".dl.%s" % artifact)
-            artifilename = os.path.join(self.settings['artifact-dir'],
-                                        artifact)
-            os.rename(tmpname, artifilename)
-
         return ret
 
 
@@ -172,7 +156,6 @@ class MorphCacheServer(cliapp.Application):
                 response.set_header('Cache-Control', 'no-cache')
                 artifacts = artifacts.split(",")
                 return self._fetch_artifacts(host, cacheid, artifacts)
-
             except Exception, e:
                 response.status = 500
                 logging.debug('%s' % e)
@@ -298,11 +281,38 @@ class MorphCacheServer(cliapp.Application):
         @app.get('/artifacts')
         def artifact():
             basename = self._unescape_parameter(request.query.filename)
-            filename = os.path.join(self.settings['artifact-dir'], basename)
-            if os.path.exists(filename):
-                return static_file(basename,
-                                   root=self.settings['artifact-dir'],
-                                   download=True)
+            cache = OSTreeArtifactCache(self.settings['artifact-dir'],
+                                        mode=self.settings['ostree-repo-mode'])
+            try:
+                cachekey, kind, name = basename.split('.', 2)
+                a = ArtifactCacheReference(basename)
+            except ValueError:
+                # We can't split the name as expected, we want metadata
+                cachekey, metadata_name = basename.split('.', 1)
+                logging.debug('Looking for artifact metadata: %s'
+                              % metadata_name)
+                a = ArtifactCacheReference(cachekey)
+                if cache.has_artifact_metadata(a, metadata_name):
+                    filename = cache._artifact_metadata_filename(
+                        a, metadata_name)
+                    return static_file(basename,
+                                       root=self.settings['artifact-dir'],
+                                       download=True)
+                else:
+                    response.status = 404
+                    logging.debug('artifact metadata %s does not exist'
+                                  % metadata_name)
+
+            if cache.has(a):
+                if kind == 'stratum':
+                    logging.debug('Stratum %s is in the cache' % name)
+                    return static_file(basename,
+                                       root=self.settings['artifact-dir'],
+                                       download=True)
+                else:
+                    response.status = 500
+                    logging.error('use `ostree pull` to get non-stratum '
+                                  'artifacts')
             else:
                 response.status = 404
                 logging.debug('artifact %s does not exist' % basename)
@@ -318,24 +328,34 @@ class MorphCacheServer(cliapp.Application):
 
             logging.debug('Received a POST request for /artifacts')
 
-            for artifact in artifacts:
-                if artifact.startswith('/'):
+            cache = OSTreeArtifactCache(self.settings['artifact-dir'],
+                                        mode=self.settings['ostree-repo-mode'])
+            for basename in artifacts:
+                if basename.startswith('/'):
                     response.status = 500
                     logging.error("%s: artifact name cannot start with a '/'"
-                        % artifact)
+                        % basename)
                     return
 
-                filename = os.path.join(self.settings['artifact-dir'],
-                                        artifact)
-                results[artifact] = os.path.exists(filename)
+                a = ArtifactCacheReference(basename)
+                results[basename] = cache.has(a)
 
-                if results[artifact]:
+                if results[basename]:
                     logging.debug('%s is in the cache', artifact)
                 else:
                     logging.debug('%s is NOT in the cache', artifact)
 
             return results
 
+        @app.get('/method')
+        def method():
+            return 'ostree'
+
+        @app.get('/ostreeinfo')
+        def ostree_info():
+            logging.debug('returning %s' % self.settings['ostree-port'])
+            return str(self.settings['ostree-port'])
+
         root = Bottle()
         root.mount(app, '/1.0')