From 18a016b93261bf19f67100b86d55d8df6f9fc083 Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Fri, 27 Feb 2015 08:54:20 +0000 Subject: morph-cache-server: Add support for an OSTree artifact cache Change-Id: I8409eb5816e5fbd6416437f51b97a703e3bddac4 --- morph-cache-server | 124 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 72 insertions(+), 52 deletions(-) diff --git a/morph-cache-server b/morph-cache-server index 007cfbe8..938ecb1f 100755 --- a/morph-cache-server +++ b/morph-cache-server @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Copyright (C) 2013, 2014-2015 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -27,6 +27,9 @@ import shutil from bottle import Bottle, request, response, run, static_file from flup.server.fcgi import WSGIServer from morphcacheserver.repocache import RepoCache +from morphlib.artifactcachereference import ArtifactCacheReference +from morphlib.ostreeartifactcache import OSTreeArtifactCache +from morphlib.remoteartifactcache import RemoteArtifactCache defaults = { @@ -34,6 +37,8 @@ defaults = { 'bundle-dir': '/var/cache/morph-cache-server/bundles', 'artifact-dir': '/var/cache/morph-cache-server/artifacts', 'port': 8080, + 'ostree-port': 12324, + 'ostree-repo-mode': 'archive_z2', } @@ -44,6 +49,10 @@ class MorphCacheServer(cliapp.Application): 'port to listen on', metavar='PORTNUM', default=defaults['port']) + self.settings.integer(['ostree-port'], + 'port for accessing the ostree repo for ' + 'the artifact cache', + default=defaults['ostree-port']) self.settings.string(['port-file'], 'write port number to FILE', metavar='FILE', @@ -60,6 +69,11 @@ class MorphCacheServer(cliapp.Application): 'path to the artifact cache directory', metavar='PATH', default=defaults['artifact-dir']) + self.settings.string(['ostree-repo-mode'], + 'mode of the ostree artifact cache - either ' + 'archive_z2 or bare, for servers and users ' + 'respectively.', + default=defaults['ostree-repo-mode']) self.settings.boolean(['direct-mode'], 'cache directories are directly managed') self.settings.boolean(['enable-writes'], @@ -68,50 +82,20 @@ class MorphCacheServer(cliapp.Application): 'runs a fcgi-server', default=True) - - def _fetch_artifact(self, url, filename): - in_fh = None - try: - in_fh = urllib2.urlopen(url) - with open(filename, "w") as localtmp: - shutil.copyfileobj(in_fh, localtmp) - in_fh.close() - except Exception, e: - if in_fh is not None: - in_fh.close() - raise - else: - if in_fh is not None: - in_fh.close() - return os.stat(filename) - def _fetch_artifacts(self, server, cacheid, artifacts): ret = {} + cache = OSTreeArtifactCache(self.settings['artifact-dir'], + mode=self.settings['ostree-repo-mode']) + remote = RemoteArtifactCache('http://%s/' % server) try: for artifact in artifacts: - artifact_name = "%s.%s" % (cacheid, artifact) - tmpname = os.path.join(self.settings['artifact-dir'], - ".dl.%s" % artifact_name) - url = "http://%s/1.0/artifacts?filename=%s" % ( - server, urllib.quote(artifact_name)) - stinfo = self._fetch_artifact(url, tmpname) - ret[artifact_name] = { - "size": stinfo.st_size, - "used": stinfo.st_blocks * 512, - } + logging.debug('%s.%s' % (cacheid, artifact)) + cache_artifact = ArtifactCacheReference( + '.'.join((cacheid, artifact))) + cache.copy_from_remote(cache_artifact, remote) except Exception, e: - for artifact in ret.iterkeys(): - os.unlink(os.path.join(self.settings['artifact-dir'], - ".dl.%s" % artifact)) + logging.debug('OSTree raised an Exception: %s' % e) raise - - for artifact in ret.iterkeys(): - tmpname = os.path.join(self.settings['artifact-dir'], - ".dl.%s" % artifact) - artifilename = os.path.join(self.settings['artifact-dir'], - artifact) - os.rename(tmpname, artifilename) - return ret @@ -172,7 +156,6 @@ class MorphCacheServer(cliapp.Application): response.set_header('Cache-Control', 'no-cache') artifacts = artifacts.split(",") return self._fetch_artifacts(host, cacheid, artifacts) - except Exception, e: response.status = 500 logging.debug('%s' % e) @@ -298,11 +281,38 @@ class MorphCacheServer(cliapp.Application): @app.get('/artifacts') def artifact(): basename = self._unescape_parameter(request.query.filename) - filename = os.path.join(self.settings['artifact-dir'], basename) - if os.path.exists(filename): - return static_file(basename, - root=self.settings['artifact-dir'], - download=True) + cache = OSTreeArtifactCache(self.settings['artifact-dir'], + mode=self.settings['ostree-repo-mode']) + try: + cachekey, kind, name = basename.split('.', 2) + a = ArtifactCacheReference(basename) + except ValueError: + # We can't split the name as expected, we want metadata + cachekey, metadata_name = basename.split('.', 1) + logging.debug('Looking for artifact metadata: %s' + % metadata_name) + a = ArtifactCacheReference(cachekey) + if cache.has_artifact_metadata(a, metadata_name): + filename = cache._artifact_metadata_filename( + a, metadata_name) + return static_file(basename, + root=self.settings['artifact-dir'], + download=True) + else: + response.status = 404 + logging.debug('artifact metadata %s does not exist' + % metadata_name) + + if cache.has(a): + if kind == 'stratum': + logging.debug('Stratum %s is in the cache' % name) + return static_file(basename, + root=self.settings['artifact-dir'], + download=True) + else: + response.status = 500 + logging.error('use `ostree pull` to get non-stratum ' + 'artifacts') else: response.status = 404 logging.debug('artifact %s does not exist' % basename) @@ -318,24 +328,34 @@ class MorphCacheServer(cliapp.Application): logging.debug('Received a POST request for /artifacts') - for artifact in artifacts: - if artifact.startswith('/'): + cache = OSTreeArtifactCache(self.settings['artifact-dir'], + mode=self.settings['ostree-repo-mode']) + for basename in artifacts: + if basename.startswith('/'): response.status = 500 logging.error("%s: artifact name cannot start with a '/'" - % artifact) + % basename) return - filename = os.path.join(self.settings['artifact-dir'], - artifact) - results[artifact] = os.path.exists(filename) + a = ArtifactCacheReference(basename) + results[basename] = cache.has(a) - if results[artifact]: + if results[basename]: logging.debug('%s is in the cache', artifact) else: logging.debug('%s is NOT in the cache', artifact) return results + @app.get('/method') + def method(): + return 'ostree' + + @app.get('/ostreeinfo') + def ostree_info(): + logging.debug('returning %s' % self.settings['ostree-port']) + return str(self.settings['ostree-port']) + root = Bottle() root.mount(app, '/1.0') -- cgit v1.2.1