Merge in morph cache server

This is a runtime dependency of distbuild, so you can't exactly test
distbuild without it.

The cache server itself depends on bottle and flup, so rather than
having a distbuild stratum and a trove stratum both containing the cache
server, we should split morph out of the tools stratum and include it in
both devel and trove systems.

4 files changed, 527 insertions, 6 deletions

diff --git a/morph-cache-server b/morph-cache-server
new file mode 100755
index 00000000..a3c3c978
--- /dev/null
+++ b/morph-cache-server
@@ -0,0 +1,349 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2013, 2014 Codethink Limited
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import base64
+import cliapp
+import json
+import logging
+import os
+import urllib
+import urllib2
+import shutil
+
+from bottle import Bottle, request, response, run, static_file
+from flup.server.fcgi import WSGIServer
+from morphcacheserver.repocache import RepoCache
+
+
+defaults = {
+    'repo-dir': '/var/cache/morph-cache-server/gits',
+    'bundle-dir': '/var/cache/morph-cache-server/bundles',
+    'artifact-dir': '/var/cache/morph-cache-server/artifacts',
+    'port': 8080,
+}
+
+
+class MorphCacheServer(cliapp.Application):
+
+    def add_settings(self):
+        self.settings.integer(['port'],
+                              'port to listen on',
+                              metavar='PORTNUM',
+                              default=defaults['port'])
+        self.settings.string(['repo-dir'],
+                             'path to the repository cache directory',
+                             metavar='PATH',
+                             default=defaults['repo-dir'])
+        self.settings.string(['bundle-dir'],
+                             'path to the bundle cache directory',
+                             metavar='PATH',
+                             default=defaults['bundle-dir'])
+        self.settings.string(['artifact-dir'],
+                             'path to the artifact cache directory',
+                             metavar='PATH',
+                             default=defaults['artifact-dir'])
+        self.settings.boolean(['direct-mode'],
+                              'cache directories are directly managed')
+        self.settings.boolean(['enable-writes'],
+                              'enable the write methods (fetch and delete)')
+        self.settings.boolean(['fcgi-server'],
+                              'runs a fcgi-server',
+                              default=True)
+
+
+    def _fetch_artifact(self, url, filename):
+        in_fh = None
+        try:
+            in_fh = urllib2.urlopen(url)
+            with open(filename, "w") as localtmp:
+                shutil.copyfileobj(in_fh, localtmp)
+            in_fh.close()
+        except Exception, e:
+            if in_fh is not None:
+                in_fh.close()
+            raise
+        else:
+            if in_fh is not None:
+                in_fh.close()
+        return os.stat(filename)
+
+    def _fetch_artifacts(self, server, cacheid, artifacts):
+        ret = {}
+        try:
+            for artifact in artifacts:
+                artifact_name = "%s.%s" % (cacheid, artifact)
+                tmpname = os.path.join(self.settings['artifact-dir'],
+                                       ".dl.%s" % artifact_name)
+                url = "http://%s/1.0/artifacts?filename=%s" % (
+                    server, urllib.quote(artifact_name))
+                stinfo = self._fetch_artifact(url, tmpname)
+                ret[artifact_name] = {
+                    "size": stinfo.st_size,
+                    "used": stinfo.st_blocks * 512,
+                    }
+        except Exception, e:
+            for artifact in ret.iterkeys():
+                os.unlink(os.path.join(self.settings['artifact-dir'],
+                                       ".dl.%s" % artifact))
+            raise
+
+        for artifact in ret.iterkeys():
+            tmpname = os.path.join(self.settings['artifact-dir'],
+                                   ".dl.%s" % artifact)
+            artifilename = os.path.join(self.settings['artifact-dir'],
+                                        artifact)
+            os.rename(tmpname, artifilename)
+
+        return ret
+
+
+    def process_args(self, args):
+        app = Bottle()
+
+        repo_cache = RepoCache(self,
+                               self.settings['repo-dir'],
+                               self.settings['bundle-dir'],
+                               self.settings['direct-mode'])
+
+        def writable(prefix):
+            """Selectively enable bottle prefixes.
+
+            prefix -- The path prefix we are enabling
+
+            If the runtime configuration setting --enable-writes is provided
+            then we return the app.get() decorator for the given path prefix
+            otherwise we return a lambda which passes the function through
+            undecorated.
+
+            This has the effect of being a runtime-enablable @app.get(...)
+
+            """
+            if self.settings['enable-writes']:
+                return app.get(prefix)
+            return lambda fn: fn
+
+        @writable('/list')
+        def list():
+            response.set_header('Cache-Control', 'no-cache')
+            results = {}
+            files = {}
+            results["files"] = files
+            for artifactdir, __, filenames in \
+                    os.walk(self.settings['artifact-dir']):
+                fsstinfo = os.statvfs(artifactdir)
+                results["freespace"] = fsstinfo.f_bsize * fsstinfo.f_bavail
+                for fname in filenames:
+                    if not fname.startswith(".dl."):
+                        try:
+                            stinfo = os.stat("%s/%s" % (artifactdir, fname))
+                            files[fname] = {
+                                "atime": stinfo.st_atime,
+                                "size": stinfo.st_size,
+                                "used": stinfo.st_blocks * 512,
+                                }
+                        except Exception, e:
+                            print(e)
+            return results
+
+        @writable('/fetch')
+        def fetch():
+            host = self._unescape_parameter(request.query.host)
+            cacheid = self._unescape_parameter(request.query.cacheid)
+            artifacts = self._unescape_parameter(request.query.artifacts)
+            try:
+                response.set_header('Cache-Control', 'no-cache')
+                artifacts = artifacts.split(",")
+                return self._fetch_artifacts(host, cacheid, artifacts)
+
+            except Exception, e:
+                response.status = 500
+                logging.debug('%s' % e)
+
+        @writable('/delete')
+        def delete():
+            artifact = self._unescape_parameter(request.query.artifact)
+            try:
+                os.unlink('%s/%s' % (self.settings['artifact-dir'],
+                                     artifact))
+                return { "status": 0, "reason": "success" }
+            except OSError, ose:
+                return { "status": ose.errno, "reason": ose.strerror }
+            except Exception, e:
+                response.status = 500
+                logging.debug('%s' % e)
+
+        @app.get('/sha1s')
+        def sha1():
+            repo = self._unescape_parameter(request.query.repo)
+            ref = self._unescape_parameter(request.query.ref)
+            try:
+                response.set_header('Cache-Control', 'no-cache')
+                sha1, tree = repo_cache.resolve_ref(repo, ref)
+                return {
+                    'repo': '%s' % repo,
+                    'ref': '%s' % ref,
+                    'sha1': '%s' % sha1,
+                    'tree': '%s' % tree
+                }
+            except Exception, e:
+                response.status = 404
+                logging.debug('%s' % e)
+
+        @app.post('/sha1s')
+        def sha1s():
+            result = []
+            for pair in request.json:
+                repo = pair['repo']
+                ref = pair['ref']
+                try:
+                    sha1, tree = repo_cache.resolve_ref(repo, ref)
+                    result.append({
+                        'repo': '%s' % repo,
+                        'ref': '%s' % ref,
+                        'sha1': '%s' % sha1,
+                        'tree': '%s' % tree
+                    })
+                except Exception, e:
+                    logging.debug('%s' % e)
+                    result.append({
+                        'repo': '%s' % repo,
+                        'ref': '%s' % ref,
+                        'error': '%s' % e
+                    })
+            response.set_header('Cache-Control', 'no-cache')
+            response.set_header('Content-Type', 'application/json')
+            return json.dumps(result)
+        
+        @app.get('/files')
+        def file():
+            repo = self._unescape_parameter(request.query.repo)
+            ref = self._unescape_parameter(request.query.ref)
+            filename = self._unescape_parameter(request.query.filename)
+            try:
+                content = repo_cache.cat_file(repo, ref, filename)
+                response.set_header('Content-Type', 'application/octet-stream')
+                return content
+            except Exception, e:
+                response.status = 404
+                logging.debug('%s' % e)
+
+        @app.post('/files')
+        def files():
+            result = []
+            for pair in request.json:
+                repo = pair['repo']
+                ref = pair['ref']
+                filename = pair['filename']
+                try:
+                    content = repo_cache.cat_file(repo, ref, filename)
+                    result.append({
+                        'repo': '%s' % repo,
+                        'ref': '%s' % ref,
+                        'filename': '%s' % filename,
+                        'data': '%s' % base64.b64encode(content),
+                    })
+                except Exception, e:
+                    logging.debug('%s' % e)
+                    result.append({
+                        'repo': '%s' % repo,
+                        'ref': '%s' % ref,
+                        'filename': '%s' % filename,
+                        'error': '%s' % e
+                    })
+            response.set_header('Content-Type', 'application/json')
+            return json.dumps(result)
+
+        @app.get('/trees')
+        def tree():
+            repo = self._unescape_parameter(request.query.repo)
+            ref = self._unescape_parameter(request.query.ref)
+            path = self._unescape_parameter(request.query.path)
+            try:
+                tree = repo_cache.ls_tree(repo, ref, path)
+                return {
+                    'repo': '%s' % repo,
+                    'ref': '%s' % ref,
+                    'tree': tree,
+                }
+            except Exception, e:
+                response.status = 404
+                logging.debug('%s' % e)
+
+        @app.get('/bundles')
+        def bundle():
+            repo = self._unescape_parameter(request.query.repo)
+            filename = repo_cache.get_bundle_filename(repo)
+            dirname = os.path.dirname(filename)
+            basename = os.path.basename(filename)
+            return static_file(basename, root=dirname, download=True)
+
+        @app.get('/artifacts')
+        def artifact():
+            basename = self._unescape_parameter(request.query.filename)
+            filename = os.path.join(self.settings['artifact-dir'], basename)
+            if os.path.exists(filename):
+                return static_file(basename,
+                                   root=self.settings['artifact-dir'],
+                                   download=True)
+            else:
+                response.status = 404
+                logging.debug('artifact %s does not exist' % basename)
+
+        @app.post('/artifacts')
+        def post_artifacts():
+            if request.content_type != 'application/json':
+                logging.warning('Content-type is not json: '
+                    'expecting a json post request')
+
+            artifacts = json.load(request.body)
+            results = {}
+
+            logging.debug('Received a POST request for /artifacts')
+
+            for artifact in artifacts:
+                if artifact.startswith('/'):
+                    response.status = 500
+                    logging.error("%s: artifact name cannot start with a '/'"
+                        % artifact)
+                    return
+
+                filename = os.path.join(self.settings['artifact-dir'], artifact)
+                results[artifact] = os.path.exists(filename)
+
+                if results[artifact]:
+                    logging.debug('%s is in the cache', artifact)
+                else:
+                    logging.debug('%s is NOT in the cache', artifact)
+
+            return results
+
+        root = Bottle()
+        root.mount(app, '/1.0')
+
+
+        if self.settings['fcgi-server']:
+            WSGIServer(root).run()
+        else:
+            run(root, host='0.0.0.0', port=self.settings['port'], reloader=True)
+
+    def _unescape_parameter(self, param):
+        return urllib.unquote(param)
+
+
+if __name__ == '__main__':
+    MorphCacheServer().run()
diff --git a/morphcacheserver/__init__.py b/morphcacheserver/__init__.py
new file mode 100644
index 00000000..2c25ce28
--- /dev/null
+++ b/morphcacheserver/__init__.py
@@ -0,0 +1,17 @@
+# Copyright (C) 2013  Codethink Limited
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import repocache
diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py
new file mode 100644
index 00000000..0e4d909e
--- /dev/null
+++ b/morphcacheserver/repocache.py
@@ -0,0 +1,158 @@
+# Copyright (C) 2013  Codethink Limited
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import cliapp
+import os
+import re
+import string
+import urlparse
+
+
+class RepositoryNotFoundError(cliapp.AppException):
+
+    def __init__(self, repo):
+        cliapp.AppException.__init__(
+                self, 'Repository %s does not exist in the cache' % repo)
+
+
+class InvalidReferenceError(cliapp.AppException):
+    
+    def __init__(self, repo, ref):
+        cliapp.AppException.__init__(
+                self, 'Ref %s is an invalid reference for repo %s' %
+                (ref, repo))
+
+
+class UnresolvedNamedReferenceError(cliapp.AppException):
+
+    def __init__(self, repo, ref):
+        cliapp.AppException.__init__(
+                self, 'Ref %s is not a SHA1 ref for repo %s' %
+                (ref, repo))
+
+
+class RepoCache(object):
+    
+    def __init__(self, app, repo_cache_dir, bundle_cache_dir, direct_mode):
+        self.app = app
+        self.repo_cache_dir = repo_cache_dir
+        self.bundle_cache_dir = bundle_cache_dir
+        self.direct_mode = direct_mode
+
+    def resolve_ref(self, repo_url, ref):
+        quoted_url = self._quote_url(repo_url)
+        repo_dir = os.path.join(self.repo_cache_dir, quoted_url)
+        if not os.path.exists(repo_dir):
+            repo_dir = "%s.git" % repo_dir
+            if not os.path.exists(repo_dir):
+                raise RepositoryNotFoundError(repo_url)
+        try:
+            if re.match('^[0-9a-fA-F]{40}$', ref):
+                sha1 = ref
+            else:
+                if (not self.direct_mode and
+                    not ref.startswith('refs/origin/')):
+                    ref = 'refs/origin/' + ref
+                sha1 = self._rev_parse(repo_dir, ref)
+            return sha1, self._tree_from_commit(repo_dir, sha1)
+
+        except cliapp.AppException:
+            raise
+
+    def _tree_from_commit(self, repo_dir, commitsha):
+        commit_info = self.app.runcmd(['git', 'log', '-1',
+                                       '--format=format:%T', commitsha],
+                                      cwd=repo_dir)
+        return commit_info.strip()
+
+    def cat_file(self, repo_url, ref, filename):
+        quoted_url = self._quote_url(repo_url)
+        repo_dir = os.path.join(self.repo_cache_dir, quoted_url)
+        if not os.path.exists(repo_dir):
+            repo_dir = "%s.git" % repo_dir
+            if not os.path.exists(repo_dir):
+                raise RepositoryNotFoundError(repo_url)
+        if not self._is_valid_sha1(ref):
+            raise UnresolvedNamedReferenceError(repo_url, ref)
+        if not os.path.exists(repo_dir):
+            raise RepositoryNotFoundError(repo_url)
+        try:
+            sha1 = self._rev_parse(repo_dir, ref)
+        except:
+            raise InvalidReferenceError(repo_url, ref)
+
+        return self._cat_file(repo_dir, sha1, filename)
+
+    def ls_tree(self, repo_url, ref, path):
+        quoted_url = self._quote_url(repo_url)
+        repo_dir = os.path.join(self.repo_cache_dir, quoted_url)
+        if not os.path.exists(repo_dir):
+            repo_dir = "%s.git" % repo_dir
+            if not os.path.exists(repo_dir):
+                raise RepositoryNotFoundError(repo_url)
+        if not self._is_valid_sha1(ref):
+            raise UnresolvedNamedReferenceError(repo_url, ref)
+        if not os.path.exists(repo_dir):
+            raise RepositoryNotFoundError(repo_url)
+
+        try:
+            sha1 = self._rev_parse(repo_dir, ref)
+        except:
+            raise InvalidReferenceError(repo_url, ref)
+
+        lines = self._ls_tree(repo_dir, sha1, path).strip()
+        lines = lines.splitlines()
+        data = {}
+        for line in lines:
+            elements = line.split()
+            basename = elements[3]
+            data[basename] = {
+                'mode': elements[0],
+                'kind': elements[1],
+                'sha1': elements[2],
+            }
+        return data
+
+    def get_bundle_filename(self, repo_url):
+        quoted_url = self._quote_url(repo_url, True)
+        return os.path.join(self.bundle_cache_dir, '%s.bndl' % quoted_url)
+        
+    def _quote_url(self, url, always_indirect=False):
+        if self.direct_mode and not always_indirect:
+            quoted_url = urlparse.urlparse(url)[2]
+            while quoted_url.startswith("/"):
+                quoted_url = quoted_url[1:]
+            return quoted_url
+        else:
+            valid_chars = string.digits + string.letters + '%_'
+            transl = lambda x: x if x in valid_chars else '_'
+            return ''.join([transl(x) for x in url])
+
+    def _rev_parse(self, repo_dir, ref):
+        return self.app.runcmd(['git', 'rev-parse', '--verify', ref],
+                               cwd=repo_dir)[0:40]
+
+    def _cat_file(self, repo_dir, sha1, filename):
+        return self.app.runcmd(
+                ['git', 'cat-file', 'blob', '%s:%s' % (sha1, filename)],
+                cwd=repo_dir)
+
+    def _ls_tree(self, repo_dir, sha1, path):
+        return self.app.runcmd(['git', 'ls-tree', sha1, path], cwd=repo_dir)
+
+    def _is_valid_sha1(self, ref):
+        valid_chars = 'abcdefABCDEF0123456789'
+        return len(ref) == 40 and all([x in valid_chars for x in ref])
diff --git a/setup.py b/setup.py
index 8125788f..60926779 100644
--- a/setup.py
+++ b/setup.py
@@ -133,10 +133,6 @@ class Check(Command):
 
 
 setup(name='morph',
-      description='FIXME',
-      long_description='''\
-FIXME
-''',
       classifiers=[
           'Development Status :: 2 - Pre-Alpha',
           'Environment :: Console',
@@ -152,8 +148,9 @@ FIXME
       author='Codethink Limited',
       author_email='baserock-dev@baserock.org',
       url='http://www.baserock.org/',
-      scripts=['morph', 'distbuild-helper'],
-      packages=['morphlib', 'morphlib.plugins', 'distbuild'],
+      scripts=['morph', 'distbuild-helper', 'morph-cache-server'],
+      packages=['morphlib', 'morphlib.plugins', 'distbuild',
+                'morphcacheserver'],
       package_data={
           'morphlib': [
               'xfer-hole',