diff options
-rwxr-xr-x | morph-cache-server | 349 | ||||
-rw-r--r-- | morphcacheserver/__init__.py | 17 | ||||
-rw-r--r-- | morphcacheserver/repocache.py | 158 | ||||
-rw-r--r-- | setup.py | 9 |
4 files changed, 527 insertions, 6 deletions
diff --git a/morph-cache-server b/morph-cache-server new file mode 100755 index 00000000..a3c3c978 --- /dev/null +++ b/morph-cache-server @@ -0,0 +1,349 @@ +#!/usr/bin/env python +# +# Copyright (C) 2013, 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import base64 +import cliapp +import json +import logging +import os +import urllib +import urllib2 +import shutil + +from bottle import Bottle, request, response, run, static_file +from flup.server.fcgi import WSGIServer +from morphcacheserver.repocache import RepoCache + + +defaults = { + 'repo-dir': '/var/cache/morph-cache-server/gits', + 'bundle-dir': '/var/cache/morph-cache-server/bundles', + 'artifact-dir': '/var/cache/morph-cache-server/artifacts', + 'port': 8080, +} + + +class MorphCacheServer(cliapp.Application): + + def add_settings(self): + self.settings.integer(['port'], + 'port to listen on', + metavar='PORTNUM', + default=defaults['port']) + self.settings.string(['repo-dir'], + 'path to the repository cache directory', + metavar='PATH', + default=defaults['repo-dir']) + self.settings.string(['bundle-dir'], + 'path to the bundle cache directory', + metavar='PATH', + default=defaults['bundle-dir']) + self.settings.string(['artifact-dir'], + 'path to the artifact cache directory', + metavar='PATH', + default=defaults['artifact-dir']) + self.settings.boolean(['direct-mode'], + 'cache directories are directly managed') + self.settings.boolean(['enable-writes'], + 'enable the write methods (fetch and delete)') + self.settings.boolean(['fcgi-server'], + 'runs a fcgi-server', + default=True) + + + def _fetch_artifact(self, url, filename): + in_fh = None + try: + in_fh = urllib2.urlopen(url) + with open(filename, "w") as localtmp: + shutil.copyfileobj(in_fh, localtmp) + in_fh.close() + except Exception, e: + if in_fh is not None: + in_fh.close() + raise + else: + if in_fh is not None: + in_fh.close() + return os.stat(filename) + + def _fetch_artifacts(self, server, cacheid, artifacts): + ret = {} + try: + for artifact in artifacts: + artifact_name = "%s.%s" % (cacheid, artifact) + tmpname = os.path.join(self.settings['artifact-dir'], + ".dl.%s" % artifact_name) + url = "http://%s/1.0/artifacts?filename=%s" % ( + server, urllib.quote(artifact_name)) + stinfo = self._fetch_artifact(url, tmpname) + ret[artifact_name] = { + "size": stinfo.st_size, + "used": stinfo.st_blocks * 512, + } + except Exception, e: + for artifact in ret.iterkeys(): + os.unlink(os.path.join(self.settings['artifact-dir'], + ".dl.%s" % artifact)) + raise + + for artifact in ret.iterkeys(): + tmpname = os.path.join(self.settings['artifact-dir'], + ".dl.%s" % artifact) + artifilename = os.path.join(self.settings['artifact-dir'], + artifact) + os.rename(tmpname, artifilename) + + return ret + + + def process_args(self, args): + app = Bottle() + + repo_cache = RepoCache(self, + self.settings['repo-dir'], + self.settings['bundle-dir'], + self.settings['direct-mode']) + + def writable(prefix): + """Selectively enable bottle prefixes. + + prefix -- The path prefix we are enabling + + If the runtime configuration setting --enable-writes is provided + then we return the app.get() decorator for the given path prefix + otherwise we return a lambda which passes the function through + undecorated. + + This has the effect of being a runtime-enablable @app.get(...) + + """ + if self.settings['enable-writes']: + return app.get(prefix) + return lambda fn: fn + + @writable('/list') + def list(): + response.set_header('Cache-Control', 'no-cache') + results = {} + files = {} + results["files"] = files + for artifactdir, __, filenames in \ + os.walk(self.settings['artifact-dir']): + fsstinfo = os.statvfs(artifactdir) + results["freespace"] = fsstinfo.f_bsize * fsstinfo.f_bavail + for fname in filenames: + if not fname.startswith(".dl."): + try: + stinfo = os.stat("%s/%s" % (artifactdir, fname)) + files[fname] = { + "atime": stinfo.st_atime, + "size": stinfo.st_size, + "used": stinfo.st_blocks * 512, + } + except Exception, e: + print(e) + return results + + @writable('/fetch') + def fetch(): + host = self._unescape_parameter(request.query.host) + cacheid = self._unescape_parameter(request.query.cacheid) + artifacts = self._unescape_parameter(request.query.artifacts) + try: + response.set_header('Cache-Control', 'no-cache') + artifacts = artifacts.split(",") + return self._fetch_artifacts(host, cacheid, artifacts) + + except Exception, e: + response.status = 500 + logging.debug('%s' % e) + + @writable('/delete') + def delete(): + artifact = self._unescape_parameter(request.query.artifact) + try: + os.unlink('%s/%s' % (self.settings['artifact-dir'], + artifact)) + return { "status": 0, "reason": "success" } + except OSError, ose: + return { "status": ose.errno, "reason": ose.strerror } + except Exception, e: + response.status = 500 + logging.debug('%s' % e) + + @app.get('/sha1s') + def sha1(): + repo = self._unescape_parameter(request.query.repo) + ref = self._unescape_parameter(request.query.ref) + try: + response.set_header('Cache-Control', 'no-cache') + sha1, tree = repo_cache.resolve_ref(repo, ref) + return { + 'repo': '%s' % repo, + 'ref': '%s' % ref, + 'sha1': '%s' % sha1, + 'tree': '%s' % tree + } + except Exception, e: + response.status = 404 + logging.debug('%s' % e) + + @app.post('/sha1s') + def sha1s(): + result = [] + for pair in request.json: + repo = pair['repo'] + ref = pair['ref'] + try: + sha1, tree = repo_cache.resolve_ref(repo, ref) + result.append({ + 'repo': '%s' % repo, + 'ref': '%s' % ref, + 'sha1': '%s' % sha1, + 'tree': '%s' % tree + }) + except Exception, e: + logging.debug('%s' % e) + result.append({ + 'repo': '%s' % repo, + 'ref': '%s' % ref, + 'error': '%s' % e + }) + response.set_header('Cache-Control', 'no-cache') + response.set_header('Content-Type', 'application/json') + return json.dumps(result) + + @app.get('/files') + def file(): + repo = self._unescape_parameter(request.query.repo) + ref = self._unescape_parameter(request.query.ref) + filename = self._unescape_parameter(request.query.filename) + try: + content = repo_cache.cat_file(repo, ref, filename) + response.set_header('Content-Type', 'application/octet-stream') + return content + except Exception, e: + response.status = 404 + logging.debug('%s' % e) + + @app.post('/files') + def files(): + result = [] + for pair in request.json: + repo = pair['repo'] + ref = pair['ref'] + filename = pair['filename'] + try: + content = repo_cache.cat_file(repo, ref, filename) + result.append({ + 'repo': '%s' % repo, + 'ref': '%s' % ref, + 'filename': '%s' % filename, + 'data': '%s' % base64.b64encode(content), + }) + except Exception, e: + logging.debug('%s' % e) + result.append({ + 'repo': '%s' % repo, + 'ref': '%s' % ref, + 'filename': '%s' % filename, + 'error': '%s' % e + }) + response.set_header('Content-Type', 'application/json') + return json.dumps(result) + + @app.get('/trees') + def tree(): + repo = self._unescape_parameter(request.query.repo) + ref = self._unescape_parameter(request.query.ref) + path = self._unescape_parameter(request.query.path) + try: + tree = repo_cache.ls_tree(repo, ref, path) + return { + 'repo': '%s' % repo, + 'ref': '%s' % ref, + 'tree': tree, + } + except Exception, e: + response.status = 404 + logging.debug('%s' % e) + + @app.get('/bundles') + def bundle(): + repo = self._unescape_parameter(request.query.repo) + filename = repo_cache.get_bundle_filename(repo) + dirname = os.path.dirname(filename) + basename = os.path.basename(filename) + return static_file(basename, root=dirname, download=True) + + @app.get('/artifacts') + def artifact(): + basename = self._unescape_parameter(request.query.filename) + filename = os.path.join(self.settings['artifact-dir'], basename) + if os.path.exists(filename): + return static_file(basename, + root=self.settings['artifact-dir'], + download=True) + else: + response.status = 404 + logging.debug('artifact %s does not exist' % basename) + + @app.post('/artifacts') + def post_artifacts(): + if request.content_type != 'application/json': + logging.warning('Content-type is not json: ' + 'expecting a json post request') + + artifacts = json.load(request.body) + results = {} + + logging.debug('Received a POST request for /artifacts') + + for artifact in artifacts: + if artifact.startswith('/'): + response.status = 500 + logging.error("%s: artifact name cannot start with a '/'" + % artifact) + return + + filename = os.path.join(self.settings['artifact-dir'], artifact) + results[artifact] = os.path.exists(filename) + + if results[artifact]: + logging.debug('%s is in the cache', artifact) + else: + logging.debug('%s is NOT in the cache', artifact) + + return results + + root = Bottle() + root.mount(app, '/1.0') + + + if self.settings['fcgi-server']: + WSGIServer(root).run() + else: + run(root, host='0.0.0.0', port=self.settings['port'], reloader=True) + + def _unescape_parameter(self, param): + return urllib.unquote(param) + + +if __name__ == '__main__': + MorphCacheServer().run() diff --git a/morphcacheserver/__init__.py b/morphcacheserver/__init__.py new file mode 100644 index 00000000..2c25ce28 --- /dev/null +++ b/morphcacheserver/__init__.py @@ -0,0 +1,17 @@ +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import repocache diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py new file mode 100644 index 00000000..0e4d909e --- /dev/null +++ b/morphcacheserver/repocache.py @@ -0,0 +1,158 @@ +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import cliapp +import os +import re +import string +import urlparse + + +class RepositoryNotFoundError(cliapp.AppException): + + def __init__(self, repo): + cliapp.AppException.__init__( + self, 'Repository %s does not exist in the cache' % repo) + + +class InvalidReferenceError(cliapp.AppException): + + def __init__(self, repo, ref): + cliapp.AppException.__init__( + self, 'Ref %s is an invalid reference for repo %s' % + (ref, repo)) + + +class UnresolvedNamedReferenceError(cliapp.AppException): + + def __init__(self, repo, ref): + cliapp.AppException.__init__( + self, 'Ref %s is not a SHA1 ref for repo %s' % + (ref, repo)) + + +class RepoCache(object): + + def __init__(self, app, repo_cache_dir, bundle_cache_dir, direct_mode): + self.app = app + self.repo_cache_dir = repo_cache_dir + self.bundle_cache_dir = bundle_cache_dir + self.direct_mode = direct_mode + + def resolve_ref(self, repo_url, ref): + quoted_url = self._quote_url(repo_url) + repo_dir = os.path.join(self.repo_cache_dir, quoted_url) + if not os.path.exists(repo_dir): + repo_dir = "%s.git" % repo_dir + if not os.path.exists(repo_dir): + raise RepositoryNotFoundError(repo_url) + try: + if re.match('^[0-9a-fA-F]{40}$', ref): + sha1 = ref + else: + if (not self.direct_mode and + not ref.startswith('refs/origin/')): + ref = 'refs/origin/' + ref + sha1 = self._rev_parse(repo_dir, ref) + return sha1, self._tree_from_commit(repo_dir, sha1) + + except cliapp.AppException: + raise + + def _tree_from_commit(self, repo_dir, commitsha): + commit_info = self.app.runcmd(['git', 'log', '-1', + '--format=format:%T', commitsha], + cwd=repo_dir) + return commit_info.strip() + + def cat_file(self, repo_url, ref, filename): + quoted_url = self._quote_url(repo_url) + repo_dir = os.path.join(self.repo_cache_dir, quoted_url) + if not os.path.exists(repo_dir): + repo_dir = "%s.git" % repo_dir + if not os.path.exists(repo_dir): + raise RepositoryNotFoundError(repo_url) + if not self._is_valid_sha1(ref): + raise UnresolvedNamedReferenceError(repo_url, ref) + if not os.path.exists(repo_dir): + raise RepositoryNotFoundError(repo_url) + try: + sha1 = self._rev_parse(repo_dir, ref) + except: + raise InvalidReferenceError(repo_url, ref) + + return self._cat_file(repo_dir, sha1, filename) + + def ls_tree(self, repo_url, ref, path): + quoted_url = self._quote_url(repo_url) + repo_dir = os.path.join(self.repo_cache_dir, quoted_url) + if not os.path.exists(repo_dir): + repo_dir = "%s.git" % repo_dir + if not os.path.exists(repo_dir): + raise RepositoryNotFoundError(repo_url) + if not self._is_valid_sha1(ref): + raise UnresolvedNamedReferenceError(repo_url, ref) + if not os.path.exists(repo_dir): + raise RepositoryNotFoundError(repo_url) + + try: + sha1 = self._rev_parse(repo_dir, ref) + except: + raise InvalidReferenceError(repo_url, ref) + + lines = self._ls_tree(repo_dir, sha1, path).strip() + lines = lines.splitlines() + data = {} + for line in lines: + elements = line.split() + basename = elements[3] + data[basename] = { + 'mode': elements[0], + 'kind': elements[1], + 'sha1': elements[2], + } + return data + + def get_bundle_filename(self, repo_url): + quoted_url = self._quote_url(repo_url, True) + return os.path.join(self.bundle_cache_dir, '%s.bndl' % quoted_url) + + def _quote_url(self, url, always_indirect=False): + if self.direct_mode and not always_indirect: + quoted_url = urlparse.urlparse(url)[2] + while quoted_url.startswith("/"): + quoted_url = quoted_url[1:] + return quoted_url + else: + valid_chars = string.digits + string.letters + '%_' + transl = lambda x: x if x in valid_chars else '_' + return ''.join([transl(x) for x in url]) + + def _rev_parse(self, repo_dir, ref): + return self.app.runcmd(['git', 'rev-parse', '--verify', ref], + cwd=repo_dir)[0:40] + + def _cat_file(self, repo_dir, sha1, filename): + return self.app.runcmd( + ['git', 'cat-file', 'blob', '%s:%s' % (sha1, filename)], + cwd=repo_dir) + + def _ls_tree(self, repo_dir, sha1, path): + return self.app.runcmd(['git', 'ls-tree', sha1, path], cwd=repo_dir) + + def _is_valid_sha1(self, ref): + valid_chars = 'abcdefABCDEF0123456789' + return len(ref) == 40 and all([x in valid_chars for x in ref]) @@ -133,10 +133,6 @@ class Check(Command): setup(name='morph', - description='FIXME', - long_description='''\ -FIXME -''', classifiers=[ 'Development Status :: 2 - Pre-Alpha', 'Environment :: Console', @@ -152,8 +148,9 @@ FIXME author='Codethink Limited', author_email='baserock-dev@baserock.org', url='http://www.baserock.org/', - scripts=['morph', 'distbuild-helper'], - packages=['morphlib', 'morphlib.plugins', 'distbuild'], + scripts=['morph', 'distbuild-helper', 'morph-cache-server'], + packages=['morphlib', 'morphlib.plugins', 'distbuild', + 'morphcacheserver'], package_data={ 'morphlib': [ 'xfer-hole', |