Merge branch 'danielsilverstone/updates'

3 files changed, 137 insertions, 14 deletions

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 00000000..0d20b648
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+*.pyc
diff --git a/morph-cache-server b/morph-cache-server
index 3f72c186..b4f8fa1a 100755
--- a/morph-cache-server
+++ b/morph-cache-server
@@ -20,6 +20,7 @@ import cliapp
 import logging
 import os
 import urllib
+import shutil
 
 from bottle import Bottle, request, response, run, static_file
 
@@ -30,12 +31,17 @@ defaults = {
     'repo-dir': '/var/cache/morph-cache-server/gits',
     'bundle-dir': '/var/cache/morph-cache-server/bundles',
     'artifact-dir': '/var/cache/morph-cache-server/artifacts',
+    'port': 8080,
 }
 
 
 class MorphCacheServer(cliapp.Application):
 
     def add_settings(self):
+        self.settings.integer(['port'],
+                              'port to listen on',
+                              metavar='PORTNUM',
+                              default=defaults['port'])
         self.settings.string(['repo-dir'],
                              'path to the repository cache directory',
                              metavar='PATH',
@@ -48,13 +54,101 @@ class MorphCacheServer(cliapp.Application):
                              'path to the artifact cache directory',
                              metavar='PATH',
                              default=defaults['artifact-dir'])
+        self.settings.boolean(['direct-mode'],
+                              'cache directories are directly managed')
+        self.settings.boolean(['enable-writes'],
+                              'enable the write methods (fetch and delete)')
 
     def process_args(self, args):
         app = Bottle()
 
         repo_cache = RepoCache(self,
                                self.settings['repo-dir'],
-                               self.settings['bundle-dir'])
+                               self.settings['bundle-dir'],
+                               self.settings['direct-mode'])
+
+        def writable(prefix):
+            """Selectively enable bottle prefixes.
+
+            prefix -- The path prefix we are enabling
+
+            If the runtime configuration setting --enable-writes is provided
+            then we return the app.get() decorator for the given path prefix
+            otherwise we return a lambda which passes the function through
+            undecorated.
+
+            This has the effect of being a runtime-enablable @app.get(...)
+
+            """
+            if self.settings['enable-writes']:
+                return app.get(prefix)
+            return lambda fn: fn
+
+        @writable('/list')
+        def list():
+            response.set_header('Cache-Control', 'no-cache')
+            results = {}
+            files = {}
+            results["files"] = files
+            for artifactdir, __, filenames in \
+                    os.walk(self.settings['artifact-dir']):
+                fsstinfo = os.statvfs(artifactdir)
+                results["freespace"] = fsstinfo.f_bsize * fsstinfo.f_bavail
+                for fname in filenames:
+                    if not fname.startswith(".dl."):
+                        try:
+                            stinfo = os.stat("%s/%s" % (artifactdir, fname))
+                            files[fname] = {
+                                "atime": stinfo.st_atime,
+                                "size": stinfo.st_size,
+                                "used": stinfo.st_blocks * 512,
+                                }
+                        except Exception, e:
+                            print(e)
+            return results
+
+        @writable('/fetch')
+        def fetch():
+            host = self._unescape_parameter(request.query.host)
+            artifact = self._unescape_parameter(request.query.artifact)
+            try:
+                response.set_header('Cache-Control', 'no-cache')
+                in_fh = urllib.urlopen("http://%s/artifacts?basename=%s" %
+                                       (host, urllib.quote(artifact)))
+                tmpname = "%s/.dl.%s" % (
+                    self.settings['artifact-dir'],
+                    artifact)
+                localtmp = open(tmpname, "w")
+                shutil.copyfileobj(in_fh, localtmp)
+                localtmp.close()
+                in_fh.close()
+                artifilename = "%s/%s" % (self.settings['artifact-dir'],
+                                          artifact)
+                os.rename(tmpname, artifilename)
+                stinfo = os.stat(artifilename)
+                ret = {}
+                ret[artifact] = {
+                    "size": stinfo.st_size,
+                    "used": stinfo.st_blocks * 512
+                    }
+                return ret
+
+            except Exception, e:
+                response.status = 500
+                logging.debug('%s' % e)
+
+        @writable('/delete')
+        def delete():
+            artifact = self._unescape_parameter(request.query.artifact)
+            try:
+                os.unlink('%s/%s' % (self.settings['artifact-dir'],
+                                     artifact))
+                return { "status": 0, "reason": "success" }
+            except OSError, ose:
+                return { "status": ose.errno, "reason": ose.strerror }
+            except Exception, e:
+                response.status = 500
+                logging.debug('%s' % e)
 
         @app.get('/sha1s')
         def sha1():
@@ -62,11 +156,12 @@ class MorphCacheServer(cliapp.Application):
             ref = self._unescape_parameter(request.query.ref)
             try:
                 response.set_header('Cache-Control', 'no-cache')
-                sha1 = repo_cache.resolve_ref(repo, ref)
+                sha1, tree = repo_cache.resolve_ref(repo, ref)
                 return {
                     'repo': '%s' % repo,
                     'ref': '%s' % ref,
-                    'sha1': '%s' % sha1
+                    'sha1': '%s' % sha1,
+                    'tree': '%s' % tree
                 }
             except Exception, e:
                 response.status = 404
@@ -124,7 +219,7 @@ class MorphCacheServer(cliapp.Application):
         root = Bottle()
         root.mount(app, '/1.0')
         
-        run(root, host='0.0.0.0', port=8080, reloader=True)
+        run(root, host='0.0.0.0', port=self.settings['port'], reloader=True)
 
     def _unescape_parameter(self, param):
         return urllib.unquote(param)
diff --git a/morphcacheserver/repocache.py b/morphcacheserver/repocache.py
index 7061508d..b55692f2 100644
--- a/morphcacheserver/repocache.py
+++ b/morphcacheserver/repocache.py
@@ -17,6 +17,7 @@
 import cliapp
 import os
 import string
+import urlparse
 
 
 class RepositoryNotFoundError(cliapp.AppException):
@@ -44,32 +45,51 @@ class UnresolvedNamedReferenceError(cliapp.AppException):
 
 class RepoCache(object):
     
-    def __init__(self, app, repo_cache_dir, bundle_cache_dir):
+    def __init__(self, app, repo_cache_dir, bundle_cache_dir, direct_mode):
         self.app = app
         self.repo_cache_dir = repo_cache_dir
         self.bundle_cache_dir = bundle_cache_dir
+        self.direct_mode = direct_mode
 
     def resolve_ref(self, repo_url, ref):
         quoted_url = self._quote_url(repo_url)
         repo_dir = os.path.join(self.repo_cache_dir, quoted_url)
         if not os.path.exists(repo_dir):
-            raise RepositoryNotFoundError(repo_url)
+            repo_dir = "%s.git" % repo_dir
+            if not os.path.exists(repo_dir):
+                raise RepositoryNotFoundError(repo_url)
         try:
             refs = self._show_ref(repo_dir, ref).split('\n')
-            refs = [x.split() for x in refs if 'origin' in x]
-            return refs[0][0]
+            if self.direct_mode:
+                refs = [x.split() for x in refs]
+            else:
+                refs = [x.split() for x in refs if 'origin' in x]
+            return refs[0][0], self._tree_from_commit(repo_dir, refs[0][0])
+
         except cliapp.AppException:
             pass
+
         if not self._is_valid_sha1(ref):
             raise InvalidReferenceError(repo_url, ref)
         try:
-            return self._rev_list(ref).strip()
+            sha = self._rev_list(ref).strip()
+            return sha, self._tree_from_commit(repo_dir, sha)
         except:
             raise InvalidReferenceError(repo_url, ref)
 
+    def _tree_from_commit(self, repo_dir, commitsha):
+        commit_info = self.app.runcmd(['git', 'log', '-1',
+                                       '--format=format:%T', commitsha],
+                                      cwd=repo_dir)
+        return commit_info.strip()
+
     def cat_file(self, repo_url, ref, filename):
         quoted_url = self._quote_url(repo_url)
         repo_dir = os.path.join(self.repo_cache_dir, quoted_url)
+        if not os.path.exists(repo_dir):
+            repo_dir = "%s.git" % repo_dir
+            if not os.path.exists(repo_dir):
+                raise RepositoryNotFoundError(repo_url)
         if not self._is_valid_sha1(ref):
             raise UnresolvedNamedReferenceError(repo_url, ref)
         if not os.path.exists(repo_dir):
@@ -84,6 +104,10 @@ class RepoCache(object):
     def ls_tree(self, repo_url, ref, path):
         quoted_url = self._quote_url(repo_url)
         repo_dir = os.path.join(self.repo_cache_dir, quoted_url)
+        if not os.path.exists(repo_dir):
+            repo_dir = "%s.git" % repo_dir
+            if not os.path.exists(repo_dir):
+                raise RepositoryNotFoundError(repo_url)
         if not self._is_valid_sha1(ref):
             raise UnresolvedNamedReferenceError(repo_url, ref)
         if not os.path.exists(repo_dir):
@@ -108,13 +132,16 @@ class RepoCache(object):
         return data
 
     def get_bundle_filename(self, repo_url):
-        quoted_url = self._quote_url(repo_url)
+        quoted_url = self._quote_url(repo_url, True)
         return os.path.join(self.bundle_cache_dir, '%s.bndl' % quoted_url)
         
-    def _quote_url(self, url):
-        valid_chars = string.digits + string.letters + '%_'
-        transl = lambda x: x if x in valid_chars else '_'
-        return ''.join([transl(x) for x in url])
+    def _quote_url(self, url, always_indirect=False):
+        if self.direct_mode and not always_indirect:
+            return urlparse.urlparse(url)[2]
+        else:
+            valid_chars = string.digits + string.letters + '%_'
+            transl = lambda x: x if x in valid_chars else '_'
+            return ''.join([transl(x) for x in url])
 
     def _show_ref(self, repo_dir, ref):
         return self.app.runcmd(['git', 'show-ref', ref], cwd=repo_dir)