path: root/morph-cache-server

#!/usr/bin/env python
#
# Copyright (C) 2013-2015 Codethink Limited
# 
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License along
# with this program.  If not, see <http://www.gnu.org/licenses/>.


import base64
import cliapp
import json
import logging
import os
import urllib
import urllib2
import shutil

from bottle import Bottle, request, response, run, static_file
from flup.server.fcgi import WSGIServer
from morphcacheserver.repocache import RepoCache
from morphlib.artifactcachereference import ArtifactCacheReference
from morphlib.ostreeartifactcache import OSTreeArtifactCache
from morphlib.remoteartifactcache import RemoteArtifactCache


defaults = {
    'repo-dir': '/var/cache/morph-cache-server/gits',
    'bundle-dir': '/var/cache/morph-cache-server/bundles',
    'artifact-dir': '/var/cache/morph-cache-server/artifacts',
    'port': 8080,
    'ostree-port': 12324,
    'ostree-repo-mode': 'archive_z2',
}


class MorphCacheServer(cliapp.Application):

    def add_settings(self):
        self.settings.integer(['port'],
                              'port to listen on',
                              metavar='PORTNUM',
                              default=defaults['port'])
        self.settings.integer(['ostree-port'],
                              'port for accessing the ostree repo for '
                              'the artifact cache',
                              default=defaults['ostree-port'])
        self.settings.string(['port-file'],
                              'write port number to FILE',
                              metavar='FILE',
                              default='')
        self.settings.string(['repo-dir'],
                             'path to the repository cache directory',
                             metavar='PATH',
                             default=defaults['repo-dir'])
        self.settings.string(['bundle-dir'],
                             'path to the bundle cache directory',
                             metavar='PATH',
                             default=defaults['bundle-dir'])
        self.settings.string(['artifact-dir'],
                             'path to the artifact cache directory',
                             metavar='PATH',
                             default=defaults['artifact-dir'])
        self.settings.string(['ostree-repo-mode'],
                             'mode of the ostree artifact cache - either '
                             'archive_z2 or bare, for servers and users '
                             'respectively.',
                             default=defaults['ostree-repo-mode'])
        self.settings.boolean(['direct-mode'],
                              'cache directories are directly managed')
        self.settings.boolean(['enable-writes'],
                              'enable the write methods (fetch and delete)')
        self.settings.boolean(['fcgi-server'],
                              'runs a fcgi-server',
                              default=True)

    def _fetch_artifacts(self, server, cacheid, artifacts):
        ret = {}
        cache = OSTreeArtifactCache(self.settings['artifact-dir'],
                                    mode=self.settings['ostree-repo-mode'])
        remote = RemoteArtifactCache('http://%s/' % server)
        try:
            for artifact in artifacts:
                logging.debug('%s.%s' % (cacheid, artifact))
                cache_artifact = ArtifactCacheReference(
                    '.'.join((cacheid, artifact)))
                cache.copy_from_remote(cache_artifact, remote)
        except Exception, e:
            logging.debug('OSTree raised an Exception: %s' % e)
            raise
        return ret


    def process_args(self, args):
        app = Bottle()

        repo_cache = RepoCache(self,
                               self.settings['repo-dir'],
                               self.settings['bundle-dir'],
                               self.settings['direct-mode'])

        def writable(prefix):
            """Selectively enable bottle prefixes.

            prefix -- The path prefix we are enabling

            If the runtime configuration setting --enable-writes is provided
            then we return the app.get() decorator for the given path prefix
            otherwise we return a lambda which passes the function through
            undecorated.

            This has the effect of being a runtime-enablable @app.get(...)

            """
            if self.settings['enable-writes']:
                return app.get(prefix)
            return lambda fn: fn

        @writable('/list')
        def list():
            response.set_header('Cache-Control', 'no-cache')
            results = {}
            files = {}
            results["files"] = files
            for artifactdir, __, filenames in \
                    os.walk(self.settings['artifact-dir']):
                fsstinfo = os.statvfs(artifactdir)
                results["freespace"] = fsstinfo.f_bsize * fsstinfo.f_bavail
                for fname in filenames:
                    if not fname.startswith(".dl."):
                        try:
                            stinfo = os.stat("%s/%s" % (artifactdir, fname))
                            files[fname] = {
                                "atime": stinfo.st_atime,
                                "size": stinfo.st_size,
                                "used": stinfo.st_blocks * 512,
                                }
                        except Exception, e:
                            print(e)
            return results

        @writable('/fetch')
        def fetch():
            host = self._unescape_parameter(request.query.host)
            cacheid = self._unescape_parameter(request.query.cacheid)
            artifacts = self._unescape_parameter(request.query.artifacts)
            try:
                response.set_header('Cache-Control', 'no-cache')
                artifacts = artifacts.split(",")
                return self._fetch_artifacts(host, cacheid, artifacts)
            except Exception, e:
                response.status = 500
                logging.debug('%s' % e)

        @writable('/delete')
        def delete():
            artifact = self._unescape_parameter(request.query.artifact)
            try:
                os.unlink('%s/%s' % (self.settings['artifact-dir'],
                                     artifact))
                return { "status": 0, "reason": "success" }
            except OSError, ose:
                return { "status": ose.errno, "reason": ose.strerror }
            except Exception, e:
                response.status = 500
                logging.debug('%s' % e)

        @app.get('/sha1s')
        def sha1():
            repo = self._unescape_parameter(request.query.repo)
            ref = self._unescape_parameter(request.query.ref)
            try:
                response.set_header('Cache-Control', 'no-cache')
                sha1, tree = repo_cache.resolve_ref(repo, ref)
                return {
                    'repo': '%s' % repo,
                    'ref': '%s' % ref,
                    'sha1': '%s' % sha1,
                    'tree': '%s' % tree
                }
            except Exception, e:
                response.status = 404
                logging.debug('%s' % e)

        @app.post('/sha1s')
        def sha1s():
            result = []
            for pair in request.json:
                repo = pair['repo']
                ref = pair['ref']
                try:
                    sha1, tree = repo_cache.resolve_ref(repo, ref)
                    result.append({
                        'repo': '%s' % repo,
                        'ref': '%s' % ref,
                        'sha1': '%s' % sha1,
                        'tree': '%s' % tree
                    })
                except Exception, e:
                    logging.debug('%s' % e)
                    result.append({
                        'repo': '%s' % repo,
                        'ref': '%s' % ref,
                        'error': '%s' % e
                    })
            response.set_header('Cache-Control', 'no-cache')
            response.set_header('Content-Type', 'application/json')
            return json.dumps(result)
        
        @app.get('/files')
        def file():
            repo = self._unescape_parameter(request.query.repo)
            ref = self._unescape_parameter(request.query.ref)
            filename = self._unescape_parameter(request.query.filename)
            try:
                content = repo_cache.cat_file(repo, ref, filename)
                response.set_header('Content-Type', 'application/octet-stream')
                return content
            except Exception, e:
                response.status = 404
                logging.debug('%s' % e)

        @app.post('/files')
        def files():
            result = []
            for pair in request.json:
                repo = pair['repo']
                ref = pair['ref']
                filename = pair['filename']
                try:
                    content = repo_cache.cat_file(repo, ref, filename)
                    result.append({
                        'repo': '%s' % repo,
                        'ref': '%s' % ref,
                        'filename': '%s' % filename,
                        'data': '%s' % base64.b64encode(content),
                    })
                except Exception, e:
                    logging.debug('%s' % e)
                    result.append({
                        'repo': '%s' % repo,
                        'ref': '%s' % ref,
                        'filename': '%s' % filename,
                        'error': '%s' % e
                    })
            response.set_header('Content-Type', 'application/json')
            return json.dumps(result)

        @app.get('/trees')
        def tree():
            repo = self._unescape_parameter(request.query.repo)
            ref = self._unescape_parameter(request.query.ref)
            path = self._unescape_parameter(request.query.path)
            try:
                tree = repo_cache.ls_tree(repo, ref, path)
                return {
                    'repo': '%s' % repo,
                    'ref': '%s' % ref,
                    'tree': tree,
                }
            except Exception, e:
                response.status = 404
                logging.debug('%s' % e)

        @app.get('/bundles')
        def bundle():
            repo = self._unescape_parameter(request.query.repo)
            filename = repo_cache.get_bundle_filename(repo)
            dirname = os.path.dirname(filename)
            basename = os.path.basename(filename)
            return static_file(basename, root=dirname, download=True)

        @app.get('/artifacts')
        def artifact():
            basename = self._unescape_parameter(request.query.filename)
            cache = OSTreeArtifactCache(self.settings['artifact-dir'],
                                        mode=self.settings['ostree-repo-mode'])
            try:
                cachekey, kind, name = basename.split('.', 2)
                a = ArtifactCacheReference(basename)
            except ValueError:
                # We can't split the name as expected, we want metadata
                cachekey, metadata_name = basename.split('.', 1)
                logging.debug('Looking for artifact metadata: %s'
                              % metadata_name)
                a = ArtifactCacheReference(cachekey)
                if cache.has_artifact_metadata(a, metadata_name):
                    filename = cache._artifact_metadata_filename(
                        a, metadata_name)
                    return static_file(basename,
                                       root=self.settings['artifact-dir'],
                                       download=True)
                else:
                    response.status = 404
                    logging.debug('artifact metadata %s does not exist'
                                  % metadata_name)

            if cache.has(a):
                if kind == 'stratum':
                    logging.debug('Stratum %s is in the cache' % name)
                    return static_file(basename,
                                       root=self.settings['artifact-dir'],
                                       download=True)
                else:
                    response.status = 500
                    logging.error('use `ostree pull` to get non-stratum '
                                  'artifacts')
            else:
                response.status = 404
                logging.debug('artifact %s does not exist' % basename)

        @app.post('/artifacts')
        def post_artifacts():
            if request.content_type != 'application/json':
                logging.warning('Content-type is not json: '
                    'expecting a json post request')

            artifacts = json.load(request.body)
            results = {}

            logging.debug('Received a POST request for /artifacts')

            cache = OSTreeArtifactCache(self.settings['artifact-dir'],
                                        mode=self.settings['ostree-repo-mode'])
            for basename in artifacts:
                if basename.startswith('/'):
                    response.status = 500
                    logging.error("%s: artifact name cannot start with a '/'"
                        % basename)
                    return

                a = ArtifactCacheReference(basename)
                results[basename] = cache.has(a)

                if results[basename]:
                    logging.debug('%s is in the cache', artifact)
                else:
                    logging.debug('%s is NOT in the cache', artifact)

            return results

        @app.get('/method')
        def method():
            return 'ostree'

        @app.get('/ostreeinfo')
        def ostree_info():
            logging.debug('returning %s' % self.settings['ostree-port'])
            return str(self.settings['ostree-port'])

        root = Bottle()
        root.mount(app, '/1.0')


        if self.settings['fcgi-server']:
            WSGIServer(root).run()
        elif self.settings['port-file']:
            import wsgiref.simple_server

            server_port_file = self.settings['port-file']
            class DebugServer(wsgiref.simple_server.WSGIServer):
                '''WSGI-like server that uses an ephemeral port.
                
                Rather than use a specified port, or default, the
                DebugServer binds to an ephemeral port on 127.0.0.1
                and writes its number to port-file, so a non-racy
                temporary port can be used.
                
                '''
                
                def __init__(self, (host, port), *args, **kwargs):
                    wsgiref.simple_server.WSGIServer.__init__(
                        self, ('127.0.0.1', 0), *args, **kwargs)
                    with open(server_port_file, 'w') as f:
                        f.write(str(self.server_port) + '\n')
            run(root, server_class=DebugServer, debug=True)
        else:
            run(root, host='0.0.0.0', port=self.settings['port'],
                reloader=True)

    def _unescape_parameter(self, param):
        return urllib.unquote(param)


if __name__ == '__main__':
    MorphCacheServer().run()