#!/usr/bin/env python # # Copyright (C) 2014-2017 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import logging import os import wsgiref.simple_server import bottle import cliapp from flup.server.fcgi import WSGIServer import lorrycontroller ONE_MINUTE = 60 class WEBAPP(cliapp.Application): def add_settings(self): self.settings.string( ['statedb'], 'use FILE as the state database', metavar='FILE') self.settings.string( ['configuration-directory'], 'use DIR as the configuration directory', metavar='DIR', default='.') self.settings.string( ['confgit-url'], 'get CONFGIT from URL', metavar='URL') self.settings.string( ['confgit-branch'], 'get git branch BRANCH in CONFGIT', metavar='URL', default='master') self.settings.boolean( ['debug-real-confgit'], 'if true, do real git operations on the configuration directory; ' 'if false, do no git operations on it and just what is there', default=True) self.settings.string( ['status-html'], 'write a static HTML page to FILE to describe overall status', metavar='FILE', default='/dev/null') self.settings.boolean( ['wsgi'], 'run in wsgi mode (default is debug mode, for development)') self.settings.integer( ['debug-port'], 'use PORT in debugging mode ' '(i.e., when not running under WSGI); ' 'note that using this to non-zero disables --debug-port-file', metavar='PORT', default=0) self.settings.string( ['debug-port-file'], 'write listening port to FILE when in debug mode ' '(i.e., not running under WSGI)', metavar='FILE', default='webapp.port') self.settings.string( ['debug-host'], 'listen on HOST when in debug mode (i.e., not running under WSGI)', metavar='HOST', default='0.0.0.0') self.settings.string_list( ['debug-fake-trove'], 'fake access to remote Troves (to do gitano ls, etc) ' 'using local files: get ls listing for TROVE from $PATH, ' 'where PATH names a file in JSON with the necessary info; ' 'may be used multiple times', metavar='TROVE=PATH') self.settings.string( ['templates'], 'find HTML page templates (*.tpl) in DIR', metavar='DIR', default='/usr/share/lorry-controller/templates') self.settings.string( ['static-files'], 'server static files from DIR', metavar='DIR', default='/usr/share/lorry-controller/static') # The default value of ten minutes for the ghost-timeout # setting was chosen arbitrarily, by Lars Wirzenius. The value # needs to be long enough that there's no realistic danger of # hitting it just because a host is a bit overloaded, but # still short enough that ghost jobs do get removed often # enough, especially right after boot, when all jobs are # ghosts. Experience may show that a different value would # actually be better, and if so, the code and this comment # should be changed accordingly. self.settings.integer( ['ghost-timeout'], 'running jobs should get an update from their ' 'MINION within this time or they will be considered ' 'ghosts and be removed from STATEDB (in seconds)', default=10*ONE_MINUTE) self.settings.choice( ['git-server-type'], ['gitano', 'gerrit', 'gitlab'], 'what API the local Git server speaks') self.settings.string( ['gitlab-private-token'], 'private token for GitLab API access') self.settings.boolean( ['publish-failures'], 'make the status page show failure logs from lorry') def find_routes(self): '''Return all classes that are API routes. This is a generator. ''' # This is a bit tricky and magic. Find all subclasses of # LorryControllerRoute in the lorrycontroller package. # This saves us from having to maintain a list of them # manually, but the introspective code is not necessarily # the most obvious. for name in dir(lorrycontroller): x = getattr(lorrycontroller, name) is_route = ( type(x) == type and # it must be class, for issubclass issubclass(x, lorrycontroller.LorryControllerRoute) and x != lorrycontroller.LorryControllerRoute) if is_route: yield x def process_args(self, args): self.settings.require('statedb') if (self.settings['git-server-type'] == 'gitlab' and not self.settings['gitlab-private-token']): logging.error('A private token must be provided to create ' 'repositories on a GitLab instance.') self.settings.require('gitlab-private-token') self.setup_proxy() templates = self.load_templates() webapp = bottle.Bottle() for route_class in self.find_routes(): route = route_class(self.settings, templates) webapp.route( path=route.path, method=route.http_method, callback=route.run) logging.info('Initialising database') statedb = lorrycontroller.StateDB(self.settings['statedb']) statedb.initialise_db() logging.info('Starting server') if self.settings['wsgi']: self.run_wsgi_server(webapp) else: self.run_debug_server(webapp) def load_templates(self): templates = {} for basename in os.listdir(self.settings['templates']): if basename.endswith('.tpl'): name = basename[:-len('.tpl')] pathname = os.path.join(self.settings['templates'], basename) with open(pathname) as f: templates[name] = f.read() return templates def run_wsgi_server(self, webapp): WSGIServer(webapp).run() def run_debug_server(self, webapp): if self.settings['debug-port']: self.run_debug_server_on_given_port(webapp) else: self.run_debug_server_on_random_port(webapp) def run_debug_server_on_given_port(self, webapp): bottle.run( webapp, host=self.settings['debug-host'], port=self.settings['debug-port'], quiet=True, debug=True) def run_debug_server_on_random_port(self, webapp): server_port_file = self.settings['debug-port-file'] class DebugServer(wsgiref.simple_server.WSGIServer): '''WSGI-like server that uses an ephemeral port. Rather than use a specified port, or default, the DebugServer connects to an ephemeral port and writes its number to debug-port-file, so a non-racy temporary port can be used. ''' def __init__(self, (host, port), *args, **kwargs): wsgiref.simple_server.WSGIServer.__init__( self, (host, 0), *args, **kwargs) with open(server_port_file, 'w') as f: f.write(str(self.server_port) + '\n') bottle.run( webapp, host=self.settings['debug-host'], server_class=DebugServer, quiet=True, debug=True) def setup_proxy(self): """Tell urllib2 to use a proxy for http action by lorry-controller. Load the proxy information from the JSON file given by proxy_def, then set urllib2's url opener to open urls via an authenticated proxy. """ config_filename = os.path.join( self.settings['configuration-directory'], 'proxy.conf') lorrycontroller.setup_proxy(config_filename) WEBAPP().run()