From 716ad28c18ac00c52797dc42c843569b1834fb88 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 10 Apr 2014 12:47:13 +0000 Subject: Remove old Lorry Controller to make room for new one --- lorry-controller | 355 ---------------------------------------- lorry-controller.conf | 20 --- lorrycontroller/__init__.py | 18 -- lorrycontroller/confparser.py | 335 ------------------------------------- lorrycontroller/htmlstatus.py | 286 -------------------------------- lorrycontroller/workingstate.py | 127 -------------- setup.py | 19 --- 7 files changed, 1160 deletions(-) delete mode 100755 lorry-controller delete mode 100644 lorry-controller.conf delete mode 100644 lorrycontroller/__init__.py delete mode 100644 lorrycontroller/confparser.py delete mode 100644 lorrycontroller/htmlstatus.py delete mode 100644 lorrycontroller/workingstate.py delete mode 100644 setup.py diff --git a/lorry-controller b/lorry-controller deleted file mode 100755 index 0ae4ceb..0000000 --- a/lorry-controller +++ /dev/null @@ -1,355 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (C) 2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import cliapp -import json -import logging -import os -import time -import re -import urllib -import urllib2 - - -from lorrycontroller.confparser import LorryControllerConfig -from lorrycontroller.workingstate import WorkingStateManager -from lorrycontroller.htmlstatus import HTMLStatusManager - - -defaults = { - 'work-area': '/home/lorry/controller-area', - 'config-name': 'lorry-controller.conf', - 'lorry': 'lorry', -} - - -token_finder = re.compile("([0-9a-f]{40})") - - -class LorryController(cliapp.Application): - - def add_settings(self): - self.settings.string(['work-area'], - 'path to the area for the controller to work in', - metavar='PATH', - default=defaults['work-area']) - self.settings.boolean(['dry-run'], - "do a dry-run and don't actually do anything " - "beyond updating the git tree", - default=False) - self.settings.string(['lorry'], - 'path to the lorry binary to use', - metavar='LORRY', - default=defaults['lorry']) - self.settings.string(['config-name'], - 'configuration leafname. Defaults to ' - 'lorry-controller.conf', - metavar='CONFNAME', - default=defaults['config-name']) - self.settings.boolean(['lorry-verbose'], - 'Whether to pass --verbose to lorry', - default=False) - self.settings.string(['lorry-log'], - 'Log file name for lorry if wanted', - metavar='LORRYLOG', - default=None) - self.settings.string(['html-file'], - 'HTML filename for lorry controller status', - metavar='HTMLFILE', - default=None) - - def process_args(self, args): - logging.info("Starting to control lorry") - try: - os.chdir(self.settings['work-area']) - except OSError, e: - logging.error("Unable to chdir() to %s" % - self.settings['work-area']) - raise SystemExit(2) - if not os.path.isdir("git"): - logging.error("Unable to find git checkout") - raise SystemExit(3) - if not os.path.isdir("work"): - os.mkdir("work") - - logging.info("Updating configuration checkout") - self.rungit(['remote', 'update', 'origin']) - self.rungit(['reset', '--hard', 'origin/master']) - self.rungit(['clean', '-fdx']) - - self.lorrycmd=[self.settings['lorry']] - if self.settings['lorry-verbose']: - self.lorrycmd += ["--verbose"] - if self.settings['lorry-log'] is not None: - self.lorrycmd += ["--log", self.settings['lorry-log']] - - if not os.path.exists(os.path.join('git', - self.settings['config-name'])): - logging.error("Unable to find lorry-controller.conf in git") - raise SystemExit(4) - - if os.path.isfile('git/proxy.conf'): - self.set_proxy('git/proxy.conf') - logging.info('Loaded proxy information') - self.conf = LorryControllerConfig(self, 'git/lorry-controller.conf') - self.html = HTMLStatusManager(self) - if self.settings['dry-run']: - self.html.series = 0 - self.html.write_out_status() - self.conf.parse_config() - - with WorkingStateManager(self) as mgr: - # Update any troves - self.html.set_mgr(mgr) - self.html.bump_state() - self.conf.update_troves(mgr) - prev_lorries = set(mgr.lorry_state.keys()) - cur_lorries = set(self.conf.lorries.keys()) - logging.info("Starting processing. Previously %d lorries " - "were handled. We currently have %d defined." % ( - len(prev_lorries), len(cur_lorries))) - - # 1. Handle deletes for any old lorries we no longer want - self.html.bump_state() - logging.info("Delete any old lorries...") - for dead_lorry in prev_lorries - cur_lorries: - self.html.set_processing(dead_lorry) - logging.info("Dead lorry: %s" % dead_lorry) - conf_uuid = mgr.lorry_state[dead_lorry]['conf'] - if conf_uuid in self.conf.configs: - should_delete = self.conf.configs[conf_uuid]['destroy'] - else: - # Could not find UUID in config, switch to 'never' - should_delete = "never" - want_destroy = (should_delete == "always") - if should_delete == "unchanged": - exit, out, err = self.maybe_runcmd( - ['git', 'ls-remote', 'ssh://git@localhost/%s.git' % - dead_lorry], dry=True) - if exit != 0: - logging.error("Unable to ls-remote to decide if " - "unchanged. Assuming it is changed.") - else: - logging.debug("TODO: Should decide if unchanged!") - - if want_destroy: - exit, out, err = self.maybe_runcmd(['ssh', 'git@localhost', - 'destroy', dead_lorry], - dry=True) - if exit != 0: - logging.error("Unable to destroy %s" % dead_lorry) - else: - token = token_finder.match(out).group(1) - exit, out, err = self.maybe_runcmd( - ['ssh', 'git@localhost', 'destroy', dead_lorry, - token]) - if exit != 0: - logging.error("Unable to destroy %s despite having" - " the token %s" % - (dead_lorry, token)) - else: - logging.debug("Destroyed") - del mgr.lorry_state[dead_lorry] - - # 2. Handle creates for any new lorries we now want - self.html.bump_state() - logging.info("Create any new lorries...") - for new_lorry in cur_lorries - prev_lorries: - self.html.set_processing(new_lorry) - logging.info("New lorry: %s" % new_lorry) - lorry = self.conf.lorries[new_lorry] - conf_uuid = lorry['controller-uuid'] - conf = self.conf.configs[conf_uuid] - nextdue = self.conf.duetimes[new_lorry] - # Make new lorries overdue. - nextdue -= conf['interval-parsed'] - should_create = conf['create'] == "always" - store_state = True - if should_create: - exit, out, err = self.maybe_runcmd(["ssh", "git@localhost", - "create", new_lorry]) - if exit != 0: - if ' already exists' in err: - logging.warn("Repository %s already exists" % - new_lorry) - else: - logging.error("Unable to create repository %s" % - new_lorry) - logging.error(err) - store_state = False - if store_state: - self.maybe_runcmd(["ssh", "git@localhost", "set-head", - new_lorry, lorry['source-HEAD']]) - mgr.lorry_state[new_lorry] = { - 'destroy': conf['destroy'], - 'conf': conf_uuid, - 'lorry': lorry, - 'next-due': nextdue, - } - else: - # Remove this from cur_lorries so we don't run it - cur_lorries.remove(new_lorry) - - # 3. For every lorry we have, update the settings if necessary. - # and reset the next-due as appropriate. - self.html.bump_state() - logging.info("Update active lorry configurations...") - updated_count = 0 - for upd_lorry in cur_lorries: - if mgr.lorry_state[upd_lorry]['lorry'] != \ - self.conf.lorries[upd_lorry]: - lorry = self.conf.lorries[upd_lorry] - old_lorry = mgr.lorry_state[upd_lorry]["lorry"] - if lorry["source-HEAD"] != \ - old_lorry.get("source-HEAD", "refs/heads/master"): - self.maybe_runcmd(['ssh', 'git@localhost', 'set-head', - upd_lorry, lorry["source-HEAD"]]) - conf_uuid = lorry['controller-uuid'] - conf = self.conf.configs[conf_uuid] - nextdue = self.conf.duetimes[upd_lorry] - mgr.lorry_state[upd_lorry] = { - 'destroy': conf['destroy'], - 'conf': conf_uuid, - 'lorry': lorry, - 'next-due': nextdue, - } - updated_count += 1 - logging.info("Result: %d/%d lorries needed updating" % ( - updated_count, len(cur_lorries))) - - # 3. Iterate all active lorries and see if they're due - logging.info("Iterate active lorries looking for work...") - now = time.time() - lorried = 0 - earliest_due = None - what_early_due = "" - lorries_to_run = [] - for lorry in cur_lorries: - state = mgr.lorry_state[lorry] - conf_uuid = state['conf'] - conf = self.conf.configs[conf_uuid] - due = state['next-due'] - if now >= due: - lorries_to_run.append(lorry) - lorries_to_run.sort() - for lorry in lorries_to_run: - state = mgr.lorry_state[lorry] - conf_uuid = state['conf'] - conf = self.conf.configs[conf_uuid] - due = state['next-due'] - lorried += 1 - logging.info("Running %d/%d. Lorrying: %s" % ( - lorried, len(lorries_to_run),lorry)) - self.html.set_processing(lorry) - # Before we run lorry, make sure that Git doesn't verify - # SSL certificates. This is a workaround for the fact that - # we don't yet have a solution for proper SSL certificates - # in Trove yet. - os.environ['GIT_SSL_NO_VERIFY'] = 'true' - with mgr.runner(lorry) as runner: - runner.run_lorry(*self.lorrycmd) - while state['next-due'] <= now: - state['next-due'] += conf['interval-parsed'] - - for lorry in cur_lorries: - state = mgr.lorry_state[lorry] - due = state['next-due'] - if earliest_due is None or due < earliest_due: - earliest_due = due - what_early_due = lorry - - if earliest_due is None: - logging.info("Lorried %d. No idea what's next." % lorried) - else: - logging.info("Lorried %d. %s due in %d seconds" % ( - lorried, what_early_due, int(earliest_due - now))) - logging.info("All done.") - self.html.bump_state() - - def rungit(self, args): - self.runcmd(['git']+args, cwd=os.path.join(self.settings['work-area'], - 'git')) - - def maybe_http_request(self, url, auth=None, dry=False): - """If not a dry run, make an HTTP request and return its output.""" - if (not self.settings['dry-run']) or dry: - return self.http_request(url, auth) - else: - logging.debug('DRY-RUN: Not sending a request to %s' % url) - return 0, 'DRY-RUN', 'DRY-RUN' - - def maybe_runcmd(self, cmdline, dry=False, *args, **kwargs): - if (not self.settings['dry-run']) or dry: - return self.runcmd_unchecked(cmdline, *args, **kwargs) - else: - logging.debug("DRY-RUN: Not running %r" % cmdline) - return 0, 'DRY-RUN', 'DRY-RUN' - - def http_request(self, url, auth=None): - """Make an HTTP request to the given url, return the output. - - Make an HTTP request to `url`. If the request succeeds (response code - 200) then return an exit code 0, the data from the response and the - response code. Otherwise return the response code, any data in the - repsonse and a string containing the response code. - - """ - request = urllib2.Request(url, None, {}) - if auth: - password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm() - password_mgr.add_password( - None, url, auth['username'], auth['password']) - auth_handler = urllib2.HTTPBasicAuthHandler(password_mgr) - opener = urllib2.build_opener(auth_handler) - response = opener.open(url) - else: - response = urllib2.urlopen(request) - code = response.getcode() - if code == 200: - return 0, response.read(), '200' - else: - return code, response.read(), str(code) - - def set_proxy(self, proxy_def): - """Tell urllib2 to use a proxy for http action by lorry-controller. - - Load the proxy information from the JSON file given by proxy_def, then - set urllib2's url opener to open urls via an authenticated proxy. - - """ - with open(proxy_def, 'r') as proxy_info: - proxy = json.load(proxy_info) - - # set the required environment variables - hostname = urllib.quote(proxy['hostname']) - user = '%s:%s' % (proxy['username'], proxy['password']) - url = '%s:%s' % (hostname, proxy['port']) - os.environ['http_proxy'] = 'http://%s@%s' % (user, url) - os.environ['https_proxy'] = 'https://%s@%s' % (user, url) - - # create a ProxyHandler - proxies = {'http_proxy': 'http://%s@%s' % (user, url), - 'https_proxy': 'https://%s@%s' % (user, url)} - proxy_handler = urllib2.ProxyHandler(proxies) - - # install an opener to use the proxy - opener = urllib2.build_opener(proxy_handler) - urllib2.install_opener(opener) - -if __name__ == '__main__': - LorryController(version='1').run() diff --git a/lorry-controller.conf b/lorry-controller.conf deleted file mode 100644 index 3d46b23..0000000 --- a/lorry-controller.conf +++ /dev/null @@ -1,20 +0,0 @@ -[ - { - "type": "trove", - "uuid": "default-staggered-short", - "trovehost": "git.baserock.org", - "protocol": "ssh", - "ls-interval": "1H", - "prefixmap": { - "baserock": "baserock" - }, - "ignore": [ - "baserock/lorries", - "baserock/tests/*" - ], - "create": "never", - "destroy": "unchanged", - "interval": "30M", - "stagger": true - } -] diff --git a/lorrycontroller/__init__.py b/lorrycontroller/__init__.py deleted file mode 100644 index 0fe0b33..0000000 --- a/lorrycontroller/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright (C) 2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import confparser -import workingstate diff --git a/lorrycontroller/confparser.py b/lorrycontroller/confparser.py deleted file mode 100644 index 403b768..0000000 --- a/lorrycontroller/confparser.py +++ /dev/null @@ -1,335 +0,0 @@ -# Copyright (C) 2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import json -import logging -import re -import glob -import os -import time -import fnmatch -import urllib - -default_values = [ - ( u'create', u'never' ), - ( u'destroy', u'never' ), - ( u'interval', u'1m' ), - ( u'stagger', False ), - ( u'tarball', u'never' ), - ( u'type', u'invalid_type' ), -] - -valid_interval = re.compile(r"^([1-9][0-9]*)([mhd])?$") -interval_mults = { - None: 1, - 'm': 60, - 'h': 60 * 60, - 'd': 60 * 60 * 24, -} -class LorryControllerConfig(object): - '''This encapsulates the configuration for lorry-controller.''' - - def __init__(self, app, confpath): - self.app = app - self.confpath = confpath - self.lorries = {} - self.configs = {} - self.duetimes = {} - self.troves = [] - - def parse_config(self): - confpath = os.path.join(self.app.settings['work-area'], self.confpath) - logging.info("Parsing configuration: %s" % confpath) - try: - with open(confpath, "r") as fh: - self._raw_conf = json.load(fh) - except Exception, e: - logging.error("Unable to parse: %r" % e) - raise - logging.debug("Validating configuration semantics") - self._validate__raw_conf() - logging.info("Configuration loaded") - - def _validate__raw_conf(self): - '''Validate the entire raw config.''' - if type(self._raw_conf) != list: - self._give_up("Configuration was not a list.") - for entry in self._raw_conf: - if type(entry) != dict: - self._give_up("Configuration entry was not a dict.") - if type(entry.get('type', None)) != unicode: - self._give_up("Configuration entry lacked a suitable 'type' " - "field.") - # Set the defaults - for key, defval in default_values: - entry[key] = entry.get(key, defval) - # And validate the generic values - self._validate__generics(entry) - # Now validate the rest - validator = getattr(self, '_validate_' + entry['type'], None) - if validator is None: - self._give_up("Configuration entry had unknown type: %s" % - entry['type']) - validator(entry) - - def _validate__generics(self, entry): - '''Validate the generic entries such as 'uuid'.''' - if type(entry.get('uuid', None)) != unicode: - self._give_up("UUID missing, cannot reconcile without it!") - if entry['uuid'] in self.configs: - self._give_up("UUID is not unique") - self.configs[entry['uuid']] = entry - for key, defval in default_values: - if type(defval) != type(entry[key]): - self._give_up("Invalid type for '%s': %r" % (key, entry[key])) - self._validate__when(entry, 'create', ["always", "never"]) - self._validate__when(entry, 'destroy', - ["always", "never", "unchanged"]) - self._validate__when(entry, 'tarball', ["always", "never", "first"]) - entry['interval-parsed'] = self._parse_interval(entry['interval']) - if 'ls-interval' in entry: - entry['ls-interval-parsed'] = \ - self._parse_interval(entry['ls-interval']) - - def _validate__when(self, entry, key, valid_whens): - if entry[key] not in valid_whens: - self._give_up("Invalid value for %s: %s" % (key, entry[key])) - - def _parse_interval(self, interval): - m = valid_interval.match(interval.lower()) - if m is None: - self._give_up("Unable to parse '%s' as an interval" % interval) - num, mult = m.groups() - num = int(num) - mult = interval_mults.get(mult, None) - if mult is None: - self._give_up("Somehow, '%s' managed to appear as a multiplier!" % - m.group(2)) - logging.debug("Converted interval %r to %r", interval, (num * mult)) - return num * mult - - def _validate_lorries(self, entry): - '''Validate a 'lorries' stanza.''' - if type(entry.get('globs', None)) != list: - self._give_up("Lorries stanzas need lists for their 'globs'") - if entry.get('prefix', None) is None: - entry['prefix'] = u"" - if type(entry['prefix']) != unicode: - self._give_up("Lorry prefixes should be strings.") - my_lorries = set() - git_base = os.path.join(self.app.settings['work-area'], 'git') - for glob_entry in entry['globs']: - if type(glob_entry) != unicode: - self._give_up("Lorries globs should be strings") - fullglob = os.path.join(git_base, glob_entry) - my_lorries = my_lorries.union(set(glob.iglob(fullglob))) - for lorry in my_lorries: - if not lorry.startswith(git_base): - self._give_up("Glob found %s which is outside the git base") - - logging.debug("Expanded globs in entry to %d lorry files" % - len(my_lorries)) - logging.debug("Loading lorries into memory, please wait...") - - my_lorry_names = set() - for lorry in my_lorries: - try: - with open(lorry, "r") as fh: - lorry_json = json.load(fh) - for name, content in lorry_json.iteritems(): - fullname = os.path.join(entry['prefix'], name) - if self.lorries.get(fullname, None) is not None: - self._give_up("Lorry repeated: %s" % fullname) - content['controller-uuid'] = entry['uuid'] - if not content.has_key('source-HEAD'): - content['source-HEAD'] = 'refs/heads/master' - my_lorry_names.add(fullname) - self.lorries[fullname] = content - except Exception, e: - logging.warning("Unable to parse %s, because of %s. " - "Moving on" % (lorry, e)) - - # Now calculate the 'next due' time for every lorry we just parsed - starttime = time.time() - 1 - endtime = starttime + entry['interval-parsed'] - step = 0 - if entry['stagger']: - step = (endtime - starttime) / (len(my_lorry_names) + 1) - for lorry_name in my_lorry_names: - self.duetimes[lorry_name] = starttime - starttime += step - - logging.debug("Now loaded %d lorries" % len(self.lorries.keys())) - - def _validate_trove(self, entry): - # Validate top levels - if type(entry.get('trovehost', None)) != unicode: - self._give_up("Trove host %r is not a string" % - entry.get('trovehost', None)) - if 'ls-interval-parsed' not in entry: - self._give_up("No ls-interval specified for %s" % - entry['trovehost']) - if type(entry.get('prefixmap', None)) != dict: - self._give_up("Prefixmap not a dict for %s" % - entry['trovehost']) - if type(entry.get('ignore', [])) != list: - self._give_up("Ignore is not a list for %s" % - entry['trovehost']) - protocol = entry.get('protocol') - auth = entry.get('auth') - if protocol == 'https' and not auth: - self._give_up('Trove access protocol requires authorisation ' - 'details but none were defined.') - elif not protocol: - self._give_up('Trove access protocol not defined.') - # Validate prefixmap - for local, remote in entry['prefixmap'].iteritems(): - if type(local) != unicode: - self._give_up("Local part of prefixmap is not a string: %r" % - local) - if type(remote) != unicode: - self._give_up("Remote part of prefixmap is not a string: %r" % - remote) - # Validate ignore - for ign in entry.get('ignore', []): - if type(ign) != unicode: - self._give_up("Part of ignore list is not a string: %r" % ign) - - self.troves.append(entry) - - def update_trove(self, trove, state): - logging.info("Processing trove %s (%s)" % (trove['trovehost'], - trove['uuid'])) - # 1. Ensure that if we need to 'ls' the trove, we do it - now = time.time() - state['next-vls'] = state.get('next-vls', now - 1) - if state['next-vls'] < now: - exit, out, err = self.run_gitano_command(trove, True, 'ls', '--verbose') - if exit == 0: - repo_info = {} - for entry in [x for x in out.split("\n") if x != ""]: - while entry.find(" ") > -1: - entry = entry.replace(" ", " ") - elems = entry.split(" ") - this_repo = { - "perm": elems[0], - "name": elems[1], - "head": elems[2], - "desc": " ".join(elems[3:]), - } - repo_info[elems[1]] = this_repo - state['last-ls-output'] = repo_info - logging.info("ls interval %d" % trove['ls-interval-parsed']) - logging.info("next-vls was %s" % time.asctime(time.gmtime(state['next-vls']))) - while state['next-vls'] < now: - state['next-vls'] += trove['ls-interval-parsed'] - logging.info("next-vls now %s" % time.asctime(time.gmtime(state['next-vls']))) - else: - # Pass through unchanged - state['last-ls-output'] = state.get('last-ls-output', {}) - - def ignored(reponame): - for pattern in trove['ignore']: - if fnmatch.fnmatch(reponame, pattern): - return True - return False - - # 2. For every entry in last-ls-output, construct a lorry if we want it - lorries_made = set() - for remotereponame, info in state['last-ls-output'].iteritems(): - localreponame = None - for local, remote in trove['prefixmap'].iteritems(): - if remotereponame.startswith(remote+"/"): - localreponame = "%s/%s" % (local, - remotereponame[len(remote)+1:]) - if ((not ignored(remotereponame)) and (localreponame is not None)): - # Make the url in the correct form for the given protocol - if trove['protocol'] == 'ssh': - url = 'ssh://git@%s/%s.git' % (trove['trovehost'], - remotereponame) - elif trove['protocol'] == 'https': - auth = trove['auth'] - url = 'https://%s:%s@%s/git/%s.git' % (auth['username'], - auth['password'], - trove['trovehost'], - remotereponame) - else: - url = 'http://%s/git/%s.git' % (trove['trovehost'], - remotereponame) - # Construct a lorry for this one. - lorry = { - "type": "git", - "url": url, - "controller-uuid": trove['uuid'], - "source-HEAD": info["head"], - "refspecs": [ "+refs/heads/*:refs/heads/*", - "+refs/tags/*:refs/tags/*" ] - } - if localreponame in self.lorries: - logging.warn("Skipping %s (%s from %s) because we already " - "have something for that." % ( - localreponame, remotereponame, trove['trovehost'])) - else: - self.lorries[localreponame] = lorry - lorries_made.add(localreponame) - - # 3. Now schedule all those lorries in case they're new - starttime = time.time() - 1 - endtime = starttime + trove['interval-parsed'] - step = 0 - if trove['stagger']: - step = (endtime - starttime) / (len(lorries_made)+1) - for lorry_name in lorries_made: - self.duetimes[lorry_name] = starttime - starttime += step - - logging.debug("Generated %d lorries from that trove" % - len(lorries_made)) - - def update_troves(self, statemgr): - # Now that we have a state manager we can look at the trove data. - for trove in self.troves: - self.app.html.set_processing(trove['uuid']) - trove_state = statemgr.get_trove(trove['uuid']) - self.update_trove(trove, trove_state) - - def run_gitano_command(self, trove, dry, command, *args): - """Run a gitano command on the trove, and return the output.""" - if trove['protocol'] == 'ssh': - # construct list to run command over ssh - cmdargs = ['ssh', - '-oStrictHostKeyChecking=no', - '-oBatchMode=yes', - 'git@%s' % trove['trovehost'], - command] - cmdargs.extend(args) - # run the command - exit, out, err = self.app.maybe_runcmd(cmdargs, dry=dry) - else: - # construct a url which will return the command output - query_string = '%s %s' % (command, ' '.join(args)) - query_string = urllib.quote(query_string) - trovehost = urllib.quote(trove['trovehost']) - url = '%s://%s/gitano-command.cgi?cmd=%s' % ( - trove['protocol'], trovehost, query_string) - auth = trove.get('auth', None) - # make an http request to the url - exit, out, err = self.app.maybe_http_request(url, auth=auth, dry=dry) - return exit, out, err - - def _give_up(self, *args, **kwargs): - logging.error(*args, **kwargs) - raise SystemExit(5) diff --git a/lorrycontroller/htmlstatus.py b/lorrycontroller/htmlstatus.py deleted file mode 100644 index 30b52f0..0000000 --- a/lorrycontroller/htmlstatus.py +++ /dev/null @@ -1,286 +0,0 @@ -# Copyright (C) 2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import os -import time -from cgi import escape - -state_names = [ - "Initialisation", - "Load Troves", - "Remove old repos", - "Create new repos", - "Process Lorries", - "Finished" - ] - -def format_time(time_t): - return time.strftime("%Y-%m-%d %H:%M:%S UTC", time.gmtime(time_t)) - -class HTMLStatusManager(object): - '''Manage the HTML status page for lorry-controller.''' - - - def __init__(self, app): - self.app = app - self.state = 0 - self.series = None - self.filename = self.app.settings['html-file'] - self.mgr = None - self.all_processing = set() - self.processing = None - self.processing_time = time.time() - self.failing = None - self.all_lorries_ever = set() - self.bump_time = time.time() - - def set_failing(self, failmsg): - self.failing = failmsg - self.write_out_status() - - def set_mgr(self, mgr): - self.mgr = mgr - - def set_processing(self, proc): - if self.processing is not None: - self.all_processing.add(self.processing) - self.processing = proc - self.processing_time = time.time() - self.write_out_status() - - def bump_state(self): - self.state = self.state + 1 - self.bump_time = time.time() - self.processing = None - self.write_out_status() - - def write_out_status(self): - if self.filename is None: return - try: - with open(self.filename + ".new", "w") as ofh: - ofh.write("\n") - ofh.write(self.gen_html()) - ofh.write("\n") - target = self.filename - if self.series is not None: - target += ".%d" % self.series - self.series += 1 - os.rename(self.filename + ".new", target) - except: - os.unlink(self.filename + ".new") - raise - - def gen_html(self): - head = self.gen_head() - body = self.gen_body() - return self.tag("html", content=head+"\n"+body, gap=True) - - def gen_head(self): - title = self.tag("title", content="Lorry Controller") - css = self.tag("link", href="trove.css", rel="stylesheet", - type="text/css") - script = self.tag( - "script", type="text/javascript", src="/table.js", content="") - return self.tag("head", content=title+css+script) - - def gen_body(self): - # 1. Rough header, branded as trove - curtime = format_time(time.time()) - link = "/" - if self.series is not None: - link = self.filename + ".%d" % (self.series + 1) - header = ''' - - - -''' % (link, curtime) - # 2. List of steps and where we are currently - steps = self.gen_steps() - - # 4. Main content - content = self.gen_content() - - # 5. footer - footer = self.gen_footer() - - # magic args - return self.tag("body", content=self.tag( - "div", content=(header+steps+content+footer), - Class="lorrycontroller")) - - def gen_content(self): - if self.failing is not None: - return self.tag("div", Class="failure", content=self.failing) - # 1. Troves known - troves = self.gen_troves() - # 2. Lorries known - lorries = self.gen_lorries() - - return self.tag("div", Class="content", content= - self.tag("div", id="troves", content=troves) + - self.tag("div", id="lorries", content=lorries)) - - def gen_troves(self): - troves = [] - now = time.time() - for trove in self.app.conf.troves: - troveinfo = {} - if self.mgr is not None: - troveinfo = self.mgr.trove_state.get(trove['uuid'], {}) - uuid = self.tag("td", content=escape(trove['uuid'])) - state = "Up to date" - if self.processing == trove['uuid']: - state = "Processing since " + \ - format_time(self.processing_time) - elif troveinfo.get('next-vls', now - 1) < now: - if self.state < len(state_names) - 1: - state = "Due to be checked this run." - else: - state = "Due to be checked on the next run" - state = self.tag("td", content=escape(state)) - nextdue = self.tag("td", content=escape(format_time( - troveinfo.get('next-vls', now - 1)))) - lorrycount = len([l for l in self.app.conf.lorries.itervalues() - if l['controller-uuid'] == trove['uuid']]) - lorrycount = self.tag("td", content=str(lorrycount)) - - troves.append(self.tag("tr", content= - uuid+state+nextdue+lorrycount)) - if len(troves) == 0: - content = "No troves detected" - else: - header = self.tag("tr", content= - self.tag("th", content="Trove UUID") + - self.tag("th", content="Status") + - self.tag("th", content="Next due") + - self.tag("th", content="Lorries created")) - content = self.tag("table", content= - header + "".join(troves)) - - return content - - def gen_lorries(self): - lorries = [] - now = time.time() - all_lorry_names = set(self.app.conf.lorries.keys()) - if self.mgr is not None: - all_lorry_names.update(set(self.mgr.lorry_state.keys())) - self.all_lorries_ever.update(all_lorry_names) - all_lorry_names = list(self.all_lorries_ever) - all_lorry_names.sort() - for lorry_name in all_lorry_names: - lorry = self.app.conf.lorries.get(lorry_name, None) - dead_lorry = False - dead_and_gone = False - if lorry is None: - lorrystate = self.mgr.lorry_state.get(lorry_name, None) - if lorrystate is None: - dead_and_gone = True - lorry = {} - else: - lorry = lorrystate['lorry'] - dead_lorry = True - lorryinfo = {} - if self.mgr is not None: - lorryinfo = self.mgr.lorry_state.get(lorry_name, {}) - uuid = self.tag("td", content= - escape(lorry.get('controller-uuid', 'Dead'))) - state = "Waiting " - if self.processing == lorry_name: - state = "Processing since " + \ - format_time(self.processing_time) - elif lorryinfo.get('next-due', now - 1) < self.bump_time: - if self.state < len(state_names) - 1: - state = "Due to be checked this run." - else: - state = "Due to be checked on the next run" - if self.mgr is not None: - if self.mgr.lorry_state.get(lorry_name, None) is None: - state = "Needs creating" - elif lorry_name in self.all_processing: - state = "Processed" - if dead_lorry: - state = "Dead - To be removed" - if dead_and_gone: - state = "Dead" - if self.processing == lorry_name: - state = "Removing since " + \ - format_time(self.processing_time) - state = self.tag("td", content=escape(state)) - lastresult = self.tag("td", content=self.tag( - "pre", content=escape(lorryinfo.get('result', '-')))) - nextdue = self.tag("td", content=escape(format_time( - lorryinfo.get('next-due', now - 1)))) - lorryname = self.tag("td", content=escape(lorry_name)) - lorries.append(self.tag("tr", content= - lorryname+uuid+state+lastresult+nextdue)) - if len(lorries) == 0: - content = "No lorries detected yet" - else: - header = self.tag("tr", content= - self.tag("th", - Class="table-sortable:alphanumeric", - content="Lorry Name") + - self.tag("th", - Class="table-sortable:alphanumeric", - content="Comes From") + - self.tag("th", - Class="table-sortable:alphanumeric", - content="Status") + - self.tag("th", - Class="table-sortable:alphanumeric", - content="Last result") + - self.tag("th", - Class="table-sortable:alphanumeric", - content="Next due")) - header = self.tag("thead", content=header) - content = self.tag("table", Class="table-autosort:4", content= - header + "\n" + "\n".join(lorries)) - - return content - - - def gen_footer(self): - curtime = format_time(time.time()) - return self.tag("div", Class="footer", content= - "Generated by Lorry Controller at " + curtime) - - def gen_steps(self): - steps = [] - for idx in xrange(len(state_names)): - if idx < self.state: - Class = "donestep" - elif idx == self.state: - Class = "activestep" - else: - Class = "pendingstep" - steps.append(self.tag("span", Class=Class, - content=state_names[idx])) - return self.tag("table", Class="steps", content= - self.tag("tr", content= - self.tag("td", content= - "".join(steps)))) - - def tag(self, tagname, content=None, gap=False, **kwargs): - tagval = " ".join([tagname] + - ["%s=%r" % (k.lower(), v) for k, v in kwargs.iteritems()]) - gap = "\n" if gap else "" - if content is None: - return "<%s />" % tagval - else: - return "<%s>%s%s%s" % (tagval, gap, content, gap, tagname) - diff --git a/lorrycontroller/workingstate.py b/lorrycontroller/workingstate.py deleted file mode 100644 index b8dc751..0000000 --- a/lorrycontroller/workingstate.py +++ /dev/null @@ -1,127 +0,0 @@ -# Copyright (C) 2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import json -import os -import logging -import string - -class LorryFileRunner(object): - def __init__(self, mgr, lorryname): - self.mgr = mgr - self.lorryname = lorryname - self.lorryfile = os.path.join(self.mgr.workdir, - self._esc(lorryname) + ".lorry") - - def _esc(self, name): - valid_chars = string.digits + string.letters + '%_' - transl = lambda x: x if x in valid_chars else '_' - return ''.join([transl(x) for x in name]) - - def __enter__(self): - lorry_obj = { self.lorryname: - self.mgr.lorry_state[self.lorryname]['lorry'] } - with open(self.lorryfile, "w") as fh: - json.dump(lorry_obj, fh) - fh.write("\n") - return self - - def __exit__(self, exctype, excvalue, exctraceback): - os.unlink(self.lorryfile) - - def run_lorry(self, *args): - cmdargs = list(args) - cmdargs.append(self.lorryfile) - conf_uuid = self.mgr.lorry_state[self.lorryname]['conf'] - conf = self.mgr.app.conf.configs[conf_uuid] - cmdargs.append("--tarball=%s" % conf['tarball']) - exit, out, err = self.mgr.app.maybe_runcmd(cmdargs) - if exit == 0: - logging.debug("Lorry of %s succeeded: %s" % (self.lorryname, out)) - self.mgr.lorry_state[self.lorryname]['result'] = "OK" - else: - logging.warn("Lorry of %s failed: %s" % (self.lorryname, err)) - self.mgr.lorry_state[self.lorryname]['result'] = err - -class WorkingStateManager(object): - '''Manage the working state of lorry-controller''' - - def __init__(self, app): - self.app = app - self.workdir = os.path.join(self.app.settings['work-area'], 'work') - - def __enter__(self): - self._load_state() - return self - - def __exit__(self, exctype, excvalue, exctraceback): - self.purge_dead_troves() - if not self.app.settings['dry-run']: - self.save_state() - else: - logging.debug("DRY-RUN: Not saving state again") - - def purge_dead_troves(self): - old_trove_count = len(self.trove_state.keys()) - all_troves = self.trove_state - self.trove_state = {} - new_trove_count = 0 - for uuid, trove in all_troves.iteritems(): - self.trove_state[uuid] = trove - new_trove_count += 1 - if old_trove_count != new_trove_count: - trove_diff = old_trove_count - new_trove_count - logging.info("Purged %d dead trove entr%s from the state file" % ( - trove_diff, ("y" if trove_diff == 1 else "ies"))) - - def _load_state(self): - self.lorry_state_file = os.path.join(self.workdir, - "last-lorry-state.json") - self.trove_state_file = os.path.join(self.workdir, - "last-trove-state.json") - if os.path.exists(self.lorry_state_file): - logging.info("Loading lorry state file: %s" % - self.lorry_state_file) - with open(self.lorry_state_file, "r") as fh: - self.lorry_state = json.load(fh) - else: - self.lorry_state = dict() - - if os.path.exists(self.trove_state_file): - logging.info("Loading trove state file: %s" % - self.trove_state_file) - with open(self.trove_state_file, "r") as fh: - self.trove_state = json.load(fh) - else: - self.trove_state = dict() - - def save_state(self): - logging.info("Serialising lorry state: %s" % self.lorry_state_file) - with open(self.lorry_state_file, "w") as fh: - json.dump(self.lorry_state, fh, sort_keys=True, indent=4) - fh.write("\n") - logging.info("Serialising trove state: %s" % self.trove_state_file) - with open(self.trove_state_file, "w") as fh: - json.dump(self.trove_state, fh, sort_keys=True, indent=4) - fh.write("\n") - - def get_trove(self, troveuuid): - if troveuuid not in self.trove_state: - self.trove_state[troveuuid] = {} - return self.trove_state[troveuuid] - - def runner(self, lorryname): - return LorryFileRunner(self, lorryname) diff --git a/setup.py b/setup.py deleted file mode 100644 index b27b9d5..0000000 --- a/setup.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/python -# -# Copyright (C) 2012 Codethink Limited - - -from distutils.core import setup - - -setup(name='lorry-controller', - description='FIXME', - long_description='''\ -FIXME -''', - author='Daniel Silverstone', - author_email='daniel.silverstne@codethink.co.uk', - url='http://www.baserock.com/', - scripts=['lorry-controller'], - packages=['lorrycontroller'], - ) -- cgit v1.2.1