diff options
author | Daniel Silverstone <daniel.silverstone@codethink.co.uk> | 2012-10-02 15:28:17 +0100 |
---|---|---|
committer | Daniel Silverstone <daniel.silverstone@codethink.co.uk> | 2012-10-02 15:28:17 +0100 |
commit | 151305ae730be72fc16af6a55b3ed2e8e675eb6c (patch) | |
tree | 2ab2ab1a31fa830c8aed41f11e2e30390a5dabb1 | |
parent | 704695b55eedc77eec565719faa11f404754a5af (diff) | |
download | lorry-controller-151305ae730be72fc16af6a55b3ed2e8e675eb6c.tar.gz |
Initial crack at trove support
-rwxr-xr-x | lorry-controller | 3 | ||||
-rw-r--r-- | lorrycontroller/confparser.py | 70 | ||||
-rw-r--r-- | lorrycontroller/workingstate.py | 5 |
3 files changed, 71 insertions, 7 deletions
diff --git a/lorry-controller b/lorry-controller index ce60989..2a18d8c 100755 --- a/lorry-controller +++ b/lorry-controller @@ -75,8 +75,7 @@ class LorryController(cliapp.Application): logging.error("Unable to find lorry-controller.conf in git") raise SystemExit(4) - self.conf = LorryControllerConfig(self.settings, - 'git/lorry-controller.conf') + self.conf = LorryControllerConfig(self, 'git/lorry-controller.conf') with WorkingStateManager(self) as mgr: # Update any troves diff --git a/lorrycontroller/confparser.py b/lorrycontroller/confparser.py index cc8f0e5..a5f1e3e 100644 --- a/lorrycontroller/confparser.py +++ b/lorrycontroller/confparser.py @@ -28,12 +28,13 @@ interval_mults = { class LorryControllerConfig(object): '''This encapsulates the configuration for lorry-controller.''' - def __init__(self, settings, confpath): - self.settings = settings + def __init__(self, app, confpath): + self.app = app self.lorries = {} self.configs = {} self.duetimes = {} - confpath = os.path.join(settings['work-area'], confpath) + self.troves = [] + confpath = os.path.join(app.settings['work-area'], confpath) logging.debug("Parsing configuration: %s" % confpath) with open(confpath, "r") as fh: self._raw_conf = json.load(fh) @@ -105,7 +106,7 @@ class LorryControllerConfig(object): if type(entry['prefix']) != unicode: self._give_up("Lorry prefixes should be strings.") my_lorries = set() - git_base = os.path.join(self.settings['work-area'], 'git') + git_base = os.path.join(self.app.settings['work-area'], 'git') for glob_entry in entry['globs']: if type(glob_entry) != unicode: self._give_up("Lorries globs should be strings") @@ -175,9 +176,68 @@ class LorryControllerConfig(object): if type(ign) != unicode: self._give_up("Part of ignore list is not a string: %r" % ign) + self.troves.append(entry) + + def update_trove(self, trove, state): + logging.debug("Processing trove %s (%s)" % (trove['trovehost'], + trove['uuid'])) + # 1. Ensure that if we need to 'ls' the trove, we do it + now = time.time() + listcmdargs = ["ssh", "git@" + trove['trovehost'], "ls"] + state['next-ls'] = state.get('next-ls', now - 1) + if state['next-ls'] < now: + exit, out, err = self.app.maybe_runcmd(listcmdargs) + if exit == 0: + state['last-ls-output'] = [x[3:] for x in out.split("\n")] + while state['next-ls'] < now: + state['next-ls'] += trove['ls-interval-parsed'] + else: + # Pass through unchanged + state['last-ls-output'] = state.get('last-ls-output', []) + + # 2. For every entry in last-ls-output, construct a lorry if we want it + lorries_made = set() + for remotereponame in state['last-ls-output']: + localreponame = None + for local, remote in trove['prefixmap'].iteritems(): + if remotereponame.startswith(remote+"/"): + localreponame = "%s/%s" % (local, + remotereponame[len(remote)+1:]) + if ((remotereponame not in trove['ignore']) and + (localreponame is not None)): + # Construct a lorry for this one. + lorry = { + "type": "git", + "url": "ssh://git@%s/%s.git" % (trove['trovehost'], + remotereponame) + } + if localreponame in self.lorries: + logging.warn("Skipping %s (%s from %s) because we already " + "have something for that." % ( + localreponame, remotereponame, trove['trovehost'])) + else: + self.lorries[localreponame] = lorry + self.configs[localreponame] = trove + lorries_made.add(localreponame) + + # 3. Now schedule all those lorries in case they're new + starttime = time.time() - 1 + endtime = starttime + trove['interval-parsed'] + step = 0 + if trove['stagger']: + step = (endtime - starttime) / len(lorries_made) + for lorry_name in lorries_made: + self.duetimes[lorry_name] = starttime + starttime += step + + logging.debug("Generated %d lorries from that trove" % + len(lorries_made)) + def update_troves(self, statemgr): # Now that we have a state manager we can look at the trove data. - pass + for trove in self.troves: + trove_state = statemgr.get_trove(trove['uuid']) + self.update_trove(trove, trove_state) def _give_up(self, *args, **kwargs): logging.error(*args, **kwargs) diff --git a/lorrycontroller/workingstate.py b/lorrycontroller/workingstate.py index c4b031f..1ef2e9b 100644 --- a/lorrycontroller/workingstate.py +++ b/lorrycontroller/workingstate.py @@ -84,5 +84,10 @@ class WorkingStateManager(object): json.dump(self.trove_state, fh, sort_keys=True, indent=4) fh.write("\n") + def get_trove(self, troveuuid): + if troveuuid not in self.trove_state: + self.trove_state[troveuuid] = {} + return self.trove_state[troveuuid] + def runner(self, lorryname): return LorryFileRunner(self, lorryname) |