diff options
author | Jannis Pohlmann <jannis.pohlmann@codethink.co.uk> | 2012-01-24 14:23:34 +0000 |
---|---|---|
committer | Jannis Pohlmann <jannis.pohlmann@codethink.co.uk> | 2012-01-24 14:40:08 +0000 |
commit | 14e768ce2e5693ebf6418e7a6e94fe4fb9a71844 (patch) | |
tree | 6c629c56aa6e32092a5745bc6de3b50d5c86fbc5 | |
parent | 10122174305ac5c6040212a8b526a77ee473577b (diff) | |
download | morph-14e768ce2e5693ebf6418e7a6e94fe4fb9a71844.tar.gz |
Share functionality in BuildWorker, implement SSH, add name/ident concept.
All BuildWorker subclasses are likely to make use of the multiprocessing
library anyway, so most of the functionality can be shared.
All workers now have an ident (e.g. "user@hostname" or "local") which
is used by the BuildController to generate enumerated names for them
(e.g. "user@hostname-1", "user@hostname-2" or "local-1" and "local-2"),
which makes it easier to identify who does what.
The RemoteBuildWorker now runs "ssh HOSTNAME fakeroot morph" for
building stuff remotely.
-rwxr-xr-x | morph | 16 | ||||
-rw-r--r-- | morphlib/buildcontroller.py | 4 | ||||
-rw-r--r-- | morphlib/buildworker.py | 87 |
3 files changed, 71 insertions, 36 deletions
@@ -24,6 +24,8 @@ import logging import os import morphlib +from morphlib import buildworker +from morphlib import buildcontroller from morphlib.morphologyloader import MorphologyLoader from morphlib.builddependencygraph import BuildDependencyGraph @@ -226,17 +228,19 @@ class Morph(cliapp.Application): source_manager = morphlib.sourcemanager.SourceManager(self) # create a build controller - controller = morphlib.buildcontroller.BuildController(self, tempdir) + controller = buildcontroller.BuildController(self, tempdir) # create and add the build workers if len(self.settings['worker']) == 0: - worker = morphlib.buildworker.LocalBuildWorker("local-1", self) - controller.add_worker(worker) - worker = morphlib.buildworker.LocalBuildWorker("local-2", self) - controller.add_worker(worker) + num_workers = morphlib.util.make_concurrency() + for i in range(num_workers): + name = controller.generate_worker_name('local') + worker = buildworker.LocalBuildWorker(name, 'local', self) + controller.add_worker(worker) else: for worker in self.settings['worker']: - worker = morphlib.buildworker.RemoteBuildWorker(self) + name = controller.generate_worker_name(worker) + worker = buildworker.RemoteBuildWorker(name, worker, self) controller.add_worker(worker) result = [] diff --git a/morphlib/buildcontroller.py b/morphlib/buildcontroller.py index ea9719c0..f794928c 100644 --- a/morphlib/buildcontroller.py +++ b/morphlib/buildcontroller.py @@ -43,6 +43,10 @@ class BuildController(object): spaces = ' ' * self.indent self.real_msg('%s%s' % (spaces, text)) + def generate_worker_name(self, ident): + similar_workers = [x for x in self.workers if x.ident == ident] + return '%s-%s' % (ident, len(similar_workers) + 1) + def add_worker(self, worker): self.workers.add(worker) self.mark_idle(worker) diff --git a/morphlib/buildworker.py b/morphlib/buildworker.py index ad951466..f75177b3 100644 --- a/morphlib/buildworker.py +++ b/morphlib/buildworker.py @@ -22,15 +22,15 @@ import morphlib class BuildWorker(object): - def __init__(self, name, app): + def __init__(self, name, ident, app): self.name = name + self.ident = ident self.settings = app.settings self.real_msg = app.msg self.indent = 2 self.idle_since = datetime.datetime.now() - - def __str__(self): - return self.name + self.manager = Manager() + self.reset() def indent_more(self): self.indent += 1 @@ -42,30 +42,48 @@ class BuildWorker(object): spaces = ' ' * self.indent self.real_msg('%s%s' % (spaces, text)) + def reset(self): + self.process = None + self.blob = None + self._output = self.manager.list() + def build(self, blob): raise NotImplementedError def check_complete(self, timeout): - raise NotImplementedError + if self.process: + self.process.join(timeout) + if self.process.is_alive(): + return False + else: + self.idle_since = datetime.datetime.now() + return True + else: + return True + @property + def output(self): + try: + return self._output[0] + except IndexError: + return None -class LocalBuildWorker(BuildWorker): + def __str__(self): + return self.name - def __init__(self, name, app): - BuildWorker.__init__(self, name, app) - self.manager = Manager() - self.reset() - def reset(self): - self.process = None - self.blob = None - self._output = self.manager.list() +class LocalBuildWorker(BuildWorker): + + def __init__(self, name, ident, app): + BuildWorker.__init__(self, name, ident, app) def run(self, repo, ref, filename, output): ex = morphlib.execute.Execute('.', self.msg) stdout = ex.runv(['./morph', '--verbose', '--keep-path', 'build', repo, ref, filename]) output.append(stdout) + + # TODO report errors back to the caller def build(self, blob): self.reset() @@ -77,23 +95,32 @@ class LocalBuildWorker(BuildWorker): self.process = Process(group=None, target=self.run, args=args) self.process.start() - def check_complete(self, timeout): - if self.process: - self.process.join(timeout) - if self.process.is_alive(): - return False - else: - self.idle_since = datetime.datetime.now() - return True - else: - return True - @property - def output(self): - return self._output[0] +class RemoteBuildWorker(BuildWorker): + def __init__(self, name, ident, app): + BuildWorker.__init__(self, name, ident, app) + self.hostname = ident -class RemoteBuildWorker(BuildWorker): + def run(self, repo, ref, filename, output): + ex = morphlib.execute.Execute('.', self.msg) + + # generate command line options + cmdline = ['ssh', self.hostname] + cmdline.extend(['fakeroot', 'morph', 'build', repo, ref, filename]) + + # run morph on the other machine + stdout = ex.runv(cmdline) + output.append(stdout) + + # TODO report errors back to the caller - def __init__(self, app): - BuildWorker.__init__(self, app) + def build(self, blob): + self.reset() + self.blob = blob + args = (blob.morph.treeish.original_repo, + blob.morph.treeish.ref, + blob.morph.filename, + self._output) + self.process = Process(group=None, target=self.run, args=args) + self.process.start() |