diff options
| -rwxr-xr-x | morph | 16 | ||||
| -rw-r--r-- | morphlib/buildcontroller.py | 4 | ||||
| -rw-r--r-- | morphlib/buildworker.py | 87 |
3 files changed, 71 insertions, 36 deletions
@@ -24,6 +24,8 @@ import logging import os import morphlib +from morphlib import buildworker +from morphlib import buildcontroller from morphlib.morphologyloader import MorphologyLoader from morphlib.builddependencygraph import BuildDependencyGraph @@ -226,17 +228,19 @@ class Morph(cliapp.Application): source_manager = morphlib.sourcemanager.SourceManager(self) # create a build controller - controller = morphlib.buildcontroller.BuildController(self, tempdir) + controller = buildcontroller.BuildController(self, tempdir) # create and add the build workers if len(self.settings['worker']) == 0: - worker = morphlib.buildworker.LocalBuildWorker("local-1", self) - controller.add_worker(worker) - worker = morphlib.buildworker.LocalBuildWorker("local-2", self) - controller.add_worker(worker) + num_workers = morphlib.util.make_concurrency() + for i in range(num_workers): + name = controller.generate_worker_name('local') + worker = buildworker.LocalBuildWorker(name, 'local', self) + controller.add_worker(worker) else: for worker in self.settings['worker']: - worker = morphlib.buildworker.RemoteBuildWorker(self) + name = controller.generate_worker_name(worker) + worker = buildworker.RemoteBuildWorker(name, worker, self) controller.add_worker(worker) result = [] diff --git a/morphlib/buildcontroller.py b/morphlib/buildcontroller.py index ea9719c0..f794928c 100644 --- a/morphlib/buildcontroller.py +++ b/morphlib/buildcontroller.py @@ -43,6 +43,10 @@ class BuildController(object): spaces = ' ' * self.indent self.real_msg('%s%s' % (spaces, text)) + def generate_worker_name(self, ident): + similar_workers = [x for x in self.workers if x.ident == ident] + return '%s-%s' % (ident, len(similar_workers) + 1) + def add_worker(self, worker): self.workers.add(worker) self.mark_idle(worker) diff --git a/morphlib/buildworker.py b/morphlib/buildworker.py index ad951466..f75177b3 100644 --- a/morphlib/buildworker.py +++ b/morphlib/buildworker.py @@ -22,15 +22,15 @@ import morphlib class BuildWorker(object): - def __init__(self, name, app): + def __init__(self, name, ident, app): self.name = name + self.ident = ident self.settings = app.settings self.real_msg = app.msg self.indent = 2 self.idle_since = datetime.datetime.now() - - def __str__(self): - return self.name + self.manager = Manager() + self.reset() def indent_more(self): self.indent += 1 @@ -42,30 +42,48 @@ class BuildWorker(object): spaces = ' ' * self.indent self.real_msg('%s%s' % (spaces, text)) + def reset(self): + self.process = None + self.blob = None + self._output = self.manager.list() + def build(self, blob): raise NotImplementedError def check_complete(self, timeout): - raise NotImplementedError + if self.process: + self.process.join(timeout) + if self.process.is_alive(): + return False + else: + self.idle_since = datetime.datetime.now() + return True + else: + return True + @property + def output(self): + try: + return self._output[0] + except IndexError: + return None -class LocalBuildWorker(BuildWorker): + def __str__(self): + return self.name - def __init__(self, name, app): - BuildWorker.__init__(self, name, app) - self.manager = Manager() - self.reset() - def reset(self): - self.process = None - self.blob = None - self._output = self.manager.list() +class LocalBuildWorker(BuildWorker): + + def __init__(self, name, ident, app): + BuildWorker.__init__(self, name, ident, app) def run(self, repo, ref, filename, output): ex = morphlib.execute.Execute('.', self.msg) stdout = ex.runv(['./morph', '--verbose', '--keep-path', 'build', repo, ref, filename]) output.append(stdout) + + # TODO report errors back to the caller def build(self, blob): self.reset() @@ -77,23 +95,32 @@ class LocalBuildWorker(BuildWorker): self.process = Process(group=None, target=self.run, args=args) self.process.start() - def check_complete(self, timeout): - if self.process: - self.process.join(timeout) - if self.process.is_alive(): - return False - else: - self.idle_since = datetime.datetime.now() - return True - else: - return True - @property - def output(self): - return self._output[0] +class RemoteBuildWorker(BuildWorker): + def __init__(self, name, ident, app): + BuildWorker.__init__(self, name, ident, app) + self.hostname = ident -class RemoteBuildWorker(BuildWorker): + def run(self, repo, ref, filename, output): + ex = morphlib.execute.Execute('.', self.msg) + + # generate command line options + cmdline = ['ssh', self.hostname] + cmdline.extend(['fakeroot', 'morph', 'build', repo, ref, filename]) + + # run morph on the other machine + stdout = ex.runv(cmdline) + output.append(stdout) + + # TODO report errors back to the caller - def __init__(self, app): - BuildWorker.__init__(self, app) + def build(self, blob): + self.reset() + self.blob = blob + args = (blob.morph.treeish.original_repo, + blob.morph.treeish.ref, + blob.morph.filename, + self._output) + self.process = Process(group=None, target=self.run, args=args) + self.process.start() |