Share functionality in BuildWorker, implement SSH, add name/ident concept.

All BuildWorker subclasses are likely to make use of the multiprocessing
library anyway, so most of the functionality can be shared.

All workers now have an ident (e.g. "user@hostname" or "local") which
is used by the BuildController to generate enumerated names for them
(e.g. "user@hostname-1", "user@hostname-2" or "local-1" and "local-2"),
which makes it easier to identify who does what.

The RemoteBuildWorker now runs "ssh HOSTNAME fakeroot morph" for
building stuff remotely.

3 files changed, 71 insertions, 36 deletions

diff --git a/morph b/morph
index 42aa6f38..cbebc46d 100755
--- a/morph
+++ b/morph
@@ -24,6 +24,8 @@ import logging
 import os
 
 import morphlib
+from morphlib import buildworker
+from morphlib import buildcontroller
 from morphlib.morphologyloader import MorphologyLoader
 from morphlib.builddependencygraph import BuildDependencyGraph
 
@@ -226,17 +228,19 @@ class Morph(cliapp.Application):
         source_manager = morphlib.sourcemanager.SourceManager(self)
 
         # create a build controller
-        controller = morphlib.buildcontroller.BuildController(self, tempdir)
+        controller = buildcontroller.BuildController(self, tempdir)
 
         # create and add the build workers
         if len(self.settings['worker']) == 0:
-            worker = morphlib.buildworker.LocalBuildWorker("local-1", self)
-            controller.add_worker(worker)
-            worker = morphlib.buildworker.LocalBuildWorker("local-2", self)
-            controller.add_worker(worker)
+            num_workers = morphlib.util.make_concurrency()
+            for i in range(num_workers):
+                name = controller.generate_worker_name('local')
+                worker = buildworker.LocalBuildWorker(name, 'local', self)
+                controller.add_worker(worker)
         else:
             for worker in self.settings['worker']:
-                worker = morphlib.buildworker.RemoteBuildWorker(self)
+                name = controller.generate_worker_name(worker)
+                worker = buildworker.RemoteBuildWorker(name, worker, self)
                 controller.add_worker(worker)
 
         result = []
diff --git a/morphlib/buildcontroller.py b/morphlib/buildcontroller.py
index ea9719c0..f794928c 100644
--- a/morphlib/buildcontroller.py
+++ b/morphlib/buildcontroller.py
@@ -43,6 +43,10 @@ class BuildController(object):
         spaces = '  ' * self.indent
         self.real_msg('%s%s' % (spaces, text))
 
+    def generate_worker_name(self, ident):
+        similar_workers = [x for x in self.workers if x.ident == ident]
+        return '%s-%s' % (ident, len(similar_workers) + 1)
+
     def add_worker(self, worker):
         self.workers.add(worker)
         self.mark_idle(worker)
diff --git a/morphlib/buildworker.py b/morphlib/buildworker.py
index ad951466..f75177b3 100644
--- a/morphlib/buildworker.py
+++ b/morphlib/buildworker.py
@@ -22,15 +22,15 @@ import morphlib
 
 class BuildWorker(object):
 
-    def __init__(self, name, app):
+    def __init__(self, name, ident, app):
         self.name = name
+        self.ident = ident
         self.settings = app.settings
         self.real_msg = app.msg
         self.indent = 2
         self.idle_since = datetime.datetime.now()
-
-    def __str__(self):
-        return self.name
+        self.manager = Manager()
+        self.reset()
 
     def indent_more(self):
         self.indent += 1
@@ -42,30 +42,48 @@ class BuildWorker(object):
         spaces = '  ' * self.indent
         self.real_msg('%s%s' % (spaces, text))
 
+    def reset(self):
+        self.process = None
+        self.blob = None
+        self._output = self.manager.list()
+
     def build(self, blob):
         raise NotImplementedError
 
     def check_complete(self, timeout):
-        raise NotImplementedError
+        if self.process:
+            self.process.join(timeout)
+            if self.process.is_alive():
+                return False
+            else:
+                self.idle_since = datetime.datetime.now()
+                return True
+        else:
+            return True
 
+    @property
+    def output(self):
+        try:
+            return self._output[0]
+        except IndexError:
+            return None
 
-class LocalBuildWorker(BuildWorker):
+    def __str__(self):
+        return self.name
 
-    def __init__(self, name, app):
-        BuildWorker.__init__(self, name, app)
-        self.manager = Manager()
-        self.reset()
 
-    def reset(self):
-        self.process = None
-        self.blob = None
-        self._output = self.manager.list()
+class LocalBuildWorker(BuildWorker):
+
+    def __init__(self, name, ident, app):
+        BuildWorker.__init__(self, name, ident, app)
 
     def run(self, repo, ref, filename, output):
         ex = morphlib.execute.Execute('.', self.msg)
         stdout = ex.runv(['./morph', '--verbose', '--keep-path',
                           'build', repo, ref, filename])
         output.append(stdout)
+        
+        # TODO report errors back to the caller
 
     def build(self, blob):
         self.reset()
@@ -77,23 +95,32 @@ class LocalBuildWorker(BuildWorker):
         self.process = Process(group=None, target=self.run, args=args)
         self.process.start()
 
-    def check_complete(self, timeout):
-        if self.process:
-            self.process.join(timeout)
-            if self.process.is_alive():
-                return False
-            else:
-                self.idle_since = datetime.datetime.now()
-                return True
-        else:
-            return True
 
-    @property
-    def output(self):
-        return self._output[0]
+class RemoteBuildWorker(BuildWorker):
 
+    def __init__(self, name, ident, app):
+        BuildWorker.__init__(self, name, ident, app)
+        self.hostname = ident
 
-class RemoteBuildWorker(BuildWorker):
+    def run(self, repo, ref, filename, output):
+        ex = morphlib.execute.Execute('.', self.msg)
+
+        # generate command line options
+        cmdline = ['ssh', self.hostname]
+        cmdline.extend(['fakeroot', 'morph', 'build', repo, ref, filename])
+
+        # run morph on the other machine
+        stdout = ex.runv(cmdline)
+        output.append(stdout)
+
+        # TODO report errors back to the caller
 
-    def __init__(self, app):
-        BuildWorker.__init__(self, app)
+    def build(self, blob):
+        self.reset()
+        self.blob = blob
+        args = (blob.morph.treeish.original_repo,
+                blob.morph.treeish.ref,
+                blob.morph.filename,
+                self._output)
+        self.process = Process(group=None, target=self.run, args=args)
+        self.process.start()