summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJannis Pohlmann <jannis.pohlmann@codethink.co.uk>2012-01-23 17:52:37 +0000
committerJannis Pohlmann <jannis.pohlmann@codethink.co.uk>2012-01-23 18:26:51 +0000
commitd8f4dbdfe07df8cbb576e32f653c86190f07c392 (patch)
treef3844aece4a8ff488504b040d6b773102367042c
parent9d59ed4d78747902cd141f743a6aeabd9e531dc8 (diff)
downloadmorph-d8f4dbdfe07df8cbb576e32f653c86190f07c392.tar.gz
Add controller, worker classes and a new "build-distributed" command.
This commit introduces four new classes: BuildController: * takes an app instance and a tempdir * allows to add BuildWorker objects * provides a build() method that takes a set of blobs and a build order that is then built by assigning work to the build workers as needed * the build() method takes care of polling the workers for their state, moving them between busy and idle states reliably, collect and print their output in a non-confusing order, and makes sure to wait for all workers to finish before processing the next group in the build order. * at this point, when waiting for one or more workers to become idle to assign them another blob to build, the controller always picks the worker that has been idling for the longest period of time. this can be changed later. BuildWorker: * base class for all worker classes * takes a name and an app instance * has a idle_since datetime property * provides a build() method that takes a Blob object and builds it in whatever way the subclasses implement it * provides a check_complete(timeout) method that checks whether the worker has finished building the blob yet or not LocalBuildWorker: * worker class for local builds that don't go through SSH * it uses morphlib.execute.Execute to run morph in a child process in build() * at the moment, this class executes "./morph" instead of "morph" as it assumes the user to run morph from its source tree. obviously, this will have to be fixed later. RemoteBuildWorker: * doesn't implement anything yet, will be used for distributing work to other machines running morph via SSH Notes: * At the moment, there is a degree of undesired redundancy when building a stratum in a worker, as this will cause the worker to rebuild all its dependencies. This will have to be fixed as it is avoidable and wastes a lot of time and processing power.
-rwxr-xr-xmorph68
-rw-r--r--morphlib/__init__.py2
-rw-r--r--morphlib/buildcontroller.py119
-rw-r--r--morphlib/buildworker.py99
-rw-r--r--morphlib/sourcemanager.py4
-rw-r--r--without-test-modules2
6 files changed, 284 insertions, 10 deletions
diff --git a/morph b/morph
index c039a4c8..42aa6f38 100755
--- a/morph
+++ b/morph
@@ -67,6 +67,12 @@ class Morph(cliapp.Application):
metavar='TIMEOUT',
default=10)
+ self.settings.string_list(['worker'],
+ 'IP or host name of a machine to distribute '
+ 'build work to',
+ metavar='HOSTNAME')
+
+
def cmd_build(self, args):
'''Build a binary from a morphology.
@@ -102,7 +108,7 @@ class Morph(cliapp.Application):
# build things in this order
ret.append(builder.build(blobs, order))
-
+
# we may not have permission to tempdir.remove()
ex = morphlib.execute.Execute('.', lambda msg: None)
ex.runv(["rm", "-rf", tempdir.dirname])
@@ -111,20 +117,20 @@ class Morph(cliapp.Application):
raise cliapp.AppException('Extra args on command line: %s' % args)
return ret
-
+
def cmd_testsysimg(self, args):
'''Run tests for a built system image.
-
+
Command line arguments are the filename of the system image,
and the filenames of the Python modules that contain the test
"stories". Each module must have a variable called "story",
which is a list of tuples. Each tuple is either two strings
(one to send, the other a regular expression for what is expected
in return), or two strings and a timeout in seconds.
-
+
testsysimg runs the image under KVM, and accesses it via a
serial console, and runs the test stories, one by one.
-
+
'''
if not args:
@@ -145,11 +151,11 @@ class Morph(cliapp.Application):
def cmd_test(self, args):
'''Build and test a system morphology.
-
+
The tests are specified in the morphology's test-stories field.
-
+
'''
-
+
for morph, built in self.cmd_build(args):
if morph.kind == 'system':
self.msg('running tests on system %s' % morph.name)
@@ -214,6 +220,52 @@ class Morph(cliapp.Application):
source_manager)
builder.get_cache_id(blob)
+ def cmd_build_distributed(self, args):
+ tempdir = morphlib.tempdir.Tempdir()
+ morph_loader = MorphologyLoader(self.settings)
+ source_manager = morphlib.sourcemanager.SourceManager(self)
+
+ # create a build controller
+ controller = morphlib.buildcontroller.BuildController(self, tempdir)
+
+ # create and add the build workers
+ if len(self.settings['worker']) == 0:
+ worker = morphlib.buildworker.LocalBuildWorker("local-1", self)
+ controller.add_worker(worker)
+ worker = morphlib.buildworker.LocalBuildWorker("local-2", self)
+ controller.add_worker(worker)
+ else:
+ for worker in self.settings['worker']:
+ worker = morphlib.buildworker.RemoteBuildWorker(self)
+ controller.add_worker(worker)
+
+ result = []
+
+ while len(args) >= 3:
+ # read the build tuple from the command line
+ repo, ref, filename = args[:3]
+ args = args[3:]
+
+ # derive a build order from the dependency graph
+ graph = BuildDependencyGraph(source_manager, morph_loader,
+ repo, ref, filename)
+ graph.resolve()
+ blobs, order = graph.build_order()
+
+ self.msg('Building %s|%s|%s' % (repo, ref, filename))
+
+ # build the tuple and all its dependencies
+ result.append(controller.build(blobs, order))
+
+ # we may not have permission to tempdir.remove()
+ ex = morphlib.execute.Execute('.', lambda msg: None)
+ ex.runv(["rm", "-rf", tempdir.dirname])
+
+ if args:
+ raise cliapp.AppException('Extra args on command line: %s' % args)
+
+ return result
+
def msg(self, msg):
'''Show a message to the user about what is going on.'''
logging.debug(msg)
diff --git a/morphlib/__init__.py b/morphlib/__init__.py
index d4346826..11b6cc9a 100644
--- a/morphlib/__init__.py
+++ b/morphlib/__init__.py
@@ -22,7 +22,9 @@ __version__ = '0.0'
import bins
import blobs
+import buildcontroller
import builddependencygraph
+import buildworker
import builder
import cachedir
import execute
diff --git a/morphlib/buildcontroller.py b/morphlib/buildcontroller.py
new file mode 100644
index 00000000..a53bea06
--- /dev/null
+++ b/morphlib/buildcontroller.py
@@ -0,0 +1,119 @@
+# Copyright (C) 2012 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import collections
+import time
+
+
+class BuildController(object):
+
+ def __init__(self, app, tempdir):
+ self.settings = app.settings
+ self.real_msg = app.msg
+ self.tempdir = tempdir
+ self.indent = 1
+
+ self.workers = set()
+ self.busy_workers = set()
+ self.idle_workers = set()
+
+ self.blobs = set()
+ self.build_order = collections.deque()
+
+ def indent_more(self):
+ self.indent += 1
+
+ def indent_less(self):
+ self.indent -= 1
+
+ def msg(self, text):
+ spaces = ' ' * self.indent
+ self.real_msg('%s%s' % (spaces, text))
+
+ def add_worker(self, worker):
+ self.workers.add(worker)
+ self.mark_idle(worker)
+
+ def wait_for_workers(self, need_idle=False, timeout=100):
+ # first, check if any of the busy workers are finished
+ while all(not x.check_complete(timeout) for x in self.busy_workers):
+ # wait and repeat if they are all busy and we have no idle workers
+ if need_idle and len(self.idle_workers) == 0:
+ self.msg('Waiting for idle workers...')
+ time.sleep(0.250)
+ else:
+ break
+
+ # get a list of all finished busy workers
+ finished = [x for x in self.busy_workers if x.check_complete(0)]
+
+ # log the result of all workers that we are moving from busy to idle
+ for worker in finished:
+ self.msg('Built %s using worker %s' % (worker.blob, worker))
+ for line in worker.output.split('\n'):
+ self.msg('> %s' % line)
+
+ # mark all finished workers as being idle
+ for worker in finished:
+ self.mark_idle(worker)
+
+ def wait_for_worker(self):
+ # wait for at least one worker to be idle
+ self.wait_for_workers(need_idle = True)
+
+ # sort idle workers by their idle timestamps (ascending)
+ idle_workers = sorted(self.idle_workers, key=lambda x: x.idle_since)
+
+ # return the worker that has been idling for the longest period of time
+ return idle_workers[0]
+
+ def build(self, blobs, build_order):
+ self.blobs = blobs
+ self.build_order = build_order
+
+ result = []
+
+ while len(build_order) > 0:
+ group = build_order.popleft()
+ group_str = ', '.join([x.morph.filename for x in group])
+ self.msg('Building parallel group %s' % group_str)
+ self.indent_more()
+
+ while len(group) > 0:
+ blob = group.pop()
+
+ worker = self.wait_for_worker()
+ self.msg('Distributing %s to worker %s' % (blob, worker))
+ self.mark_busy(worker)
+ worker.build(blob)
+
+ self.wait_for_workers(need_idle = False, timeout = None)
+
+ self.indent_less()
+
+ return result
+
+ def mark_idle(self, worker):
+ if worker not in self.idle_workers:
+ self.idle_workers.add(worker)
+ if worker in self.busy_workers:
+ self.busy_workers.remove(worker)
+
+ def mark_busy(self, worker):
+ if worker not in self.busy_workers:
+ self.busy_workers.add(worker)
+ if worker in self.idle_workers:
+ self.idle_workers.remove(worker)
diff --git a/morphlib/buildworker.py b/morphlib/buildworker.py
new file mode 100644
index 00000000..ad951466
--- /dev/null
+++ b/morphlib/buildworker.py
@@ -0,0 +1,99 @@
+# Copyright (C) 2012 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import datetime
+from multiprocessing import Manager, Process
+
+import morphlib
+
+
+class BuildWorker(object):
+
+ def __init__(self, name, app):
+ self.name = name
+ self.settings = app.settings
+ self.real_msg = app.msg
+ self.indent = 2
+ self.idle_since = datetime.datetime.now()
+
+ def __str__(self):
+ return self.name
+
+ def indent_more(self):
+ self.indent += 1
+
+ def indent_less(self):
+ self.indent -= 1
+
+ def msg(self, text):
+ spaces = ' ' * self.indent
+ self.real_msg('%s%s' % (spaces, text))
+
+ def build(self, blob):
+ raise NotImplementedError
+
+ def check_complete(self, timeout):
+ raise NotImplementedError
+
+
+class LocalBuildWorker(BuildWorker):
+
+ def __init__(self, name, app):
+ BuildWorker.__init__(self, name, app)
+ self.manager = Manager()
+ self.reset()
+
+ def reset(self):
+ self.process = None
+ self.blob = None
+ self._output = self.manager.list()
+
+ def run(self, repo, ref, filename, output):
+ ex = morphlib.execute.Execute('.', self.msg)
+ stdout = ex.runv(['./morph', '--verbose', '--keep-path',
+ 'build', repo, ref, filename])
+ output.append(stdout)
+
+ def build(self, blob):
+ self.reset()
+ self.blob = blob
+ args = (blob.morph.treeish.original_repo,
+ blob.morph.treeish.ref,
+ blob.morph.filename,
+ self._output)
+ self.process = Process(group=None, target=self.run, args=args)
+ self.process.start()
+
+ def check_complete(self, timeout):
+ if self.process:
+ self.process.join(timeout)
+ if self.process.is_alive():
+ return False
+ else:
+ self.idle_since = datetime.datetime.now()
+ return True
+ else:
+ return True
+
+ @property
+ def output(self):
+ return self._output[0]
+
+
+class RemoteBuildWorker(BuildWorker):
+
+ def __init__(self, app):
+ BuildWorker.__init__(self, app)
diff --git a/morphlib/sourcemanager.py b/morphlib/sourcemanager.py
index c25f7331..a5ae9124 100644
--- a/morphlib/sourcemanager.py
+++ b/morphlib/sourcemanager.py
@@ -69,11 +69,11 @@ class SourceManager(object):
location = self.cache_dir + '/' + name
if os.path.exists(location):
- self.msg('Have a cached clone, update origin')
+ self.msg('Cached clone exists, updating origin')
morphlib.git.update_remote(location, "origin", self.msg)
return True, location
else:
- self.msg('Need to clone from %s' % repo)
+ self.msg('No cached clone found, fetching from %s' % repo)
success = False
diff --git a/without-test-modules b/without-test-modules
index 4d716837..52a7e602 100644
--- a/without-test-modules
+++ b/without-test-modules
@@ -1,5 +1,7 @@
morphlib/__init__.py
+morphlib/buildcontroller.py
morphlib/builddependencygraph.py
+morphlib/buildworker.py
morphlib/builder.py
morphlib/tester.py
morphlib/git.py