diff options
| -rwxr-xr-x | morph | 68 | ||||
| -rw-r--r-- | morphlib/__init__.py | 2 | ||||
| -rw-r--r-- | morphlib/buildcontroller.py | 119 | ||||
| -rw-r--r-- | morphlib/buildworker.py | 99 | ||||
| -rw-r--r-- | morphlib/sourcemanager.py | 4 | ||||
| -rw-r--r-- | without-test-modules | 2 |
6 files changed, 284 insertions, 10 deletions
@@ -67,6 +67,12 @@ class Morph(cliapp.Application): metavar='TIMEOUT', default=10) + self.settings.string_list(['worker'], + 'IP or host name of a machine to distribute ' + 'build work to', + metavar='HOSTNAME') + + def cmd_build(self, args): '''Build a binary from a morphology. @@ -102,7 +108,7 @@ class Morph(cliapp.Application): # build things in this order ret.append(builder.build(blobs, order)) - + # we may not have permission to tempdir.remove() ex = morphlib.execute.Execute('.', lambda msg: None) ex.runv(["rm", "-rf", tempdir.dirname]) @@ -111,20 +117,20 @@ class Morph(cliapp.Application): raise cliapp.AppException('Extra args on command line: %s' % args) return ret - + def cmd_testsysimg(self, args): '''Run tests for a built system image. - + Command line arguments are the filename of the system image, and the filenames of the Python modules that contain the test "stories". Each module must have a variable called "story", which is a list of tuples. Each tuple is either two strings (one to send, the other a regular expression for what is expected in return), or two strings and a timeout in seconds. - + testsysimg runs the image under KVM, and accesses it via a serial console, and runs the test stories, one by one. - + ''' if not args: @@ -145,11 +151,11 @@ class Morph(cliapp.Application): def cmd_test(self, args): '''Build and test a system morphology. - + The tests are specified in the morphology's test-stories field. - + ''' - + for morph, built in self.cmd_build(args): if morph.kind == 'system': self.msg('running tests on system %s' % morph.name) @@ -214,6 +220,52 @@ class Morph(cliapp.Application): source_manager) builder.get_cache_id(blob) + def cmd_build_distributed(self, args): + tempdir = morphlib.tempdir.Tempdir() + morph_loader = MorphologyLoader(self.settings) + source_manager = morphlib.sourcemanager.SourceManager(self) + + # create a build controller + controller = morphlib.buildcontroller.BuildController(self, tempdir) + + # create and add the build workers + if len(self.settings['worker']) == 0: + worker = morphlib.buildworker.LocalBuildWorker("local-1", self) + controller.add_worker(worker) + worker = morphlib.buildworker.LocalBuildWorker("local-2", self) + controller.add_worker(worker) + else: + for worker in self.settings['worker']: + worker = morphlib.buildworker.RemoteBuildWorker(self) + controller.add_worker(worker) + + result = [] + + while len(args) >= 3: + # read the build tuple from the command line + repo, ref, filename = args[:3] + args = args[3:] + + # derive a build order from the dependency graph + graph = BuildDependencyGraph(source_manager, morph_loader, + repo, ref, filename) + graph.resolve() + blobs, order = graph.build_order() + + self.msg('Building %s|%s|%s' % (repo, ref, filename)) + + # build the tuple and all its dependencies + result.append(controller.build(blobs, order)) + + # we may not have permission to tempdir.remove() + ex = morphlib.execute.Execute('.', lambda msg: None) + ex.runv(["rm", "-rf", tempdir.dirname]) + + if args: + raise cliapp.AppException('Extra args on command line: %s' % args) + + return result + def msg(self, msg): '''Show a message to the user about what is going on.''' logging.debug(msg) diff --git a/morphlib/__init__.py b/morphlib/__init__.py index d4346826..11b6cc9a 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -22,7 +22,9 @@ __version__ = '0.0' import bins import blobs +import buildcontroller import builddependencygraph +import buildworker import builder import cachedir import execute diff --git a/morphlib/buildcontroller.py b/morphlib/buildcontroller.py new file mode 100644 index 00000000..a53bea06 --- /dev/null +++ b/morphlib/buildcontroller.py @@ -0,0 +1,119 @@ +# Copyright (C) 2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import collections +import time + + +class BuildController(object): + + def __init__(self, app, tempdir): + self.settings = app.settings + self.real_msg = app.msg + self.tempdir = tempdir + self.indent = 1 + + self.workers = set() + self.busy_workers = set() + self.idle_workers = set() + + self.blobs = set() + self.build_order = collections.deque() + + def indent_more(self): + self.indent += 1 + + def indent_less(self): + self.indent -= 1 + + def msg(self, text): + spaces = ' ' * self.indent + self.real_msg('%s%s' % (spaces, text)) + + def add_worker(self, worker): + self.workers.add(worker) + self.mark_idle(worker) + + def wait_for_workers(self, need_idle=False, timeout=100): + # first, check if any of the busy workers are finished + while all(not x.check_complete(timeout) for x in self.busy_workers): + # wait and repeat if they are all busy and we have no idle workers + if need_idle and len(self.idle_workers) == 0: + self.msg('Waiting for idle workers...') + time.sleep(0.250) + else: + break + + # get a list of all finished busy workers + finished = [x for x in self.busy_workers if x.check_complete(0)] + + # log the result of all workers that we are moving from busy to idle + for worker in finished: + self.msg('Built %s using worker %s' % (worker.blob, worker)) + for line in worker.output.split('\n'): + self.msg('> %s' % line) + + # mark all finished workers as being idle + for worker in finished: + self.mark_idle(worker) + + def wait_for_worker(self): + # wait for at least one worker to be idle + self.wait_for_workers(need_idle = True) + + # sort idle workers by their idle timestamps (ascending) + idle_workers = sorted(self.idle_workers, key=lambda x: x.idle_since) + + # return the worker that has been idling for the longest period of time + return idle_workers[0] + + def build(self, blobs, build_order): + self.blobs = blobs + self.build_order = build_order + + result = [] + + while len(build_order) > 0: + group = build_order.popleft() + group_str = ', '.join([x.morph.filename for x in group]) + self.msg('Building parallel group %s' % group_str) + self.indent_more() + + while len(group) > 0: + blob = group.pop() + + worker = self.wait_for_worker() + self.msg('Distributing %s to worker %s' % (blob, worker)) + self.mark_busy(worker) + worker.build(blob) + + self.wait_for_workers(need_idle = False, timeout = None) + + self.indent_less() + + return result + + def mark_idle(self, worker): + if worker not in self.idle_workers: + self.idle_workers.add(worker) + if worker in self.busy_workers: + self.busy_workers.remove(worker) + + def mark_busy(self, worker): + if worker not in self.busy_workers: + self.busy_workers.add(worker) + if worker in self.idle_workers: + self.idle_workers.remove(worker) diff --git a/morphlib/buildworker.py b/morphlib/buildworker.py new file mode 100644 index 00000000..ad951466 --- /dev/null +++ b/morphlib/buildworker.py @@ -0,0 +1,99 @@ +# Copyright (C) 2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import datetime +from multiprocessing import Manager, Process + +import morphlib + + +class BuildWorker(object): + + def __init__(self, name, app): + self.name = name + self.settings = app.settings + self.real_msg = app.msg + self.indent = 2 + self.idle_since = datetime.datetime.now() + + def __str__(self): + return self.name + + def indent_more(self): + self.indent += 1 + + def indent_less(self): + self.indent -= 1 + + def msg(self, text): + spaces = ' ' * self.indent + self.real_msg('%s%s' % (spaces, text)) + + def build(self, blob): + raise NotImplementedError + + def check_complete(self, timeout): + raise NotImplementedError + + +class LocalBuildWorker(BuildWorker): + + def __init__(self, name, app): + BuildWorker.__init__(self, name, app) + self.manager = Manager() + self.reset() + + def reset(self): + self.process = None + self.blob = None + self._output = self.manager.list() + + def run(self, repo, ref, filename, output): + ex = morphlib.execute.Execute('.', self.msg) + stdout = ex.runv(['./morph', '--verbose', '--keep-path', + 'build', repo, ref, filename]) + output.append(stdout) + + def build(self, blob): + self.reset() + self.blob = blob + args = (blob.morph.treeish.original_repo, + blob.morph.treeish.ref, + blob.morph.filename, + self._output) + self.process = Process(group=None, target=self.run, args=args) + self.process.start() + + def check_complete(self, timeout): + if self.process: + self.process.join(timeout) + if self.process.is_alive(): + return False + else: + self.idle_since = datetime.datetime.now() + return True + else: + return True + + @property + def output(self): + return self._output[0] + + +class RemoteBuildWorker(BuildWorker): + + def __init__(self, app): + BuildWorker.__init__(self, app) diff --git a/morphlib/sourcemanager.py b/morphlib/sourcemanager.py index c25f7331..a5ae9124 100644 --- a/morphlib/sourcemanager.py +++ b/morphlib/sourcemanager.py @@ -69,11 +69,11 @@ class SourceManager(object): location = self.cache_dir + '/' + name if os.path.exists(location): - self.msg('Have a cached clone, update origin') + self.msg('Cached clone exists, updating origin') morphlib.git.update_remote(location, "origin", self.msg) return True, location else: - self.msg('Need to clone from %s' % repo) + self.msg('No cached clone found, fetching from %s' % repo) success = False diff --git a/without-test-modules b/without-test-modules index 4d716837..52a7e602 100644 --- a/without-test-modules +++ b/without-test-modules @@ -1,5 +1,7 @@ morphlib/__init__.py +morphlib/buildcontroller.py morphlib/builddependencygraph.py +morphlib/buildworker.py morphlib/builder.py morphlib/tester.py morphlib/git.py |