#!/usr/bin/python # # WARNING: THIS IS HIGHLY EXPERIMENTAL CODE RIGHT NOW. JUST PROOF OF CONCEPT. # DO NOT RUN UNTIL YOU KNOW WHAT YOU ARE DOING. # # Copyright (C) 2011-2012 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import cliapp import logging import os import morphlib from morphlib import buildworker from morphlib import buildcontroller from morphlib.morphologyloader import MorphologyLoader from morphlib.builddependencygraph import BuildDependencyGraph class Morph(cliapp.Application): def add_settings(self): self.settings.boolean(['verbose', 'v'], 'show what is happening') self.settings.string_list(['git-base-url'], 'prepend URL to git repos that are not URLs', metavar='URL') self.settings.string(['bundle-server'], 'base URL to download bundles', metavar='URL') self.settings.string(['cachedir'], 'put build results in DIR (default: %default)', metavar='DIR', default='.') self.settings.boolean(['no-ccache'], 'do not use ccache') self.settings.boolean(['no-distcc'], 'do not use distcc') self.settings.integer(['max-jobs'], 'run at most N parallel jobs with make (default ' 'is to a value based on the number of CPUs ' 'in the machine running morph', metavar='N', default=0) self.settings.boolean(['keep-path'], 'do not touch the PATH environment variable ' '(use with tests ONLY)') self.settings.boolean(['bootstrap'], 'build stuff in bootstrap mode; this is ' 'DANGEROUS and will install stuff on your ' 'system') self.settings.boolean(['test-console'], 'show what the system outputs on the serial ' 'console during tests') self.settings.integer(['test-timeout'], 'abort test if system doesn\'t produce ' 'expected output in TIMEOUT seconds ' '(default: %default)', metavar='TIMEOUT', default=10) self.settings.string_list(['worker'], 'IP or host name of a machine to distribute ' 'build work to', metavar='HOSTNAME') def cmd_build(self, args): '''Build a binary from a morphology. Command line arguments are the repository, git tree-ish reference, and morphology filename. The binary gets put into the cache. (The triplet of command line arguments may be repeated as many times as necessary.) ''' tempdir = morphlib.tempdir.Tempdir() morph_loader = MorphologyLoader(self.settings) source_manager = morphlib.sourcemanager.SourceManager(self) builder = morphlib.builder.Builder(tempdir, self, morph_loader, source_manager) if not os.path.exists(self.settings['cachedir']): os.mkdir(self.settings['cachedir']) ret = [] while len(args) >= 3: repo, ref, filename = args[:3] args = args[3:] # derive a build order from the dependency graph graph = BuildDependencyGraph(source_manager, morph_loader, repo, ref, filename) graph.resolve() blobs, order = graph.build_order() self.msg('Building %s|%s|%s' % (repo, ref, filename)) # build things in this order ret.append(builder.build(blobs, order)) # we may not have permission to tempdir.remove() ex = morphlib.execute.Execute('.', lambda msg: None) ex.runv(["rm", "-rf", tempdir.dirname]) if args: raise cliapp.AppException('Extra args on command line: %s' % args) return ret def cmd_testsysimg(self, args): '''Run tests for a built system image. Command line arguments are the filename of the system image, and the filenames of the Python modules that contain the test "stories". Each module must have a variable called "story", which is a list of tuples. Each tuple is either two strings (one to send, the other a regular expression for what is expected in return), or two strings and a timeout in seconds. testsysimg runs the image under KVM, and accesses it via a serial console, and runs the test stories, one by one. ''' if not args: raise cliapp.AppException('Missing command line arguments. ' 'Run with --help to see usage.') system = morphlib.tester.KvmSystem(args[0], verbose=self.settings['test-console'], timeout=self.settings['test-timeout']) for filename in args[1:]: self.msg('Running %s' % filename) module = morphlib.tester.load_module(filename) story_steps = getattr(module, 'story') story = morphlib.tester.TestStory(system, story_steps, self.msg) story.run() self.msg('Finished OK.') def cmd_test(self, args): '''Build and test a system morphology. The tests are specified in the morphology's test-stories field. ''' for morph, built in self.cmd_build(args): if morph.kind == 'system': self.msg('running tests on system %s' % morph.name) assert len(built) == 1 image_filename = built.values()[0] morphdir = os.path.dirname(morph.filename) stories = [os.path.join(morphdir, x) for x in morph.test_stories] self.cmd_testsysimg([image_filename] + stories) else: self.msg('not testing %s %s (not a system)' % (morph.kind, morph.name)) def cmd_show_dependencies(self, args): '''Dumps the dependency tree of all input morphologies.''' morph_loader = MorphologyLoader(self.settings) source_manager = morphlib.sourcemanager.SourceManager(self) while len(args) >= 3: # read the build tuple from the command line repo, ref, filename = args[:3] args = args[3:] # create a dependency graph for the morphology graph = BuildDependencyGraph(source_manager, morph_loader, repo, ref, filename) graph.resolve() # print the graph self.output.write('dependency tree:\n') for blob in sorted(graph.blobs, key=str): self.output.write(' %s\n' % blob) for dependency in sorted(blob.dependencies, key=str): self.output.write(' -> %s\n' % dependency) # compute a build order from the graph blobs, order = graph.build_order() self.output.write('build order:\n') for group in order: self.output.write(' group:\n') for blob in sorted(group, key=str): self.output.write(' %s\n' % blob) def cmd_update_gits(self, args): tempdir = morphlib.tempdir.Tempdir() morph_loader = MorphologyLoader(self.settings) source_manager = morphlib.sourcemanager.SourceManager(self) while len(args) >= 3: # read the build tuple from the command line repo, ref, filename = args[:3] args = args[3:] # first step: clone the corresponding repo treeish = source_manager.get_treeish(repo, ref) morph = morph_loader.load(treeish, filename) blob = morphlib.blobs.Blob.create_blob(morph) # second step: compute the cache ID, which will implicitly # clone all repositories needed to build the blob builder = morphlib.builder.Builder(tempdir, self, morph_loader, source_manager) builder.get_cache_id(blob) def cmd_build_distributed(self, args): tempdir = morphlib.tempdir.Tempdir() morph_loader = MorphologyLoader(self.settings) source_manager = morphlib.sourcemanager.SourceManager(self) # create a build controller controller = buildcontroller.BuildController(self, tempdir) # create and add the build workers if len(self.settings['worker']) == 0: num_workers = morphlib.util.make_concurrency() for i in range(num_workers): name = controller.generate_worker_name('local') worker = buildworker.LocalBuildWorker(name, 'local', self) controller.add_worker(worker) else: for worker in self.settings['worker']: name = controller.generate_worker_name(worker) worker = buildworker.RemoteBuildWorker(name, worker, self) controller.add_worker(worker) result = [] while len(args) >= 3: # read the build tuple from the command line repo, ref, filename = args[:3] args = args[3:] # derive a build order from the dependency graph graph = BuildDependencyGraph(source_manager, morph_loader, repo, ref, filename) graph.resolve() blobs, order = graph.build_order() self.msg('Building %s|%s|%s' % (repo, ref, filename)) # build the tuple and all its dependencies result.append(controller.build(blobs, order)) # we may not have permission to tempdir.remove() ex = morphlib.execute.Execute('.', lambda msg: None) ex.runv(["rm", "-rf", tempdir.dirname]) if args: raise cliapp.AppException('Extra args on command line: %s' % args) return result def msg(self, msg): '''Show a message to the user about what is going on.''' logging.debug(msg) if self.settings['verbose']: self.output.write('%s\n' % msg) self.output.flush() if __name__ == '__main__': Morph().run()