#!/usr/bin/python # # Copyright (C) 2011-2012 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import cliapp import logging import os import morphlib from morphlib import buildworker from morphlib import buildcontroller from morphlib.morphologyloader import MorphologyLoader from morphlib.builddependencygraph import BuildDependencyGraph defaults = { 'git-base-url': [ 'git://gitorious.org/baserock-morphs/', 'git://gitorious.org/baserock/', ], 'bundle-server': 'http://roadtrain.codethink.co.uk/bundles/', 'cachedir': os.path.expanduser('~/.cache/morph'), 'max-jobs': morphlib.util.make_concurrency(), } class Morph(cliapp.Application): def add_settings(self): self.settings.boolean(['verbose', 'v'], 'show what is happening') self.settings.string_list(['git-base-url'], 'prepend URL to git repos that are not URLs', metavar='URL', default=defaults['git-base-url']) self.settings.string(['bundle-server'], 'base URL to download bundles', metavar='URL', default=defaults['bundle-server']) self.settings.string(['cachedir'], 'put build results in DIR', metavar='DIR', default=defaults['cachedir']) self.settings.string(['tempdir'], 'temporary directory to use for builds ' '(this is separate from just setting $TMPDIR ' 'or /tmp because those are used internally ' 'by things that cannot be on NFS, but ' 'this setting can point at a directory in ' 'NFS)', metavar='DIR', default=os.environ.get('TMPDIR')) self.settings.boolean(['no-ccache'], 'do not use ccache') self.settings.boolean(['no-distcc'], 'do not use distcc') self.settings.integer(['max-jobs'], 'run at most N parallel jobs with make (default ' 'is to a value based on the number of CPUs ' 'in the machine running morph', metavar='N', default=defaults['max-jobs']) self.settings.boolean(['keep-path'], 'do not touch the PATH environment variable') self.settings.boolean(['bootstrap'], 'build stuff in bootstrap mode; this is ' 'DANGEROUS and will install stuff on your ' 'system') self.settings.boolean(['ignore-submodules'], 'do not cache repositories of git submodules ' 'or unpack them into the build directory') self.settings.boolean(['no-git-update'], 'do not update the cached git repositories ' 'during a build (user must have done that ' 'already using the update-gits subcommand)') self.settings.string_list(['staging-filler'], 'unpack BLOB into staging area for ' 'non-bootstrap builds (this will ' 'eventually be replaced with proper ' 'build dependencies)', metavar='BLOB') self.settings.boolean(['staging-chroot'], 'build things in a staging chroot ' '(require real root to use)') self.settings.string_list(['worker'], 'IP or host name of a machine to distribute ' 'build work to', metavar='HOSTNAME') def _itertriplets(self, args): '''Generate repo, ref, filename triples from args.''' if (len(args) % 3) != 0: raise cliapp.AppException('Argument list must have full triplets') while args: assert len(args) >= 2, args yield args[0], args[1], args[2] args = args[3:] def cmd_build(self, args): '''Build a binary from a morphology. Command line arguments are the repository, git tree-ish reference, and morphology filename. Morph takes care of building all dependencies before building the morphology. All generated binaries are put into the cache. (The triplet of command line arguments may be repeated as many times as necessary.) ''' if not os.path.exists(self.settings['cachedir']): os.mkdir(self.settings['cachedir']) for repo, ref, filename in self._itertriplets(args): tempdir = morphlib.tempdir.Tempdir(self.settings['tempdir']) morph_loader = MorphologyLoader(self.settings) source_manager = morphlib.sourcemanager.SourceManager(self, update=not self.settings['no-git-update']) factory = morphlib.builder.Factory(tempdir) builder = morphlib.builder.Builder(tempdir, self, morph_loader, source_manager, factory) # Unpack manually specified build dependencies. factory.create_staging() for bin in self.settings['staging-filler']: factory.unpack_binary_from_file(bin) # derive a build order from the dependency graph graph = BuildDependencyGraph(source_manager, morph_loader, repo, ref, filename) graph.resolve() blobs, order = graph.build_order() self.msg('Building %s|%s|%s' % (repo, ref, filename)) # build things in this order builder.build(blobs, order) tempdir.remove() def cmd_show_dependencies(self, args): '''Dumps the dependency tree of all input morphologies.''' morph_loader = MorphologyLoader(self.settings) source_manager = morphlib.sourcemanager.SourceManager(self, update=not self.settings['no-git-update']) for repo, ref, filename in self._itertriplets(args): # create a dependency graph for the morphology graph = BuildDependencyGraph(source_manager, morph_loader, repo, ref, filename) graph.resolve() # print the graph self.output.write('dependency tree:\n') for blob in sorted(graph.blobs, key=str): self.output.write(' %s\n' % blob) for dependency in sorted(blob.dependencies, key=str): self.output.write(' -> %s\n' % dependency) # compute a build order from the graph blobs, order = graph.build_order() self.output.write('build order:\n') for group in order: self.output.write(' group:\n') for blob in sorted(group, key=str): self.output.write(' %s\n' % blob) def cmd_update_gits(self, args): '''Update cached git repositories. Parse the given morphologies, and their dependencies, and update all the git repositories referred to by them in the morph cache directory. ''' morph_loader = MorphologyLoader(self.settings) source_manager = morphlib.sourcemanager.SourceManager(self) for repo, ref, filename in self._itertriplets(args): # resolve the dependency graph, which will implicitly # clone all repositories needed to build the blob graph = BuildDependencyGraph(source_manager, morph_loader, repo, ref, filename) graph.resolve() def cmd_build_single(self, args): '''Build a binary from a morphology but do not build its dependencies. To build a system or stratum morphology, simply specify the repository, a git tree-ish reference and the morphology filename on the command line. To build a chunk morphology, first provide the repository, reference and morphology filename of its surrounding stratum and then provide the same information for the chunk morphology itself. This is needed to resolve the dependencies of the chunk using the stratum as the build context and to unpack these dependencies in the staging area prior to building the chunk. This command differs from 'build' in that it only builds the morphology specified on the command line. Dependencies are assumed to have been built prior to running this command and are only unpacked into the staging area. ''' tempdir = morphlib.tempdir.Tempdir(self.settings['tempdir']) morph_loader = MorphologyLoader(self.settings) source_manager = morphlib.sourcemanager.SourceManager(self, update=False) factory = morphlib.builder.Factory(tempdir) builder = morphlib.builder.Builder(tempdir, self, morph_loader, source_manager, factory) if not os.path.exists(self.settings['cachedir']): os.mkdir(self.settings['cachedir']) factory.create_staging() if len(args) >= 3: repo, ref, filename = args[:3] args = args[3:] # derive a build order from the dependency graph graph = BuildDependencyGraph(source_manager, morph_loader, repo, ref, filename) first_blob = graph.resolve() blobs, order = graph.build_order() # parse the second blob, if there is one second_blob = None if len(args) >= 3: repo, ref, filename = args[:3] args = args[3:] # load the blob manually treeish = source_manager.get_treeish(repo, ref) morphology = morph_loader.load(treeish, filename) second_blob = morphlib.blobs.Blob.create_blob(morphology) try: # find the corresponding blob object in the blobs # returned from the dependency graph second_blob = [x for x in blobs if x == second_blob][0] except IndexError: raise cliapp.AppException('%s and %s are unrelated' % (first_blob, second_blob)) # build the single blob if second_blob: # verify that the two input blobs are valid if first_blob.morph.kind != 'stratum': raise cliapp.AppException('The first tuple %s needs to ' 'refer to a stratum' % first_blob) if second_blob.morph.kind != 'chunk': raise cliapp.AppException('The first tuple %s needs to ' 'refer to a chunk' % second_blob) # build now self.msg('Building %s' % second_blob) builder.build_single(second_blob, blobs, order) else: # build the blob now self.msg('Building %s' % first_blob) builder.build_single(first_blob, blobs, order) tempdir.remove() if args: raise cliapp.AppException('Extra args on command line: %s' % args) def cmd_build_distributed(self, args): '''Build a binary from a morphology, distributing work where possible. Command line arguments are the repository, git tree-ish reference, and morphology filename. Morph takes care of building all dependencies before building the morphology. All generated binaries are put into the cache. This command differs from 'build' in that it distributes work to other machines where possible. This may include preparing the source tree, preparing the staging area, building individual chunks and caching the built items. Distributing currently works by running 'morph build-single' on other machines via SSH. The machines to be used for this are specified using the '-w/--worker' command line options. (The triplet of command line arguments may be repeated as many times as necessary.) ''' tempdir = morphlib.tempdir.Tempdir(self.settings['tempdir']) morph_loader = MorphologyLoader(self.settings) source_manager = morphlib.sourcemanager.SourceManager(self, update=not self.settings['no-git-update']) # create a build controller controller = buildcontroller.BuildController(self, tempdir) # create and add the build workers if len(self.settings['worker']) == 0: num_workers = morphlib.util.make_concurrency() for i in range(num_workers): name = controller.generate_worker_name('local') worker = buildworker.LocalBuildWorker(name, 'local', self) controller.add_worker(worker) else: for worker in self.settings['worker']: name = controller.generate_worker_name(worker) worker = buildworker.RemoteBuildWorker(name, worker, self) controller.add_worker(worker) result = [] for repo, ref, filename in self._itertriplets(args): # derive a build order from the dependency graph graph = BuildDependencyGraph(source_manager, morph_loader, repo, ref, filename) graph.resolve() blobs, order = graph.build_order() self.msg('Building %s|%s|%s' % (repo, ref, filename)) # build the tuple and all its dependencies result.append(controller.build(blobs, order)) tempdir.remove() def msg(self, msg): '''Show a message to the user about what is going on.''' logging.debug(msg) if self.settings['verbose']: self.output.write('%s\n' % msg) self.output.flush() def runcmd(self, argv, *args, **kwargs): # check which msg function to use msg = self.msg if 'msg' in kwargs: msg = kwargs['msg'] del kwargs['msg'] # convert the command line arguments into a string commands = [argv] + list(args) for command in commands: if isinstance(command, list): for i in xrange(0, len(command)): command[i] = str(command[i]) commands = [' '.join(command) for command in commands] # print the command line msg('# %s' % ' | '.join(commands)) # run the command line cliapp.Application.runcmd(self, argv, *args, **kwargs) if __name__ == '__main__': Morph().run()