path: root/morph

#!/usr/bin/python
#
# Copyright (C) 2011-2012  Codethink Limited
# 
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.


import cliapp
import logging
import os

import morphlib
from morphlib import buildworker
from morphlib import buildcontroller
from morphlib.morphologyloader import MorphologyLoader
from morphlib.builddependencygraph import BuildDependencyGraph


defaults = {
    'git-base-url': [
        'git://gitorious.org/baserock-morphs/',
        'git://gitorious.org/baserock/',
    ],
    'bundle-server': 'http://roadtrain.codethink.co.uk/bundles/',
    'cachedir': os.path.expanduser('~/.cache/morph'),
    'max-jobs': morphlib.util.make_concurrency(),
}


class Morph(cliapp.Application):

    def add_settings(self):
        self.settings.boolean(['verbose', 'v'], 'show what is happening')
        self.settings.string_list(['git-base-url'],
                             'prepend URL to git repos that are not URLs',
                             metavar='URL',
                             default=defaults['git-base-url'])
        self.settings.string(['bundle-server'],
                             'base URL to download bundles',
                             metavar='URL',
                             default=defaults['bundle-server'])
        self.settings.string(['cachedir'], 
                             'put build results in DIR',
                             metavar='DIR', 
                             default=defaults['cachedir'])
        self.settings.string(['tempdir'],
                             'temporary directory to use for builds '
                                '(this is separate from just setting $TMPDIR '
                                'or /tmp because those are used internally '
                                'by things that cannot be on NFS, but '
                                'this setting can point at a directory in '
                                'NFS)',
                             metavar='DIR', 
                             default=os.environ.get('TMPDIR'))
        self.settings.boolean(['no-ccache'], 'do not use ccache')
        self.settings.boolean(['no-distcc'], 'do not use distcc')
        self.settings.integer(['max-jobs'], 
                              'run at most N parallel jobs with make (default '
                                'is to a value based on the number of CPUs '
                                'in the machine running morph',
                              metavar='N',
                              default=defaults['max-jobs'])
        self.settings.boolean(['keep-path'], 
                              'do not touch the PATH environment variable')
        self.settings.boolean(['bootstrap'], 
                              'build stuff in bootstrap mode; this is '
                                'DANGEROUS and will install stuff on your '
                                'system')
        self.settings.boolean(['ignore-submodules'],
                              'do not cache repositories of git submodules '
                              'or unpack them into the build directory')

        self.settings.boolean(['no-git-update'],
                              'do not update the cached git repositories '
                                'during a build (user must have done that '
                                'already using the update-gits subcommand)')

        self.settings.string_list(['staging-filler'],
                                  'unpack BLOB into staging area for '
                                    'non-bootstrap builds (this will '
                                    'eventually be replaced with proper '
                                    'build dependencies)',
                                   metavar='BLOB')
        self.settings.boolean(['staging-chroot'],
                              'build things in a staging chroot '
                                '(require real root to use)')

        self.settings.string_list(['worker'],
                                  'IP or host name of a machine to distribute '
                                  'build work to',
                                  metavar='HOSTNAME')

    def _itertriplets(self, args):
        '''Generate repo, ref, filename triples from args.'''
        
        if (len(args) % 3) != 0:
            raise cliapp.AppException('Argument list must have full triplets')
        
        while args:
            assert len(args) >= 2, args
            yield args[0], args[1], args[2]
            args = args[3:]


    def cmd_build(self, args):
        '''Build a binary from a morphology.
        
        Command line arguments are the repository, git tree-ish reference,
        and morphology filename. Morph takes care of building all dependencies
        before building the morphology. All generated binaries are put into the
        cache.
        
        (The triplet of command line arguments may be repeated as many
        times as necessary.)
        
        '''


        if not os.path.exists(self.settings['cachedir']):
            os.mkdir(self.settings['cachedir'])

        for repo, ref, filename in self._itertriplets(args):
            tempdir = morphlib.tempdir.Tempdir(self.settings['tempdir'])
            morph_loader = MorphologyLoader(self.settings)
            source_manager = morphlib.sourcemanager.SourceManager(self,
                                update=not self.settings['no-git-update'])
            factory = morphlib.builder.Factory(tempdir)
            builder = morphlib.builder.Builder(tempdir, self, morph_loader,
                                               source_manager, factory)

            # Unpack manually specified build dependencies.
            factory.create_staging()
            for bin in self.settings['staging-filler']:
                factory.unpack_binary_from_file(bin)

            # derive a build order from the dependency graph
            graph = BuildDependencyGraph(source_manager, morph_loader,
                                         repo, ref, filename)
            graph.resolve()
            blobs, order = graph.build_order()

            self.msg('Building %s|%s|%s' % (repo, ref, filename))

            # build things in this order
            builder.build(blobs, order)

            tempdir.remove()

    def cmd_show_dependencies(self, args):
        '''Dumps the dependency tree of all input morphologies.'''

        morph_loader = MorphologyLoader(self.settings)
        source_manager = morphlib.sourcemanager.SourceManager(self,
                            update=not self.settings['no-git-update'])

        for repo, ref, filename in self._itertriplets(args):
            # create a dependency graph for the morphology
            graph = BuildDependencyGraph(source_manager, morph_loader,
                                         repo, ref, filename)
            graph.resolve()

            # print the graph
            self.output.write('dependency tree:\n')
            for blob in sorted(graph.blobs, key=str):
                self.output.write('  %s\n' % blob)
                for dependency in sorted(blob.dependencies, key=str):
                    self.output.write('    -> %s\n' % dependency)

            # compute a build order from the graph
            blobs, order = graph.build_order()
            self.output.write('build order:\n')
            for group in order:
                self.output.write('  group:\n')
                for blob in sorted(group, key=str):
                    self.output.write('    %s\n' % blob)

    def cmd_update_gits(self, args):
        '''Update cached git repositories.

        Parse the given morphologies, and their dependencies, and
        update all the git repositories referred to by them in the
        morph cache directory.
        
        '''

        morph_loader = MorphologyLoader(self.settings)
        source_manager = morphlib.sourcemanager.SourceManager(self)

        for repo, ref, filename in self._itertriplets(args):
            # resolve the dependency graph, which will implicitly
            # clone all repositories needed to build the blob
            graph = BuildDependencyGraph(source_manager, morph_loader,
                                         repo, ref, filename)
            graph.resolve()

    def cmd_build_single(self, args):
        '''Build a binary from a morphology but do not build its dependencies.
        
        To build a system or stratum morphology, simply specify the repository,
        a git tree-ish reference and the morphology filename on the command
        line. To build a chunk morphology, first provide the repository,
        reference and morphology filename of its surrounding stratum and then
        provide the same information for the chunk morphology itself. This is
        needed to resolve the dependencies of the chunk using the stratum as
        the build context and to unpack these dependencies in the staging area
        prior to building the chunk.

        This command differs from 'build' in that it only builds the morphology
        specified on the command line. Dependencies are assumed to have been
        built prior to running this command and are only unpacked into the
        staging area.

        '''
        tempdir = morphlib.tempdir.Tempdir(self.settings['tempdir'])
        morph_loader = MorphologyLoader(self.settings)
        source_manager = morphlib.sourcemanager.SourceManager(self,
                                                              update=False)
        factory = morphlib.builder.Factory(tempdir)
        builder = morphlib.builder.Builder(tempdir, self, morph_loader,
                                           source_manager, factory)

        if not os.path.exists(self.settings['cachedir']):
            os.mkdir(self.settings['cachedir'])

        factory.create_staging()

        if len(args) >= 3:
            repo, ref, filename = args[:3]
            args = args[3:]

            # derive a build order from the dependency graph
            graph = BuildDependencyGraph(source_manager, morph_loader,
                                         repo, ref, filename)
            first_blob = graph.resolve()
            blobs, order = graph.build_order()

            # parse the second blob, if there is one
            second_blob = None
            if len(args) >= 3:
                repo, ref, filename = args[:3]
                args = args[3:]

                # load the blob manually
                treeish = source_manager.get_treeish(repo, ref)
                morphology = morph_loader.load(treeish, filename)
                second_blob = morphlib.blobs.Blob.create_blob(morphology)

                try:
                    # find the corresponding blob object in the blobs
                    # returned from the dependency graph
                    second_blob = [x for x in blobs if x == second_blob][0]
                except IndexError:
                    raise cliapp.AppException('%s and %s are unrelated' %
                                              (first_blob, second_blob))

            # build the single blob
            if second_blob:
                # verify that the two input blobs are valid
                if first_blob.morph.kind != 'stratum':
                    raise cliapp.AppException('The first tuple %s needs to '
                                              'refer to a stratum' %
                                              first_blob)
                if second_blob.morph.kind != 'chunk':
                    raise cliapp.AppException('The first tuple %s needs to '
                                              'refer to a chunk' % second_blob)

                # build now
                self.msg('Building %s' % second_blob)
                builder.build_single(second_blob, blobs, order)
            else:
                # build the blob now
                self.msg('Building %s' % first_blob)
                builder.build_single(first_blob, blobs, order)

        tempdir.remove()

        if args:
            raise cliapp.AppException('Extra args on command line: %s' % args)

    def cmd_build_distributed(self, args):
        '''Build a binary from a morphology, distributing work where possible.
        
        Command line arguments are the repository, git tree-ish reference,
        and morphology filename. Morph takes care of building all dependencies
        before building the morphology. All generated binaries are put into the
        cache.

        This command differs from 'build' in that it distributes work to
        other machines where possible. This may include preparing the source
        tree, preparing the staging area, building individual chunks and
        caching the built items.
        
        Distributing currently works by running 'morph build-single' on other
        machines via SSH. The machines to be used for this are specified using
        the '-w/--worker' command line options.
        
        (The triplet of command line arguments may be repeated as many
        times as necessary.)

        '''
        
        tempdir = morphlib.tempdir.Tempdir(self.settings['tempdir'])
        morph_loader = MorphologyLoader(self.settings)
        source_manager = morphlib.sourcemanager.SourceManager(self,
                                update=not self.settings['no-git-update'])

        # create a build controller
        controller = buildcontroller.BuildController(self, tempdir)

        # create and add the build workers
        if len(self.settings['worker']) == 0:
            num_workers = morphlib.util.make_concurrency()
            for i in range(num_workers):
                name = controller.generate_worker_name('local')
                worker = buildworker.LocalBuildWorker(name, 'local', self)
                controller.add_worker(worker)
        else:
            for worker in self.settings['worker']:
                name = controller.generate_worker_name(worker)
                worker = buildworker.RemoteBuildWorker(name, worker, self)
                controller.add_worker(worker)

        result = []

        for repo, ref, filename in self._itertriplets(args):
            # derive a build order from the dependency graph
            graph = BuildDependencyGraph(source_manager, morph_loader,
                                         repo, ref, filename)
            graph.resolve()
            blobs, order = graph.build_order()

            self.msg('Building %s|%s|%s' % (repo, ref, filename))

            # build the tuple and all its dependencies
            result.append(controller.build(blobs, order))

        tempdir.remove()

    def msg(self, msg):
        '''Show a message to the user about what is going on.'''
        logging.debug(msg)
        if self.settings['verbose']:
            self.output.write('%s\n' % msg)
            self.output.flush()

    def runcmd(self, argv, *args, **kwargs):
        # check which msg function to use
        msg = self.msg
        if 'msg' in kwargs:
            msg = kwargs['msg']
            del kwargs['msg']

        # convert the command line arguments into a string
        commands = [argv] + list(args)
        for command in commands:
            if isinstance(command, list):
                for i in xrange(0, len(command)):
                    command[i] = str(command[i])
        commands = [' '.join(command) for command in commands]

        # print the command line
        msg('# %s' % ' | '.join(commands))

        # run the command line
        cliapp.Application.runcmd(self, argv, *args, **kwargs)


if __name__ == '__main__':
    Morph().run()