path: root/morph

#!/usr/bin/python
#
# WARNING: THIS IS HIGHLY EXPERIMENTAL CODE RIGHT NOW. JUST PROOF OF CONCEPT.
# DO NOT RUN UNTIL YOU KNOW WHAT YOU ARE DOING.
# 
# Copyright (C) 2011-2012  Codethink Limited
# 
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.


import cliapp
import logging
import os

import morphlib
from morphlib.morphologyloader import MorphologyLoader
from morphlib.builddependencygraph import BuildDependencyGraph


class Morph(cliapp.Application):

    def add_settings(self):
        self.settings.boolean(['verbose', 'v'], 'show what is happening')
        self.settings.string_list(['git-base-url'],
                             'prepend URL to git repos that are not URLs',
                             metavar='URL')
        self.settings.string(['bundle-server'],
                             'base URL to download bundles',
                             metavar='URL')
        self.settings.string(['cachedir'], 
                             'put build results in DIR (default: %default)',
                             metavar='DIR', default='.')
        self.settings.boolean(['no-ccache'], 'do not use ccache')
        self.settings.boolean(['no-distcc'], 'do not use distcc')
        self.settings.integer(['max-jobs'], 
                              'run at most N parallel jobs with make (default '
                                'is to a value based on the number of CPUs '
                                'in the machine running morph',
                              metavar='N',
                              default=0)
        self.settings.boolean(['keep-path'], 
                              'do not touch the PATH environment variable '
                                '(use with tests ONLY)')
        self.settings.boolean(['bootstrap'], 
                              'build stuff in bootstrap mode; this is '
                                'DANGEROUS and will install stuff on your '
                                'system')

        self.settings.boolean(['test-console'], 
                              'show what the system outputs on the serial '
                                    'console during tests')
        self.settings.integer(['test-timeout'], 
                              'abort test if system doesn\'t produce '
                                'expected output in TIMEOUT seconds '
                                '(default: %default)',
                              metavar='TIMEOUT',
                              default=10)

        self.settings.string_list(['worker'],
                                  'IP or host name of a machine to distribute '
                                  'build work to',
                                  metavar='HOSTNAME')


    def cmd_build(self, args):
        '''Build a binary from a morphology.
        
        Command line arguments are the repository, git tree-ish reference,
        and morphology filename. The binary gets put into the cache.
        
        (The triplet of command line arguments may be repeated as many
        times as necessary.)
        
        '''

        tempdir = morphlib.tempdir.Tempdir()
        morph_loader = MorphologyLoader(self.settings)
        source_manager = morphlib.sourcemanager.SourceManager(self)
        builder = morphlib.builder.Builder(tempdir, self, morph_loader,
                                           source_manager)

        if not os.path.exists(self.settings['cachedir']):
            os.mkdir(self.settings['cachedir'])

        ret = []
        while len(args) >= 3:
            repo, ref, filename = args[:3]
            args = args[3:]

            # derive a build order from the dependency graph
            graph = BuildDependencyGraph(source_manager, morph_loader,
                                         repo, ref, filename)
            graph.resolve()
            blobs, order = graph.build_order()

            self.msg('Building %s|%s|%s' % (repo, ref, filename))

            # build things in this order
            ret.append(builder.build(blobs, order))

        # we may not have permission to tempdir.remove()
        ex = morphlib.execute.Execute('.', lambda msg: None)
        ex.runv(["rm", "-rf", tempdir.dirname])

        if args:
            raise cliapp.AppException('Extra args on command line: %s' % args)

        return ret

    def cmd_testsysimg(self, args):
        '''Run tests for a built system image.

        Command line arguments are the filename of the system image,
        and the filenames of the Python modules that contain the test
        "stories". Each module must have a variable called "story",
        which is a list of tuples. Each tuple is either two strings
        (one to send, the other a regular expression for what is expected
        in return), or two strings and a timeout in seconds.

        testsysimg runs the image under KVM, and accesses it via a
        serial console, and runs the test stories, one by one.

        '''

        if not args:
            raise cliapp.AppException('Missing command line arguments. '
                                      'Run with --help to see usage.')

        system = morphlib.tester.KvmSystem(args[0], 
                                        verbose=self.settings['test-console'],
                                        timeout=self.settings['test-timeout'])

        for filename in args[1:]:
            self.msg('Running %s' % filename)
            module = morphlib.tester.load_module(filename)
            story_steps = getattr(module, 'story')
            story = morphlib.tester.TestStory(system, story_steps, self.msg)
            story.run()
            self.msg('Finished OK.')

    def cmd_test(self, args):
        '''Build and test a system morphology.

        The tests are specified in the morphology's test-stories field.

        '''

        for morph, built in self.cmd_build(args):
            if morph.kind == 'system':
                self.msg('running tests on system %s' % morph.name)
                assert len(built) == 1
                image_filename = built.values()[0]
                morphdir = os.path.dirname(morph.filename)
                stories = [os.path.join(morphdir, x)
                           for x in morph.test_stories]
                self.cmd_testsysimg([image_filename] + stories)
            else:
                self.msg('not testing %s %s (not a system)' %
                         (morph.kind, morph.name))

    def cmd_show_dependencies(self, args):
        '''Dumps the dependency tree of all input morphologies.'''

        morph_loader = MorphologyLoader(self.settings)
        source_manager = morphlib.sourcemanager.SourceManager(self)

        while len(args) >= 3:
            # read the build tuple from the command line
            repo, ref, filename = args[:3]
            args = args[3:]

            # create a dependency graph for the morphology
            graph = BuildDependencyGraph(source_manager, morph_loader,
                                         repo, ref, filename)
            graph.resolve()

            # print the graph
            self.output.write('dependency tree:\n')
            for blob in sorted(graph.blobs, key=str):
                self.output.write('  %s\n' % blob)
                for dependency in sorted(blob.dependencies, key=str):
                    self.output.write('    -> %s\n' % dependency)

            # compute a build order from the graph
            blobs, order = graph.build_order()
            self.output.write('build order:\n')
            for group in order:
                self.output.write('  group:\n')
                for blob in sorted(group, key=str):
                    self.output.write('    %s\n' % blob)

    def cmd_update_gits(self, args):
        tempdir = morphlib.tempdir.Tempdir()
        morph_loader = MorphologyLoader(self.settings)
        source_manager = morphlib.sourcemanager.SourceManager(self)
        while len(args) >= 3:
            # read the build tuple from the command line
            repo, ref, filename = args[:3]
            args = args[3:]

            # first step: clone the corresponding repo
            treeish = source_manager.get_treeish(repo, ref)
            morph = morph_loader.load(treeish, filename)
            blob = morphlib.blobs.Blob.create_blob(morph)

            # second step: compute the cache ID, which will implicitly
            # clone all repositories needed to build the blob
            builder = morphlib.builder.Builder(tempdir, self, morph_loader,
                                               source_manager)
            builder.get_cache_id(blob)

    def cmd_build_distributed(self, args):
        tempdir = morphlib.tempdir.Tempdir()
        morph_loader = MorphologyLoader(self.settings)
        source_manager = morphlib.sourcemanager.SourceManager(self)

        # create a build controller
        controller = morphlib.buildcontroller.BuildController(self, tempdir)

        # create and add the build workers
        if len(self.settings['worker']) == 0:
            worker = morphlib.buildworker.LocalBuildWorker("local-1", self)
            controller.add_worker(worker)
            worker = morphlib.buildworker.LocalBuildWorker("local-2", self)
            controller.add_worker(worker)
        else:
            for worker in self.settings['worker']:
                worker = morphlib.buildworker.RemoteBuildWorker(self)
                controller.add_worker(worker)

        result = []

        while len(args) >= 3:
            # read the build tuple from the command line
            repo, ref, filename = args[:3]
            args = args[3:]

            # derive a build order from the dependency graph
            graph = BuildDependencyGraph(source_manager, morph_loader,
                                         repo, ref, filename)
            graph.resolve()
            blobs, order = graph.build_order()

            self.msg('Building %s|%s|%s' % (repo, ref, filename))

            # build the tuple and all its dependencies
            result.append(controller.build(blobs, order))

        # we may not have permission to tempdir.remove()
        ex = morphlib.execute.Execute('.', lambda msg: None)
        ex.runv(["rm", "-rf", tempdir.dirname])

        if args:
            raise cliapp.AppException('Extra args on command line: %s' % args)

        return result

    def msg(self, msg):
        '''Show a message to the user about what is going on.'''
        logging.debug(msg)
        if self.settings['verbose']:
            self.output.write('%s\n' % msg)
            self.output.flush()


if __name__ == '__main__':
    Morph().run()