path: root/morph

#!/usr/bin/python
#
# Copyright (C) 2011-2012  Codethink Limited
# 
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.


import cliapp
import collections
import json
import logging
import os
import shutil
import tempfile

import morphlib
from morphlib import buildworker
from morphlib import buildcontroller
from morphlib.morphologyloader import MorphologyLoader
from morphlib.builddependencygraph import BuildDependencyGraph


defaults = {
    'git-base-url': [
        'git://gitorious.org/baserock-morphs/',
        'git://gitorious.org/baserock/',
    ],
    'cachedir': os.path.expanduser('~/.cache/morph'),
    'max-jobs': morphlib.util.make_concurrency(),
    'prefix': '/usr',
    'toolchain-target': '%s-baserock-linux-gnu' % os.uname()[4],
    'ccache-remotedir': '',
    'ccache-remotenlevels': 2,
}


class Morph(cliapp.Application):

    def add_settings(self):
        self.settings.boolean(['verbose', 'v'], 'show what is happening')
        self.settings.string_list(['git-base-url'],
                             'prepend URL to git repos that are not URLs',
                             metavar='URL',
                             default=defaults['git-base-url'])
        self.settings.string(['bundle-server'],
                             'base URL to download bundles',
                             metavar='URL',
                             default=None)
        self.settings.string(['cache-server'],
                             'HTTP URL of the morph cache server to use',
                             metavar='URL',
                             default=None)
        self.settings.string(['cachedir'], 
                             'put build results in DIR',
                             metavar='DIR', 
                             default=defaults['cachedir'])
        self.settings.string(['prefix'],
                             'build chunks with prefix PREFIX',
                             metavar='PREFIX', default=defaults['prefix'])
        self.settings.string(['toolchain-target'],
                             'set the TOOLCHAIN_TARGET variable which is used '
                             'in some chunks to determine which architecture '
                             'to build tools for',
                             metavar='TOOLCHAIN_TARGET',
                             default=defaults['toolchain-target'])
        self.settings.string(['target-cflags'],
                             'inject additional CFLAGS into the environment '
                             'that is used to build chunks',
                             metavar='CFLAGS',
                             default='')
        self.settings.string(['tempdir'],
                             'temporary directory to use for builds '
                                '(this is separate from just setting $TMPDIR '
                                'or /tmp because those are used internally '
                                'by things that cannot be on NFS, but '
                                'this setting can point at a directory in '
                                'NFS)',
                             metavar='DIR', 
                             default=os.environ.get('TMPDIR'))
        self.settings.boolean(['no-ccache'], 'do not use ccache')
        self.settings.string(['ccache-remotedir'],
                             'allow ccache to download objects from REMOTEDIR '
                               'if they are not cached locally',
                             metavar='REMOTEDIR',
                             default=defaults['ccache-remotedir'])
        self.settings.integer(['ccache-remotenlevels'],
                              'assume ccache directory objects are split into '
                                'NLEVELS levels of subdirectories',
                              metavar='NLEVELS',
                              default=defaults['ccache-remotenlevels'])
        self.settings.boolean(['no-distcc'], 'do not use distcc')
        self.settings.integer(['max-jobs'], 
                              'run at most N parallel jobs with make (default '
                                'is to a value based on the number of CPUs '
                                'in the machine running morph',
                              metavar='N',
                              default=defaults['max-jobs'])
        self.settings.boolean(['keep-path'], 
                              'do not touch the PATH environment variable')
        self.settings.boolean(['bootstrap'], 
                              'build stuff in bootstrap mode; this is '
                                'DANGEROUS and will install stuff on your '
                                'system')
        self.settings.boolean(['ignore-submodules'],
                              'do not cache repositories of git submodules '
                              'or unpack them into the build directory')

        self.settings.boolean(['no-git-update'],
                              'do not update the cached git repositories '
                                'during a build (user must have done that '
                                'already using the update-gits subcommand)')

        self.settings.string_list(['staging-filler'],
                                  'unpack BLOB into staging area for '
                                    'non-bootstrap builds (this will '
                                    'eventually be replaced with proper '
                                    'build dependencies)',
                                   metavar='BLOB')
        self.settings.boolean(['staging-chroot'],
                              'build things in a staging chroot '
                                '(require real root to use)')

        self.settings.string_list(['worker'],
                                  'IP or host name of a machine to distribute '
                                  'build work to',
                                  metavar='HOSTNAME')

    def _itertriplets(self, args):
        '''Generate repo, ref, filename triples from args.'''
        
        if (len(args) % 3) != 0:
            raise cliapp.AppException('Argument list must have full triplets')
        
        while args:
            assert len(args) >= 2, args
            yield args[0], args[1], args[2]
            args = args[3:]

    def _create_source_pool(
            self, local_repo_cache, remote_repo_cache,
            triplet):
        pool = morphlib.sourcepool.SourcePool()
        def add_to_pool(reponame, ref, filename, absref, morphology):
            source = morphlib.source.Source(reponame, ref, absref,
                                            morphology, filename)
            pool.add(source)

        self._traverse_morphs([triplet], local_repo_cache, remote_repo_cache,
                              update=not self.settings['no-git-update'],
                              visit=add_to_pool)
        return pool

    def cmd_build(self, args):
        '''Build a binary from a morphology.
        
        Command line arguments are the repository, git tree-ish reference,
        and morphology filename. Morph takes care of building all dependencies
        before building the morphology. All generated binaries are put into the
        cache.
        
        (The triplet of command line arguments may be repeated as many
        times as necessary.)
        
        '''

        logging.debug('cmd_build starting')

        cachedir = self.settings['cachedir']
        if not os.path.exists(cachedir):
            os.mkdir(cachedir)

        build_env = morphlib.buildenvironment.BuildEnvironment(self.settings)
        ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env)
        lac = morphlib.localartifactcache.LocalArtifactCache(cachedir)
        lrc = morphlib.localrepocache.LocalRepoCache(
                os.path.join(cachedir, 'gits'),
                self.settings['git-base-url'],
                bundle_base_url=self.settings['bundle-server'])
        rrc = None

        for repo_name, ref, filename in self._itertriplets(args):
            logging.debug('cmd_build: %s %s %s' % (repo_name, ref, filename))

            logging.debug('cmd_build: creating source pool')
            srcpool = self._create_source_pool(lrc, rrc, (repo_name, ref,
                                               filename))
            logging.debug('cmd_build: creating artifact resolver')
            ar = morphlib.artifactresolver.ArtifactResolver()
            logging.debug('cmd_build: resolving artifacts')
            artifacts = ar.resolve_artifacts(srcpool)
            logging.debug('cmd_build: computing cache keys')
            for artifact in artifacts:
                artifact.cache_key = ckc.compute_key(artifact)
                artifact.cache_id = ckc.get_cache_id(artifact)
            logging.debug('cmd_build: computing build order')
            order = morphlib.buildorder.BuildOrder(artifacts)

            logging.debug('cmd_build: finding what needs to be built')
            needed = []
            for group in order.groups:
                for artifact in group:
                    if not lac.has(artifact):
                        needed.append(artifact)

            logging.debug('cmd_build: cloning/updating cached repos')
            for artifact in needed:
                artifact.source.repo = lrc.cache_repo(
                                            artifact.source.repo_name)
                if not self.settings['no-git-update']:
                    artifact.source.repo.update()

            if self.settings['bootstrap']:
                staging_root = '/'
                staging_temp = tempfile.mkdtemp()
                install_chunks = True
                setup_proc = False
            elif self.settings['staging-chroot']:
                staging_root = tempfile.mkdtemp()
                staging_temp = staging_root
                install_chunks = True
                setup_proc = True
            else:
                staging_root = '/'
                staging_temp = tempfile.mkdtemp()
                install_chunks = False
                setup_proc = False

            staging_area = morphlib.stagingarea.StagingArea(staging_root,
                                                            staging_temp)
            if self.settings['staging-chroot']:
                self._install_initial_staging(staging_area)

            builder = morphlib.builder2.Builder(staging_area, lac, lrc,
                                                build_env,
                                                self.settings['max-jobs'])
            if setup_proc:
                builder.setup_proc = True

            for group in order.groups:
                for artifact in group:
                    if artifact in needed:
                        logging.debug('Need to build %s' % artifact.name)
                        self.msg('Building %s' % artifact.name)
                        builder.build_and_cache(artifact)
                    else:
                        logging.debug('No need to build %s' % artifact.name)
                        self.msg('Using cached %s' % artifact.name)

                if install_chunks:
                    logging.debug('installing chunks from just-built group')
                    # install chunks only
                    chunk_artifacts = [x 
                                       for x in group 
                                       if x.source.morphology['kind'] == 
                                            'chunk']
                    for artifact in chunk_artifacts:
                        handle = lac.get(artifact)
                        staging_area.install_artifact(handle)

            if staging_root != '/':
                staging_area.remove()
            if staging_temp != '/' and os.path.exists(staging_temp):
                shutil.rmtree(staging_temp)

    def _install_initial_staging(self, staging_area):
        logging.debug('Pre-populating staging area %s' % staging_area.dirname)
        logging.debug('Fillers: %s' % repr(self.settings['staging-filler']))
        for filename in self.settings['staging-filler']:
            with open(filename, 'rb') as f:
                staging_area.install_artifact(f)

    def cmd_show_dependencies(self, args):
        '''Dumps the dependency tree of all input morphologies.'''

        if not os.path.exists(self.settings['cachedir']):
            os.mkdir(self.settings['cachedir'])
        cachedir = os.path.join(self.settings['cachedir'], 'gits')
        baseurls = self.settings['git-base-url']
        bundle_base_url = self.settings['bundle-server']
        local_repo_cache = morphlib.localrepocache.LocalRepoCache(
                cachedir, baseurls, bundle_base_url)
        if self.settings['cache-server']:
            remote_repo_cache = morphlib.remoterepocache.RemoteRepoCache(
                    self.settings['cache-server'], baseurls)
        else:
            remote_repo_cache = None

        # traverse the morphs to list all the sources
        for repo, ref, filename in self._itertriplets(args):
            repo, ref, filename = args[:3]
            pool = self._create_source_pool(
                    local_repo_cache, remote_repo_cache,
                    (repo, ref, filename))

            resolver = morphlib.artifactresolver.ArtifactResolver()
            artifacts = resolver.resolve_artifacts(pool)

            self.output.write('dependency graph for %s|%s|%s:\n' % 
                              (repo, ref, filename))
            for artifact in sorted(artifacts, key=str):
                self.output.write('  %s\n' % artifact)
                for dependency in sorted(artifact.dependencies, key=str):
                    self.output.write('    -> %s\n' % dependency)

            order = morphlib.buildorder.BuildOrder(artifacts)
            self.output.write('build order for %s|%s|%s:\n' %
                              (repo, ref, filename))
            for group in order.groups:
                self.output.write('  group:\n')
                for artifact in group:
                    self.output.write('    %s\n' % artifact)

    def _resolveref(self, lrc, rrc, reponame, ref, update=True):
        '''Resolves the sha1 of the ref in reponame and returns it.

        If update is True then this has the side-effect of updating
        or cloning the repository into the local repo cache.  

        '''
        absref = None
        if lrc.has_repo(reponame):
            repo = lrc.get_repo(reponame)
            if update:
                repo.update()
            absref = repo.resolve_ref(ref)
        elif rrc != None:
            try:
                absref = rrc.resolve_ref(reponame, ref)
            except:
                pass
        if absref == None:
            if update:
                repo = lrc.cache_repo(reponame)
                repo.update()
            else:
                repo = lrc.get_repo(reponame)
            absref = repo.resolve_ref(ref)
        return absref

    def _traverse_morphs(self, triplets, lrc, rrc, update=True,
                         visit=lambda rn, rf, fn, arf, m: None):
        morph_factory = morphlib.morphologyfactory.MorphologyFactory(lrc, rrc)
        queue = collections.deque(triplets)

        while queue:
            reponame, ref, filename = queue.popleft()
            absref = self._resolveref(lrc, rrc, reponame, ref, update)
            morphology = morph_factory.get_morphology(
                            reponame, absref, filename)
            visit(reponame, ref, filename, absref, morphology)
            if morphology['kind'] == 'system':
                queue.extend((reponame, ref, '%s.morph' % s)
                             for s in morphology['strata'])
            elif morphology['kind'] == 'stratum':
                if morphology['build-depends']:
                    queue.extend((reponame, ref, '%s.morph' % s)
                                 for s in morphology['build-depends'])
                queue.extend((c['repo'], c['ref'], '%s.morph' % c['morph']) 
                             for c in morphology['sources'])

    def cmd_update_gits(self, args):
        '''Update cached git repositories.

        Parse the given morphologies, and their dependencies, and
        update all the git repositories referred to by them in the
        morph cache directory.
        
        '''

        if not os.path.exists(self.settings['cachedir']):
            os.mkdir(self.settings['cachedir'])
        cachedir = os.path.join(self.settings['cachedir'], 'gits')
        baseurls = self.settings['git-base-url']
        bundle_base_url = self.settings['bundle-server']
        cache = morphlib.localrepocache.LocalRepoCache(
                cachedir, baseurls, bundle_base_url)

        subs_to_process = set()
        
        def visit(reponame, ref, filename, absref, morphology):
            self.msg('Updating %s|%s|%s' % (reponame, ref, filename))
            assert cache.has_repo(reponame)
            cached_repo = cache.get_repo(reponame)
            try:
                submodules = morphlib.git.Submodules(cached_repo.path, ref)
                submodules.load()
            except morphlib.git.NoModulesFileError:
                pass
            else:
                for submod in submodules:
                    subs_to_process.add((submod.url, submod.commit))
            
        self._traverse_morphs(self._itertriplets(args), cache, None,
                              update=True, visit=visit)

        done = set()
        while subs_to_process:
            url, ref = subs_to_process.pop()
            done.add((url, ref))
            cached_repo = cache.cache_repo(url)
            cached_repo.update()

            try:
                submodules = morphlib.git.Submodules(cached_repo.path, ref)
                submodules.load()
            except morphlib.git.NoModulesFileError:
                pass
            else:
                for submod in submodules:
                    if (submod.url, submod.commit) not in done:
                        subs_to_process.add((submod.url, submod.commit))

    def cmd_build_single(self, args):
        '''Build a binary from a morphology but do not build its dependencies.
        
        To build a system or stratum morphology, simply specify the repository,
        a git tree-ish reference and the morphology filename on the command
        line. To build a chunk morphology, first provide the repository,
        reference and morphology filename of its surrounding stratum and then
        provide the same information for the chunk morphology itself. This is
        needed to resolve the dependencies of the chunk using the stratum as
        the build context and to unpack these dependencies in the staging area
        prior to building the chunk.

        This command differs from 'build' in that it only builds the morphology
        specified on the command line. Dependencies are assumed to have been
        built prior to running this command and are only unpacked into the
        staging area.

        '''
        tempdir = morphlib.tempdir.Tempdir(self.settings['tempdir'])
        morph_loader = MorphologyLoader(self.settings)
        source_manager = morphlib.sourcemanager.SourceManager(self,
                                                              update=False)
        factory = morphlib.builder.Factory(tempdir)
        builder = morphlib.builder.Builder(tempdir, self, morph_loader,
                                           source_manager, factory)

        if not os.path.exists(self.settings['cachedir']):
            os.mkdir(self.settings['cachedir'])

        factory.create_staging()

        if len(args) >= 3:
            repo, ref, filename = args[:3]
            args = args[3:]

            # derive a build order from the dependency graph
            graph = BuildDependencyGraph(source_manager, morph_loader,
                                         repo, ref, filename)
            first_blob = graph.resolve()
            blobs, order = graph.build_order()

            # parse the second blob, if there is one
            second_blob = None
            if len(args) >= 3:
                repo, ref, filename = args[:3]
                args = args[3:]

                # load the blob manually
                treeish = source_manager.get_treeish(repo, ref)
                morphology = morph_loader.load(treeish, filename)
                second_blob = morphlib.blobs.Blob.create_blob(morphology)

                try:
                    # find the corresponding blob object in the blobs
                    # returned from the dependency graph
                    second_blob = [x for x in blobs if x == second_blob][0]
                except IndexError:
                    raise cliapp.AppException('%s and %s are unrelated' %
                                              (first_blob, second_blob))

            # build the single blob
            if second_blob:
                # verify that the two input blobs are valid
                if first_blob.morph.kind != 'stratum':
                    raise cliapp.AppException('The first tuple %s needs to '
                                              'refer to a stratum' %
                                              first_blob)
                if second_blob.morph.kind != 'chunk':
                    raise cliapp.AppException('The first tuple %s needs to '
                                              'refer to a chunk' % second_blob)

                # build now
                self.msg('Building %s' % second_blob)
                builder.build_single(second_blob, order)
            else:
                # build the blob now
                self.msg('Building %s' % first_blob)
                builder.build_single(first_blob, order)

        tempdir.remove()

        if args:
            raise cliapp.AppException('Extra args on command line: %s' % args)

    def cmd_build_distributed(self, args):
        '''Build a binary from a morphology, distributing work where possible.
        
        Command line arguments are the repository, git tree-ish reference,
        and morphology filename. Morph takes care of building all dependencies
        before building the morphology. All generated binaries are put into the
        cache.

        This command differs from 'build' in that it distributes work to
        other machines where possible. This may include preparing the source
        tree, preparing the staging area, building individual chunks and
        caching the built items.
        
        Distributing currently works by running 'morph build-single' on other
        machines via SSH. The machines to be used for this are specified using
        the '-w/--worker' command line options.
        
        (The triplet of command line arguments may be repeated as many
        times as necessary.)

        '''
        
        tempdir = morphlib.tempdir.Tempdir(self.settings['tempdir'])
        morph_loader = MorphologyLoader(self.settings)
        source_manager = morphlib.sourcemanager.SourceManager(self,
                                update=not self.settings['no-git-update'])

        # create a build controller
        controller = buildcontroller.BuildController(self, tempdir)

        # create and add the build workers
        if len(self.settings['worker']) == 0:
            num_workers = morphlib.util.make_concurrency()
            for i in range(num_workers):
                name = controller.generate_worker_name('local')
                worker = buildworker.LocalBuildWorker(name, 'local', self)
                controller.add_worker(worker)
        else:
            for worker in self.settings['worker']:
                name = controller.generate_worker_name(worker)
                worker = buildworker.RemoteBuildWorker(name, worker, self)
                controller.add_worker(worker)

        result = []

        for repo, ref, filename in self._itertriplets(args):
            # derive a build order from the dependency graph
            graph = BuildDependencyGraph(source_manager, morph_loader,
                                         repo, ref, filename)
            graph.resolve()
            blobs, order = graph.build_order()

            self.msg('Building %s|%s|%s' % (repo, ref, filename))

            # build the tuple and all its dependencies
            result.append(controller.build(blobs, order))

        tempdir.remove()

    def cmd_make_patch(self, args):
        '''Create a Trebuchet patch between two system images.'''
        
        logging.debug('cmd_make_patch starting')
        
        if len(args) != 7:
            raise cliapp.AppException('make-patch requires arguments: '
                                      'name of output file plus two triplest')

        output = args[0]
        repo_name1, ref1, filename1 = args[1:4]
        repo_name2, ref2, filename2 = args[4:7]

        self.settings.require('cachedir')
        cachedir = self.settings['cachedir']
        build_env = morphlib.buildenvironment.BuildEnvironment(self.settings)
        ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env)
        lac = morphlib.localartifactcache.LocalArtifactCache(cachedir)
        lrc = morphlib.localrepocache.LocalRepoCache(
                os.path.join(cachedir, 'gits'),
                self.settings['git-base-url'],
                bundle_base_url=self.settings['bundle-server'])
        rrc = None

        def the_one(source, repo_name, ref, filename):
            return (source.repo_name == repo_name and
                    source.original_ref == ref and
                    source.filename == filename)

        def get_artifact(repo_name, ref, filename):
            srcpool = self._create_source_pool(lrc, rrc, 
                                               (repo_name, ref, filename))
            ar = morphlib.artifactresolver.ArtifactResolver()
            artifacts = ar.resolve_artifacts(srcpool)
            for artifact in artifacts:
                artifact.cache_key = ckc.compute_key(artifact)
                if the_one(artifact.source, repo_name, ref, filename):
                    return artifact

        artifact1 = get_artifact(repo_name1, ref1, filename1)
        artifact2 = get_artifact(repo_name2, ref2, filename2)

        image_path_1 = lac.get(artifact1).name
        image_path_2 = lac.get(artifact2).name
        
        def setup(path):
            part = morphlib.fsutils.setup_device_mapping(ex, path)
            mount_point = tempfile.mkdtemp()
            morphlib.fsutils.mount(ex, part, mount_point)
            return mount_point

        def cleanup(path, mount_point):
            try:
                morphlib.fsutils.unmount(ex, mount_point)
            except:
                pass
            try:
                morphlib.fsutils.undo_device_mapping(ex, path)
            except:
                pass
            try:
                os.rmdir(mount_point)
            except:
                pass

        ex = morphlib.execute.Execute('.', logging.debug)
        try:
            mount_point_1 = setup(image_path_1)
            mount_point_2 = setup(image_path_2)

            ex.runv(['tbdiff-create', output,
                     os.path.join(mount_point_1, 'factory'),
                     os.path.join(mount_point_2, 'factory')])
        except BaseException:
            raise
        finally:
            cleanup(image_path_1, mount_point_1)
            cleanup(image_path_2, mount_point_2)
        
    def cmd_init(self, args):
        '''Initialize a mine.'''
        
        if not args:
            args = ['.']
        elif len(args) > 1:
            raise cliapp.AppException('init must get at most one argument')

        dirname = args[0]

        if os.path.exists(dirname):
            if os.listdir(dirname) != []:
                raise cliapp.AppException('can only initialize empty '
                                          'directory: %s' % dirname)
        else:
            raise cliapp.AppException('can only initialize an existing '
                                        'empty directory: %s' % dirname)

        os.mkdir(os.path.join(dirname, '.morph'))

    def _deduce_mine_directory(self):
        dirname = os.getcwd()
        while dirname != '/':
            dot_morph = os.path.join(dirname, '.morph')
            if os.path.isdir(dot_morph):
                return dirname
            dirname = os.path.dirname(dirname)
        return None

    def cmd_minedir(self, args):
        '''Find morph mine directory from current working directory.'''
        
        dirname = self._deduce_mine_directory()
        if dirname is None:
            raise cliapp.AppException("Can't find the mine directory")
        self.output.write('%s\n' % dirname)
    
    def _clone_to_directory(self, dirname, repo, ref):
        '''Clone a repository below a directory.
        
        As a side effect, clone it into the morph repository.
        
        '''

        # Get the repository into the cache.
        tempdir = morphlib.tempdir.Tempdir(self.settings['tempdir'])
        morph_loader = MorphologyLoader(self.settings)
        source_manager = morphlib.sourcemanager.SourceManager(self,
                                update=not self.settings['no-git-update'])
        treeish = source_manager.get_treeish(repo, ref)
        
        # Clone it from cache to target directory.
        morphlib.git.clone(dirname, treeish.repo, self.msg)
        
        # Set the origin to point at the original repository.
        morphlib.git.set_remote(dirname, 'origin', treeish.original_repo)
        
        # Update remotes.
        self.runcmd(['git', 'remote', 'update'], cwd=dirname)
    
    def cmd_branch(self, args):
        '''Branch the whole system.'''
        
        if len(args) != 1:
            raise cliapp.AppException('morph branch needs name of branch '
                                        'as parameter')

        new_branch = args[0]
        repo = 'morphs'
        commit = 'master'

        # Create the system branch directory.
        os.makedirs(new_branch)

        # Clone into system branch directory.
        new_repo = os.path.join(new_branch, os.path.basename(repo))
        self._clone_to_directory(new_repo, repo, commit)
        
        # Create a new branch in the local morphs repository.
        self.runcmd(['git', 'checkout', '-b', new_branch, commit],
                    cwd=new_repo)

    def cmd_checkout(self, args):
        '''Check out an existing system branch.'''

        if len(args) != 1:
            raise cliapp.AppException('morph checkout needs name of '
                                        'branch as parameter')

        system_branch = args[0]
        repo = 'morphs'

        # Create the system branch directory.
        os.makedirs(system_branch)

        # Clone into system branch directory.
        new_repo = os.path.join(system_branch, os.path.basename(repo))
        self._clone_to_directory(new_repo, repo, system_branch)

    def _deduce_system_branch(self):
        minedir = self._deduce_mine_directory()
        if minedir is None:
            return None

        if not minedir.endswith('/'):
            minedir += '/'
        
        cwd = os.getcwd()
        if not cwd.startswith(minedir):
            return None
            
        return os.path.dirname(cwd[len(minedir):])
    
    def cmd_show_system_branch(self, args):
        '''Print name of current system branch.
        
        This must be run in the system branch's ``morphs`` repository.
        
        '''
        
        system_branch = self._deduce_system_branch()
        if system_branch is None:
            raise cliapp.AppException("Can't determine system branch")
        self.output.write('%s\n' % system_branch)
    
    def cmd_edit(self, args):
        '''Edit a component in a system branch.'''
        
        if len(args) != 2:
            raise cliapp.AppException('morph edit must get a repository name '
                                        'and commit ref as argument')

        repo = args[0]
        ref = args[1]

        mine_directory = self._deduce_mine_directory()
        system_branch = self._deduce_system_branch()
        new_repo = os.path.join(mine_directory, system_branch, 
                                os.path.basename(repo))
        self._clone_to_directory(new_repo, repo, ref)
        
        system_branch = self._deduce_system_branch()
        if system_branch == ref:
            self.runcmd(['git', 'checkout', system_branch],
                        cwd=new_repo)
        else:
            self.runcmd(['git', 'checkout', '-b', system_branch, ref],
                        cwd=new_repo)

    def cmd_merge(self, args):
        '''Merge specified repositories from another system branch.'''
        
        if len(args) == 0:
            raise cliapp.AppException('morph merge must get a branch name '
                                        'and some repo names as arguments')

        other_branch = args[0]
        mine = self._deduce_mine_directory()
        this_branch = self._deduce_system_branch()
        
        for repo in args[1:]:
            basename = os.path.basename(repo)
            pull_from = os.path.join(mine, other_branch, basename)
            repo_dir = os.path.join(mine, this_branch, basename)
            self.runcmd(['git', 'pull', pull_from, other_branch], cwd=repo_dir)

    def cmd_petrify(self, args):
        '''Make refs to chunks be absolute SHA-1s.'''
        
        morph_loader = MorphologyLoader(self.settings)
        source_manager = morphlib.sourcemanager.SourceManager(self,
                            update=not self.settings['no-git-update'])

        for filename in args:
            with open(filename) as f:
                morph = json.load(f)
            
            if morph['kind'] != 'stratum':
                self.msg('Not a stratum: %s' % filename)
                continue

            self.msg('Petrifying %s' % filename)

            for source in morph['sources']:
                repo = source.get('repo', source['name'])
                ref = source['ref']
                self.msg('.. looking up sha1 for %s %s' % (repo, ref))
                treeish = source_manager.get_treeish(repo, ref)
                source['ref'] = treeish.sha1
            
            with open(filename, 'w') as f:
                json.dump(morph, f, indent=2)

    def msg(self, msg):
        '''Show a message to the user about what is going on.'''
        logging.debug(msg)
        if self.settings['verbose']:
            self.output.write('%s\n' % msg)
            self.output.flush()

    def runcmd(self, argv, *args, **kwargs):
        # check which msg function to use
        msg = self.msg
        if 'msg' in kwargs:
            msg = kwargs['msg']
            del kwargs['msg']

        # convert the command line arguments into a string
        commands = [argv] + list(args)
        for command in commands:
            if isinstance(command, list):
                for i in xrange(0, len(command)):
                    command[i] = str(command[i])
        commands = [' '.join(command) for command in commands]

        # print the command line
        msg('# %s' % ' | '.join(commands))

        # run the command line
        return cliapp.Application.runcmd(self, argv, *args, **kwargs)

    # This is in morph so that policy is easily visible, and not embedded
    # deep down in the call stack.
    def clean_env(self):
        '''Create a fresh set of environment variables for a clean build.
        
        Return a dict with the new environment.
        
        '''
        
        path = os.environ['PATH']
        tools = os.environ.get('BOOTSTRAP_TOOLS')
        distcc_hosts = os.environ.get('DISTCC_HOSTS')

        # copy a set of white-listed variables from the original env
        copied_vars = dict.fromkeys([
            'TMPDIR',
            'LD_PRELOAD',
            'LD_LIBRARY_PATH',
            'FAKEROOTKEY',
            'FAKED_MODE',
            'FAKEROOT_FD_BASE',
        ])
        for name in copied_vars:
            copied_vars[name] = os.environ.get(name, None)

        env = {}
        
        # apply the copied variables to the clean env
        for name in copied_vars:
            if copied_vars[name] is not None:
                env[name] = copied_vars[name]

        env['TERM'] = 'dumb'
        env['SHELL'] = '/bin/sh'
        env['USER'] = \
            env['USERNAME'] = \
            env['LOGNAME'] = 'tomjon'
        env['LC_ALL'] = 'C'
        env['HOME'] = '/tmp'

        if self.settings['keep-path'] or self.settings['bootstrap']:
            env['PATH'] = path
        else:
            env['PATH'] = '/sbin:/usr/sbin:/bin:/usr/bin'

        env['TOOLCHAIN_TARGET'] = self.settings['toolchain-target']
        env['CFLAGS'] = self.settings['target-cflags']
        env['PREFIX'] = self.settings['prefix']
        env['BOOTSTRAP'] = 'true' if self.settings['bootstrap'] else 'false'
        if tools is not None:
            env['BOOTSTRAP_TOOLS'] = tools
        if distcc_hosts is not None:
            env['DISTCC_HOSTS'] = distcc_hosts

        if not self.settings['no-ccache']:
            env['PATH'] = ('/usr/lib/ccache:%s' % env['PATH'])
# FIXME: This needs to be made working, but it doesn't really, right now:
# the tempdir is not available inside the staging chroot.
#            env['CCACHE_BASEDIR'] = self.tempdir.dirname
            env['CCACHE_EXTRAFILES'] = ':'.join(
                f for f in ('/baserock/binutils.meta',
                            '/baserock/eglibc.meta',
                            '/baserock/gcc.meta') if os.path.exists(f)
            )
            if self.settings['ccache-remotedir'] != '':
                env['CCACHE_REMOTEDIR'] = self.settings['ccache-remotedir']
            env['CCACHE_REMOTENLEVELS'] = \
                                    str(self.settings['ccache-remotenlevels'])
            if not self.settings['no-distcc']:
                env['CCACHE_PREFIX'] = 'distcc'
            
        return env


if __name__ == '__main__':
    Morph().run()