path: root/morphlib/buildcommand.py

# Copyright (C) 2011-2013  Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.


import os
import shutil
import logging
import tempfile

import morphlib


class BuildCommand(object):

    '''High level logic for building.

    This controls how the whole build process goes. This is a separate
    class to enable easy experimentation of different approaches to
    the various parts of the process.

    '''

    def __init__(self, app):
        self.app = app
        self.build_env = self.new_build_env()
        self.ckc = self.new_cache_key_computer(self.build_env)
        self.lac, self.rac = self.new_artifact_caches()
        self.lrc, self.rrc = self.new_repo_caches()
        self.supports_local_build = True

    def build(self, args):
        '''Build triplets specified on command line.'''

        self.app.status(msg='Build starts', chatty=True)

        for repo_name, ref, filename in self.app.itertriplets(args):
            self.app.status(msg='Building %(repo_name)s %(ref)s %(filename)s',
                            repo_name=repo_name, ref=ref, filename=filename)
            artifact = self.get_artifact_object(repo_name, ref, filename)
            self.build_in_order(artifact)

        self.app.status(msg='Build ends successfully')

    def new_build_env(self):
        '''Create a new BuildEnvironment instance.'''
        return morphlib.buildenvironment.BuildEnvironment(self.app.settings)

    def new_cache_key_computer(self, build_env):
        '''Create a new cache key computer.'''
        return morphlib.cachekeycomputer.CacheKeyComputer(build_env)

    def new_artifact_caches(self):
        '''Create interfaces for the build artifact caches.

        This includes creating the directories on disk if they are missing.

        '''
        return morphlib.util.new_artifact_caches(self.app.settings)

    def new_repo_caches(self):
        return morphlib.util.new_repo_caches(self.app)

    def get_artifact_object(self, repo_name, ref, filename):
        '''Create an Artifact object representing the given triplet.'''
        
        self.app.status(msg='Figuring out the right build order')

        self.app.status(msg='Creating source pool', chatty=True)
        srcpool = self.app.create_source_pool(
            self.lrc, self.rrc, (repo_name, ref, filename))

        root_kind = srcpool.lookup(repo_name, ref, filename).morphology['kind']
        if root_kind != 'system':
            raise morphlib.Error(
                'Building a %s directly is not supported' % root_kind)

        self.app.status(
            msg='Validating cross-morphology references', chatty=True)
        self._validate_cross_morphology_references(srcpool)

        self.app.status(msg='Creating artifact resolver', chatty=True)
        ar = morphlib.artifactresolver.ArtifactResolver()

        self.app.status(msg='Resolving artifacts', chatty=True)
        artifacts = ar.resolve_artifacts(srcpool)

        self.app.status(msg='Computing cache keys', chatty=True)
        for artifact in artifacts:
            artifact.cache_key = self.ckc.compute_key(artifact)
            artifact.cache_id = self.ckc.get_cache_id(artifact)

        self.app.status(msg='Computing build order', chatty=True)
        root_artifact = self._find_root_artifact(artifacts)

        return root_artifact

    def _validate_cross_morphology_references(self, srcpool):
        '''Perform validation across all morphologies involved in the build'''

        stratum_names = []

        for src in srcpool:
            kind = src.morphology['kind']

            # Verify that chunks pointed to by strata really are chunks, etc.
            method_name = '_validate_cross_refs_for_%s' % kind
            if hasattr(self, method_name):
                logging.debug('Calling %s' % method_name)
                getattr(self, method_name)(src, srcpool)
            else:
                logging.warning('No %s' % method_name)

            # Verify stratum build-depends agree with the system's contents.
            # It's not an error to build-depend on a stratum that isn't
            # included in the target system, but if it is included, the repo
            # and ref fields must match.
            if src.morphology['kind'] == 'stratum':
                name = src.morphology['name']
                if name in stratum_names:
                    raise morphlib.Error(
                        "Conflicting versions of stratum '%s' appear in the "
                        "build. Check the contents of the system against the "
                        "build-depends of the strata." % name)
                stratum_names.append(name)

    def _validate_cross_refs_for_system(self, src, srcpool):
        self._validate_cross_refs_for_xxx(
            src, srcpool, src.morphology['strata'], 'stratum')

    def _validate_cross_refs_for_stratum(self, src, srcpool):
        self._validate_cross_refs_for_xxx(
            src, srcpool, src.morphology['chunks'], 'chunk')

    def _validate_cross_refs_for_xxx(self, src, srcpool, specs, wanted):
        for spec in specs:
            repo_name = spec['repo']
            ref = spec['ref']
            filename = '%s.morph' % spec['morph']
            logging.debug(
                'Validating cross ref to %s:%s:%s' %
                    (repo_name, ref, filename))
            other = srcpool.lookup(repo_name, ref, filename)
            if other.morphology['kind'] != wanted:
                raise morphlib.Error(
                    '%s %s references %s:%s:%s which is a %s, '
                        'instead of a %s' %
                        (src.morphology['kind'],
                         src.morphology['name'],
                         repo_name,
                         ref,
                         filename,
                         other.morphology['kind'],
                         wanted))

    def _find_root_artifact(self, artifacts):
        '''Find the root artifact among a set of artifacts in a DAG.
        
        It would be nice if the ArtifactResolver would return its results in a
        more useful order to save us from needing to do this -- the root object
        is known already since that's the one the user asked us to build.
        
        '''

        maybe = set(artifacts)
        for a in artifacts:
            for dep in a.dependencies:
                if dep in maybe:
                    maybe.remove(dep)
        assert len(maybe) == 1
        return maybe.pop()

    def build_in_order(self, artifact):
        '''Build everything specified in a build order.'''
        self.app.status(msg='Building according to build ordering',
                        chatty=True)

        artifacts = artifact.walk()
        old_prefix = self.app.status_prefix
        for i, a in enumerate(artifacts):
            self.app.status_prefix = (
                old_prefix + '[Build %d/%d] ' % ((i+1), len(artifacts)))
            self.build_artifact(a)
        self.app.status_prefix = old_prefix

    def build_artifact(self, artifact):
        '''Build one artifact.

        All the dependencies are assumed to be built and available
        in either the local or remote cache already.

        '''

        self.app.status(msg='Checking if %(kind)s %(name)s needs building',
                        kind=artifact.source.morphology['kind'],
                        name=artifact.name)

        if self.is_built(artifact):
            self.app.status(msg='The %(kind)s %(name)s is already built',
                            kind=artifact.source.morphology['kind'],
                            name=artifact.name)
            self.cache_artifacts_locally([artifact])
        else:
            self.app.status(msg='Building %(kind)s %(name)s',
                            kind=artifact.source.morphology['kind'],
                            name=artifact.name)
            self.get_sources(artifact)
            deps = self.get_recursive_deps(artifact)
            self.cache_artifacts_locally(deps)
            staging_area = self.create_staging_area(artifact)
            if self.app.settings['staging-chroot']:
                if artifact.source.morphology.needs_staging_area:
                    self.install_fillers(staging_area)
                    self.install_chunk_artifacts(staging_area,
                                                 deps)
                    morphlib.builder2.ldconfig(self.app.runcmd,
                                               staging_area.tempdir)

            self.build_and_cache(staging_area, artifact)
            if self.app.settings['bootstrap']:
                self.install_chunk_artifacts(staging_area,
                                             (artifact,))
            self.remove_staging_area(staging_area)
        self.app.status(msg='%(kind)s %(name)s is cached at %(cachepath)s',
                        kind=artifact.source.morphology['kind'],
                        name=artifact.name,
                        cachepath=self.lac.artifact_filename(artifact),
                        chatty=(artifact.source.morphology['kind'] !=
                                "system"))

    def is_built(self, artifact):
        '''Does either cache already have the artifact?'''
        return self.lac.has(artifact) or (self.rac and self.rac.has(artifact))

    def get_recursive_deps(self, artifact):
        done = set()
        todo = set((artifact,))
        while todo:
            for a in todo.pop().dependencies:
                if a not in done:
                    done.add(a)
                    todo.add(a)
        return done

    def get_sources(self, artifact):
        '''Update the local git repository cache with the sources.'''

        repo_name = artifact.source.repo_name
        if self.app.settings['no-git-update']:
            self.app.status(msg='Not updating existing git repository '
                                '%(repo_name)s '
                                'because of no-git-update being set',
                            chatty=True,
                            repo_name=repo_name)
            artifact.source.repo = self.lrc.get_repo(repo_name)
            return

        if self.lrc.has_repo(repo_name):
            artifact.source.repo = self.lrc.get_repo(repo_name)
            try:
                sha1 = artifact.source.sha1
                artifact.source.repo.resolve_ref(sha1)
                self.app.status(msg='Not updating git repository '
                                    '%(repo_name)s because it '
                                    'already contains sha1 %(sha1)s',
                                chatty=True, repo_name=repo_name,
                                sha1=sha1)
            except morphlib.cachedrepo.InvalidReferenceError:
                self.app.status(msg='Updating %(repo_name)s',
                                repo_name=repo_name)
                artifact.source.repo.update()
        else:
            self.app.status(msg='Cloning %(repo_name)s',
                            repo_name=repo_name)
            artifact.source.repo = self.lrc.cache_repo(repo_name)

        # Update submodules.
        done = set()
        self.app.cache_repo_and_submodules(
            self.lrc, artifact.source.repo.url,
            artifact.source.sha1, done)

    def cache_artifacts_locally(self, artifacts):
        '''Get artifacts missing from local cache from remote cache.'''

        def copy(remote, local):
            shutil.copyfileobj(remote, local)
            remote.close()
            local.close()

        for artifact in artifacts:
            if not self.lac.has(artifact):
                self.app.status(msg='Fetching to local cache: '
                                    'artifact %(name)s',
                                name=artifact.name)
                rac_file = self.rac.get(artifact)
                lac_file = self.lac.put(artifact)
                copy(rac_file, lac_file)

            if artifact.source.morphology.needs_artifact_metadata_cached:
                if not self.lac.has_artifact_metadata(artifact, 'meta'):
                    self.app.status(msg='Fetching to local cache: '
                                        'artifact metadata %(name)s',
                                    name=artifact.name)
                    copy(self.rac.get_artifact_metadata(artifact, 'meta'),
                         self.lac.put_artifact_metadata(artifact, 'meta'))

    def create_staging_area(self, artifact):
        '''Create the staging area for building a single artifact.'''

        if self.app.settings['staging-chroot']:
            staging_root = tempfile.mkdtemp(dir=self.app.settings['tempdir'])
            staging_temp = staging_root
        else:
            staging_root = '/'
            staging_temp = tempfile.mkdtemp(dir=self.app.settings['tempdir'])

        self.app.status(msg='Creating staging area')
        staging_area = morphlib.stagingarea.StagingArea(self.app,
                                                        staging_root,
                                                        staging_temp)
        return staging_area

    def remove_staging_area(self, staging_area):
        '''Remove the staging area.'''

        if staging_area.dirname != '/':
            self.app.status(msg='Removing staging area')
            staging_area.remove()
        temp_path = staging_area.tempdir
        if temp_path != '/' and os.path.exists(temp_path):
            self.app.status(msg='Removing temporary staging directory')
            shutil.rmtree(temp_path)

    def install_fillers(self, staging_area):
        '''Install staging fillers into the staging area.

        This must not be called in bootstrap mode.

        '''

        logging.debug('Pre-populating staging area %s' % staging_area.dirname)
        logging.debug('Fillers: %s' %
                      repr(self.app.settings['staging-filler']))
        for filename in self.app.settings['staging-filler']:
            with open(filename, 'rb') as f:
                self.app.status(msg='Installing %(filename)s',
                                filename=filename)
                staging_area.install_artifact(f)

    def install_chunk_artifacts(self, staging_area, artifacts):
        '''Install chunk artifacts into staging area.

        We only ever care about chunk artifacts as build dependencies,
        so this is not a generic artifact installer into staging area.
        Any non-chunk artifacts are silently ignored.

        All artifacts MUST be in the local artifact cache already.

        '''

        for artifact in artifacts:
            if artifact.source.morphology['kind'] != 'chunk':
                continue
            self.app.status(msg='Installing chunk %(chunk_name)s',
                            chunk_name=artifact.name)
            handle = self.lac.get(artifact)
            staging_area.install_artifact(handle)

    def build_and_cache(self, staging_area, artifact):
        '''Build an artifact and put it into the local artifact cache.'''

        self.app.status(msg='Starting actual build')
        setup_mounts = self.app.settings['staging-chroot']
        builder = morphlib.builder2.Builder(
            self.app, staging_area, self.lac, self.rac, self.lrc,
            self.build_env, self.app.settings['max-jobs'], setup_mounts)
        return builder.build_and_cache(artifact)