path: root/morphlib/builder2.py

# Copyright (C) 2012  Codethink Limited
# 
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.


import json
import logging
import os
import shutil
import time

import morphlib

class BuilderBase(object):

    '''Base class for building artifacts.'''

    def __init__(self, staging_area, artifact_cache, artifact, repo_cache,
                 build_env, max_jobs, setup_proc):
        self.staging_area = staging_area
        self.artifact_cache = artifact_cache
        self.artifact = artifact
        self.repo_cache = repo_cache
        self.build_env = build_env
        self.max_jobs = max_jobs
        self.build_watch = morphlib.stopwatch.Stopwatch()
        self.setup_proc = setup_proc

    def save_build_times(self):
        '''Write the times captured by the stopwatch'''
        meta = {
            'build-times': {}
        }
        for stage in self.build_watch.ticks.iterkeys():
            meta['build-times'][stage] = {
                'start': '%s' % self.build_watch.start_time(stage),
                'stop': '%s' % self.build_watch.stop_time(stage),
                'delta': '%.4f' % self.build_watch.start_stop_seconds(stage)
            }

        logging.debug('Writing metadata to the cache')
        with self.artifact_cache.put_source_metadata(
                self.artifact.source, self.artifact.cache_key,
                'meta') as f:
            json.dump(meta, f, indent=4, sort_keys=True)
            f.write('\n')
    
    def create_metadata(self, artifact_name):
        '''Create metadata to artifact to allow it to be reproduced later.
        
        The metadata is represented as a dict, which later on will be
        written out as a JSON file.
        
        '''
        
        assert isinstance(self.artifact.source.repo, 
                          morphlib.cachedrepo.CachedRepo)
        meta = {
            'artifact-name': artifact_name,
            'source-name': self.artifact.source.morphology['name'],
            'kind': self.artifact.source.morphology['kind'],
            'description': self.artifact.source.morphology['description'],
            'repo': self.artifact.source.repo.url,
            'original_ref': self.artifact.source.original_ref,
            'sha1': self.artifact.source.sha1,
            'morphology': self.artifact.source.filename,
            'cache-key': self.artifact.cache_key,
            'cache-id': self.artifact.cache_id,
        }
        
        return meta

    # Wrapper around open() to allow it to be overridden by unit tests.
    def _open(self, filename, mode): # pragma: no cover
        dirname = os.path.dirname(filename)
        if not os.path.exists(dirname):
            os.makedirs(dirname)
        return open(filename, mode)

    def write_metadata(self, instdir, artifact_name):
        '''Write the metadata for an artifact.
        
        The file will be located under the ``baserock`` directory under
        instdir, named after ``cache_key`` with ``.meta`` as the suffix.
        It will be in JSON format.
        
        '''
        
        meta = self.create_metadata(artifact_name)

        basename = '%s.meta' % artifact_name
        filename = os.path.join(instdir, 'baserock', basename)

        # Unit tests use StringIO, which in Python 2.6 isn't usable with
        # the "with" statement. So we don't do it with "with".
        f = self._open(filename, 'w')
        f.write(json.dumps(meta, indent=4, sort_keys=True))
        f.close()
        
    def new_artifact(self, artifact_name):
        '''Return an Artifact object for something built from our source.'''
        a = morphlib.artifact.Artifact(self.artifact.source, artifact_name)
        a.cache_key = self.artifact.cache_key
        return a
        
    def runcmd(self, *args, **kwargs):
        kwargs['env'] = self.build_env.env
        return self.staging_area.runcmd(*args, **kwargs)


class ChunkBuilder(BuilderBase):

    '''Build chunk artifacts.'''
    
    def get_commands(self, which, morphology, build_system):
        '''Return the commands to run from a morphology or the build system.'''
        if morphology[which] is None:
            attr = '_'.join(which.split('-'))
            return getattr(build_system, attr)
        else:
            return morphology[which]

    def build_and_cache(self): # pragma: no cover
        with self.build_watch('overall-build'):
            mounted = self.mount_proc()
            try:
                builddir = self.staging_area.builddir(self.artifact.source)
                self.get_sources(builddir)
                destdir = self.staging_area.destdir(self.artifact.source)
                self.run_commands(builddir, destdir)
            except:
                self.umount_proc(mounted)
                raise
            self.umount_proc(mounted)
            self.assemble_chunk_artifacts(destdir)

        self.save_build_times()

    def mount_proc(self): # pragma: no cover
        logging.debug('Mounting /proc in staging area')
        path = os.path.join(self.staging_area.dirname, 'proc')
        if os.path.exists(path) and self.setup_proc:
            ex = morphlib.execute.Execute('.', logging.debug)
            ex.runv(['mount', '-t', 'proc', 'none', path])
            return path
        else:
            logging.debug('Not mounting /proc after all, %s does not exist' %
                            path)
            return None

    def umount_proc(self, mounted): # pragma: no cover
        if (mounted and self.setup_proc and mounted and 
            os.path.exists(os.path.join(mounted, 'self'))):
            logging.error('Unmounting /proc in staging area: %s' % mounted)
            ex = morphlib.execute.Execute('.', logging.debug)
            ex.runv(['umount', mounted])

    def get_sources(self, srcdir): # pragma: no cover
        '''Get sources from git to a source directory, for building.'''

        cache_dir = os.path.dirname(self.artifact.source.repo.path)

        def extract_repo(path, sha1, destdir):
            logging.debug('Extracting %s into %s' % (path, destdir))
            if not os.path.exists(destdir):
                os.mkdir(destdir)
            morphlib.git.copy_repository(path, destdir, logging.debug)
            morphlib.git.checkout_ref(destdir, sha1, logging.debug)
            morphlib.git.reset_workdir(destdir, logging.debug)
            submodules = morphlib.git.Submodules(path, sha1)
            try:
                submodules.load()
            except morphlib.git.NoModulesFileError:
                return []
            else:
                tuples = []
                for sub in submodules:
                    cached_repo = self.repo_cache.get_repo(sub.url)
                    sub_dir = os.path.join(destdir, sub.path)
                    tuples.append((cached_repo.path, sub.commit, sub_dir))
                return tuples

        s = self.artifact.source
        todo = [(s.repo.path, s.sha1, srcdir)]
        while todo:
            path, sha1, srcdir = todo.pop()
            todo += extract_repo(path, sha1, srcdir)
        self.set_mtime_recursively(srcdir)

    def set_mtime_recursively(self, root): # pragma: no cover
        '''Set the mtime for every file in a directory tree to the same.
        
        We do this because git checkout does not set the mtime to anything,
        and some projects (binutils, gperf for example) include formatted
        documentation and try to randomly build things or not because of
        the timestamps. This should help us get more reliable  builds.
        
        '''
        
        now = time.time()
        for dirname, subdirs, basenames in os.walk(root, topdown=False):
            for basename in basenames:
                pathname = os.path.join(dirname, basename)
                # we need the following check to ignore broken symlinks
                if os.path.exists(pathname):
                    os.utime(pathname, (now, now))
            os.utime(dirname, (now, now))


    def run_commands(self, builddir, destdir): # pragma: no cover
        m = self.artifact.source.morphology
        bs = morphlib.buildsystem.lookup_build_system(m['build-system'])

        relative_builddir = self.staging_area.relative(builddir)
        relative_destdir = self.staging_area.relative(destdir)
        self.build_env.env['DESTDIR'] = relative_destdir

        steps = [('configure', False), 
                 ('build', True),
                 ('test', False),
                 ('install', False)]
        for step, in_parallel in steps:
            with self.build_watch(step):
                cmds = self.get_commands('%s-commands' % step, m, bs)
                for cmd in cmds:
                    if in_parallel:
                        max_jobs = self.artifact.source.morphology['max-jobs']
                        if max_jobs is None:
                            max_jobs = self.max_jobs
                        self.build_env.env['MAKEFLAGS'] = '-j%s' % max_jobs
                    else:
                        self.build_env.env['MAKEFLAGS'] = '-j1'
                    self.runcmd(['sh', '-c', cmd], cwd=relative_builddir)

    def assemble_chunk_artifacts(self, destdir): # pragma: no cover
        with self.build_watch('create-chunks'):
            ex = None # create_chunk doesn't actually use this
            specs = self.artifact.source.morphology['chunks']
            if len(specs) == 0:
                specs = {
                    self.artifact.source.morphology['name']: ['.'],
                }
            for artifact_name in specs:
                self.write_metadata(destdir, artifact_name)
                patterns = specs[artifact_name]
                patterns += [r'baserock/%s\.' % artifact_name]
    
                artifact = self.new_artifact(artifact_name)
                with self.artifact_cache.put(artifact) as f:
                    logging.debug('assembling chunk %s' % artifact_name)
                    logging.debug('assembling into %s' % f.name)
                    morphlib.bins.create_chunk(destdir, f, patterns, ex)
    
            files = os.listdir(destdir)
            if files:
                raise Exception('DESTDIR %s is not empty: %s' %
                                (destdir, files))


class StratumBuilder(BuilderBase):

    '''Build stratum artifacts.'''

    def build_and_cache(self): # pragma: no cover
        with self.build_watch('overall-build'):
            destdir = self.staging_area.destdir(self.artifact.source)
    
            constituents = [dependency
                            for dependency in self.artifact.dependencies
                            if dependency.source.morphology['kind'] == 'chunk']
            with self.build_watch('unpack-chunks'):
                for chunk_artifact in constituents:
                    with self.artifact_cache.get(chunk_artifact) as f:
                        morphlib.bins.unpack_binary_from_file(f, destdir)

            with self.build_watch('create-binary'):    
                artifact_name = self.artifact.source.morphology['name']
                self.write_metadata(destdir, artifact_name)
                artifact = self.new_artifact(artifact_name)
                with self.artifact_cache.put(artifact) as f:
                    morphlib.bins.create_stratum(destdir, f, None)
        self.save_build_times()


class SystemBuilder(BuilderBase): # pragma: no cover

    '''Build system image artifacts.'''

    def build_and_cache(self):
        with self.build_watch('overall-build'):
            logging.debug('SystemBuilder.do_build called')
            self.ex = morphlib.execute.Execute(self.staging_area.dirname,
                                               logging.debug)
            
            image_name = os.path.join(self.staging_area.dirname,
                                      '%s.img' % self.artifact.name)
            self._create_image(image_name)
            self._partition_image(image_name)
            self._install_mbr(image_name)
            partition = self._setup_device_mapping(image_name)
    
            mount_point = None
            try:
                self._create_fs(partition)
                mount_point = self.staging_area.destdir(self.artifact.source)
                self._mount(partition, mount_point)
                factory_path = os.path.join(mount_point, 'factory')
                self._create_subvolume(factory_path)
                self._unpack_strata(factory_path)
                self._create_fstab(factory_path)
                self._create_extlinux_config(factory_path)
                self._create_subvolume_snapshot(
                        mount_point, 'factory', 'factory-run')
                factory_run_path = os.path.join(mount_point, 'factory-run')
                self._install_boot_files(factory_run_path, mount_point)
                self._install_extlinux(mount_point)
                self._unmount(mount_point)
            except BaseException, e:
                logging.error('Got error while system image building, '
                                'unmounting and device unmapping')
                self._unmount(mount_point)
                self._undo_device_mapping(image_name)
                raise
    
            self._undo_device_mapping(image_name)
            self._move_image_to_cache(image_name)
        self.save_build_times()

    def _create_image(self, image_name):
        logging.debug('Creating disk image %s' % image_name)
        with self.build_watch('create-image'):
            morphlib.fsutils.create_image(
                self.ex, image_name,
                self.artifact.source.morphology['disk-size'])

    def _partition_image(self, image_name):
        logging.debug('Partitioning disk image %s' % image_name)
        with self.build_watch('partition-image'):
            morphlib.fsutils.partition_image(self.ex, image_name)

    def _install_mbr(self, image_name):
        logging.debug('Installing mbr on disk image %s' % image_name)
        with self.build_watch('install-mbr'):
            morphlib.fsutils.install_mbr(self.ex, image_name)

    def _setup_device_mapping(self, image_name):
        logging.debug('Device mapping partitions in %s' % image_name)
        with self.build_watch('setup-device-mapper'):
            return morphlib.fsutils.setup_device_mapping(self.ex, image_name)

    def _create_fs(self, partition):
        logging.debug('Creating filesystem on %s' % partition)
        with self.build_watch('create-filesystem'):
            morphlib.fsutils.create_fs(self.ex, partition)

    def _mount(self, partition, mount_point):
        logging.debug('Mounting %s on %s' % (partition, mount_point))
        with self.build_watch('mount-filesystem'):
            morphlib.fsutils.mount(self.ex, partition, mount_point)

    def _create_subvolume(self, path):
        logging.debug('Creating subvolume %s' % path)
        with self.build_watch('create-factory-subvolume'):
            self.ex.runv(['btrfs', 'subvolume', 'create', path])

    def _unpack_strata(self, path):
        logging.debug('Unpacking strata to %s' % path)
        with self.build_watch('unpack-strata'):
            for stratum_artifact in self.artifact.dependencies:
                with self.artifact_cache.get(stratum_artifact) as f:
                    morphlib.bins.unpack_binary_from_file(f, path)
            morphlib.builder.ldconfig(self.ex, path)

    def _create_fstab(self, path):
        logging.debug('Creating fstab in %s' % path)
        with self.build_watch('create-fstab'):
            fstab = os.path.join(path, 'etc', 'fstab')
            if not os.path.exists(os.path.dirname(fstab)):# FIXME: should exist
                os.makedirs(os.path.dirname(fstab))
            with open(fstab, 'w') as f:
                f.write('proc      /proc proc  defaults          0 0\n')
                f.write('sysfs     /sys  sysfs defaults          0 0\n')
                f.write('/dev/sda1 / btrfs errors=remount-ro 0 1\n')

    def _create_extlinux_config(self, path):
        logging.debug('Creating extlinux.conf in %s' % path)
        with self.build_watch('create-extlinux-config'):
            config = os.path.join(path, 'extlinux.conf')
            with open(config, 'w') as f:
                f.write('default linux\n')
                f.write('timeout 1\n')
                f.write('label linux\n')
                f.write('kernel /boot/vmlinuz\n')
                f.write('append root=/dev/sda1 rootflags=subvol=factory-run '
                                               'init=/sbin/init quiet rw\n')
    
    def _create_subvolume_snapshot(self, path, source, target):
        logging.debug('Creating subvolume snapshot %s to %s' % 
                        (source, target))
        with self.build_watch('create-runtime-snapshot'):
            self.ex.runv(['btrfs', 'subvolume', 'snapshot', source, target],
                         cwd=path)

    def _install_boot_files(self, sourcefs, targetfs):
        logging.debug('installing boot files into root volume')
        with self.build_watch('install-boot-files'):
            shutil.copy2(os.path.join(sourcefs, 'extlinux.conf'),
                         os.path.join(targetfs, 'extlinux.conf'))
            os.mkdir(os.path.join(targetfs, 'boot'))
            shutil.copy2(os.path.join(sourcefs, 'boot', 'vmlinuz'),
                         os.path.join(targetfs, 'boot', 'vmlinuz'))
            shutil.copy2(os.path.join(sourcefs, 'boot', 'System.map'),
                         os.path.join(targetfs, 'boot', 'System.map'))

    def _install_extlinux(self, path):
        logging.debug('Installing extlinux to %s' % path)
        with self.build_watch('install-bootloader'):
            self.ex.runv(['extlinux', '--install', path])

            # FIXME this hack seems to be necessary to let extlinux finish
            self.ex.runv(['sync'])
            time.sleep(2)

    def _unmount(self, mount_point):
        logging.debug('Unmounting %s' % mount_point)
        with self.build_watch('unmount-filesystem'):
            if mount_point is not None:
                morphlib.fsutils.unmount(self.ex, mount_point)

    def _undo_device_mapping(self, image_name):
        logging.debug('Undoing device mappings for %s' % image_name)
        with self.build_watch('undo-device-mapper'):
            morphlib.fsutils.undo_device_mapping(self.ex, image_name)

    def _move_image_to_cache(self, image_name):
        logging.debug('Moving image to cache: %s' % image_name)
        # FIXME: Need to create file directly in cache to avoid costly
        # copying here.
        with self.build_watch('cache-image'):
            with self.artifact_cache.put(self.artifact) as outf:
                with open(image_name) as inf:
                    while True:
                        data = inf.read(1024**2)
                        if not data:
                            break
                        outf.write(data)


class Builder(object): # pragma: no cover

    '''Helper class to build with the right BuilderBase subclass.'''
    
    classes = {
        'chunk': ChunkBuilder,
        'stratum': StratumBuilder,
        'system': SystemBuilder,
    }

    def __init__(self, staging_area, artifact_cache, repo_cache, build_env, 
                 max_jobs):
        self.staging_area = staging_area
        self.artifact_cache = artifact_cache
        self.repo_cache = repo_cache
        self.build_env = build_env
        self.max_jobs = max_jobs
        self.setup_proc = False
        
    def build_and_cache(self, artifact):
        kind = artifact.source.morphology['kind']
        o = self.classes[kind](self.staging_area, self.artifact_cache, 
                               artifact, self.repo_cache, self.build_env, 
                               self.max_jobs, self.setup_proc)
        logging.debug('Builder.build: artifact %s with %s' %
                      (artifact.name, repr(o)))
        o.build_and_cache()
        logging.debug('Builder.build: done')