path: root/morphlib/builder.py

# Copyright (C) 2011  Codethink Limited
# 
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.


import json
import logging
import os
import shutil
import StringIO
import tarfile
import urlparse

import morphlib


class BinaryBlob(object):

    def __init__(self, morph, repo, ref):
        self.morph = morph
        self.repo = repo
        self.ref = ref
        
        # The following MUST get set by the caller.
        self.builddir = None
        self.destdir = None
        self.settings = None
        self.msg = None
        self.cache_prefix = None
        self.tempdir = None
        self.built = None

    def dict_key(self):
        return {}
    
    def needs_built(self):
        return []
    
    def build(self):
        raise NotImplemented()

    def filename(self, name):
        return '%s.%s.%s' % (self.cache_prefix, self.morph.kind, name)

    def prepare_binary_metadata(self, **kwargs):
        '''Add metadata to a binary about to be built.'''

        self.msg('Adding metadata to %s' % self.morph.name)
        meta = {
            'name': self.morph.name,
            'kind': self.morph.kind,
            'description': self.morph.description,
        }
        for key, value in kwargs.iteritems():
            meta[key] = value
        
        dirname = os.path.join(self.destdir, 'baserock')
        filename = os.path.join(dirname, '%s.meta' % self.morph.name)
        if not os.path.exists(dirname):
            os.mkdir(dirname)
        with open(filename, 'w') as f:
            json.dump(meta, f, indent=4)
            f.write('\n')


class Chunk(BinaryBlob):

    build_system = {
        'autotools': {
            'configure-commands': [
                'if [ -e autogen.sh ]; then ./autogen.sh; fi',
                './configure --prefix=/usr',
            ],
            'build-commands': [
                'make',
            ],
            'test-commands': [
            ],
            'install-commands': [
                'make DESTDIR="$DESTDIR" install',
            ],
        },
    }

    def build(self):
        self.ex = morphlib.execute.Execute(self.builddir, self.msg)
        self.ex.env['WORKAREA'] = self.tempdir.dirname
        self.ex.env['DESTDIR'] = self.destdir + '/'

        if self.morph.max_jobs:
            max_jobs = int(self.morph.max_jobs)
        elif self.settings['max-jobs']:
            max_jobs = self.settings['max-jobs']
        else:
            max_jobs = morphlib.util.make_concurrency()
        self.ex.env['MAKEFLAGS'] = '-j%d' % max_jobs

        if not self.settings['no-ccache']:
            self.ex.env['PATH'] = ('/usr/lib/ccache:%s' % 
                                    self.ex.env['PATH'])
            self.ex.env['CCACHE_BASEDIR'] = self.tempdir.dirname

        logging.debug('Creating build tree at %s' % self.builddir)
        tarball = self.cache_prefix + '.src.tar.gz'
        morphlib.git.export_sources(self.repo, self.ref, tarball)
        os.mkdir(self.builddir)
        f = tarfile.open(tarball)
        f.extractall(path=self.builddir)
        f.close()

        os.mkdir(self.destdir)
        if self.morph.build_system:
            bs = self.build_system[self.morph.build_system]
            self.ex.run(bs['configure-commands'])
            self.ex.run(bs['build-commands'])
            self.ex.run(bs['test-commands'])
            self.ex.run(bs['install-commands'])
        else:
            self.ex.run(self.morph.configure_commands)
            self.ex.run(self.morph.build_commands)
            self.ex.run(self.morph.test_commands)
            self.ex.run(self.morph.install_commands, as_fakeroot=True)

        self.prepare_binary_metadata()

        if self.morph.chunks:
            ret = {}
            for chunk_name in self.morph.chunks:
                patterns = self.morph.chunks[chunk_name]
                self.msg('Creating chunk %s' % chunk_name)
                filename = self.filename(chunk_name)
                morphlib.bins.create_chunk(self.destdir, filename, patterns)
                ret[chunk_name] = filename
            # FIXME: check that destdir is empty
            return ret
        else:
            self.msg('Creating chunk %s' % self.morph.name)
            filename = self.filename(self.morph.name)
            morphlib.bins.create_chunk(self.destdir, filename, ['.'])
            return { self.morph.name: filename }


class Stratum(BinaryBlob):
    
    def needs_built(self):
        for chunk_name, source in self.morph.sources.iteritems():
            morph_name = source['morph'] if 'morph' in source else chunk_name
            repo = source['repo']
            ref = source['ref']
            yield repo, ref, morph_name

    def build(self):
        os.mkdir(self.destdir)
        for chunk_name, filename in self.built.iteritems():
            self.msg('Unpacking chunk %s' % chunk_name)
            morphlib.bins.unpack_chunk(filename, self.destdir)
        self.prepare_binary_metadata()
        self.msg('Creating binary for %s' % self.morph.name)
        filename = self.filename(self.morph.name)
        morphlib.bins.create_stratum(self.destdir, filename)
        return { self.morph.name: filename }


class System(BinaryBlob):

    def needs_built(self):
        for stratum_name in self.morph.strata:
            yield self.repo, self.ref, stratum_name

    def build(self):
        self.ex = morphlib.execute.Execute(self.tempdir.dirname, self.msg)
        
        # Create image.
        image_name = self.tempdir.join('%s.img' % self.morph.name)
        self.ex.runv(['qemu-img', 'create', '-f', 'raw', image_name,
                      self.morph.disk_size])

        # Partition it.
        self.ex.runv(['parted', '-s', image_name, 'mklabel', 'msdos'],
                     as_root=True)
        self.ex.runv(['parted', '-s', image_name, 'mkpart', 'primary', 
                      '0%', '100%'], as_root=True)
        self.ex.runv(['parted', '-s', image_name, 'set', '1', 'boot', 'on'],
                     as_root=True)

        # Install first stage boot loader into MBR.
        self.ex.runv(['install-mbr', image_name], as_root=True)

        # Setup device mapper to access the partition.
        out = self.ex.runv(['kpartx', '-av', image_name], as_root=True)
        devices = [line.split()[2]
                   for line in out.splitlines()
                   if line.startswith('add map ')]
        partition = '/dev/mapper/%s' % devices[0]

        mount_point = None
        try:
            # Create filesystem.
            self.ex.runv(['mkfs', '-t', 'ext3', partition], as_root=True)
            
            # Mount it.
            mount_point = self.tempdir.join('mnt')
            os.mkdir(mount_point)
            self.ex.runv(['mount', partition, mount_point], as_root=True)

            # Unpack all strata into filesystem.
            for name, filename in self.built.iteritems():
                self.msg('unpack %s from %s' % (name, filename))
                self.ex.runv(['tar', '-C', mount_point, '-xf', filename],
                             as_root=True)

            # Create fstab.
            fstab = self.tempdir.join('mnt/etc/fstab')
            with open(fstab, 'w') as f:
                f.write('proc /proc proc defaults 0 0\n')
                f.write('sysfs /sys sysfs defaults 0 0\n')
                f.write('/dev/sda1 / ext4 errors=remount-ro 0 1\n')

            # Install extlinux bootloader.
            conf = os.path.join(mount_point, 'extlinux.conf')
            logging.debug('configure extlinux %s' % conf)
            f = open(conf, 'w')
            f.write('''
default linux
timeout 1

label linux
kernel /vmlinuz
append root=/dev/sda1 init=/bin/sh quiet rw
''')
            f.close()

            self.ex.runv(['extlinux', '--install', mount_point], as_root=True)
            
            # Weird hack that makes extlinux work. There is a bug somewhere.
            self.ex.runv(['sync'])
            import time; time.sleep(2)

            # Unmount.
            self.ex.runv(['umount', mount_point], as_root=True)
        except BaseException, e:
            # Unmount.
            if mount_point is not None:
                try:
                    self.ex.runv(['umount', mount_point], as_root=True)
                except Exception:
                    pass

            # Undo device mapping.
            try:
                self.ex.runv(['kpartx', '-d', image_name], as_root=True)
            except Exception:
                pass
            raise

        # Undo device mapping.
        self.ex.runv(['kpartx', '-d', image_name], as_root=True)

        # Move image file to cache.
        filename = self.filename(self.morph.name)
        self.ex.runv(['mv', image_name, filename])

        return { self.morph.name: filename }

class Builder(object):

    '''Build binary objects for Baserock.
    
    The objects may be chunks or strata.'''
    
    def __init__(self, tempdir, msg, settings):
        self.tempdir = tempdir
        self.msg = msg
        self.settings = settings
        self.cachedir = morphlib.cachedir.CacheDir(settings['cachedir'])

    def build(self, repo, ref, filename):
        '''Build a binary based on a morphology.'''

        repo = urlparse.urljoin(self.settings['git-base-url'], repo)
        morph = self.get_morph_from_git(repo, ref, filename)

        if morph.kind == 'chunk':
            blob = Chunk(morph, repo, ref)
        elif morph.kind == 'stratum':
            blob = Stratum(morph, repo, ref)
        elif morph.kind == 'system':
            blob = System(morph, repo, ref)
        else:
            raise Exception('Unknown kind of morphology: %s' % morph.kind)

        dict_key = blob.dict_key()
        self.complete_dict_key(dict_key, morph.name, repo, ref)
        logging.debug('completed dict_key:\n%s' % repr(dict_key))

        blob.builddir = self.tempdir.join('%s.build' % morph.name)
        blob.destdir = self.tempdir.join('%s.inst' % morph.name)
        blob.settings = self.settings
        blob.msg = self.msg
        blob.cache_prefix = self.cachedir.name(dict_key)
        blob.tempdir = self.tempdir

        blob.built = {}
        for needed_repo, needed_ref, needed_name in blob.needs_built():
            needed_filename = '%s.morph' % needed_name
            needed_cached = self.build(needed_repo, needed_ref, 
                                       needed_filename)
            blob.built.update(needed_cached)

        self.msg('Building %s %s' % (morph.kind, morph.name))
        built = blob.build()
        for filename in built:
            self.msg('%s %s cached at %s' %
                        (morph.kind, built[filename], filename))
        return built
            
    def complete_dict_key(self, dict_key, name, repo, ref):
        '''Fill in default fields of a cache's dict key.'''

        if repo and ref:
            abs_ref = morphlib.git.get_commit_id(repo, ref)
        else:
            abs_ref = ''

        dict_key['name'] = name
        dict_key['arch'] = morphlib.util.arch()
        dict_key['repo'] = repo
        dict_key['ref'] = abs_ref

    def get_morph_from_git(self, repo, ref, filename):
        morph_text = morphlib.git.get_morph_text(repo, ref, filename)
        f = StringIO.StringIO(morph_text)
        f.name = filename
        morph = morphlib.morphology.Morphology(f, 
                                               self.settings['git-base-url'])
        return morph