path: root/morphlib/builder.py

# Copyright (C) 2011  Codethink Limited
# 
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.


import json
import logging
import os
import shutil
import StringIO
import urlparse

import morphlib


class NoMorphs(Exception):

    def __init__(self, repo, ref):
        Exception.__init__(self, 
                            'Cannot find any morpologies at %s:%s' %
                                (repo, ref))


class TooManyMorphs(Exception):

    def __init__(self, repo, ref, morphs):
        Exception.__init__(self, 
                            'Too many morphologies at %s:%s: %s' %
                                (repo, ref, ', '.join(morphs)))


class Builder(object):

    '''Build binary objects for Baserock.
    
    The objects may be chunks or strata.'''
    
    def __init__(self, tempdir, msg, settings):
        self.tempdir = tempdir
        self.msg = msg
        self.settings = settings
        self.cachedir = morphlib.cachedir.CacheDir(settings['cachedir'])

    def build(self, morph):
        '''Build a binary based on a morphology.'''
        if morph.kind == 'chunk':
            self.build_chunk(morph, self.settings['chunk-repo'], 
                             self.settings['chunk-ref'])
        elif morph.kind == 'stratum':
            self.build_stratum(morph)
        elif morph.kind == 'system':
            self.build_system(morph)
        else:
            raise Exception('Unknown kind of morphology: %s' % morph.kind)

    def build_chunk(self, morph, repo, ref):
        '''Build a chunk from a morphology.'''
        logging.debug('Building chunk')
        self.msg('Building chunk %s' % morph.name)
        filename = self.get_cached_name(morph.name, 'chunk', repo, ref)
        if os.path.exists(filename):
            self.msg('Chunk already exists: %s %s' % (repo, ref))
            self.msg('(chunk cached at %s)' % filename)
        else:
            self.ex = morphlib.execute.Execute(self._build, self.msg)
            self.ex.env['WORKAREA'] = self.tempdir.dirname
            self.ex.env['DESTDIR'] = self._inst + '/'
            self.create_build_tree(morph, repo, ref)
            self.ex.run(morph.configure_commands)
            self.ex.run(morph.build_commands)
            self.ex.run(morph.test_commands)
            os.mkdir(self._inst)
            self.ex.run(morph.install_commands, as_fakeroot=True)
            self.prepare_binary_metadata(morph, 
                    repo=repo, 
                    ref=self.get_git_commit_id(repo, ref))
            self.create_chunk(morph, repo, ref)
            self.tempdir.clear()
        
    def create_build_tree(self, morph, repo, ref):
        '''Export sources from git into the ``self._build`` directory.'''

        logging.debug('Creating build tree at %s' % self._build)
        os.mkdir(self._build)
        tarball = self.tempdir.join('sources.tar')
        self.ex.runv(['git', 'archive',
                      '--output', tarball,
                      '--remote', repo,
                      ref])
        self.ex.runv(['tar', '-C', self._build, '-xf', tarball])
        os.remove(tarball)

    def create_chunk(self, morph, repo, ref):
        '''Create a Baserock chunk from the ``self._inst`` directory.
        
        The directory must be filled in with all the relevant files already.
        
        '''

        logging.debug('Creating chunk %s' % morph.name)
        filename = self.get_cached_name(morph.name, 'chunk', repo, ref)
        morphlib.bins.create_chunk(self._inst, filename)

    def build_stratum(self, morph):
        '''Build a stratum from a morphology.'''

        for chunk_name, source in morph.sources.iteritems():
            self.msg('Want chunk %s' % chunk_name)
            repo = source['repo']
            ref = source['ref']
            chunk_morph = self.get_morph_from_git(repo, ref)
            self.build_chunk(chunk_morph, repo, ref)

        self.msg('Creating stratum %s' % morph.name)
        os.mkdir(self._inst)
        self.ex = morphlib.execute.Execute(self.tempdir.dirname, self.msg)
        for chunk_name in morph.sources:
            self.msg('Unpacking chunk %s' % chunk_name)
            source = morph.sources[chunk_name]
            chunk_repo = source['repo']
            chunk_ref = source['ref']
            logging.debug('Looking for chunk at repo=%s ref=%s' %
                            (chunk_repo, chunk_ref))
            filename = self.get_cached_name(chunk_name, 'chunk', 
                                            chunk_repo, chunk_ref)
            morphlib.bins.unpack_chunk(filename, self._inst)
        self.prepare_binary_metadata(morph)
        stratum_filename = self.create_stratum(morph)
        self.tempdir.clear()
        return stratum_filename

    def create_stratum(self, morph):
        '''Create a Baserock stratum from the ``self._inst`` directory.
        
        The directory must be filled in with all the relevant files already.
        
        '''

        # FIXME: Should put in stratum's git repo and reference here.
        filename = self.get_cached_name(morph.name, 'stratum', '', '')
        self.msg('Creating stratum %s at %s' % (morph.name, filename))
        morphlib.bins.create_stratum(self._inst, filename)
        return filename

    @property
    def _build(self):
        return self.tempdir.join('build')

    @property
    def _inst(self):
        return self.tempdir.join('inst')

    def get_cached_name(self, name, kind, repo, ref):
        '''Return the cached name of a binary blob, if and when it exists.'''
        abs_ref = self.get_git_commit_id(repo, ref)
        dict_key = {
            'name': name,
            'kind': kind,
            'arch': morphlib.util.arch(),
            'repo': repo,
            'ref': abs_ref,
        }
        return self.cachedir.name(dict_key)

    def get_git_commit_id(self, repo, ref):
        '''Return the full SHA-1 commit id for a repo+ref.'''
        if repo and ref:
            path = self.get_repo_dir(repo)
            ex = morphlib.execute.Execute(path, self.msg)
            out = ex.runv(['git', 'rev-list', '-n1', ref])
            return out.strip()
        else:
            return ''

    def get_morph_from_git(self, repo, ref):
        '''Return a morphology from a git repository.'''
        # FIXME: This implementation assume a local repo.

        path = self.get_repo_dir(repo)
        ex = morphlib.execute.Execute(path, self.msg)
        out = ex.runv(['git', 'ls-tree', '--name-only', '-z', ref])
        names = [x for x in out.split('\0') if x]
        morphs = [x for x in names if x.endswith('.morph')]
        if len(morphs) == 0:
            raise NoMorphs(repo, ref)
        if len(morphs) > 1:
            raise TooManyMorphs(repo, ref, morphs)
        out = ex.runv(['git', 'cat-file', 'blob', '%s:%s' % (ref, morphs[0])])
        
        f = StringIO.StringIO(out)
        f.name = morphs[0]
        morph = morphlib.morphology.Morphology(f, 
                                               self.settings['git-base-url'])
        return morph

    def get_repo_dir(self, repo):
        scheme, netlock, path, params, query, frag = urlparse.urlparse(repo)
        return path

    def prepare_binary_metadata(self, morph, **kwargs):
        '''Add metadata to a binary about to be built.'''

        meta = {
            'name': morph.name,
            'kind': morph.kind,
            'description': morph.description,
        }
        for key, value in kwargs.iteritems():
            meta[key] = value
        
        dirname = os.path.join(self._inst, 'baserock')
        filename = os.path.join(dirname, '%s.meta' % morph.name)
        if not os.path.exists(dirname):
            os.mkdir(dirname)
        with open(filename, 'w') as f:
            json.dump(meta, f, indent=4)
            f.write('\n')

    def build_system(self, morph):
        '''Build a system image.'''

        logging.debug('Building system image %s' % morph.name)
        self.msg('Building system %s' % morph.name)

        # Build strata.
        stratum_filenames = []
        for stratum in morph.strata:
            self.msg('Want stratum %s' % stratum)
            dirname = os.path.dirname(morph.filename)
            stratum_filename = os.path.join(dirname, '%s.morph' % stratum)
            logging.debug('Morphology should be in %s' % stratum_filename)
            with open(stratum_filename) as f:
                stratum_morph = morphlib.morphology.Morphology(f,
                                    baseurl=self.settings['git-base-url'])
            filename = self.build_stratum(stratum_morph)
            stratum_filenames.append(filename)

        self.tempdir.clear()
        self.msg('Building system image %s' % morph.name)
        self.ex = morphlib.execute.Execute(self.tempdir.dirname, self.msg)
        
        image_name = self.tempdir.join('%s.img' % morph.name)
        
        # Create image.
        self.ex.runv(['qemu-img', 'create', '-f', 'raw', image_name,
                      morph.disk_size])

        # Partition it.
        self.ex.runv(['parted', '-s', image_name, 'mklabel', 'msdos'],
                     as_root=True)
        self.ex.runv(['parted', '-s', image_name, 'mkpart', 'primary', 
                      '0%', '100%'], as_root=True)
        self.ex.runv(['parted', '-s', image_name, 'set', '1', 'boot', 'on'],
                     as_root=True)

        # Install first stage boot loader into MBR.
        self.ex.runv(['install-mbr', image_name], as_root=True)

        # Setup device mapper to access the partition.
        out = self.ex.runv(['kpartx', '-av', image_name], as_root=True)
        devices = [line.split()[2]
                   for line in out.splitlines()
                   if line.startswith('add map ')]
        partition = '/dev/mapper/%s' % devices[0]

        try:
            # Create filesystem.
            self.ex.runv(['mkfs', '-t', 'ext3', partition], as_root=True)
            
            # Mount it.
            mount_point = self.tempdir.join('mnt')
            os.mkdir(mount_point)
            self.ex.runv(['mount', partition, mount_point], as_root=True)

            # Unpack all strata into filesystem.
            for filename in stratum_filenames:
                morphlib.bins.unpack_stratum(filename, mount_point)

            # Create fstab.
            fstab = self.tempdir.join('mnt/etc/fstab')
            with open(fstab, 'w') as f:
                f.write('proc /proc proc defaults 0 0\n')
                f.write('sysfs /sys sysfs defaults 0 0\n')
                f.write('/dev/sda1 / ext4 errors=remount-ro 0 1\n')

            # Install extlinux bootloader.
            conf = os.path.join(mount_point, 'extlinux.conf')
            logging.debug('configure extlinux %s' % conf)
            f = open(conf, 'w')
            f.write('''
default linux
timeout 1

label linux
kernel /vmlinuz
append root=/dev/sda1 init=/bin/sh quiet
''')
            f.close()

            self.ex.runv(['extlinux', '--install', mount_point], as_root=True)
            
            # Weird hack that makes extlinux work. There is a bug somewhere.
            self.ex.runv(['sync'])
            import time; time.sleep(2)

            # Unmount.
            self.ex.runv(['umount', mount_point], as_root=True)
        except BaseException, e:
            # Unmount.
            try:
                self.ex.runv(['umount', mount_point], as_root=True)
            except Exception:
                pass

            # Undo device mapping.
            try:
                self.ex.runv(['kpartx', '-d', image_name], as_root=True)
            except Exception:
                pass
            raise

        # Undo device mapping.
        self.ex.runv(['kpartx', '-d', image_name], as_root=True)

        # Copy image file to cache.
        filename = self.get_cached_name(morph.name, 'system', '', '')
        self.ex.runv(['cp', '-a', image_name, filename])