diff options
author | Lars Wirzenius <lars.wirzenius@codethink.co.uk> | 2012-04-12 14:47:08 +0100 |
---|---|---|
committer | Lars Wirzenius <lars.wirzenius@codethink.co.uk> | 2012-04-18 18:29:00 +0100 |
commit | 4bb6b91088b0a0f52491deb30a020080a6cde937 (patch) | |
tree | b4776a8c78438e0421a10b19a7326f9af41d35fd /morphlib/builder2.py | |
parent | 3e8797ff2318c8d19828d869f16daba297f423a1 (diff) | |
download | morph-4bb6b91088b0a0f52491deb30a020080a6cde937.tar.gz |
Start a new builder class
Diffstat (limited to 'morphlib/builder2.py')
-rw-r--r-- | morphlib/builder2.py | 328 |
1 files changed, 328 insertions, 0 deletions
diff --git a/morphlib/builder2.py b/morphlib/builder2.py new file mode 100644 index 00000000..7eff7066 --- /dev/null +++ b/morphlib/builder2.py @@ -0,0 +1,328 @@ +# Copyright (C) 2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import json +import logging +import os + +import morphlib + + +class BuilderBase(object): + + '''Base class for building artifacts.''' + + def __init__(self, staging_area, artifact_cache, artifact, build_env, + max_jobs): + self.staging_area = staging_area + self.artifact_cache = artifact_cache + self.artifact = artifact + self.build_env = build_env + self.max_jobs = max_jobs + + def create_metadata(self, artifact_name): + '''Create metadata to artifact to allow it to be reproduced later. + + The metadata is represented as a dict, which later on will be + written out as a JSON file. + + ''' + + meta = { + 'artifact-name': artifact_name, + 'source-name': self.artifact.source.morphology['name'], + 'kind': self.artifact.source.morphology['kind'], + 'description': self.artifact.source.morphology['description'], + 'repo': self.artifact.source.repo, + 'original_ref': self.artifact.source.original_ref, + 'sha1': self.artifact.source.sha1, + 'morphology': self.artifact.source.filename, + } + + return meta + + # Wrapper around open() to allow it to be overridden by unit tests. + def _open(self, filename, mode): # pragma: no cover + dirname = os.path.dirname(filename) + if not os.path.exists(dirname): + os.makedirs(dirname) + return open(filename, mode) + + def write_metadata(self, instdir, artifact_name): + '''Write the metadata for an artifact. + + The file will be located under the ``baserock`` directory under + instdir, named after ``cache_key`` with ``.meta`` as the suffix. + It will be in JSON format. + + ''' + + meta = self.create_metadata(artifact_name) + + basename = '%s.meta' % artifact_name + filename = os.path.join(instdir, 'baserock', basename) + + # Unit tests use StringIO, which in Python 2.6 isn't usable with + # the "with" statement. So we don't do it with "with". + f = self._open(filename, 'w') + f.write(json.dumps(meta)) + f.close() + + def new_artifact(self, artifact_name): + '''Return an Artifact object for something built from our source.''' + return morphlib.artifact.Artifact(self.artifact.source, artifact_name, + self.artifact.cache_key) + + def runcmd(self, *args, **kwargs): + return self.staging_area.runcmd(*args, **kwargs) + + +class ChunkBuilder(BuilderBase): + + '''Build chunk artifacts.''' + + def get_commands(self, which, morphology, build_system): + '''Return the commands to run from a morphology or the build system.''' + if morphology[which] is None: + attr = '_'.join(which.split('-')) + return getattr(build_system, attr) + else: + return morphology[which] + + def build_and_cache(self): # pragma: no cover + destdir = self.staging_area.destdir(self.artifact.source) + self.run_commands(destdir) + self.assemble_chunk_artifacts(destdir) + + def run_commands(self, destdir): # pragma: no cover + m = self.artifact.source.morphology + bs = morphlib.buildsystem.lookup_build_system(m['build-system']) + + relative_destdir = self.staging_area.relative(destdir) + self.build_env.env['DESTDIR'] = relative_destdir + + steps = [('configure', False), + ('build', True), + ('test', False), + ('install', False)] + for step, in_parallel in steps: + cmds = self.get_commands('%s-commands' % step, m, bs) + for cmd in cmds: + if in_parallel: + max_jobs = self.artifact.source.morphology['max-jobs'] + if max_jobs is None: + max_jobs = self.max_jobs + self.build_env.env['MAKEFLAGS'] = '-j%s' % max_jobs + else: + self.build_env.env['MAKEFLAGS'] = '-j1' + self.runcmd(['sh', '-c', cmd], cwd=relative_destdir) + + def assemble_chunk_artifacts(self, destdir): # pragma: no cover + ex = None # create_chunk doesn't actually use this + specs = self.artifact.source.morphology['chunks'] + if len(specs) == 0: + specs = { + self.artifact.source.morphology['name']: ['.'], + } + for artifact_name in specs: + self.write_metadata(destdir, artifact_name) + patterns = specs[artifact_name] + patterns += [r'baserock/%s\.' % artifact_name] + + artifact = self.new_artifact(artifact_name) + with self.artifact_cache.put(artifact) as f: + logging.debug('assembling chunk %s' % artifact_name) + logging.debug('assembling into %s' % f.name) + morphlib.bins.create_chunk(destdir, f, patterns, ex) + + files = os.listdir(destdir) + if files: + raise Exception('DESTDIR %s is not empty: %s' % (destdir, files)) + + +class StratumBuilder(BuilderBase): + + '''Build stratum artifacts.''' + + def build_and_cache(self): # pragma: no cover + destdir = self.staging_area.destdir(self.artifact.source) + + constituents = [dependency + for dependency in self.artifact.dependencies + if dependency.source.morphology['kind'] == 'chunk'] + for chunk_artifact in constituents: + with self.artifact_cache.get(chunk_artifact) as f: + morphlib.bins.unpack_binary_from_file(f, destdir) + + artifact_name = self.artifact.source.morphology['name'] + self.write_metadata(destdir, artifact_name) + artifact = self.new_artifact(artifact_name) + with self.artifact_cache.put(artifact) as f: + morphlib.bins.create_stratum(destdir, f, ex) + + +class SystemBuilder(BuilderBase): # pragma: no cover + + '''Build system image artifacts.''' + + def build_and_cache(self): + logging.debug('SystemBuilder.do_build called') + self.ex = morphlib.execute.Execute(self.staging_area.dirname, + logging.debug) + + image_name = os.path.join(self.staging_area.dirname, + '%s.img' % self.artifact.name) + self._create_image(image_name) + self._partition_image(image_name) + self._install_mbr(image_name) + partition = self._setup_device_mapping(image_name) + + mount_point = None + try: + self._create_fs(partition) + mount_point = self.staging_area.destdir(self.artifact.source) + self._mount(partition, mount_point) + factory_path = os.path.join(mount_point, 'factory') + self._create_subvolume(factory_path) + self._unpack_strata(factory_path) + self._create_fstab(factory_path) + self._create_extlinux_config(factory_path) + self._create_subvolume_snapshot( + mount_point, 'factory', 'factory-run') + factory_run_path = os.path.join(mount_point, 'factory-run') + self._install_boot_files(factory_run_path, mount_point) + self._install_extlinux(mount_point) + self._unmount(mount_point) + except BaseException: + self._unmount(mount_point) + self._undo_device_mapping(image_name) + raise + + self._undo_device_mapping(image_name) + self._move_image_to_cache(image_name) + + def _create_image(self, image_name): + morphlib.fsutils.create_image(self.ex, image_name, + self.artifact.morphology['disk-size']) + + def _partition_image(self, image_name): + morphlib.fsutils.partition_image(self.ex, image_name) + + def _install_mbr(self, image_name): + morphlib.fsutils.install_mbr(self.ex, image_name) + + def _setup_device_mapping(self, image_name): + return morphlib.fsutils.setup_device_mapping(self.ex, image_name) + + def _create_fs(self, partition): + morphlib.fsutils.create_fs(self.ex, partition) + + def _mount(self, partition, mount_point): + morphlib.fsutils.mount(self.ex, partition, mount_point) + + def _create_subvolume(self, path): + self.ex.runv(['btrfs', 'subvolume', 'create', path]) + + def _unpack_strata(self, path): + for stratum_artifact in self.artifact.dependencies: + with self.artifact_cache.get(stratum_artifact) as f: + morphlib.bins.unpack_binary_from_file(f, path, self.ex) + morphlib.builder.ldconfig(self.ex, path) + + def _create_fstab(self, path): + fstab = os.path.join(path, 'etc', 'fstab') + if not os.path.exists(os.path.dirname(fstab)): # FIXME: should exist + os.makedirs(os.path.dirname(fstab)) + with open(fstab, 'w') as f: + f.write('proc /proc proc defaults 0 0\n') + f.write('sysfs /sys sysfs defaults 0 0\n') + f.write('/dev/sda1 / btrfs errors=remount-ro 0 1\n') + + def _create_extlinux_config(self, path): + config = os.path.join(path, 'extlinux.conf') + with open(config, 'w') as f: + f.write('default linux\n') + f.write('timeout 1\n') + f.write('label linux\n') + f.write('kernel /boot/vmlinuz\n') + f.write('append root=/dev/sda1 rootflags=subvol=factory-run ' + 'init=/sbin/init quiet rw\n') + + def _create_subvolume_snapshot(self, path, source, target): + self.ex.runv(['btrfs', 'subvolume', 'snapshot', source, target], + cwd=path) + + def _install_boot_files(self, sourcefs, targetfs): + logging.debug('installing boot files into root volume') + shutil.copy2(os.path.join(sourcefs, 'extlinux.conf'), + os.path.join(targetfs, 'extlinux.conf')) + os.mkdir(os.path.join(targetfs, 'boot')) + shutil.copy2(os.path.join(sourcefs, 'boot', 'vmlinuz'), + os.path.join(targetfs, 'boot', 'vmlinuz')) + shutil.copy2(os.path.join(sourcefs, 'boot', 'System.map'), + os.path.join(targetfs, 'boot', 'System.map')) + + def _install_extlinux(self, path): + self.ex.runv(['extlinux', '--install', path]) + + # FIXME this hack seems to be necessary to let extlinux finish + self.ex.runv(['sync']) + time.sleep(2) + + def _unmount(self, mount_point): + if mount_point is not None: + morphlib.fsutils.unmount(self.ex, mount_point) + + def _undo_device_mapping(self, image_name): + morphlib.fsutils.undo_device_mapping(self.ex, image_name) + + def _move_image_to_cache(self, image_name): + # FIXME: Need to create file directly in cache to avoid costly + # copying here. + with self.artifact_cache.put(self.artifact) as outf: + with open(image_name) as inf: + while True: + data = inf.read(1024**2) + if not data: + break + outf.write(data) + + +class Builder(object): # pragma: no cover + + '''Helper class to build with the right BuilderBase subclass.''' + + classes = { + 'chunk': ChunkBuilder, + 'stratum': StratumBuilder, + 'system': SystemBuilder, + } + + def __init__(self, staging_area, artifact_cache, build_env, max_jobs): + self.staging_area = staging_area + self.artifact_cache = artifact_cache + self.build_env = build_env + self.max_jobs = max_jobs + + def build_and_cache(self, artifact): + kind = artifact.source.morphology['kind'] + o = self.classes[kind](self.staging_area, self.artifact_cache, + artifact, self.build_env, self.max_jobs) + logging.debug('Builder.build: artifact %s with %s' % + (artifact.name, repr(o))) + o.build_and_cache() + logging.debug('Builder.build: done') + |