From dc7f4153ae6f74fe747117154e02a2198be6e8c2 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 2 May 2012 18:03:44 +0100 Subject: Get rid of the old internal morph APIs --- morphlib/builder.py | 699 ---------------------------------------------------- 1 file changed, 699 deletions(-) delete mode 100644 morphlib/builder.py (limited to 'morphlib/builder.py') diff --git a/morphlib/builder.py b/morphlib/builder.py deleted file mode 100644 index 336a25b3..00000000 --- a/morphlib/builder.py +++ /dev/null @@ -1,699 +0,0 @@ -# Copyright (C) 2011-2012 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import collections -import json -import logging -import os -import shutil -import time - -import morphlib - - -def ldconfig(ex, rootdir): # pragma: no cover - '''Run ldconfig for the filesystem below ``rootdir``. - - Essentially, ``rootdir`` specifies the root of a new system. - Only directories below it are considered. - - ``etc/ld.so.conf`` below ``rootdir`` is assumed to exist and - be populated by the right directories, and should assume - the root directory is ``rootdir``. Example: if ``rootdir`` - is ``/tmp/foo``, then ``/tmp/foo/etc/ld.so.conf`` should - contain ``/lib``, not ``/tmp/foo/lib``. - - The ldconfig found via ``$PATH`` is used, not the one in ``rootdir``, - since in bootstrap mode that might not yet exist, the various - implementations should be compatible enough. - - ''' - - conf = os.path.join(rootdir, 'etc', 'ld.so.conf') - if os.path.exists(conf): - logging.debug('Running ldconfig for %s' % rootdir) - cache = os.path.join(rootdir, 'etc', 'ld.so.cache') - - # The following trickery with $PATH is necessary during the Baserock - # bootstrap build: we are not guaranteed that PATH contains the - # directory (/sbin conventionally) that ldconfig is in. Then again, - # it might, and if so, we don't want to hardware a particular - # location. So we add the possible locations to the end of $PATH - # and restore that aftewards. - old_path = ex.env['PATH'] - ex.env['PATH'] = '%s:/sbin:/usr/sbin:/usr/local/sbin' % old_path - ex.runv(['ldconfig', '-r', rootdir]) - ex.env['PATH'] = old_path - else: - logging.debug('No %s, not running ldconfig' % conf) - - -class Factory(object): - - '''Build Baserock binaries.''' - - def __init__(self, tempdir): - self._tempdir = tempdir - self.staging = None - - def create_staging(self): - '''Create the staging area.''' - self.staging = self._tempdir.join('staging') - os.mkdir(self.staging) - - def remove_staging(self): - '''Remove the staging area.''' - shutil.rmtree(self.staging) - self.staging = None - - def _unpack_binary(self, binary, dirname): - '''Unpack binary into a given directory.''' - ex = morphlib.execute.Execute('/', logging.debug) - morphlib.bins.unpack_binary(binary, dirname, ex) - - def unpack_binary_from_file(self, filename): - '''Unpack a binary package from a file, given its name.''' - self._unpack_binary(filename, self.staging) - - def unpack_binary_from_file_onto_system(self, filename): - '''Unpack contents of a binary package onto the running system. - - DANGER, WILL ROBINSON! This WILL modify your running system. - It should only be used during bootstrapping. - - ''' - - self._unpack_binary(filename, '/') - - def unpack_sources(self, treeish, srcdir): - '''Get sources from to a source directory, for building. - - The git repository and revision are given via a Treeish object. - The source directory must not exist. - - ''' - - def msg(s): - pass - - def extract_treeish(treeish, destdir): - logging.debug('Extracting %s into %s' % (treeish.repo, destdir)) - if not os.path.exists(destdir): - os.mkdir(destdir) - morphlib.git.copy_repository(treeish.repo, destdir, msg) - morphlib.git.checkout_ref(destdir, treeish.ref, msg) - morphlib.git.reset_workdir(destdir, msg) - return [(sub.treeish, os.path.join(destdir, sub.path)) - for sub in treeish.submodules] - - todo = [(treeish, srcdir)] - while todo: - treeish, srcdir = todo.pop() - todo += extract_treeish(treeish, srcdir) - self.set_mtime_recursively(srcdir) - - def set_mtime_recursively(self, root): - '''Set the mtime for every file in a directory tree to the same. - - We do this because git checkout does not set the mtime to anything, - and some projects (binutils, gperf for example) include formatted - documentation and try to randomly build things or not because of - the timestamps. This should help us get more reliable builds. - - ''' - - now = time.time() - for dirname, subdirs, basenames in os.walk(root, topdown=False): - for basename in basenames: - pathname = os.path.join(dirname, basename) - # we need the following check to ignore broken symlinks - if os.path.exists(pathname): - os.utime(pathname, (now, now)) - os.utime(dirname, (now, now)) - - -class BlobBuilder(object): # pragma: no cover - - def __init__(self, blob, factory, settings, cachedir, cache_key, tempdir, - env): - self.blob = blob - self.factory = factory - self.settings = settings - self.cachedir = cachedir - self.cache_key = cache_key - self.tempdir = tempdir - self.env = env - - self.logfile = None - self.stage_items = [] - self.real_msg = lambda s: None - self.dump_memory_profile = lambda msg: None - - self.destdir = os.path.join(self.factory.staging, - '%s.inst' % blob.morph.name) - - self.build_watch = morphlib.stopwatch.Stopwatch() - - def msg(self, text): - self.real_msg(text) - if self.logfile and not self.logfile.closed: - self.logfile.write('%s\n' % text) - - def builds(self): - raise NotImplemented() - - def build(self): - self.prepare_logfile() - with self.build_watch('overall-build'): - self.do_build() - self.save_build_times() - self.save_logfile() - - def filename(self, name): - return '%s.%s.%s' % (self.cachedir.name(self.cache_key), - self.blob.morph.kind, - name) - - def prepare_binary_metadata(self, blob_name): - '''Add metadata to a binary about to be built.''' - - self.msg('Adding metadata to %s' % blob_name) - meta = { - 'name': blob_name, - 'kind': self.blob.morph.kind, - 'description': self.blob.morph.description, - } - - dirname = os.path.join(self.destdir, 'baserock') - if not os.path.exists(dirname): - os.mkdir(dirname) - - filename = os.path.join(dirname, '%s.meta' % blob_name) - with open(filename, 'w') as f: - json.dump(meta, f, indent=4) - f.write('\n') - - def save_build_times(self): - meta = { - 'build-times': {} - } - for stage in self.build_watch.ticks.iterkeys(): - meta['build-times'][stage] = { - 'start': '%s' % self.build_watch.start_time(stage), - 'stop': '%s' % self.build_watch.stop_time(stage), - 'delta': '%.4f' % self.build_watch.start_stop_seconds(stage) - } - - self.msg('Writing metadata to the cache') - with self.cachedir.open(self.cache_key, suffix='.meta') as f: - json.dump(meta, f, indent=4) - f.write('\n') - - def prepare_logfile(self): - self.logfile = self.cachedir.open(self.cache_key, suffix='.log', - mode='w+', buffering=0) - - def save_logfile(self): - self.logfile.close() - - -class ChunkBuilder(BlobBuilder): # pragma: no cover - - def builds(self): - ret = {} - for chunk_name in self.blob.chunks: - ret[chunk_name] = self.filename(chunk_name) - return ret - - def do_build(self): - self.builddir = os.path.join(self.factory.staging, - '%s.build' % self.blob.morph.name) - self.msg('Creating build tree at %s' % self.builddir) - - self.ex = morphlib.execute.Execute(self.builddir, self.msg) - self.ex.env = self.env - - self.factory.unpack_sources(self.blob.morph.treeish, self.builddir) - - os.mkdir(self.destdir) - self.build_with_system_or_commands() - self.dump_memory_profile('after building chunk') - - chunks = self.create_chunks() - self.dump_memory_profile('after creating build chunks') - - # install all built items to the staging area - for name, filename in chunks: - self.msg('Fetching cached %s %s from %s' % - (self.blob.morph.kind, name, filename)) - self.install_chunk(name, filename) - self.dump_memory_profile('after installing chunk') - - def install_chunk(self, chunk_name, chunk_filename): - ex = morphlib.execute.Execute('/', self.msg) - if self.settings['bootstrap']: - self.msg('Unpacking item %s onto system' % chunk_name) - self.factory.unpack_binary_from_file_onto_system(chunk_filename) - ldconfig(ex, '/') - else: - self.msg('Unpacking chunk %s into staging' % chunk_name) - self.factory.unpack_binary_from_file(chunk_filename) - ldconfig(ex, self.factory.staging) - - def build_with_system_or_commands(self): - '''Run explicit commands or commands from build system. - - Use explicit commands, if given, and build system commands if one - has been specified. - - ''' - - bs_name = self.blob.morph.build_system - bs = morphlib.buildsystem.lookup_build_system(bs_name) - - def run_them(runner, what): - attr = '%s_commands' % what - cmds = getattr(self.blob.morph, attr, []) or bs[attr] - runner(what, cmds) - - run_them(self.run_sequentially, 'configure') - run_them(self.run_in_parallel, 'build') - run_them(self.run_sequentially, 'test') - run_them(self.run_sequentially, 'install') - - def run_in_parallel(self, what, commands): - self.msg('commands: %s' % what) - with self.build_watch(what): - max_jobs = self.blob.morph.max_jobs - if max_jobs is None: - max_jobs = self.settings['max-jobs'] - self.ex.env['MAKEFLAGS'] = '-j%s' % max_jobs - self.run_commands(commands) - - def run_sequentially(self, what, commands): - self.msg ('commands: %s' % what) - with self.build_watch(what): - self.ex.env['MAKEFLAGS'] = '-j1' - self.run_commands(commands) - - def run_commands(self, commands): - if self.settings['staging-chroot']: - ex = morphlib.execute.Execute(self.factory.staging, self.msg) - ex.env = self.ex.env.copy() - - assert self.builddir.startswith(self.factory.staging + '/') - assert self.destdir.startswith(self.factory.staging + '/') - builddir = self.builddir[len(self.factory.staging):] - destdir = self.destdir[len(self.factory.staging):] - ex.env['DESTDIR'] = destdir - - for cmd in commands: - ex.runv(['/usr/sbin/chroot', self.factory.staging, 'sh', '-c', - 'cd "$1" && shift && eval "$@"', '--', builddir, cmd]) - else: - self.ex.env['DESTDIR'] = self.destdir - self.ex.run(commands) - - def create_chunks(self): - chunks = [] - with self.build_watch('create-chunks'): - for chunk_name in self.blob.chunks: - self.msg('Creating chunk %s' % chunk_name) - self.prepare_binary_metadata(chunk_name) - patterns = self.blob.chunks[chunk_name] - patterns += [r'baserock/%s\.' % chunk_name] - filename = self.filename(chunk_name) - self.msg('Creating binary for %s' % chunk_name) - with self.cachedir.open(filename) as f: - morphlib.bins.create_chunk(self.destdir, f, patterns, - self.ex, - self.dump_memory_profile) - chunks.append((chunk_name, filename)) - - files = os.listdir(self.destdir) - if files: - raise Exception('DESTDIR %s is not empty: %s' % - (self.destdir, files)) - return chunks - - -class StratumBuilder(BlobBuilder): # pragma: no cover - - def builds(self): - filename = self.filename(self.blob.morph.name) - return { self.blob.morph.name: filename } - - def do_build(self): - os.mkdir(self.destdir) - ex = morphlib.execute.Execute(self.destdir, self.msg) - with self.build_watch('unpack-chunks'): - for chunk_name, filename in self.stage_items: - self.msg('Unpacking chunk %s' % chunk_name) - morphlib.bins.unpack_binary(filename, self.destdir, ex) - with self.build_watch('create-binary'): - self.prepare_binary_metadata(self.blob.morph.name) - self.msg('Creating binary for %s' % self.blob.morph.name) - filename = self.filename(self.blob.morph.name) - basename = os.path.basename(filename) - with self.cachedir.open(basename) as f: - morphlib.bins.create_stratum(self.destdir, f, ex) - - -class SystemBuilder(BlobBuilder): # pragma: no cover - - def builds(self): - filename = self.filename(self.blob.morph.name) - return { self.blob.morph.name: filename } - - def do_build(self): - logging.debug('SystemBuilder.do_build called') - self.ex = morphlib.execute.Execute(self.tempdir.dirname, self.msg) - - image_name = self.tempdir.join('%s.img' % self.blob.morph.name) - self._create_image(image_name) - self._partition_image(image_name) - self._install_mbr(image_name) - partition = self._setup_device_mapping(image_name) - - mount_point = None - try: - self._create_fs(partition) - mount_point = self.tempdir.join('mnt') - self._mount(partition, mount_point) - factory_path = os.path.join(mount_point, 'factory') - self._create_subvolume(factory_path) - self._unpack_strata(factory_path) - self._create_fstab(factory_path) - self._create_extlinux_config(factory_path) - self._create_subvolume_snapshot( - mount_point, 'factory', 'factory-run') - factory_run_path = os.path.join(mount_point, 'factory-run') - self._install_boot_files(factory_run_path, mount_point) - self._install_extlinux(mount_point) - self._unmount(mount_point) - except BaseException: - self._unmount(mount_point) - self._undo_device_mapping(image_name) - raise - - self._undo_device_mapping(image_name) - self._move_image_to_cache(image_name) - - def _create_image(self, image_name): - with self.build_watch('create-image'): - morphlib.fsutils.create_image(self.ex, image_name, - self.blob.morph.disk_size) - - def _partition_image(self, image_name): - with self.build_watch('partition-image'): - morphlib.fsutils.partition_image(self.ex, image_name) - - def _install_mbr(self, image_name): - with self.build_watch('install-mbr'): - morphlib.fsutils.install_mbr(self.ex, image_name) - - def _setup_device_mapping(self, image_name): - with self.build_watch('setup-device-mapper'): - return morphlib.fsutils.setup_device_mapping(self.ex, image_name) - - def _create_fs(self, partition): - with self.build_watch('create-filesystem'): - morphlib.fsutils.create_fs(self.ex, partition) - - def _mount(self, partition, mount_point): - with self.build_watch('mount-filesystem'): - morphlib.fsutils.mount(self.ex, partition, mount_point) - - def _create_subvolume(self, path): - with self.build_watch('create-factory-subvolume'): - self.ex.runv(['btrfs', 'subvolume', 'create', path]) - - def _unpack_strata(self, path): - with self.build_watch('unpack-strata'): - for name, filename in self.stage_items: - self.msg('unpack %s from %s' % (name, filename)) - self.ex.runv(['tar', '-C', path, '-xhf', filename]) - ldconfig(self.ex, path) - - def _create_fstab(self, path): - with self.build_watch('create-fstab'): - fstab = os.path.join(path, 'etc', 'fstab') - if not os.path.exists(os.path.dirname(fstab)): - os.makedirs(os.path.dirname(fstab)) - with open(fstab, 'w') as f: - f.write('proc /proc proc defaults 0 0\n') - f.write('sysfs /sys sysfs defaults 0 0\n') - f.write('/dev/sda1 / btrfs errors=remount-ro 0 1\n') - - def _create_extlinux_config(self, path): - config = os.path.join(path, 'extlinux.conf') - with open(config, 'w') as f: - f.write('default linux\n') - f.write('timeout 1\n') - f.write('label linux\n') - f.write('kernel /boot/vmlinuz\n') - f.write('append root=/dev/sda1 rootflags=subvol=factory-run ' - 'init=/sbin/init quiet rw\n') - - def _create_subvolume_snapshot(self, path, source, target): - with self.build_watch('create-subvolume-snapshot'): - self.ex.runv(['btrfs', 'subvolume', 'snapshot', source, target], - cwd=path) - - def _install_boot_files(self, sourcefs, targetfs): - with self.build_watch('install-boot-files'): - logging.debug('installing boot files into root volume') - shutil.copy2(os.path.join(sourcefs, 'extlinux.conf'), - os.path.join(targetfs, 'extlinux.conf')) - os.mkdir(os.path.join(targetfs, 'boot')) - shutil.copy2(os.path.join(sourcefs, 'boot', 'vmlinuz'), - os.path.join(targetfs, 'boot', 'vmlinuz')) - shutil.copy2(os.path.join(sourcefs, 'boot', 'System.map'), - os.path.join(targetfs, 'boot', 'System.map')) - - def _install_extlinux(self, path): - with self.build_watch('install-bootloader'): - self.ex.runv(['extlinux', '--install', path]) - - # FIXME this hack seems to be necessary to let extlinux finish - self.ex.runv(['sync']) - time.sleep(2) - - def _unmount(self, mount_point): - if mount_point is not None: - with self.build_watch('unmount-filesystem'): - morphlib.fsutils.unmount(self.ex, mount_point) - - def _undo_device_mapping(self, image_name): - with self.build_watch('undo-device-mapper'): - morphlib.fsutils.undo_device_mapping(self.ex, image_name) - - def _move_image_to_cache(self, image_name): - with self.build_watch('cache-image'): - filename = self.filename(self.blob.morph.name) - self.ex.runv(['mv', image_name, filename]) - - -class Builder(object): # pragma: no cover - - '''Build binary objects for Baserock. - - The objects may be chunks or strata.''' - - def __init__(self, tempdir, app, morph_loader, source_manager, factory): - self.tempdir = tempdir - self.app = app - self.real_msg = app.msg - self.dump_memory_profile = app.dump_memory_profile - self.cachedir = morphlib.cachedir.CacheDir( - self.app.settings['cachedir']) - self.morph_loader = morph_loader - self.source_manager = source_manager - self.factory = factory - self.indent = 0 - # create build environment string in advance - env_names = ("USER", "USERNAME", "LOGNAME", - "TOOLCHAIN_TARGET", "PREFIX", "BOOTSTRAP", "CFLAGS") - env = app.clean_env() - self.build_env = ''.join(k + env[k] for k in env_names) - - def msg(self, text): - spaces = ' ' * self.indent - self.real_msg('%s%s' % (spaces, text)) - - def indent_more(self): - self.indent += 1 - - def indent_less(self): - self.indent -= 1 - - def build(self, build_order): - '''Build a list of groups of morphologies. Items in a group - can be built in parallel.''' - - logging.debug('Builder.build called') - self.indent_more() - - # first pass: create builders for all blobs - builders = {} - for group in build_order: - for blob in group: - builder = self.create_blob_builder(blob) - builders[blob] = builder - - # second pass: build group by group, item after item - for group in build_order: - for blob in group: - logging.debug('Building blob %s' % repr(blob)) - self.msg('Building %s' % blob) - self.indent_more() - - ## TODO this needs to be done recursively - ## make sure all dependencies are in the staging area - #for dependency in blob.dependencies: - # depbuilder = builders[dependency] - # depbuilder.stage() - - ## TODO this needs the set of recursively collected - ## dependencies - ## make sure all non-dependencies are not staged - #for nondependency in (blobs - blob.dependencies): - # depbuilder = builders[nondependency] - # depbuilder.unstage() - - builder = builders[blob] - logging.debug('builder = %s' % repr(builder)) - - # get a list of all the items we have to build for this blob - builds = builder.builds() - logging.debug('builds = %s' % repr(builds)) - - # if not all build items are in the cache, rebuild the blob - if not self.all_built(builds): - logging.debug('calling builders build method') - builders[blob].build() - - # check again, fail if not all build items were actually built - if not self.all_built(builds): - raise Exception('Not all builds results expected from %s ' - 'were actually built' % blob) - - for parent in blob.parents: - for item, filename in builds.iteritems(): - self.msg('Marking %s to be staged for %s' % - (item, parent)) - - parent_builder = builders[parent] - parent_builder.stage_items += builds.items() - - self.indent_less() - - self.indent_less() - - def all_built(self, builds): - return all(os.path.isfile(builds[name]) for name in builds) - - def build_single(self, blob, build_order): - self.indent_more() - - # first pass: create builders for all blobs - builders = {} - for group in build_order: - for b in group: - builder = self.create_blob_builder(b) - builders[b] = builder - - # second pass: mark all dependencies for staging - queue = collections.deque() - visited = set() - for dependency in blob.dependencies: - queue.append(dependency) - visited.add(dependency) - while len(queue) > 0: - dependency = queue.popleft() - built_items = builders[dependency].builds() - for name, filename in built_items.iteritems(): - self.msg('Marking %s to be staged for %s' % (name, blob)) - builders[blob].stage_items.append((name, filename)) - for dep in dependency.dependencies: - if (dependency.morph.kind == 'stratum' and - not dep.morph.kind == 'chunk'): - if dep not in visited: - queue.append(dep) - visited.add(dep) - - # build the single blob now - builders[blob].build() - self.indent_less() - - def create_blob_builder(self, blob): - if isinstance(blob, morphlib.blobs.Stratum): - klass = StratumBuilder - elif isinstance(blob, morphlib.blobs.Chunk): - klass = ChunkBuilder - elif isinstance(blob, morphlib.blobs.System): - klass = SystemBuilder - else: - raise TypeError('Blob %s has unknown type %s' % - (str(blob), type(blob))) - - builder = klass(blob, self.factory, self.app.settings, self.cachedir, - self.get_cache_id(blob.morph), self.tempdir, - self.app.clean_env()) - builder.real_msg = self.msg - builder.dump_memory_profile = self.dump_memory_profile - - return builder - - def get_cache_id(self, morph): - logging.debug('get_cache_id(%s)' % morph) - - if morph.kind == 'chunk': - kids = [] - elif morph.kind == 'stratum': - kids = [] - for source in morph.sources: - repo = source['repo'] - ref = source['ref'] - treeish = self.source_manager.get_treeish(repo, ref) - filename = (source['morph'] - if 'morph' in source - else source['name']) - filename = '%s.morph' % filename - chunk = self.morph_loader.load(treeish, filename, - chunk_name=source['name']) - cache_id = self.get_cache_id(chunk) - kids.append(cache_id) - elif morph.kind == 'system': - kids = [] - for stratum_name in morph.strata: - filename = '%s.morph' % stratum_name - stratum = self.morph_loader.load(morph.treeish, filename) - cache_id = self.get_cache_id(stratum) - kids.append(cache_id) - else: - raise NotImplementedError('unknown morph kind %s' % - morph.kind) - - dict_key = { - 'filename': morph.filename, - 'arch': morphlib.util.arch(), - 'ref': morph.treeish.sha1, - 'kids': ''.join(self.cachedir.key(k) for k in kids), - 'env': self.build_env, - } - return dict_key - -- cgit v1.2.1