From fb7eb661b2b02bf049d24fa4a232f303b5058b1b Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Fri, 19 Dec 2014 11:08:23 +0000 Subject: Rename builder2 module to builder --- morphlib/__init__.py | 2 +- morphlib/buildcommand.py | 4 +- morphlib/builder.py | 720 +++++++++++++++++++++++++++++ morphlib/builder2.py | 720 ----------------------------- morphlib/builder2_tests.py | 221 --------- morphlib/builder_tests.py | 221 +++++++++ morphlib/plugins/cross-bootstrap_plugin.py | 4 +- 7 files changed, 946 insertions(+), 946 deletions(-) create mode 100644 morphlib/builder.py delete mode 100644 morphlib/builder2.py delete mode 100644 morphlib/builder2_tests.py create mode 100644 morphlib/builder_tests.py diff --git a/morphlib/__init__.py b/morphlib/__init__.py index 8319430c..0d445304 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -56,7 +56,7 @@ import buildbranch import buildcommand import buildenvironment import buildsystem -import builder2 +import builder import cachedrepo import cachekeycomputer import extensions diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index a658fc55..68fce693 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -518,7 +518,7 @@ class BuildCommand(object): staging_area.install_artifact(handle) if target_source.build_mode == 'staging': - morphlib.builder2.ldconfig(self.app.runcmd, staging_area.dirname) + morphlib.builder.ldconfig(self.app.runcmd, staging_area.dirname) def build_and_cache(self, staging_area, source, setup_mounts): '''Build a source and put its artifacts into the local cache.''' @@ -526,7 +526,7 @@ class BuildCommand(object): self.app.status(msg='Starting actual build: %(name)s ' '%(sha1)s', name=source.name, sha1=source.sha1[:7]) - builder = morphlib.builder2.Builder( + builder = morphlib.builder.Builder( self.app, staging_area, self.lac, self.rac, self.lrc, self.app.settings['max-jobs'], setup_mounts) return builder.build_and_cache(source) diff --git a/morphlib/builder.py b/morphlib/builder.py new file mode 100644 index 00000000..9e158ea1 --- /dev/null +++ b/morphlib/builder.py @@ -0,0 +1,720 @@ +# Copyright (C) 2012-2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +from collections import defaultdict +import datetime +import errno +import json +import logging +import os +from os.path import relpath +import shutil +import stat +import tarfile +import time +import traceback +import subprocess +import tempfile +import gzip + +import cliapp + +import morphlib +from morphlib.artifactcachereference import ArtifactCacheReference +import morphlib.gitversion + +SYSTEM_INTEGRATION_PATH = os.path.join('baserock', 'system-integration') + +def extract_sources(app, repo_cache, repo, sha1, srcdir): #pragma: no cover + '''Get sources from git to a source directory, including submodules''' + + def extract_repo(repo, sha1, destdir): + app.status(msg='Extracting %(source)s into %(target)s', + source=repo.original_name, + target=destdir) + + repo.checkout(sha1, destdir) + morphlib.git.reset_workdir(app.runcmd, destdir) + submodules = morphlib.git.Submodules(app, repo.path, sha1) + try: + submodules.load() + except morphlib.git.NoModulesFileError: + return [] + else: + tuples = [] + for sub in submodules: + cached_repo = repo_cache.get_repo(sub.url) + sub_dir = os.path.join(destdir, sub.path) + tuples.append((cached_repo, sub.commit, sub_dir)) + return tuples + + todo = [(repo, sha1, srcdir)] + while todo: + repo, sha1, srcdir = todo.pop() + todo += extract_repo(repo, sha1, srcdir) + set_mtime_recursively(srcdir) + +def set_mtime_recursively(root): # pragma: no cover + '''Set the mtime for every file in a directory tree to the same. + + We do this because git checkout does not set the mtime to anything, + and some projects (binutils, gperf for example) include formatted + documentation and try to randomly build things or not because of + the timestamps. This should help us get more reliable builds. + + ''' + + now = time.time() + for dirname, subdirs, basenames in os.walk(root.encode("utf-8"), + topdown=False): + for basename in basenames: + pathname = os.path.join(dirname, basename) + # we need the following check to ignore broken symlinks + if os.path.exists(pathname): + os.utime(pathname, (now, now)) + os.utime(dirname, (now, now)) + +def ldconfig(runcmd, rootdir): # pragma: no cover + '''Run ldconfig for the filesystem below ``rootdir``. + + Essentially, ``rootdir`` specifies the root of a new system. + Only directories below it are considered. + + ``etc/ld.so.conf`` below ``rootdir`` is assumed to exist and + be populated by the right directories, and should assume + the root directory is ``rootdir``. Example: if ``rootdir`` + is ``/tmp/foo``, then ``/tmp/foo/etc/ld.so.conf`` should + contain ``/lib``, not ``/tmp/foo/lib``. + + The ldconfig found via ``$PATH`` is used, not the one in ``rootdir``, + since in bootstrap mode that might not yet exist, the various + implementations should be compatible enough. + + ''' + + # FIXME: use the version in ROOTDIR, since even in + # bootstrap it will now always exist due to being part of build-essential + + conf = os.path.join(rootdir, 'etc', 'ld.so.conf') + if os.path.exists(conf): + logging.debug('Running ldconfig for %s' % rootdir) + cache = os.path.join(rootdir, 'etc', 'ld.so.cache') + + # The following trickery with $PATH is necessary during the Baserock + # bootstrap build: we are not guaranteed that PATH contains the + # directory (/sbin conventionally) that ldconfig is in. Then again, + # it might, and if so, we don't want to hardware a particular + # location. So we add the possible locations to the end of $PATH + env = dict(os.environ) + old_path = env['PATH'] + env['PATH'] = '%s:/sbin:/usr/sbin:/usr/local/sbin' % old_path + runcmd(['ldconfig', '-r', rootdir], env=env) + else: + logging.debug('No %s, not running ldconfig' % conf) + + +def download_depends(constituents, lac, rac, metadatas=None): + for constituent in constituents: + if not lac.has(constituent): + source = rac.get(constituent) + target = lac.put(constituent) + shutil.copyfileobj(source, target) + target.close() + source.close() + if metadatas is not None: + for metadata in metadatas: + if not lac.has_artifact_metadata(constituent, metadata): + if rac.has_artifact_metadata(constituent, metadata): + src = rac.get_artifact_metadata(constituent, metadata) + dst = lac.put_artifact_metadata(constituent, metadata) + shutil.copyfileobj(src, dst) + dst.close() + src.close() + + +def get_chunk_files(f): # pragma: no cover + tar = tarfile.open(fileobj=f) + for member in tar.getmembers(): + if member.type is not tarfile.DIRTYPE: + yield member.name + tar.close() + + +def get_stratum_files(f, lac): # pragma: no cover + for ca in (ArtifactCacheReference(a) + for a in json.load(f, encoding='unicode-escape')): + cf = lac.get(ca) + for filename in get_chunk_files(cf): + yield filename + cf.close() + + +class BuilderBase(object): + + '''Base class for building artifacts.''' + + def __init__(self, app, staging_area, local_artifact_cache, + remote_artifact_cache, source, repo_cache, max_jobs, + setup_mounts): + self.app = app + self.staging_area = staging_area + self.local_artifact_cache = local_artifact_cache + self.remote_artifact_cache = remote_artifact_cache + self.source = source + self.repo_cache = repo_cache + self.max_jobs = max_jobs + self.build_watch = morphlib.stopwatch.Stopwatch() + self.setup_mounts = setup_mounts + + def save_build_times(self): + '''Write the times captured by the stopwatch''' + meta = { + 'build-times': {} + } + for stage in self.build_watch.ticks.iterkeys(): + meta['build-times'][stage] = { + 'start': '%s' % self.build_watch.start_time(stage), + 'stop': '%s' % self.build_watch.stop_time(stage), + 'delta': '%.4f' % self.build_watch.start_stop_seconds(stage) + } + + logging.debug('Writing metadata to the cache') + with self.local_artifact_cache.put_source_metadata( + self.source, self.source.cache_key, + 'meta') as f: + json.dump(meta, f, indent=4, sort_keys=True, + encoding='unicode-escape') + f.write('\n') + + def create_metadata(self, artifact_name, contents=[]): # pragma: no cover + '''Create metadata to artifact to allow it to be reproduced later. + + The metadata is represented as a dict, which later on will be + written out as a JSON file. + + ''' + + assert isinstance(self.source.repo, + morphlib.cachedrepo.CachedRepo) + meta = { + 'artifact-name': artifact_name, + 'source-name': self.source.name, + 'kind': self.source.morphology['kind'], + 'description': self.source.morphology['description'], + 'repo': self.source.repo.url, + 'repo-alias': self.source.repo_name, + 'original_ref': self.source.original_ref, + 'sha1': self.source.sha1, + 'morphology': self.source.filename, + 'cache-key': self.source.cache_key, + 'cache-id': self.source.cache_id, + 'morph-version': { + 'ref': morphlib.gitversion.ref, + 'tree': morphlib.gitversion.tree, + 'commit': morphlib.gitversion.commit, + 'version': morphlib.gitversion.version, + }, + 'contents': contents, + } + + return meta + + # Wrapper around open() to allow it to be overridden by unit tests. + def _open(self, filename, mode): # pragma: no cover + dirname = os.path.dirname(filename) + if not os.path.exists(dirname): + os.makedirs(dirname) + return open(filename, mode) + + def write_metadata(self, instdir, artifact_name, + contents=[]): # pragma: no cover + '''Write the metadata for an artifact. + + The file will be located under the ``baserock`` directory under + instdir, named after ``cache_key`` with ``.meta`` as the suffix. + It will be in JSON format. + + ''' + + meta = self.create_metadata(artifact_name, contents) + + basename = '%s.meta' % artifact_name + filename = os.path.join(instdir, 'baserock', basename) + + # Unit tests use StringIO, which in Python 2.6 isn't usable with + # the "with" statement. So we don't do it with "with". + f = self._open(filename, 'w') + json.dump(meta, f, indent=4, sort_keys=True, encoding='unicode-escape') + f.close() + + def runcmd(self, *args, **kwargs): + return self.staging_area.runcmd(*args, **kwargs) + +class ChunkBuilder(BuilderBase): + + '''Build chunk artifacts.''' + + def create_devices(self, destdir): # pragma: no cover + '''Creates device nodes if the morphology specifies them''' + morphology = self.source.morphology + perms_mask = stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO + if 'devices' in morphology and morphology['devices'] is not None: + for dev in morphology['devices']: + destfile = os.path.join(destdir, './' + dev['filename']) + mode = int(dev['permissions'], 8) & perms_mask + if dev['type'] == 'c': + mode = mode | stat.S_IFCHR + elif dev['type'] == 'b': + mode = mode | stat.S_IFBLK + else: + raise IOError('Cannot create device node %s,' + 'unrecognized device type "%s"' + % (destfile, dev['type'])) + self.app.status(msg="Creating device node %s" + % destfile) + os.mknod(destfile, mode, + os.makedev(dev['major'], dev['minor'])) + os.chown(destfile, dev['uid'], dev['gid']) + + def build_and_cache(self): # pragma: no cover + with self.build_watch('overall-build'): + + builddir, destdir = self.staging_area.chroot_open( + self.source, self.setup_mounts) + + stdout = (self.app.output + if self.app.settings['build-log-on-stdout'] else None) + + cache = self.local_artifact_cache + logpath = cache.get_source_metadata_filename( + self.source, self.source.cache_key, 'build-log') + + _, temppath = tempfile.mkstemp(dir=os.path.dirname(logpath)) + + try: + self.get_sources(builddir) + self.run_commands(builddir, destdir, temppath, stdout) + self.create_devices(destdir) + + os.rename(temppath, logpath) + except BaseException, e: + logging.error('Caught exception: %s' % str(e)) + logging.info('Cleaning up staging area') + self.staging_area.chroot_close() + if os.path.isfile(temppath): + with open(temppath) as f: + for line in f: + logging.error('OUTPUT FROM FAILED BUILD: %s' % + line.rstrip('\n')) + + os.rename(temppath, logpath) + else: + logging.error("Couldn't find build log at %s", temppath) + + self.staging_area.abort() + raise + + self.staging_area.chroot_close() + built_artifacts = self.assemble_chunk_artifacts(destdir) + + self.save_build_times() + return built_artifacts + + + def run_commands(self, builddir, destdir, + logfilepath, stdout=None): # pragma: no cover + m = self.source.morphology + bs = morphlib.buildsystem.lookup_build_system(m['build-system']) + + relative_builddir = self.staging_area.relative(builddir) + relative_destdir = self.staging_area.relative(destdir) + ccache_dir = self.staging_area.ccache_dir(self.source) + extra_env = { 'DESTDIR': relative_destdir } + + steps = [ + ('pre-configure', False), + ('configure', False), + ('post-configure', False), + ('pre-build', True), + ('build', True), + ('post-build', True), + ('pre-test', False), + ('test', False), + ('post-test', False), + ('pre-install', False), + ('install', False), + ('post-install', False), + ] + for step, in_parallel in steps: + with self.build_watch(step): + key = '%s-commands' % step + cmds = m[key] + if cmds: + with open(logfilepath, 'a') as log: + self.app.status(msg='Running %(key)s', key=key) + log.write('# %s\n' % step) + + for cmd in cmds: + if in_parallel: + max_jobs = self.source.morphology['max-jobs'] + if max_jobs is None: + max_jobs = self.max_jobs + extra_env['MAKEFLAGS'] = '-j%s' % max_jobs + else: + extra_env['MAKEFLAGS'] = '-j1' + + try: + with open(logfilepath, 'a') as log: + log.write('# # %s\n' % cmd) + + # flushing is needed because writes from python and + # writes from being the output in Popen have different + # buffers, but flush handles both + if stdout: + stdout.flush() + + self.runcmd(['sh', '-c', cmd], + extra_env=extra_env, + cwd=relative_builddir, + stdout=stdout or subprocess.PIPE, + stderr=subprocess.STDOUT, + logfile=logfilepath, + ccache_dir=ccache_dir) + + if stdout: + stdout.flush() + except cliapp.AppException, e: + if not stdout: + with open(logfilepath, 'r') as log: + self.app.output.write("%s failed\n" % step) + shutil.copyfileobj(log, self.app.output) + raise e + + def write_system_integration_commands(self, destdir, + integration_commands, artifact_name): # pragma: no cover + + rel_path = SYSTEM_INTEGRATION_PATH + dest_path = os.path.join(destdir, SYSTEM_INTEGRATION_PATH) + + scripts_created = [] + + if not os.path.exists(dest_path): + os.makedirs(dest_path) + + if artifact_name in integration_commands: + prefixes_per_artifact = integration_commands[artifact_name] + for prefix, commands in prefixes_per_artifact.iteritems(): + for index, script in enumerate(commands): + script_name = "%s-%s-%04d" % (prefix, + artifact_name, + index) + script_path = os.path.join(dest_path, script_name) + + with morphlib.savefile.SaveFile(script_path, 'w') as f: + f.write("#!/bin/sh\nset -xeu\n") + f.write(script) + os.chmod(script_path, 0555) + + rel_script_path = os.path.join(SYSTEM_INTEGRATION_PATH, + script_name) + scripts_created += [rel_script_path] + + return scripts_created + + def assemble_chunk_artifacts(self, destdir): # pragma: no cover + built_artifacts = [] + filenames = [] + source = self.source + split_rules = source.split_rules + morphology = source.morphology + sys_tag = 'system-integration' + + def filepaths(destdir): + for dirname, subdirs, basenames in os.walk(destdir): + subdirsymlinks = [os.path.join(dirname, x) for x in subdirs + if os.path.islink(os.path.join(dirname, x))] + filenames = [os.path.join(dirname, x) for x in basenames] + for relpath in (os.path.relpath(x, destdir) for x in + [dirname] + subdirsymlinks + filenames): + yield relpath + + with self.build_watch('determine-splits'): + matches, overlaps, unmatched = \ + split_rules.partition(filepaths(destdir)) + + system_integration = morphology.get(sys_tag) or {} + + with self.build_watch('create-chunks'): + for chunk_artifact_name, chunk_artifact \ + in source.artifacts.iteritems(): + file_paths = matches[chunk_artifact_name] + chunk_artifact = source.artifacts[chunk_artifact_name] + + def all_parents(path): + while path != '': + yield path + path = os.path.dirname(path) + + def parentify(filenames): + names = set() + for name in filenames: + names.update(all_parents(name)) + return sorted(names) + + extra_files = self.write_system_integration_commands( + destdir, system_integration, + chunk_artifact_name) + extra_files += ['baserock/%s.meta' % chunk_artifact_name] + parented_paths = parentify(file_paths + extra_files) + + with self.local_artifact_cache.put(chunk_artifact) as f: + self.write_metadata(destdir, chunk_artifact_name, + parented_paths) + + self.app.status(msg='Creating chunk artifact %(name)s', + name=chunk_artifact_name) + morphlib.bins.create_chunk(destdir, f, parented_paths) + built_artifacts.append(chunk_artifact) + + for dirname, subdirs, files in os.walk(destdir): + if files: + raise Exception('DESTDIR %s is not empty: %s' % + (destdir, files)) + return built_artifacts + + def get_sources(self, srcdir): # pragma: no cover + s = self.source + extract_sources(self.app, self.repo_cache, s.repo, s.sha1, srcdir) + + +class StratumBuilder(BuilderBase): + '''Build stratum artifacts.''' + + def is_constituent(self, artifact): # pragma: no cover + '''True if artifact should be included in the stratum artifact''' + return (artifact.source.morphology['kind'] == 'chunk' and \ + artifact.source.build_mode != 'bootstrap') + + def build_and_cache(self): # pragma: no cover + with self.build_watch('overall-build'): + constituents = [d for d in self.source.dependencies + if self.is_constituent(d)] + + # the only reason the StratumBuilder has to download chunks is to + # check for overlap now that strata are lists of chunks + with self.build_watch('check-chunks'): + for a_name, a in self.source.artifacts.iteritems(): + # download the chunk artifact if necessary + download_depends(constituents, + self.local_artifact_cache, + self.remote_artifact_cache) + + with self.build_watch('create-chunk-list'): + lac = self.local_artifact_cache + for a_name, a in self.source.artifacts.iteritems(): + meta = self.create_metadata( + a_name, + [x.name for x in constituents]) + with lac.put_artifact_metadata(a, 'meta') as f: + json.dump(meta, f, indent=4, sort_keys=True) + with self.local_artifact_cache.put(a) as f: + json.dump([c.basename() for c in constituents], f) + self.save_build_times() + return self.source.artifacts.values() + + +class SystemBuilder(BuilderBase): # pragma: no cover + + '''Build system image artifacts.''' + + def __init__(self, *args, **kwargs): + BuilderBase.__init__(self, *args, **kwargs) + self.args = args + self.kwargs = kwargs + + def build_and_cache(self): + self.app.status(msg='Building system %(system_name)s', + system_name=self.source.name) + + with self.build_watch('overall-build'): + arch = self.source.morphology['arch'] + + for a_name, artifact in self.source.artifacts.iteritems(): + handle = self.local_artifact_cache.put(artifact) + + try: + fs_root = self.staging_area.destdir(self.source) + self.unpack_strata(fs_root) + self.write_metadata(fs_root, a_name) + self.run_system_integration_commands(fs_root) + unslashy_root = fs_root[1:] + def uproot_info(info): + info.name = relpath(info.name, unslashy_root) + if info.islnk(): + info.linkname = relpath(info.linkname, + unslashy_root) + return info + tar = tarfile.open(fileobj=handle, mode="w", name=a_name) + self.app.status(msg='Constructing tarball of rootfs', + chatty=True) + tar.add(fs_root, recursive=True, filter=uproot_info) + tar.close() + except BaseException as e: + logging.error(traceback.format_exc()) + self.app.status(msg='Error while building system', + error=True) + handle.abort() + raise + else: + handle.close() + + self.save_build_times() + return self.source.artifacts.itervalues() + + def unpack_one_stratum(self, stratum_artifact, target): + '''Unpack a single stratum into a target directory''' + + cache = self.local_artifact_cache + with cache.get(stratum_artifact) as stratum_file: + artifact_list = json.load(stratum_file, encoding='unicode-escape') + for chunk in (ArtifactCacheReference(a) for a in artifact_list): + self.app.status(msg='Unpacking chunk %(basename)s', + basename=chunk.basename(), chatty=True) + with cache.get(chunk) as chunk_file: + morphlib.bins.unpack_binary_from_file(chunk_file, target) + + target_metadata = os.path.join( + target, 'baserock', '%s.meta' % stratum_artifact.name) + with cache.get_artifact_metadata(stratum_artifact, 'meta') as meta_src: + with morphlib.savefile.SaveFile(target_metadata, 'w') as meta_dst: + shutil.copyfileobj(meta_src, meta_dst) + + def unpack_strata(self, path): + '''Unpack strata into a directory.''' + + self.app.status(msg='Unpacking strata to %(path)s', + path=path, chatty=True) + with self.build_watch('unpack-strata'): + for a_name, a in self.source.artifacts.iteritems(): + # download the stratum artifacts if necessary + download_depends(self.source.dependencies, + self.local_artifact_cache, + self.remote_artifact_cache, + ('meta',)) + + # download the chunk artifacts if necessary + for stratum_artifact in self.source.dependencies: + f = self.local_artifact_cache.get(stratum_artifact) + chunks = [ArtifactCacheReference(c) for c in json.load(f)] + download_depends(chunks, + self.local_artifact_cache, + self.remote_artifact_cache) + f.close() + + # unpack it from the local artifact cache + for stratum_artifact in self.source.dependencies: + self.unpack_one_stratum(stratum_artifact, path) + + ldconfig(self.app.runcmd, path) + + def write_metadata(self, instdir, artifact_name): + BuilderBase.write_metadata(self, instdir, artifact_name) + + os_release_file = os.path.join(instdir, 'etc', 'os-release') + dirname = os.path.dirname(os_release_file) + if not os.path.exists(dirname): + os.makedirs(dirname) + with morphlib.savefile.SaveFile(os_release_file, 'w') as f: + f.write('NAME="Baserock"\n') + f.write('ID=baserock\n') + f.write('HOME_URL="http://wiki.baserock.org"\n') + f.write('SUPPORT_URL="http://wiki.baserock.org/mailinglist"\n') + f.write('BUG_REPORT_URL="http://wiki.baserock.org/mailinglist"\n') + + os.chmod(os_release_file, 0644) + + def run_system_integration_commands(self, rootdir): # pragma: no cover + ''' Run the system integration commands ''' + + sys_integration_dir = os.path.join(rootdir, SYSTEM_INTEGRATION_PATH) + if not os.path.isdir(sys_integration_dir): + return + + env = { + 'PATH': '/bin:/usr/bin:/sbin:/usr/sbin' + } + + self.app.status(msg='Running the system integration commands') + + to_mount = ( + ('dev/shm', 'tmpfs', 'none'), + ('tmp', 'tmpfs', 'none'), + ) + try: + for bin in sorted(os.listdir(sys_integration_dir)): + argv = [os.path.join(SYSTEM_INTEGRATION_PATH, bin)] + container_config = dict( + root=rootdir, mounts=to_mount, mount_proc=True) + cmdline = morphlib.util.containerised_cmdline( + argv, **container_config) + exit, out, err = self.app.runcmd_unchecked( + cmdline, env=env) + if exit != 0: + logging.debug('Command returned code %i', exit) + msg = error_message_for_containerised_commandline( + argv, err, container_config) + raise cliapp.AppException(msg) + except BaseException, e: + self.app.status( + msg='Error while running system integration commands', + error=True) + raise + + +class Builder(object): # pragma: no cover + + '''Helper class to build with the right BuilderBase subclass.''' + + classes = { + 'chunk': ChunkBuilder, + 'stratum': StratumBuilder, + 'system': SystemBuilder, + } + + def __init__(self, app, staging_area, local_artifact_cache, + remote_artifact_cache, repo_cache, max_jobs, setup_mounts): + self.app = app + self.staging_area = staging_area + self.local_artifact_cache = local_artifact_cache + self.remote_artifact_cache = remote_artifact_cache + self.repo_cache = repo_cache + self.max_jobs = max_jobs + self.setup_mounts = setup_mounts + + def build_and_cache(self, source): + kind = source.morphology['kind'] + o = self.classes[kind](self.app, self.staging_area, + self.local_artifact_cache, + self.remote_artifact_cache, source, + self.repo_cache, self.max_jobs, + self.setup_mounts) + self.app.status(msg='Builder.build: artifact %s with %s' % + (source.name, repr(o)), + chatty=True) + built_artifacts = o.build_and_cache() + self.app.status(msg='Builder.build: done', + chatty=True) + return built_artifacts diff --git a/morphlib/builder2.py b/morphlib/builder2.py deleted file mode 100644 index 9e158ea1..00000000 --- a/morphlib/builder2.py +++ /dev/null @@ -1,720 +0,0 @@ -# Copyright (C) 2012-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -from collections import defaultdict -import datetime -import errno -import json -import logging -import os -from os.path import relpath -import shutil -import stat -import tarfile -import time -import traceback -import subprocess -import tempfile -import gzip - -import cliapp - -import morphlib -from morphlib.artifactcachereference import ArtifactCacheReference -import morphlib.gitversion - -SYSTEM_INTEGRATION_PATH = os.path.join('baserock', 'system-integration') - -def extract_sources(app, repo_cache, repo, sha1, srcdir): #pragma: no cover - '''Get sources from git to a source directory, including submodules''' - - def extract_repo(repo, sha1, destdir): - app.status(msg='Extracting %(source)s into %(target)s', - source=repo.original_name, - target=destdir) - - repo.checkout(sha1, destdir) - morphlib.git.reset_workdir(app.runcmd, destdir) - submodules = morphlib.git.Submodules(app, repo.path, sha1) - try: - submodules.load() - except morphlib.git.NoModulesFileError: - return [] - else: - tuples = [] - for sub in submodules: - cached_repo = repo_cache.get_repo(sub.url) - sub_dir = os.path.join(destdir, sub.path) - tuples.append((cached_repo, sub.commit, sub_dir)) - return tuples - - todo = [(repo, sha1, srcdir)] - while todo: - repo, sha1, srcdir = todo.pop() - todo += extract_repo(repo, sha1, srcdir) - set_mtime_recursively(srcdir) - -def set_mtime_recursively(root): # pragma: no cover - '''Set the mtime for every file in a directory tree to the same. - - We do this because git checkout does not set the mtime to anything, - and some projects (binutils, gperf for example) include formatted - documentation and try to randomly build things or not because of - the timestamps. This should help us get more reliable builds. - - ''' - - now = time.time() - for dirname, subdirs, basenames in os.walk(root.encode("utf-8"), - topdown=False): - for basename in basenames: - pathname = os.path.join(dirname, basename) - # we need the following check to ignore broken symlinks - if os.path.exists(pathname): - os.utime(pathname, (now, now)) - os.utime(dirname, (now, now)) - -def ldconfig(runcmd, rootdir): # pragma: no cover - '''Run ldconfig for the filesystem below ``rootdir``. - - Essentially, ``rootdir`` specifies the root of a new system. - Only directories below it are considered. - - ``etc/ld.so.conf`` below ``rootdir`` is assumed to exist and - be populated by the right directories, and should assume - the root directory is ``rootdir``. Example: if ``rootdir`` - is ``/tmp/foo``, then ``/tmp/foo/etc/ld.so.conf`` should - contain ``/lib``, not ``/tmp/foo/lib``. - - The ldconfig found via ``$PATH`` is used, not the one in ``rootdir``, - since in bootstrap mode that might not yet exist, the various - implementations should be compatible enough. - - ''' - - # FIXME: use the version in ROOTDIR, since even in - # bootstrap it will now always exist due to being part of build-essential - - conf = os.path.join(rootdir, 'etc', 'ld.so.conf') - if os.path.exists(conf): - logging.debug('Running ldconfig for %s' % rootdir) - cache = os.path.join(rootdir, 'etc', 'ld.so.cache') - - # The following trickery with $PATH is necessary during the Baserock - # bootstrap build: we are not guaranteed that PATH contains the - # directory (/sbin conventionally) that ldconfig is in. Then again, - # it might, and if so, we don't want to hardware a particular - # location. So we add the possible locations to the end of $PATH - env = dict(os.environ) - old_path = env['PATH'] - env['PATH'] = '%s:/sbin:/usr/sbin:/usr/local/sbin' % old_path - runcmd(['ldconfig', '-r', rootdir], env=env) - else: - logging.debug('No %s, not running ldconfig' % conf) - - -def download_depends(constituents, lac, rac, metadatas=None): - for constituent in constituents: - if not lac.has(constituent): - source = rac.get(constituent) - target = lac.put(constituent) - shutil.copyfileobj(source, target) - target.close() - source.close() - if metadatas is not None: - for metadata in metadatas: - if not lac.has_artifact_metadata(constituent, metadata): - if rac.has_artifact_metadata(constituent, metadata): - src = rac.get_artifact_metadata(constituent, metadata) - dst = lac.put_artifact_metadata(constituent, metadata) - shutil.copyfileobj(src, dst) - dst.close() - src.close() - - -def get_chunk_files(f): # pragma: no cover - tar = tarfile.open(fileobj=f) - for member in tar.getmembers(): - if member.type is not tarfile.DIRTYPE: - yield member.name - tar.close() - - -def get_stratum_files(f, lac): # pragma: no cover - for ca in (ArtifactCacheReference(a) - for a in json.load(f, encoding='unicode-escape')): - cf = lac.get(ca) - for filename in get_chunk_files(cf): - yield filename - cf.close() - - -class BuilderBase(object): - - '''Base class for building artifacts.''' - - def __init__(self, app, staging_area, local_artifact_cache, - remote_artifact_cache, source, repo_cache, max_jobs, - setup_mounts): - self.app = app - self.staging_area = staging_area - self.local_artifact_cache = local_artifact_cache - self.remote_artifact_cache = remote_artifact_cache - self.source = source - self.repo_cache = repo_cache - self.max_jobs = max_jobs - self.build_watch = morphlib.stopwatch.Stopwatch() - self.setup_mounts = setup_mounts - - def save_build_times(self): - '''Write the times captured by the stopwatch''' - meta = { - 'build-times': {} - } - for stage in self.build_watch.ticks.iterkeys(): - meta['build-times'][stage] = { - 'start': '%s' % self.build_watch.start_time(stage), - 'stop': '%s' % self.build_watch.stop_time(stage), - 'delta': '%.4f' % self.build_watch.start_stop_seconds(stage) - } - - logging.debug('Writing metadata to the cache') - with self.local_artifact_cache.put_source_metadata( - self.source, self.source.cache_key, - 'meta') as f: - json.dump(meta, f, indent=4, sort_keys=True, - encoding='unicode-escape') - f.write('\n') - - def create_metadata(self, artifact_name, contents=[]): # pragma: no cover - '''Create metadata to artifact to allow it to be reproduced later. - - The metadata is represented as a dict, which later on will be - written out as a JSON file. - - ''' - - assert isinstance(self.source.repo, - morphlib.cachedrepo.CachedRepo) - meta = { - 'artifact-name': artifact_name, - 'source-name': self.source.name, - 'kind': self.source.morphology['kind'], - 'description': self.source.morphology['description'], - 'repo': self.source.repo.url, - 'repo-alias': self.source.repo_name, - 'original_ref': self.source.original_ref, - 'sha1': self.source.sha1, - 'morphology': self.source.filename, - 'cache-key': self.source.cache_key, - 'cache-id': self.source.cache_id, - 'morph-version': { - 'ref': morphlib.gitversion.ref, - 'tree': morphlib.gitversion.tree, - 'commit': morphlib.gitversion.commit, - 'version': morphlib.gitversion.version, - }, - 'contents': contents, - } - - return meta - - # Wrapper around open() to allow it to be overridden by unit tests. - def _open(self, filename, mode): # pragma: no cover - dirname = os.path.dirname(filename) - if not os.path.exists(dirname): - os.makedirs(dirname) - return open(filename, mode) - - def write_metadata(self, instdir, artifact_name, - contents=[]): # pragma: no cover - '''Write the metadata for an artifact. - - The file will be located under the ``baserock`` directory under - instdir, named after ``cache_key`` with ``.meta`` as the suffix. - It will be in JSON format. - - ''' - - meta = self.create_metadata(artifact_name, contents) - - basename = '%s.meta' % artifact_name - filename = os.path.join(instdir, 'baserock', basename) - - # Unit tests use StringIO, which in Python 2.6 isn't usable with - # the "with" statement. So we don't do it with "with". - f = self._open(filename, 'w') - json.dump(meta, f, indent=4, sort_keys=True, encoding='unicode-escape') - f.close() - - def runcmd(self, *args, **kwargs): - return self.staging_area.runcmd(*args, **kwargs) - -class ChunkBuilder(BuilderBase): - - '''Build chunk artifacts.''' - - def create_devices(self, destdir): # pragma: no cover - '''Creates device nodes if the morphology specifies them''' - morphology = self.source.morphology - perms_mask = stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO - if 'devices' in morphology and morphology['devices'] is not None: - for dev in morphology['devices']: - destfile = os.path.join(destdir, './' + dev['filename']) - mode = int(dev['permissions'], 8) & perms_mask - if dev['type'] == 'c': - mode = mode | stat.S_IFCHR - elif dev['type'] == 'b': - mode = mode | stat.S_IFBLK - else: - raise IOError('Cannot create device node %s,' - 'unrecognized device type "%s"' - % (destfile, dev['type'])) - self.app.status(msg="Creating device node %s" - % destfile) - os.mknod(destfile, mode, - os.makedev(dev['major'], dev['minor'])) - os.chown(destfile, dev['uid'], dev['gid']) - - def build_and_cache(self): # pragma: no cover - with self.build_watch('overall-build'): - - builddir, destdir = self.staging_area.chroot_open( - self.source, self.setup_mounts) - - stdout = (self.app.output - if self.app.settings['build-log-on-stdout'] else None) - - cache = self.local_artifact_cache - logpath = cache.get_source_metadata_filename( - self.source, self.source.cache_key, 'build-log') - - _, temppath = tempfile.mkstemp(dir=os.path.dirname(logpath)) - - try: - self.get_sources(builddir) - self.run_commands(builddir, destdir, temppath, stdout) - self.create_devices(destdir) - - os.rename(temppath, logpath) - except BaseException, e: - logging.error('Caught exception: %s' % str(e)) - logging.info('Cleaning up staging area') - self.staging_area.chroot_close() - if os.path.isfile(temppath): - with open(temppath) as f: - for line in f: - logging.error('OUTPUT FROM FAILED BUILD: %s' % - line.rstrip('\n')) - - os.rename(temppath, logpath) - else: - logging.error("Couldn't find build log at %s", temppath) - - self.staging_area.abort() - raise - - self.staging_area.chroot_close() - built_artifacts = self.assemble_chunk_artifacts(destdir) - - self.save_build_times() - return built_artifacts - - - def run_commands(self, builddir, destdir, - logfilepath, stdout=None): # pragma: no cover - m = self.source.morphology - bs = morphlib.buildsystem.lookup_build_system(m['build-system']) - - relative_builddir = self.staging_area.relative(builddir) - relative_destdir = self.staging_area.relative(destdir) - ccache_dir = self.staging_area.ccache_dir(self.source) - extra_env = { 'DESTDIR': relative_destdir } - - steps = [ - ('pre-configure', False), - ('configure', False), - ('post-configure', False), - ('pre-build', True), - ('build', True), - ('post-build', True), - ('pre-test', False), - ('test', False), - ('post-test', False), - ('pre-install', False), - ('install', False), - ('post-install', False), - ] - for step, in_parallel in steps: - with self.build_watch(step): - key = '%s-commands' % step - cmds = m[key] - if cmds: - with open(logfilepath, 'a') as log: - self.app.status(msg='Running %(key)s', key=key) - log.write('# %s\n' % step) - - for cmd in cmds: - if in_parallel: - max_jobs = self.source.morphology['max-jobs'] - if max_jobs is None: - max_jobs = self.max_jobs - extra_env['MAKEFLAGS'] = '-j%s' % max_jobs - else: - extra_env['MAKEFLAGS'] = '-j1' - - try: - with open(logfilepath, 'a') as log: - log.write('# # %s\n' % cmd) - - # flushing is needed because writes from python and - # writes from being the output in Popen have different - # buffers, but flush handles both - if stdout: - stdout.flush() - - self.runcmd(['sh', '-c', cmd], - extra_env=extra_env, - cwd=relative_builddir, - stdout=stdout or subprocess.PIPE, - stderr=subprocess.STDOUT, - logfile=logfilepath, - ccache_dir=ccache_dir) - - if stdout: - stdout.flush() - except cliapp.AppException, e: - if not stdout: - with open(logfilepath, 'r') as log: - self.app.output.write("%s failed\n" % step) - shutil.copyfileobj(log, self.app.output) - raise e - - def write_system_integration_commands(self, destdir, - integration_commands, artifact_name): # pragma: no cover - - rel_path = SYSTEM_INTEGRATION_PATH - dest_path = os.path.join(destdir, SYSTEM_INTEGRATION_PATH) - - scripts_created = [] - - if not os.path.exists(dest_path): - os.makedirs(dest_path) - - if artifact_name in integration_commands: - prefixes_per_artifact = integration_commands[artifact_name] - for prefix, commands in prefixes_per_artifact.iteritems(): - for index, script in enumerate(commands): - script_name = "%s-%s-%04d" % (prefix, - artifact_name, - index) - script_path = os.path.join(dest_path, script_name) - - with morphlib.savefile.SaveFile(script_path, 'w') as f: - f.write("#!/bin/sh\nset -xeu\n") - f.write(script) - os.chmod(script_path, 0555) - - rel_script_path = os.path.join(SYSTEM_INTEGRATION_PATH, - script_name) - scripts_created += [rel_script_path] - - return scripts_created - - def assemble_chunk_artifacts(self, destdir): # pragma: no cover - built_artifacts = [] - filenames = [] - source = self.source - split_rules = source.split_rules - morphology = source.morphology - sys_tag = 'system-integration' - - def filepaths(destdir): - for dirname, subdirs, basenames in os.walk(destdir): - subdirsymlinks = [os.path.join(dirname, x) for x in subdirs - if os.path.islink(os.path.join(dirname, x))] - filenames = [os.path.join(dirname, x) for x in basenames] - for relpath in (os.path.relpath(x, destdir) for x in - [dirname] + subdirsymlinks + filenames): - yield relpath - - with self.build_watch('determine-splits'): - matches, overlaps, unmatched = \ - split_rules.partition(filepaths(destdir)) - - system_integration = morphology.get(sys_tag) or {} - - with self.build_watch('create-chunks'): - for chunk_artifact_name, chunk_artifact \ - in source.artifacts.iteritems(): - file_paths = matches[chunk_artifact_name] - chunk_artifact = source.artifacts[chunk_artifact_name] - - def all_parents(path): - while path != '': - yield path - path = os.path.dirname(path) - - def parentify(filenames): - names = set() - for name in filenames: - names.update(all_parents(name)) - return sorted(names) - - extra_files = self.write_system_integration_commands( - destdir, system_integration, - chunk_artifact_name) - extra_files += ['baserock/%s.meta' % chunk_artifact_name] - parented_paths = parentify(file_paths + extra_files) - - with self.local_artifact_cache.put(chunk_artifact) as f: - self.write_metadata(destdir, chunk_artifact_name, - parented_paths) - - self.app.status(msg='Creating chunk artifact %(name)s', - name=chunk_artifact_name) - morphlib.bins.create_chunk(destdir, f, parented_paths) - built_artifacts.append(chunk_artifact) - - for dirname, subdirs, files in os.walk(destdir): - if files: - raise Exception('DESTDIR %s is not empty: %s' % - (destdir, files)) - return built_artifacts - - def get_sources(self, srcdir): # pragma: no cover - s = self.source - extract_sources(self.app, self.repo_cache, s.repo, s.sha1, srcdir) - - -class StratumBuilder(BuilderBase): - '''Build stratum artifacts.''' - - def is_constituent(self, artifact): # pragma: no cover - '''True if artifact should be included in the stratum artifact''' - return (artifact.source.morphology['kind'] == 'chunk' and \ - artifact.source.build_mode != 'bootstrap') - - def build_and_cache(self): # pragma: no cover - with self.build_watch('overall-build'): - constituents = [d for d in self.source.dependencies - if self.is_constituent(d)] - - # the only reason the StratumBuilder has to download chunks is to - # check for overlap now that strata are lists of chunks - with self.build_watch('check-chunks'): - for a_name, a in self.source.artifacts.iteritems(): - # download the chunk artifact if necessary - download_depends(constituents, - self.local_artifact_cache, - self.remote_artifact_cache) - - with self.build_watch('create-chunk-list'): - lac = self.local_artifact_cache - for a_name, a in self.source.artifacts.iteritems(): - meta = self.create_metadata( - a_name, - [x.name for x in constituents]) - with lac.put_artifact_metadata(a, 'meta') as f: - json.dump(meta, f, indent=4, sort_keys=True) - with self.local_artifact_cache.put(a) as f: - json.dump([c.basename() for c in constituents], f) - self.save_build_times() - return self.source.artifacts.values() - - -class SystemBuilder(BuilderBase): # pragma: no cover - - '''Build system image artifacts.''' - - def __init__(self, *args, **kwargs): - BuilderBase.__init__(self, *args, **kwargs) - self.args = args - self.kwargs = kwargs - - def build_and_cache(self): - self.app.status(msg='Building system %(system_name)s', - system_name=self.source.name) - - with self.build_watch('overall-build'): - arch = self.source.morphology['arch'] - - for a_name, artifact in self.source.artifacts.iteritems(): - handle = self.local_artifact_cache.put(artifact) - - try: - fs_root = self.staging_area.destdir(self.source) - self.unpack_strata(fs_root) - self.write_metadata(fs_root, a_name) - self.run_system_integration_commands(fs_root) - unslashy_root = fs_root[1:] - def uproot_info(info): - info.name = relpath(info.name, unslashy_root) - if info.islnk(): - info.linkname = relpath(info.linkname, - unslashy_root) - return info - tar = tarfile.open(fileobj=handle, mode="w", name=a_name) - self.app.status(msg='Constructing tarball of rootfs', - chatty=True) - tar.add(fs_root, recursive=True, filter=uproot_info) - tar.close() - except BaseException as e: - logging.error(traceback.format_exc()) - self.app.status(msg='Error while building system', - error=True) - handle.abort() - raise - else: - handle.close() - - self.save_build_times() - return self.source.artifacts.itervalues() - - def unpack_one_stratum(self, stratum_artifact, target): - '''Unpack a single stratum into a target directory''' - - cache = self.local_artifact_cache - with cache.get(stratum_artifact) as stratum_file: - artifact_list = json.load(stratum_file, encoding='unicode-escape') - for chunk in (ArtifactCacheReference(a) for a in artifact_list): - self.app.status(msg='Unpacking chunk %(basename)s', - basename=chunk.basename(), chatty=True) - with cache.get(chunk) as chunk_file: - morphlib.bins.unpack_binary_from_file(chunk_file, target) - - target_metadata = os.path.join( - target, 'baserock', '%s.meta' % stratum_artifact.name) - with cache.get_artifact_metadata(stratum_artifact, 'meta') as meta_src: - with morphlib.savefile.SaveFile(target_metadata, 'w') as meta_dst: - shutil.copyfileobj(meta_src, meta_dst) - - def unpack_strata(self, path): - '''Unpack strata into a directory.''' - - self.app.status(msg='Unpacking strata to %(path)s', - path=path, chatty=True) - with self.build_watch('unpack-strata'): - for a_name, a in self.source.artifacts.iteritems(): - # download the stratum artifacts if necessary - download_depends(self.source.dependencies, - self.local_artifact_cache, - self.remote_artifact_cache, - ('meta',)) - - # download the chunk artifacts if necessary - for stratum_artifact in self.source.dependencies: - f = self.local_artifact_cache.get(stratum_artifact) - chunks = [ArtifactCacheReference(c) for c in json.load(f)] - download_depends(chunks, - self.local_artifact_cache, - self.remote_artifact_cache) - f.close() - - # unpack it from the local artifact cache - for stratum_artifact in self.source.dependencies: - self.unpack_one_stratum(stratum_artifact, path) - - ldconfig(self.app.runcmd, path) - - def write_metadata(self, instdir, artifact_name): - BuilderBase.write_metadata(self, instdir, artifact_name) - - os_release_file = os.path.join(instdir, 'etc', 'os-release') - dirname = os.path.dirname(os_release_file) - if not os.path.exists(dirname): - os.makedirs(dirname) - with morphlib.savefile.SaveFile(os_release_file, 'w') as f: - f.write('NAME="Baserock"\n') - f.write('ID=baserock\n') - f.write('HOME_URL="http://wiki.baserock.org"\n') - f.write('SUPPORT_URL="http://wiki.baserock.org/mailinglist"\n') - f.write('BUG_REPORT_URL="http://wiki.baserock.org/mailinglist"\n') - - os.chmod(os_release_file, 0644) - - def run_system_integration_commands(self, rootdir): # pragma: no cover - ''' Run the system integration commands ''' - - sys_integration_dir = os.path.join(rootdir, SYSTEM_INTEGRATION_PATH) - if not os.path.isdir(sys_integration_dir): - return - - env = { - 'PATH': '/bin:/usr/bin:/sbin:/usr/sbin' - } - - self.app.status(msg='Running the system integration commands') - - to_mount = ( - ('dev/shm', 'tmpfs', 'none'), - ('tmp', 'tmpfs', 'none'), - ) - try: - for bin in sorted(os.listdir(sys_integration_dir)): - argv = [os.path.join(SYSTEM_INTEGRATION_PATH, bin)] - container_config = dict( - root=rootdir, mounts=to_mount, mount_proc=True) - cmdline = morphlib.util.containerised_cmdline( - argv, **container_config) - exit, out, err = self.app.runcmd_unchecked( - cmdline, env=env) - if exit != 0: - logging.debug('Command returned code %i', exit) - msg = error_message_for_containerised_commandline( - argv, err, container_config) - raise cliapp.AppException(msg) - except BaseException, e: - self.app.status( - msg='Error while running system integration commands', - error=True) - raise - - -class Builder(object): # pragma: no cover - - '''Helper class to build with the right BuilderBase subclass.''' - - classes = { - 'chunk': ChunkBuilder, - 'stratum': StratumBuilder, - 'system': SystemBuilder, - } - - def __init__(self, app, staging_area, local_artifact_cache, - remote_artifact_cache, repo_cache, max_jobs, setup_mounts): - self.app = app - self.staging_area = staging_area - self.local_artifact_cache = local_artifact_cache - self.remote_artifact_cache = remote_artifact_cache - self.repo_cache = repo_cache - self.max_jobs = max_jobs - self.setup_mounts = setup_mounts - - def build_and_cache(self, source): - kind = source.morphology['kind'] - o = self.classes[kind](self.app, self.staging_area, - self.local_artifact_cache, - self.remote_artifact_cache, source, - self.repo_cache, self.max_jobs, - self.setup_mounts) - self.app.status(msg='Builder.build: artifact %s with %s' % - (source.name, repr(o)), - chatty=True) - built_artifacts = o.build_and_cache() - self.app.status(msg='Builder.build: done', - chatty=True) - return built_artifacts diff --git a/morphlib/builder2_tests.py b/morphlib/builder2_tests.py deleted file mode 100644 index 4fd0807a..00000000 --- a/morphlib/builder2_tests.py +++ /dev/null @@ -1,221 +0,0 @@ -# Copyright (C) 2012-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import json -import os -import StringIO -import unittest - -import morphlib - - -class FakeBuildSystem(object): - - def __init__(self): - self.build_commands = ['buildsys-it'] - - -class FakeApp(object): - def __init__(self, runcmd=None): - self.runcmd = runcmd - - -class FakeStagingArea(object): - - def __init__(self, runcmd, build_env): - self.runcmd = runcmd - self.env = build_env.env - - -class FakeSource(object): - - def __init__(self): - self.morphology = { - 'name': 'a', - 'kind': 'b', - 'description': 'c', - } - self.name = 'a' - - self.repo = morphlib.cachedrepo.CachedRepo(FakeApp(), 'repo', - 'url', 'path') - self.repo_name = 'url' - self.original_ref = 'e' - self.sha1 = 'f' - self.filename = 'g' - - -class FakeArtifact(object): - - def __init__(self, name): - self.name = name - self.source = FakeSource() - self.cache_key = 'blahblah' - self.cache_id = {} - - -class FakeBuildEnv(object): - - def __init__(self): - self.arch = 'le-arch' - self.env = { - 'PATH': '/le-bin:/le-bon:/le-bin-bon', - } - - -class FakeFileHandle(object): - - def __init__(self, cache, key): - self._string = "" - self._cache = cache - self._key = key - - def __enter__(self): - return self - - def _writeback(self): - self._cache._cached[self._key] = self._string - - def __exit__(self, type, value, traceback): - self._writeback() - - def close(self): - self._writeback() - - def write(self, string): - self._string += string - - -class FakeArtifactCache(object): - - def __init__(self): - self._cached = {} - - def put(self, artifact): - return FakeFileHandle(self, (artifact.cache_key, artifact.name)) - - def put_artifact_metadata(self, artifact, name): - return FakeFileHandle(self, (artifact.cache_key, artifact.name, name)) - - def put_source_metadata(self, source, cachekey, name): - return FakeFileHandle(self, (cachekey, name)) - - def get(self, artifact): - return StringIO.StringIO( - self._cached[(artifact.cache_key, artifact.name)]) - - def get_artifact_metadata(self, artifact, name): - return StringIO.StringIO( - self._cached[(artifact.cache_key, artifact.name, name)]) - - def get_source_metadata(self, source, cachekey, name): - return StringIO.StringIO(self._cached[(cachekey, name)]) - - def has(self, artifact): - return (artifact.cache_key, artifact.name) in self._cached - - def has_artifact_metadata(self, artifact, name): - return (artifact.cache_key, artifact.name, name) in self._cached - - def has_source_metadata(self, source, cachekey, name): - return (cachekey, name) in self._cached - - -class BuilderBaseTests(unittest.TestCase): - - def fake_runcmd(self, argv, *args, **kwargs): - self.commands_run.append(argv) - - def fake_open(self, filename, mode): - self.open_filename = filename - self.open_handle = StringIO.StringIO() - self.open_handle.close = lambda: None - return self.open_handle - - def setUp(self): - self.commands_run = [] - self.app = FakeApp(self.fake_runcmd) - self.staging_area = FakeStagingArea(self.fake_runcmd, FakeBuildEnv()) - self.artifact_cache = FakeArtifactCache() - self.artifact = FakeArtifact('le-artifact') - self.repo_cache = None - self.build_env = FakeBuildEnv() - self.max_jobs = 1 - self.builder = morphlib.builder2.BuilderBase(self.app, - self.staging_area, - self.artifact_cache, - None, - self.artifact, - self.repo_cache, - self.max_jobs, - False) - - def test_runs_desired_command(self): - self.builder.runcmd(['foo', 'bar']) - self.assertEqual(self.commands_run, [['foo', 'bar']]) - - def test_writes_build_times(self): - with self.builder.build_watch('nothing'): - pass - self.builder.save_build_times() - self.assertTrue(self.artifact_cache.has_source_metadata( - self.artifact.source, self.artifact.cache_key, 'meta')) - - def test_watched_events_in_cache(self): - events = ["configure", "build", "install"] - for event in events: - with self.builder.build_watch(event): - pass - self.builder.save_build_times() - meta = json.load(self.artifact_cache.get_source_metadata( - self.artifact.source, self.artifact.cache_key, 'meta')) - self.assertEqual(sorted(events), - sorted(meta['build-times'].keys())) - - def test_downloads_depends(self): - lac = FakeArtifactCache() - rac = FakeArtifactCache() - afacts = [FakeArtifact(name) for name in ('a', 'b', 'c')] - for a in afacts: - fh = rac.put(a) - fh.write(a.name) - fh.close() - morphlib.builder2.download_depends(afacts, lac, rac) - self.assertTrue(all(lac.has(a) for a in afacts)) - - def test_downloads_depends_metadata(self): - lac = FakeArtifactCache() - rac = FakeArtifactCache() - afacts = [FakeArtifact(name) for name in ('a', 'b', 'c')] - for a in afacts: - fh = rac.put(a) - fh.write(a.name) - fh.close() - fh = rac.put_artifact_metadata(a, 'meta') - fh.write('metadata') - fh.close() - morphlib.builder2.download_depends(afacts, lac, rac, ('meta',)) - self.assertTrue(all(lac.has(a) for a in afacts)) - self.assertTrue(all(lac.has_artifact_metadata(a, 'meta') - for a in afacts)) - - -class ChunkBuilderTests(unittest.TestCase): - - def setUp(self): - self.app = FakeApp() - self.build = morphlib.builder2.ChunkBuilder(self.app, None, None, - None, None, None, 1, False) diff --git a/morphlib/builder_tests.py b/morphlib/builder_tests.py new file mode 100644 index 00000000..2087c942 --- /dev/null +++ b/morphlib/builder_tests.py @@ -0,0 +1,221 @@ +# Copyright (C) 2012-2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import json +import os +import StringIO +import unittest + +import morphlib + + +class FakeBuildSystem(object): + + def __init__(self): + self.build_commands = ['buildsys-it'] + + +class FakeApp(object): + def __init__(self, runcmd=None): + self.runcmd = runcmd + + +class FakeStagingArea(object): + + def __init__(self, runcmd, build_env): + self.runcmd = runcmd + self.env = build_env.env + + +class FakeSource(object): + + def __init__(self): + self.morphology = { + 'name': 'a', + 'kind': 'b', + 'description': 'c', + } + self.name = 'a' + + self.repo = morphlib.cachedrepo.CachedRepo(FakeApp(), 'repo', + 'url', 'path') + self.repo_name = 'url' + self.original_ref = 'e' + self.sha1 = 'f' + self.filename = 'g' + + +class FakeArtifact(object): + + def __init__(self, name): + self.name = name + self.source = FakeSource() + self.cache_key = 'blahblah' + self.cache_id = {} + + +class FakeBuildEnv(object): + + def __init__(self): + self.arch = 'le-arch' + self.env = { + 'PATH': '/le-bin:/le-bon:/le-bin-bon', + } + + +class FakeFileHandle(object): + + def __init__(self, cache, key): + self._string = "" + self._cache = cache + self._key = key + + def __enter__(self): + return self + + def _writeback(self): + self._cache._cached[self._key] = self._string + + def __exit__(self, type, value, traceback): + self._writeback() + + def close(self): + self._writeback() + + def write(self, string): + self._string += string + + +class FakeArtifactCache(object): + + def __init__(self): + self._cached = {} + + def put(self, artifact): + return FakeFileHandle(self, (artifact.cache_key, artifact.name)) + + def put_artifact_metadata(self, artifact, name): + return FakeFileHandle(self, (artifact.cache_key, artifact.name, name)) + + def put_source_metadata(self, source, cachekey, name): + return FakeFileHandle(self, (cachekey, name)) + + def get(self, artifact): + return StringIO.StringIO( + self._cached[(artifact.cache_key, artifact.name)]) + + def get_artifact_metadata(self, artifact, name): + return StringIO.StringIO( + self._cached[(artifact.cache_key, artifact.name, name)]) + + def get_source_metadata(self, source, cachekey, name): + return StringIO.StringIO(self._cached[(cachekey, name)]) + + def has(self, artifact): + return (artifact.cache_key, artifact.name) in self._cached + + def has_artifact_metadata(self, artifact, name): + return (artifact.cache_key, artifact.name, name) in self._cached + + def has_source_metadata(self, source, cachekey, name): + return (cachekey, name) in self._cached + + +class BuilderBaseTests(unittest.TestCase): + + def fake_runcmd(self, argv, *args, **kwargs): + self.commands_run.append(argv) + + def fake_open(self, filename, mode): + self.open_filename = filename + self.open_handle = StringIO.StringIO() + self.open_handle.close = lambda: None + return self.open_handle + + def setUp(self): + self.commands_run = [] + self.app = FakeApp(self.fake_runcmd) + self.staging_area = FakeStagingArea(self.fake_runcmd, FakeBuildEnv()) + self.artifact_cache = FakeArtifactCache() + self.artifact = FakeArtifact('le-artifact') + self.repo_cache = None + self.build_env = FakeBuildEnv() + self.max_jobs = 1 + self.builder = morphlib.builder.BuilderBase(self.app, + self.staging_area, + self.artifact_cache, + None, + self.artifact, + self.repo_cache, + self.max_jobs, + False) + + def test_runs_desired_command(self): + self.builder.runcmd(['foo', 'bar']) + self.assertEqual(self.commands_run, [['foo', 'bar']]) + + def test_writes_build_times(self): + with self.builder.build_watch('nothing'): + pass + self.builder.save_build_times() + self.assertTrue(self.artifact_cache.has_source_metadata( + self.artifact.source, self.artifact.cache_key, 'meta')) + + def test_watched_events_in_cache(self): + events = ["configure", "build", "install"] + for event in events: + with self.builder.build_watch(event): + pass + self.builder.save_build_times() + meta = json.load(self.artifact_cache.get_source_metadata( + self.artifact.source, self.artifact.cache_key, 'meta')) + self.assertEqual(sorted(events), + sorted(meta['build-times'].keys())) + + def test_downloads_depends(self): + lac = FakeArtifactCache() + rac = FakeArtifactCache() + afacts = [FakeArtifact(name) for name in ('a', 'b', 'c')] + for a in afacts: + fh = rac.put(a) + fh.write(a.name) + fh.close() + morphlib.builder.download_depends(afacts, lac, rac) + self.assertTrue(all(lac.has(a) for a in afacts)) + + def test_downloads_depends_metadata(self): + lac = FakeArtifactCache() + rac = FakeArtifactCache() + afacts = [FakeArtifact(name) for name in ('a', 'b', 'c')] + for a in afacts: + fh = rac.put(a) + fh.write(a.name) + fh.close() + fh = rac.put_artifact_metadata(a, 'meta') + fh.write('metadata') + fh.close() + morphlib.builder.download_depends(afacts, lac, rac, ('meta',)) + self.assertTrue(all(lac.has(a) for a in afacts)) + self.assertTrue(all(lac.has_artifact_metadata(a, 'meta') + for a in afacts)) + + +class ChunkBuilderTests(unittest.TestCase): + + def setUp(self): + self.app = FakeApp() + self.build = morphlib.builder.ChunkBuilder(self.app, None, None, + None, None, None, 1, False) diff --git a/morphlib/plugins/cross-bootstrap_plugin.py b/morphlib/plugins/cross-bootstrap_plugin.py index 7b53a4a5..2b5c8911 100644 --- a/morphlib/plugins/cross-bootstrap_plugin.py +++ b/morphlib/plugins/cross-bootstrap_plugin.py @@ -46,7 +46,7 @@ def escape_source_name(source): # Most of this is ripped from RootfsTarballBuilder, and should be reconciled # with it. -class BootstrapSystemBuilder(morphlib.builder2.BuilderBase): +class BootstrapSystemBuilder(morphlib.builder.BuilderBase): '''Build a bootstrap system tarball The bootstrap system image contains a minimal cross-compiled toolchain @@ -104,7 +104,7 @@ class BootstrapSystemBuilder(morphlib.builder2.BuilderBase): source_dir = os.path.join(path, 'src', escaped_source) if not os.path.exists(source_dir): os.makedirs(source_dir) - morphlib.builder2.extract_sources( + morphlib.builder.extract_sources( self.app, self.repo_cache, s.repo, s.sha1, source_dir) name = s.name -- cgit v1.2.1