diff options
| -rw-r--r-- | morphlib/__init__.py | 3 | ||||
| -rw-r--r-- | morphlib/buildcommand.py | 31 | ||||
| -rw-r--r-- | morphlib/builder2.py | 117 | ||||
| -rw-r--r-- | morphlib/builder2_tests.py | 18 | ||||
| -rw-r--r-- | morphlib/git.py | 2 | ||||
| -rw-r--r-- | morphlib/morph2.py | 9 | ||||
| -rw-r--r-- | morphlib/morph2_tests.py | 35 | ||||
| -rw-r--r-- | morphlib/morphologyfactory.py | 7 | ||||
| -rw-r--r-- | morphlib/plugins/cross-bootstrap_plugin.py | 311 | ||||
| -rw-r--r-- | morphlib/plugins/deploy_plugin.py | 3 | ||||
| -rw-r--r-- | morphlib/plugins/graphing_plugin.py | 6 | ||||
| -rw-r--r-- | morphlib/plugins/show_dependencies_plugin.py | 5 | ||||
| -rwxr-xr-x | tests.build/bootstrap-mode.script | 119 | ||||
| -rw-r--r-- | tests.build/cross-bootstrap-only-to-supported-archs.exit | 1 | ||||
| -rwxr-xr-x | tests.build/cross-bootstrap-only-to-supported-archs.script | 25 | ||||
| -rw-r--r-- | tests.build/cross-bootstrap-only-to-supported-archs.stderr | 1 | ||||
| -rwxr-xr-x | tests.build/cross-bootstrap.script | 27 | ||||
| -rwxr-xr-x | tests.build/setup-build-essential | 137 | ||||
| -rw-r--r-- | without-test-modules | 1 |
19 files changed, 641 insertions, 217 deletions
diff --git a/morphlib/__init__.py b/morphlib/__init__.py index b20c3f01..1ebf972a 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -37,6 +37,9 @@ import gitversion __version__ = gitversion.version +# List of architectures that Morph supports +valid_archs = ['armv7l', 'armv7b', 'x86_32', 'x86_64'] + class Error(cliapp.AppException): '''Base for all morph exceptions that cause user-visible messages.''' diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index bfc64edc..fe6e0721 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -32,7 +32,7 @@ class BuildCommand(object): ''' - def __init__(self, app): + def __init__(self, app, build_env = None): self.supports_local_build = True self.app = app @@ -47,8 +47,10 @@ class BuildCommand(object): for repo_name, ref, filename in self.app.itertriplets(args): self.app.status(msg='Building %(repo_name)s %(ref)s %(filename)s', repo_name=repo_name, ref=ref, filename=filename) - artifact = self.get_artifact_object(repo_name, ref, filename) - self.build_in_order(artifact) + self.app.status(msg='Figuring out the right build order') + srcpool = self.create_source_pool(repo_name, ref, filename) + root_artifact = self.resolve_artifacts(srcpool) + self.build_in_order(root_artifact) self.app.status(msg='Build ends successfully') @@ -68,11 +70,14 @@ class BuildCommand(object): return morphlib.buildenvironment.BuildEnvironment(self.app.settings, arch) - def get_artifact_object(self, repo_name, ref, filename): - '''Create an Artifact object representing the given triplet.''' - - self.app.status(msg='Figuring out the right build order') + def create_source_pool(self, repo_name, ref, filename): + '''Find the source objects required for building a the given artifact + + The SourcePool will contain every stratum and chunk dependency of the + given artifact (which must be a system) but will not take into account + any Git submodules which are required in the build. + ''' self.app.status(msg='Creating source pool', chatty=True) srcpool = self.app.create_source_pool( self.lrc, self.rrc, (repo_name, ref, filename)) @@ -81,6 +86,11 @@ class BuildCommand(object): msg='Validating cross-morphology references', chatty=True) self._validate_cross_morphology_references(srcpool) + return srcpool + + def resolve_artifacts(self, srcpool): + '''Resolve the artifacts that will be built for a set of sources''' + self.app.status(msg='Creating artifact resolver', chatty=True) ar = morphlib.artifactresolver.ArtifactResolver() @@ -241,6 +251,7 @@ class BuildCommand(object): deps = self.get_recursive_deps(artifact) self.cache_artifacts_locally(deps) + use_chroot = False setup_mounts = False if artifact.source.morphology['kind'] == 'chunk': build_mode = artifact.source.build_mode @@ -255,7 +266,10 @@ class BuildCommand(object): (build_mode, artifact.name)) build_mode = 'staging' - use_chroot = build_mode=='staging' + if build_mode == 'staging': + use_chroot = True + setup_mounts = True + staging_area = self.create_staging_area(build_env, use_chroot, extra_env=extra_env, @@ -404,7 +418,6 @@ class BuildCommand(object): self.app.status(msg='Starting actual build: %(name)s ' '%(sha1)s', name=artifact.name, sha1=artifact.source.sha1[:7]) - setup_mounts = self.app.settings['staging-chroot'] builder = morphlib.builder2.Builder( self.app, staging_area, self.lac, self.rac, self.lrc, self.app.settings['max-jobs'], setup_mounts) diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 82aaab00..62fd5480 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -14,6 +14,7 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +from collections import defaultdict import datetime import errno import json @@ -22,9 +23,8 @@ import os from os.path import relpath import shutil import stat -import time -from collections import defaultdict import tarfile +import time import traceback import subprocess import tempfile @@ -36,6 +36,54 @@ import morphlib from morphlib.artifactcachereference import ArtifactCacheReference import morphlib.gitversion +def extract_sources(app, repo_cache, repo, sha1, srcdir): #pragma: no cover + '''Get sources from git to a source directory, including submodules''' + + def extract_repo(repo, sha1, destdir): + app.status(msg='Extracting %(source)s into %(target)s', + source=repo.original_name, + target=destdir) + + repo.checkout(sha1, destdir) + morphlib.git.reset_workdir(app.runcmd, destdir) + submodules = morphlib.git.Submodules(app, repo.path, sha1) + try: + submodules.load() + except morphlib.git.NoModulesFileError: + return [] + else: + tuples = [] + for sub in submodules: + cached_repo = repo_cache.get_repo(sub.url) + sub_dir = os.path.join(destdir, sub.path) + tuples.append((cached_repo, sub.commit, sub_dir)) + return tuples + + todo = [(repo, sha1, srcdir)] + while todo: + repo, sha1, srcdir = todo.pop() + todo += extract_repo(repo, sha1, srcdir) + set_mtime_recursively(srcdir) + +def set_mtime_recursively(root): # pragma: no cover + '''Set the mtime for every file in a directory tree to the same. + + We do this because git checkout does not set the mtime to anything, + and some projects (binutils, gperf for example) include formatted + documentation and try to randomly build things or not because of + the timestamps. This should help us get more reliable builds. + + ''' + + now = time.time() + for dirname, subdirs, basenames in os.walk(root.encode("utf-8"), + topdown=False): + for basename in basenames: + pathname = os.path.join(dirname, basename) + # we need the following check to ignore broken symlinks + if os.path.exists(pathname): + os.utime(pathname, (now, now)) + os.utime(dirname, (now, now)) def ldconfig(runcmd, rootdir): # pragma: no cover '''Run ldconfig for the filesystem below ``rootdir``. @@ -256,19 +304,10 @@ class BuilderBase(object): def runcmd(self, *args, **kwargs): return self.staging_area.runcmd(*args, **kwargs) - class ChunkBuilder(BuilderBase): '''Build chunk artifacts.''' - def get_commands(self, which, morphology, build_system): - '''Return the commands to run from a morphology or the build system.''' - if morphology[which] is None: - attr = '_'.join(which.split('-')) - return getattr(build_system, attr) - else: - return morphology[which] - def create_devices(self, destdir): # pragma: no cover '''Creates device nodes if the morphology specifies them''' morphology = self.artifact.source.morphology @@ -326,57 +365,6 @@ class ChunkBuilder(BuilderBase): self.save_build_times() return built_artifacts - def get_sources(self, srcdir): # pragma: no cover - '''Get sources from git to a source directory, for building.''' - - cache_dir = os.path.dirname(self.artifact.source.repo.path) - - def extract_repo(repo, sha1, destdir): - self.app.status(msg='Extracting %(source)s into %(target)s', - source=repo.original_name, - target=destdir) - - repo.checkout(sha1, destdir) - morphlib.git.reset_workdir(self.app.runcmd, destdir) - submodules = morphlib.git.Submodules(self.app, repo.path, sha1) - try: - submodules.load() - except morphlib.git.NoModulesFileError: - return [] - else: - tuples = [] - for sub in submodules: - cached_repo = self.repo_cache.get_repo(sub.url) - sub_dir = os.path.join(destdir, sub.path) - tuples.append((cached_repo, sub.commit, sub_dir)) - return tuples - - s = self.artifact.source - todo = [(s.repo, s.sha1, srcdir)] - while todo: - repo, sha1, srcdir = todo.pop() - todo += extract_repo(repo, sha1, srcdir) - self.set_mtime_recursively(srcdir) - - def set_mtime_recursively(self, root): # pragma: no cover - '''Set the mtime for every file in a directory tree to the same. - - We do this because git checkout does not set the mtime to anything, - and some projects (binutils, gperf for example) include formatted - documentation and try to randomly build things or not because of - the timestamps. This should help us get more reliable builds. - - ''' - - now = time.time() - for dirname, subdirs, basenames in os.walk(root.encode("utf-8"), - topdown=False): - for basename in basenames: - pathname = os.path.join(dirname, basename) - # we need the following check to ignore broken symlinks - if os.path.exists(pathname): - os.utime(pathname, (now, now)) - os.utime(dirname, (now, now)) def run_commands(self, builddir, destdir, logfile): # pragma: no cover m = self.artifact.source.morphology @@ -403,7 +391,7 @@ class ChunkBuilder(BuilderBase): for step, in_parallel in steps: with self.build_watch(step): key = '%s-commands' % step - cmds = self.get_commands(key, m, bs) + cmds = m.get_commands(key) if cmds: self.app.status(msg='Running %(key)s', key=key) logfile.write('# %s\n' % step) @@ -467,6 +455,9 @@ class ChunkBuilder(BuilderBase): (destdir, files)) return built_artifacts + def get_sources(self, srcdir): # pragma: no cover + s = self.artifact.source + extract_sources(self.app, self.repo_cache, s.repo, s.sha1, srcdir) class StratumBuilder(BuilderBase): diff --git a/morphlib/builder2_tests.py b/morphlib/builder2_tests.py index c0da3cd9..d0d56b17 100644 --- a/morphlib/builder2_tests.py +++ b/morphlib/builder2_tests.py @@ -253,21 +253,3 @@ class ChunkBuilderTests(unittest.TestCase): self.app = FakeApp() self.build = morphlib.builder2.ChunkBuilder(self.app, None, None, None, None, None, 1, False) - - def test_uses_morphology_commands_when_given(self): - m = {'build-commands': ['build-it']} - bs = FakeBuildSystem() - cmds = self.build.get_commands('build-commands', m, bs) - self.assertEqual(cmds, ['build-it']) - - def test_uses_build_system_commands_when_morphology_doesnt(self): - m = {'build-commands': None} - bs = FakeBuildSystem() - cmds = self.build.get_commands('build-commands', m, bs) - self.assertEqual(cmds, ['buildsys-it']) - - def test_uses_morphology_commands_when_morphology_has_empty_list(self): - m = {'build-commands': []} - bs = FakeBuildSystem() - cmds = self.build.get_commands('build-commands', m, bs) - self.assertEqual(cmds, []) diff --git a/morphlib/git.py b/morphlib/git.py index 88c6dae3..ff5e5c7d 100644 --- a/morphlib/git.py +++ b/morphlib/git.py @@ -21,6 +21,8 @@ import logging import os import re import StringIO +import time + import cliapp diff --git a/morphlib/morph2.py b/morphlib/morph2.py index a8e1d7d3..d949c696 100644 --- a/morphlib/morph2.py +++ b/morphlib/morph2.py @@ -85,6 +85,15 @@ class Morphology(object): def __contains__(self, key): return key in self._dict + def get_commands(self, which): + '''Return the commands to run from a morphology or the build system''' + if self[which] is None: + attr = '_'.join(which.split('-')) + bs = morphlib.buildsystem.lookup_build_system(self['build-system']) + return getattr(bs, attr) + else: + return self[which] + def keys(self): return self._dict.keys() diff --git a/morphlib/morph2_tests.py b/morphlib/morph2_tests.py index 1a0b4822..9de23e56 100644 --- a/morphlib/morph2_tests.py +++ b/morphlib/morph2_tests.py @@ -344,3 +344,38 @@ class MorphologyTests(unittest.TestCase): dummy = Morphology(self.stratum_text) output_dict = dummy._apply_changes(live_dict, original_dict) self.assertEqual(original_dict, output_dict) + + def test_uses_morphology_commands_when_given(self): + m = Morphology(''' + { + 'name': 'foo', + 'kind': 'chunk', + 'build-system': 'dummy', + 'build-commands': ['build-it'] + } + ''') + cmds = m.get_commands('build-commands') + self.assertEqual(cmds, ['build-it']) + + def test_uses_build_system_commands_when_morphology_doesnt(self): + m = Morphology(''' + { + 'name': 'foo', + 'kind': 'chunk', + 'build-system': 'dummy', + } + ''') + cmds = m.get_commands('build-commands') + self.assertEqual(cmds, ['echo dummy build']) + + def test_uses_morphology_commands_when_morphology_has_empty_list(self): + m = Morphology(''' + { + 'name': 'foo', + 'kind': 'chunk', + 'build-system': 'dummy', + 'build-commands': [] + } + ''') + cmds = m.get_commands('build-commands') + self.assertEqual(cmds, []) diff --git a/morphlib/morphologyfactory.py b/morphlib/morphologyfactory.py index 7f5cd94a..d0d9ea04 100644 --- a/morphlib/morphologyfactory.py +++ b/morphlib/morphologyfactory.py @@ -106,15 +106,14 @@ class MorphologyFactory(object): '(arch is a mandatory field)' % filename) - valid_archs = ['armv7l', 'armv7b', 'x86_32', 'x86_64'] - if morphology['arch'] == 'armv7': morphology._dict['arch'] = 'armv7l' - if morphology['arch'] not in valid_archs: + if morphology['arch'] not in morphlib.valid_archs: raise morphlib.Error('Unknown arch %s. This version of Morph ' 'supports the following architectures: %s' % - (morphology['arch'], ', '.join(valid_archs))) + (morphology['arch'], + ', '.join(morphlib.valid_archs))) kind = morphology['system-kind'] if kind == 'rootfs-tarball': # pragma: no cover diff --git a/morphlib/plugins/cross-bootstrap_plugin.py b/morphlib/plugins/cross-bootstrap_plugin.py new file mode 100644 index 00000000..8593b076 --- /dev/null +++ b/morphlib/plugins/cross-bootstrap_plugin.py @@ -0,0 +1,311 @@ +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import cliapp +import logging +import os.path +import re +import tarfile +import traceback + +import morphlib + +driver_header = '''#!/bin/sh +echo "Morph native bootstrap script" +echo "Generated by Morph version %s\n" + +set -eu + +export PATH=/tools/bin:$PATH +export SRCDIR=/src + +''' % morphlib.__version__ + +driver_footer = ''' + +echo "Complete!" +echo "Bootstrapped system rootfs is in $DESTDIR." +''' + +def escape_source_name(source): + repo_name = source.repo.original_name + ref = source.original_ref + source_name = '%s__%s' % (repo_name, ref) + return re.sub(':/', '_', source_name) + +# Most of this is ripped from RootfsTarballBuilder, and should be reconciled +# with it. +class BootstrapSystemBuilder(morphlib.builder2.BuilderBase): + '''Build a bootstrap system tarball + + The bootstrap system image contains a minimal cross-compiled toolchain + and a set of extracted sources for the rest of the system, with shell + scripts to run the required morphology commands. This allows new + architectures to be bootstrapped without needing to build Python, Git, + Perl and all of Morph's other dependencies first. + ''' + + def build_and_cache(self): + with self.build_watch('overall-build'): + handle = self.local_artifact_cache.put(self.artifact) + fs_root = self.staging_area.destdir(self.artifact.source) + try: + self.unpack_binary_chunks(fs_root) + self.unpack_sources(fs_root) + self.write_build_script(fs_root) + system_name = self.artifact.source.morphology['name'] + self.create_tarball(handle, fs_root, system_name) + except BaseException, e: + logging.error(traceback.format_exc()) + self.app.status(msg='Error while building bootstrap image', + error=True) + handle.abort() + raise + + handle.close() + + self.save_build_times() + return [self.artifact] + + def unpack_binary_chunks(self, dest): + cache = self.local_artifact_cache + for chunk_artifact in self.artifact.source.cross_chunks: + with cache.get(chunk_artifact) as chunk_file: + try: + morphlib.bins.unpack_binary_from_file(chunk_file, dest) + except BaseException, e: + self.app.status( + msg='Error unpacking binary chunk %(name)s', + name=chunk_artifact.name, + error=True) + raise + + def unpack_sources(self, path): + # Multiple chunks sources may be built from the same repo ('linux' + # and 'linux-api-headers' are a good example), so we check out the + # sources once per repository. + # + # It might be neater to build these as "source artifacts" individually, + # but that would waste huge amounts of space in the artifact cache. + for a in self.artifact.walk(): + if a in self.artifact.source.cross_chunks: + continue + if a.source.morphology['kind'] != 'chunk': + continue + + escaped_source = escape_source_name(a.source) + source_dir = os.path.join(path, 'src', escaped_source) + if not os.path.exists(source_dir): + os.makedirs(source_dir) + morphlib.builder2.extract_sources( + self.app, self.repo_cache, a.source.repo, a.source.sha1, + source_dir) + + name = a.source.morphology['name'] + chunk_script = os.path.join(path, 'src', 'build-%s' % name) + with morphlib.savefile.SaveFile(chunk_script, 'w') as f: + self.write_chunk_build_script(a, f) + os.chmod(chunk_script, 0777) + + def write_build_script(self, path): + '''Output a script to run build on the bootstrap target''' + + driver_script = os.path.join(path, 'native-bootstrap') + with morphlib.savefile.SaveFile(driver_script, 'w') as f: + f.write(driver_header) + + f.write('echo Setting up build environment...\n') + for k,v in self.staging_area.env.iteritems(): + f.write('export %s="%s"\n' % (k, v)) + + # FIXME: really, of course, we need to iterate the sources not the + # artifacts ... this will break when we have chunk splitting! + for a in self.artifact.walk(): + if a in self.artifact.source.cross_chunks: + continue + if a.source.morphology['kind'] != 'chunk': + continue + + name = a.source.morphology['name'] + f.write('\necho Building %s\n' % name) + f.write('$SRCDIR/build-%s\n' % name) + + f.write(driver_footer) + os.chmod(driver_script, 0777) + + def write_chunk_build_script(self, chunk, f): + m = chunk.source.morphology + f.write('#!/bin/sh\n') + f.write('# Build script generated by morph\n') + f.write('chunk_name=%s\n' % m['name']) + + repo = escape_source_name(chunk.source) + f.write('cp -a $SRCDIR/%s $DESTDIR/$chunk_name.build\n' % repo) + f.write('cd $DESTDIR/$chunk_name.build\n') + f.write('export PREFIX=%s\n' % chunk.source.prefix) + + bs = morphlib.buildsystem.lookup_build_system(m['build-system']) + + # FIXME: merge some of this with Morphology + steps = [ + ('pre-configure', False), + ('configure', False), + ('post-configure', False), + ('pre-build', True), + ('build', True), + ('post-build', True), + ('pre-test', False), + ('test', False), + ('post-test', False), + ('pre-install', False), + ('install', False), + ('post-install', False), + ] + + for step, in_parallel in steps: + key = '%s-commands' % step + cmds = m.get_commands(key) + for cmd in cmds: + if in_parallel: + max_jobs = m['max-jobs'] + if max_jobs is None: + max_jobs = self.max_jobs + f.write('MAKEFLAGS=-j%s ' % max_jobs) + f.write(cmd + '\n') + + f.write('rm -Rf $DESTDIR/$chunk_name.build') + + def create_tarball(self, handle, fs_root, system_name): + unslashy_root = fs_root[1:] + def uproot_info(info): + info.name = os.path.relpath(info.name, unslashy_root) + if info.islnk(): + info.linkname = os.path.relpath(info.linkname, unslashy_root) + return info + + tar = tarfile.TarFile.gzopen(fileobj=handle, mode="w", + compresslevel=1, + name=system_name) + self.app.status(msg='Constructing tarball of root filesystem', + chatty=True) + tar.add(fs_root, recursive=True, filter=uproot_info) + tar.close() + + +class CrossBootstrapPlugin(cliapp.Plugin): + + def enable(self): + self.app.add_subcommand('cross-bootstrap', + self.cross_bootstrap, + arg_synopsis='TARGET REPO REF SYSTEM-MORPH') + + def disable(self): + pass + + def cross_bootstrap(self, args): + '''Cross-bootstrap a system from a different architecture.''' + + # A brief overview of this process: the goal is to native build as much + # of the system as possible because that's easier, but in order to do + # so we need at least 'build-essential'. 'morph cross-bootstrap' will + # cross-build any bootstrap-mode chunks in the given system and + # will then prepare a large rootfs tarball which, when booted, will + # build the rest of the chunks in the system using the cross-built + # build-essential. + # + # This approach saves us from having to run Morph, Git, Python, Perl, + # or anything else complex and difficult to cross-build on the target + # until it is bootstrapped. The user of this command needs to provide + # a kernel and handle booting the system themselves (the generated + # tarball contains a /bin/sh that can be used as 'init'. + # + # This function is a variant of the BuildCommand() class in morphlib. + + # To do: make it work on a system branch instead of repo/ref/morph + # triplet. + + if len(args) < 4: + raise cliapp.AppException( + 'cross-bootstrap requires 4 arguments: target archicture, and ' + 'repo, ref and and name of the system morphology') + + arch = args[0] + root_repo, ref, system_name = args[1:4] + + if arch not in morphlib.valid_archs: + raise morphlib.Error('Unsupported architecture "%s"' % arch) + + # Get system artifact + + build_env = morphlib.buildenvironment.BuildEnvironment( + self.app.settings, arch) + build_command = morphlib.buildcommand.BuildCommand(self.app, build_env) + + morph_name = system_name + '.morph' + builds_artifacts = [system_name + '-bootstrap-rootfs'] + srcpool = build_command.create_source_pool(root_repo, ref, morph_name) + + system_source = srcpool.lookup(root_repo, ref, morph_name) + system_source.morphology.builds_artifacts = builds_artifacts + + system_artifact = build_command.resolve_artifacts(srcpool) + + # Calculate build order + # This is basically a hacked version of BuildCommand.build_in_order() + artifacts = system_artifact.walk() + cross_chunks = [] + native_chunks = [] + for a in artifacts: + if a.source.morphology['kind'] == 'chunk': + if a.source.build_mode == 'bootstrap': + cross_chunks.append(a) + else: + native_chunks.append(a) + + if len(cross_chunks) == 0: + raise morphlib.Error( + 'Nothing to cross-compile. Only chunks built in \'bootstrap\' ' + 'mode can be cross-compiled.') + + # FIXME: merge with build-command's code + for i, a in enumerate(cross_chunks): + if build_command.is_built(a): + self.app.status(msg='The %(kind)s %(name)s is already built', + kind=a.source.morphology['kind'], + name=a.name) + build_command.cache_artifacts_locally([a]) + else: + self.app.status(msg='Cross-building %(kind)s %(name)s', + kind=a.source.morphology['kind'], + name=a.name) + build_command.build_artifact(a, build_env) + + for i, a in enumerate(native_chunks): + build_command.get_sources(a) + + # Install those to the output tarball ... + self.app.status(msg='Building final bootstrap system image') + system_artifact.source.cross_chunks = cross_chunks + staging_area = build_command.create_staging_area( + build_env, use_chroot=False) + builder = BootstrapSystemBuilder( + self.app, staging_area, build_command.lac, build_command.rac, + system_artifact, build_command.lrc, 1, False) + builder.build_and_cache() + + self.app.status( + msg='Bootstrap tarball for %(name)s is cached at %(cachepath)s', + name=system_artifact.name, + cachepath=build_command.lac.artifact_filename(system_artifact)) diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index f9560b91..4e588eaa 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -271,10 +271,11 @@ class DeployPlugin(cliapp.Plugin): # Run the build. build_ref = build_repos[branch_root]['build-ref'] - artifact = build_command.get_artifact_object( + srcpool = build_command.create_source_pool( build_branch_root, build_ref, system_name + '.morph') + artifact = build_command.resolve_artifacts(srcpool) if push: self.other.delete_remote_build_refs(build_repos) diff --git a/morphlib/plugins/graphing_plugin.py b/morphlib/plugins/graphing_plugin.py index 75514eec..b57a3f3e 100644 --- a/morphlib/plugins/graphing_plugin.py +++ b/morphlib/plugins/graphing_plugin.py @@ -65,7 +65,9 @@ class GraphingPlugin(cliapp.Plugin): '%(repo_name)s %(ref)s %(filename)s', repo_name=repo_name, ref=ref, filename=filename) builder = morphlib.buildcommand.BuildCommand(self.app) - artifact = builder.get_artifact_object(repo_name, ref, filename) + srcpool = build_command.create_source_pool(repo_name, ref, + filename) + root_artifact = build_command.resolve_artifacts(srcpool) basename, ext = os.path.splitext(filename) dot_filename = basename + '.gv' @@ -81,7 +83,7 @@ class GraphingPlugin(cliapp.Plugin): with open(dot_filename, 'w') as f: f.write('digraph "%s" {\n' % basename) - for a in artifact.walk(): + for a in root_artifact.walk(): f.write( ' "%s" [shape=%s];\n' % (a.name, diff --git a/morphlib/plugins/show_dependencies_plugin.py b/morphlib/plugins/show_dependencies_plugin.py index 856a9361..9fdb7c60 100644 --- a/morphlib/plugins/show_dependencies_plugin.py +++ b/morphlib/plugins/show_dependencies_plugin.py @@ -66,9 +66,10 @@ class ShowDependenciesPlugin(cliapp.Plugin): self.app.output.write('dependency graph for %s|%s|%s:\n' % (repo, ref, morph)) - artifact = build_command.get_artifact_object(repo, ref, filename) + srcpool = build_command.create_source_pool(repo, ref, filename) + root_artifact = build_command.resolve_artifacts(srcpool) - for artifact in reversed(artifact.walk()): + for artifact in reversed(root_artifact.walk()): self.app.output.write(' %s\n' % artifact) for dependency in sorted(artifact.dependencies, key=str): self.app.output.write(' -> %s\n' % dependency) diff --git a/tests.build/bootstrap-mode.script b/tests.build/bootstrap-mode.script index f4ff0a36..923fb21f 100755 --- a/tests.build/bootstrap-mode.script +++ b/tests.build/bootstrap-mode.script @@ -23,124 +23,7 @@ set -eu # Create a fake 'compiler' chunk to go into build-essential stratum - -mkdir -p "$DATADIR/cc-repo" -cd "$DATADIR/cc-repo" - -cat <<EOF > "morph-test-cc" -#!/bin/sh -echo "I'm a compiler!" -EOF -chmod +x morph-test-cc - -cat <<EOF > "stage1-cc.morph" -{ - "name": "stage1-cc", - "kind": "chunk", - "install-commands": [ - "install -d \"\$DESTDIR\$PREFIX/bin\"", - "install -m 755 morph-test-cc \"\$DESTDIR\$PREFIX/bin/morph-test-cc\"" - ] -} -EOF - -cat <<EOF > "cc.morph" -{ - "name": "cc", - "kind": "chunk", - "configure-commands": [ - "[ -e ../tools/bin/morph-test-cc ]" - ], - "install-commands": [ - "install -d \"\$DESTDIR\$PREFIX/bin\"", - "install -m 755 morph-test-cc \"\$DESTDIR\$PREFIX/bin/morph-test-cc\"" - ] -} -EOF - -git init -q -git add morph-test-cc cc.morph stage1-cc.morph -git commit -q -m "Create compiler chunk" - -# Require 'cc' in hello-chunk. We should have the second version available -# but *not* the first one. -cd "$DATADIR/chunk-repo" -git checkout -q farrokh -cat <<EOF > "hello.morph" -{ - "name": "hello", - "kind": "chunk", - "configure-commands": [ - "[ ! -e ../tools/bin/morph-test-cc ]", - "[ -e ../usr/bin/morph-test-cc ]" - ], - "build-commands": [ - "../usr/bin/morph-test-cc > hello" - ], - "install-commands": [ - "install -d \"\$DESTDIR\$PREFIX/bin\"", - "install hello \"\$DESTDIR\$PREFIX/bin/hello\"" - ] -} -EOF -git add hello.morph -git commit -q -m "Make 'hello' require our mock compiler" - -# Add 'build-essential' stratum and make hello-stratum depend upon it. Only -# the *second* 'cc' chunk should make it into the build-essential stratum -# artifact, and neither should make it into the system. -cd "$DATADIR/morphs-repo" -cat <<EOF > "build-essential.morph" -{ - "name": "build-essential", - "kind": "stratum", - "chunks": [ - { - "name": "stage1-cc", - "repo": "test:cc-repo", - "ref": "master", - "build-depends": [], - "build-mode": "bootstrap", - "prefix": "/tools" - }, - { - "name": "cc", - "repo": "test:cc-repo", - "ref": "master", - "build-depends": [ - "stage1-cc" - ], - "build-mode": "test" - } - ] -} -EOF - -cat <<EOF > "hello-stratum.morph" -{ - "name": "hello-stratum", - "kind": "stratum", - "build-depends": [ - { - "morph": "build-essential", - "repo": "test:morphs-repo", - "ref": "master" - } - ], - "chunks": [ - { - "name": "hello", - "repo": "test:chunk-repo", - "ref": "farrokh", - "build-depends": [], - "build-mode": "test" - } - ] -} -EOF - -git add build-essential.morph hello-stratum.morph hello-system.morph -git commit -q -m "Add fake build-essential stratum" +"$SRCDIR/tests.build/setup-build-essential" "$SRCDIR/scripts/test-morph" build-morphology \ test:morphs-repo master hello-system diff --git a/tests.build/cross-bootstrap-only-to-supported-archs.exit b/tests.build/cross-bootstrap-only-to-supported-archs.exit new file mode 100644 index 00000000..d00491fd --- /dev/null +++ b/tests.build/cross-bootstrap-only-to-supported-archs.exit @@ -0,0 +1 @@ +1 diff --git a/tests.build/cross-bootstrap-only-to-supported-archs.script b/tests.build/cross-bootstrap-only-to-supported-archs.script new file mode 100755 index 00000000..872acd9f --- /dev/null +++ b/tests.build/cross-bootstrap-only-to-supported-archs.script @@ -0,0 +1,25 @@ +#!/bin/bash +# +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +# Test that "morph cross-bootstrap" works only for the architectures that +# Morph explicitly supports. + +set -eu + +"$SRCDIR/scripts/test-morph" cross-bootstrap \ + unknown-archicture test:morphs-repo master hello-system -v diff --git a/tests.build/cross-bootstrap-only-to-supported-archs.stderr b/tests.build/cross-bootstrap-only-to-supported-archs.stderr new file mode 100644 index 00000000..61c0fe0d --- /dev/null +++ b/tests.build/cross-bootstrap-only-to-supported-archs.stderr @@ -0,0 +1 @@ +ERROR: Unsupported architecture "unknown-archicture" diff --git a/tests.build/cross-bootstrap.script b/tests.build/cross-bootstrap.script new file mode 100755 index 00000000..4de0f1ae --- /dev/null +++ b/tests.build/cross-bootstrap.script @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +# Test "morph cross-bootstrap", up to the point of the tarball it generates +# for the target + +set -eu + +"$SRCDIR/tests.build/setup-build-essential" + +"$SRCDIR/scripts/test-morph" cross-bootstrap \ + x86_32 test:morphs-repo master hello-system diff --git a/tests.build/setup-build-essential b/tests.build/setup-build-essential new file mode 100755 index 00000000..778716f1 --- /dev/null +++ b/tests.build/setup-build-essential @@ -0,0 +1,137 @@ +#!/bin/sh +# +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# Set up a stratum which resembles Baserock's 'build-essential' slightly. Used +# for testing 'morph cross-bootstrap' and the 'bootstrap' build mode. + +mkdir -p "$DATADIR/cc-repo" +cd "$DATADIR/cc-repo" + +cat <<EOF > "morph-test-cc" +#!/bin/sh +echo "I'm a compiler!" +EOF +chmod +x morph-test-cc + +cat <<EOF > "stage1-cc.morph" +{ + "name": "stage1-cc", + "kind": "chunk", + "install-commands": [ + "install -d \"\$DESTDIR\$PREFIX/bin\"", + "install -m 755 morph-test-cc \"\$DESTDIR\$PREFIX/bin/morph-test-cc\"" + ] +} +EOF + +cat <<EOF > "cc.morph" +{ + "name": "cc", + "kind": "chunk", + "configure-commands": [ + "[ -e ../tools/bin/morph-test-cc ]" + ], + "install-commands": [ + "install -d \"\$DESTDIR\$PREFIX/bin\"", + "install -m 755 morph-test-cc \"\$DESTDIR\$PREFIX/bin/morph-test-cc\"" + ] +} +EOF + +git init -q +git add morph-test-cc cc.morph stage1-cc.morph +git commit -q -m "Create compiler chunk" + +# Require 'cc' in hello-chunk. We should have the second version available +# but *not* the first one. +cd "$DATADIR/chunk-repo" +git checkout -q farrokh +cat <<EOF > "hello.morph" +{ + "name": "hello", + "kind": "chunk", + "configure-commands": [ + "[ ! -e ../tools/bin/morph-test-cc ]", + "[ -e ../usr/bin/morph-test-cc ]" + ], + "build-commands": [ + "../usr/bin/morph-test-cc > hello" + ], + "install-commands": [ + "install -d \"\$DESTDIR\$PREFIX/bin\"", + "install hello \"\$DESTDIR\$PREFIX/bin/hello\"" + ] +} +EOF +git add hello.morph +git commit -q -m "Make 'hello' require our mock compiler" + +# Add 'build-essential' stratum and make hello-stratum depend upon it. Only +# the *second* 'cc' chunk should make it into the build-essential stratum +# artifact, and neither should make it into the system. +cd "$DATADIR/morphs-repo" +cat <<EOF > "build-essential.morph" +{ + "name": "build-essential", + "kind": "stratum", + "chunks": [ + { + "name": "stage1-cc", + "repo": "test:cc-repo", + "ref": "master", + "build-depends": [], + "build-mode": "bootstrap", + "prefix": "/tools" + }, + { + "name": "cc", + "repo": "test:cc-repo", + "ref": "master", + "build-depends": [ + "stage1-cc" + ], + "build-mode": "test" + } + ] +} +EOF + +cat <<EOF > "hello-stratum.morph" +{ + "name": "hello-stratum", + "kind": "stratum", + "build-depends": [ + { + "morph": "build-essential", + "repo": "test:morphs-repo", + "ref": "master" + } + ], + "chunks": [ + { + "name": "hello", + "repo": "test:chunk-repo", + "ref": "farrokh", + "build-depends": [], + "build-mode": "test" + } + ] +} +EOF + +git add build-essential.morph hello-stratum.morph hello-system.morph +git commit -q -m "Add fake build-essential stratum" diff --git a/without-test-modules b/without-test-modules index a69b2c5c..c5d59442 100644 --- a/without-test-modules +++ b/without-test-modules @@ -8,6 +8,7 @@ morphlib/app.py morphlib/mountableimage.py morphlib/extractedtarball.py morphlib/plugins/artifact_inspection_plugin.py +morphlib/plugins/cross-bootstrap_plugin.py morphlib/plugins/hello_plugin.py morphlib/plugins/graphing_plugin.py morphlib/plugins/syslinux-disk-systembuilder_plugin.py |