From 1202d2a452b3aefed351d08b1ffe2f68391b7416 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 10 Sep 2014 16:05:18 +0000 Subject: Build per-source rather than per-artifact --- morphlib/plugins/cross-bootstrap_plugin.py | 121 +++++++++++++---------------- 1 file changed, 55 insertions(+), 66 deletions(-) (limited to 'morphlib/plugins/cross-bootstrap_plugin.py') diff --git a/morphlib/plugins/cross-bootstrap_plugin.py b/morphlib/plugins/cross-bootstrap_plugin.py index 95e991f4..7b53a4a5 100644 --- a/morphlib/plugins/cross-bootstrap_plugin.py +++ b/morphlib/plugins/cross-bootstrap_plugin.py @@ -58,38 +58,39 @@ class BootstrapSystemBuilder(morphlib.builder2.BuilderBase): def build_and_cache(self): with self.build_watch('overall-build'): - handle = self.local_artifact_cache.put(self.artifact) - fs_root = self.staging_area.destdir(self.artifact.source) - try: - self.unpack_binary_chunks(fs_root) - self.unpack_sources(fs_root) - self.write_build_script(fs_root) - system_name = self.artifact.source.morphology['name'] - self.create_tarball(handle, fs_root, system_name) - except BaseException, e: - logging.error(traceback.format_exc()) - self.app.status(msg='Error while building bootstrap image', - error=True) - handle.abort() - raise - - handle.close() + for system_name, artifact in self.source.artifacts.iteritems(): + handle = self.local_artifact_cache.put(artifact) + fs_root = self.staging_area.destdir(self.source) + try: + self.unpack_binary_chunks(fs_root) + self.unpack_sources(fs_root) + self.write_build_script(fs_root) + self.create_tarball(handle, fs_root, system_name) + except BaseException, e: + logging.error(traceback.format_exc()) + self.app.status(msg='Error while building bootstrap image', + error=True) + handle.abort() + raise + + handle.close() self.save_build_times() - return [self.artifact] + return self.source.artifacts.items() def unpack_binary_chunks(self, dest): cache = self.local_artifact_cache - for chunk_artifact in self.artifact.source.cross_chunks: - with cache.get(chunk_artifact) as chunk_file: - try: - morphlib.bins.unpack_binary_from_file(chunk_file, dest) - except BaseException, e: - self.app.status( - msg='Error unpacking binary chunk %(name)s', - name=chunk_artifact.name, - error=True) - raise + for chunk_source in self.source.cross_sources: + for chunk_artifact in chunk_source.artifacts.itervalues(): + with cache.get(chunk_artifact) as chunk_file: + try: + morphlib.bins.unpack_binary_from_file(chunk_file, dest) + except BaseException, e: + self.app.status( + msg='Error unpacking binary chunk %(name)s', + name=chunk_artifact.name, + error=True) + raise def unpack_sources(self, path): # Multiple chunks sources may be built from the same repo ('linux' @@ -98,24 +99,18 @@ class BootstrapSystemBuilder(morphlib.builder2.BuilderBase): # # It might be neater to build these as "source artifacts" individually, # but that would waste huge amounts of space in the artifact cache. - for a in self.artifact.walk(): - if a in self.artifact.source.cross_chunks: - continue - if a.source.morphology['kind'] != 'chunk': - continue - - escaped_source = escape_source_name(a.source) + for s in self.source.native_sources: + escaped_source = escape_source_name(s) source_dir = os.path.join(path, 'src', escaped_source) if not os.path.exists(source_dir): os.makedirs(source_dir) morphlib.builder2.extract_sources( - self.app, self.repo_cache, a.source.repo, a.source.sha1, - source_dir) + self.app, self.repo_cache, s.repo, s.sha1, source_dir) - name = a.source.morphology['name'] + name = s.name chunk_script = os.path.join(path, 'src', 'build-%s' % name) with morphlib.savefile.SaveFile(chunk_script, 'w') as f: - self.write_chunk_build_script(a, f) + self.write_chunk_build_script(s, f) os.chmod(chunk_script, 0777) def write_build_script(self, path): @@ -130,15 +125,8 @@ class BootstrapSystemBuilder(morphlib.builder2.BuilderBase): if k != 'PATH': f.write('export %s="%s"\n' % (k, v)) - # FIXME: really, of course, we need to iterate the sources not the - # artifacts ... this will break when we have chunk splitting! - for a in self.artifact.walk(): - if a in self.artifact.source.cross_chunks: - continue - if a.source.morphology['kind'] != 'chunk': - continue - - name = a.source.morphology['name'] + for s in self.source.native_sources: + name = s.name f.write('\necho Building %s\n' % name) f.write('mkdir /%s.inst\n' % name) f.write('env DESTDIR=/%s.inst $SRCDIR/build-%s\n' @@ -150,17 +138,17 @@ class BootstrapSystemBuilder(morphlib.builder2.BuilderBase): f.write(driver_footer) os.chmod(driver_script, 0777) - def write_chunk_build_script(self, chunk, f): - m = chunk.source.morphology + def write_chunk_build_script(self, source, f): + m = source.morphology f.write('#!/bin/sh\n') f.write('# Build script generated by morph\n') f.write('set -e\n') f.write('chunk_name=%s\n' % m['name']) - repo = escape_source_name(chunk.source) + repo = escape_source_name(source) f.write('cp -a $SRCDIR/%s $DESTDIR/$chunk_name.build\n' % repo) f.write('cd $DESTDIR/$chunk_name.build\n') - f.write('export PREFIX=%s\n' % chunk.source.prefix) + f.write('export PREFIX=%s\n' % source.prefix) bs = morphlib.buildsystem.lookup_build_system(m['build-system']) @@ -280,35 +268,36 @@ class CrossBootstrapPlugin(cliapp.Plugin): # Calculate build order # This is basically a hacked version of BuildCommand.build_in_order() - artifacts = system_artifact.walk() - cross_chunks = [] - native_chunks = [] - for a in artifacts: - if a.source.morphology['kind'] == 'chunk': - if a.source.build_mode == 'bootstrap': - cross_chunks.append(a) + sources = build_command.get_ordered_sources(system_artifact.walk()) + cross_sources = [] + native_sources = [] + for s in sources: + if s.morphology['kind'] == 'chunk': + if s.build_mode == 'bootstrap': + cross_sources.append(s) else: - native_chunks.append(a) + native_sources.append(s) - if len(cross_chunks) == 0: + if len(cross_sources) == 0: raise morphlib.Error( 'Nothing to cross-compile. Only chunks built in \'bootstrap\' ' 'mode can be cross-compiled.') - for i, a in enumerate(cross_chunks): - build_command.cache_or_build_artifact(a, build_env) + for s in cross_sources: + build_command.cache_or_build_source(s, build_env) - for i, a in enumerate(native_chunks): - build_command.fetch_sources(a) + for s in native_sources: + build_command.fetch_sources(s) # Install those to the output tarball ... self.app.status(msg='Building final bootstrap system image') - system_artifact.source.cross_chunks = cross_chunks + system_artifact.source.cross_sources = cross_sources + system_artifact.source.native_sources = native_sources staging_area = build_command.create_staging_area( build_env, use_chroot=False) builder = BootstrapSystemBuilder( self.app, staging_area, build_command.lac, build_command.rac, - system_artifact, build_command.lrc, 1, False) + system_artifact.source, build_command.lrc, 1, False) builder.build_and_cache() self.app.status( -- cgit v1.2.1