diff options
Diffstat (limited to 'morphlib/builder2.py')
| -rw-r--r-- | morphlib/builder2.py | 158 |
1 files changed, 83 insertions, 75 deletions
diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 5820dc44..6edb7c61 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -30,7 +30,7 @@ import morphlib from morphlib.artifactcachereference import ArtifactCacheReference -def ldconfig(runcmd, rootdir): # pragma: no cover +def ldconfig(runcmd, rootdir): # pragma: no cover '''Run ldconfig for the filesystem below ``rootdir``. Essentially, ``rootdir`` specifies the root of a new system. @@ -41,9 +41,9 @@ def ldconfig(runcmd, rootdir): # pragma: no cover the root directory is ``rootdir``. Example: if ``rootdir`` is ``/tmp/foo``, then ``/tmp/foo/etc/ld.so.conf`` should contain ``/lib``, not ``/tmp/foo/lib``. - + The ldconfig found via ``$PATH`` is used, not the one in ``rootdir``, - since in bootstrap mode that might not yet exist, the various + since in bootstrap mode that might not yet exist, the various implementations should be compatible enough. ''' @@ -52,7 +52,7 @@ def ldconfig(runcmd, rootdir): # pragma: no cover if os.path.exists(conf): logging.debug('Running ldconfig for %s' % rootdir) cache = os.path.join(rootdir, 'etc', 'ld.so.cache') - + # The following trickery with $PATH is necessary during the Baserock # bootstrap build: we are not guaranteed that PATH contains the # directory (/sbin conventionally) that ldconfig is in. Then again, @@ -65,6 +65,7 @@ def ldconfig(runcmd, rootdir): # pragma: no cover else: logging.debug('No %s, not running ldconfig' % conf) + def download_depends(constituents, lac, rac, metadatas=None): for constituent in constituents: if not lac.has(constituent): @@ -75,29 +76,32 @@ def download_depends(constituents, lac, rac, metadatas=None): source.close() if metadatas is not None: for metadata in metadatas: - if (not lac.has_artifact_metadata(constituent, metadata) - and rac.has_artifact_metadata(constituent, metadata)): + if not lac.has_artifact_metadata(constituent, metadata): + if rac.has_artifact_metadata(constituent, metadata): src = rac.get_artifact_metadata(constituent, metadata) dst = lac.put_artifact_metadata(constituent, metadata) shutil.copyfileobj(src, dst) dst.close() src.close() -def get_chunk_files(f): # pragma: no cover + +def get_chunk_files(f): # pragma: no cover tar = tarfile.open(fileobj=f) for member in tar.getmembers(): if member.type is not tarfile.DIRTYPE: yield member.name tar.close() -def get_stratum_files(f, lac): # pragma: no cover + +def get_stratum_files(f, lac): # pragma: no cover for ca in (ArtifactCacheReference(a) for a in json.load(f)): cf = lac.get(ca) for filename in get_chunk_files(cf): yield filename cf.close() -def get_overlaps(artifact, constituents, lac): #pragma: no cover + +def get_overlaps(artifact, constituents, lac): # pragma: no cover # check whether strata overlap installed = defaultdict(set) for dep in constituents: @@ -107,7 +111,7 @@ def get_overlaps(artifact, constituents, lac): #pragma: no cover installed[filename].add(dep) elif artifact.source.morphology['kind'] == 'system': for filename in get_stratum_files(handle, lac): - installed[filename].add(dep) + installed[filename].add(dep) handle.close() overlaps = defaultdict(set) for filename, artifacts in installed.iteritems(): @@ -115,25 +119,28 @@ def get_overlaps(artifact, constituents, lac): #pragma: no cover overlaps[frozenset(artifacts)].add(filename) return overlaps -def log_overlaps(overlaps): #pragma: no cover + +def log_overlaps(overlaps): # pragma: no cover for overlapping, files in sorted(overlaps.iteritems()): logging.warning(' Artifacts %s overlap with files:' % - ', '.join(sorted(a.name for a in overlapping)) - ) + ', '.join(sorted(a.name for a in overlapping))) for filename in sorted(files): logging.warning(' %s' % filename) -def write_overlap_metadata(artifact, overlaps, lac): #pragma: no cover + +def write_overlap_metadata(artifact, overlaps, lac): # pragma: no cover f = lac.put_artifact_metadata(artifact, 'overlaps') # the big list comprehension is because json can't serialize # artifacts, sets or dicts with non-string keys - json.dump([ - [ - [a.name for a in afs], list(files) - ] for afs, files in overlaps.iteritems() - ], f, indent=4) + json.dump( + [ + [ + [a.name for a in afs], list(files) + ] for afs, files in overlaps.iteritems() + ], f, indent=4) f.close() + class BuilderBase(object): '''Base class for building artifacts.''' @@ -170,16 +177,16 @@ class BuilderBase(object): 'meta') as f: json.dump(meta, f, indent=4, sort_keys=True) f.write('\n') - + def create_metadata(self, artifact_name): '''Create metadata to artifact to allow it to be reproduced later. - + The metadata is represented as a dict, which later on will be written out as a JSON file. - + ''' - - assert isinstance(self.artifact.source.repo, + + assert isinstance(self.artifact.source.repo, morphlib.cachedrepo.CachedRepo) meta = { 'artifact-name': artifact_name, @@ -193,11 +200,11 @@ class BuilderBase(object): 'cache-key': self.artifact.cache_key, 'cache-id': self.artifact.cache_id, } - + return meta # Wrapper around open() to allow it to be overridden by unit tests. - def _open(self, filename, mode): # pragma: no cover + def _open(self, filename, mode): # pragma: no cover dirname = os.path.dirname(filename) if not os.path.exists(dirname): os.makedirs(dirname) @@ -205,13 +212,13 @@ class BuilderBase(object): def write_metadata(self, instdir, artifact_name): '''Write the metadata for an artifact. - + The file will be located under the ``baserock`` directory under instdir, named after ``cache_key`` with ``.meta`` as the suffix. It will be in JSON format. - + ''' - + meta = self.create_metadata(artifact_name) basename = '%s.meta' % artifact_name @@ -222,13 +229,13 @@ class BuilderBase(object): f = self._open(filename, 'w') f.write(json.dumps(meta, indent=4, sort_keys=True)) f.close() - + def new_artifact(self, artifact_name): '''Return an Artifact object for something built from our source.''' a = morphlib.artifact.Artifact(self.artifact.source, artifact_name) a.cache_key = self.artifact.cache_key return a - + def runcmd(self, *args, **kwargs): kwargs['env'] = self.build_env.env return self.staging_area.runcmd(*args, **kwargs) @@ -237,7 +244,7 @@ class BuilderBase(object): class ChunkBuilder(BuilderBase): '''Build chunk artifacts.''' - + def get_commands(self, which, morphology, build_system): '''Return the commands to run from a morphology or the build system.''' if morphology[which] is None: @@ -246,7 +253,7 @@ class ChunkBuilder(BuilderBase): else: return morphology[which] - def build_and_cache(self): # pragma: no cover + def build_and_cache(self): # pragma: no cover with self.build_watch('overall-build'): mounted = self.do_mounts() log_name = None @@ -264,8 +271,8 @@ class ChunkBuilder(BuilderBase): if log_name: with open(log_name) as f: for line in f: - logging.error('OUTPUT FROM FAILED BUILD: %s' % - line.rstrip('\n')) + logging.error('OUTPUT FROM FAILED BUILD: %s' % + line.rstrip('\n')) raise self.do_unmounts(mounted) built_artifacts = self.assemble_chunk_artifacts(destdir) @@ -277,7 +284,8 @@ class ChunkBuilder(BuilderBase): ('proc', 'proc', 'none'), ('dev/shm', 'tmpfs', 'none'), ) - def do_mounts(self): # pragma: no cover + + def do_mounts(self): # pragma: no cover mounted = [] if not self.setup_mounts: return mounted @@ -290,12 +298,12 @@ class ChunkBuilder(BuilderBase): mounted.append(path) return mounted - def do_unmounts(self, mounted): # pragma: no cover + def do_unmounts(self, mounted): # pragma: no cover for path in mounted: logging.debug('Unmounting %s in staging area' % path) morphlib.fsutils.unmount(self.app.runcmd, path) - def get_sources(self, srcdir): # pragma: no cover + def get_sources(self, srcdir): # pragma: no cover '''Get sources from git to a source directory, for building.''' cache_dir = os.path.dirname(self.artifact.source.repo.path) @@ -329,16 +337,16 @@ class ChunkBuilder(BuilderBase): todo += extract_repo(path, sha1, srcdir) self.set_mtime_recursively(srcdir) - def set_mtime_recursively(self, root): # pragma: no cover + def set_mtime_recursively(self, root): # pragma: no cover '''Set the mtime for every file in a directory tree to the same. - + We do this because git checkout does not set the mtime to anything, and some projects (binutils, gperf for example) include formatted documentation and try to randomly build things or not because of the timestamps. This should help us get more reliable builds. - + ''' - + now = time.time() for dirname, subdirs, basenames in os.walk(root, topdown=False): for basename in basenames: @@ -348,7 +356,7 @@ class ChunkBuilder(BuilderBase): os.utime(pathname, (now, now)) os.utime(dirname, (now, now)) - def run_commands(self, builddir, destdir, logfile): # pragma: no cover + def run_commands(self, builddir, destdir, logfile): # pragma: no cover m = self.artifact.source.morphology bs = morphlib.buildsystem.lookup_build_system(m['build-system']) @@ -356,7 +364,7 @@ class ChunkBuilder(BuilderBase): relative_destdir = self.staging_area.relative(destdir) self.build_env.env['DESTDIR'] = relative_destdir - steps = [('configure', False), + steps = [('configure', False), ('build', True), ('test', False), ('install', False)] @@ -393,7 +401,7 @@ class ChunkBuilder(BuilderBase): shutil.copyfileobj(readlog, self.app.output) raise e - def assemble_chunk_artifacts(self, destdir): # pragma: no cover + def assemble_chunk_artifacts(self, destdir): # pragma: no cover built_artifacts = [] with self.build_watch('create-chunks'): specs = self.artifact.source.morphology['chunks'] @@ -407,7 +415,7 @@ class ChunkBuilder(BuilderBase): self.write_metadata(destdir, artifact_name) patterns = specs[artifact_name] patterns += [r'baserock/%s\.' % artifact_name] - + artifact = self.new_artifact(artifact_name) with self.local_artifact_cache.put(artifact) as f: logging.debug('assembling chunk %s' % artifact_name) @@ -416,7 +424,7 @@ class ChunkBuilder(BuilderBase): name=artifact.name) morphlib.bins.create_chunk(destdir, f, patterns) built_artifacts.append(artifact) - + files = os.listdir(destdir) if files: raise Exception('DESTDIR %s is not empty: %s' % @@ -428,12 +436,12 @@ class StratumBuilder(BuilderBase): '''Build stratum artifacts.''' - def build_and_cache(self): # pragma: no cover + def build_and_cache(self): # pragma: no cover with self.build_watch('overall-build'): constituents = [dependency for dependency in self.artifact.dependencies if dependency.source.morphology['kind'] == 'chunk'] - # the only reason the StratumBuilder has to download chunks is to + # the only reason the StratumBuilder has to download chunks is to # check for overlap now that strata are lists of chunks with self.build_watch('check-chunks'): # download the chunk artifact if necessary @@ -467,13 +475,13 @@ class StratumBuilder(BuilderBase): return [artifact] -class SystemKindBuilder(BuilderBase): # pragma: no cover +class SystemKindBuilder(BuilderBase): # pragma: no cover '''Build a specific kind of a system. - + Subclasses should set the ``system_kind`` attribute to the kind of system they build. - + ''' def unpack_strata(self, path): @@ -520,10 +528,10 @@ class SystemKindBuilder(BuilderBase): # pragma: no cover chunk_handle.close() f.close() meta = self.local_artifact_cache.get_artifact_metadata( - stratum_artifact, 'meta') + stratum_artifact, 'meta') dst = morphlib.savefile.SaveFile( - os.path.join(path, 'baserock', - '%s.meta' % stratum_artifact.name), 'w') + os.path.join(path, 'baserock', + '%s.meta' % stratum_artifact.name), 'w') shutil.copyfileobj(meta, dst) dst.close() meta.close() @@ -532,18 +540,19 @@ class SystemKindBuilder(BuilderBase): # pragma: no cover def create_fstab(self, path): '''Create an /etc/fstab inside a system tree. - + The fstab is created using assumptions of the disk layout. If the assumptions are wrong, extend this code so it can deal with other cases. - + ''' self.app.status(msg='Creating fstab in %(path)s', path=path, chatty=True) with self.build_watch('create-fstab'): fstab = os.path.join(path, 'etc', 'fstab') - if not os.path.exists(os.path.dirname(fstab)):# FIXME: should exist + # FIXME: should exist + if not os.path.exists(os.path.dirname(fstab)): os.makedirs(os.path.dirname(fstab)) with open(fstab, 'w') as f: f.write('proc /proc proc defaults 0 0\n') @@ -555,11 +564,11 @@ class SystemKindBuilder(BuilderBase): # pragma: no cover The kernel image will be a separate artifact from the root filesystem/disk image/whatever. This is sometimes useful with - funky bootloaders or virtualisation. - + funky bootloaders or virtualisation. + ''' - name = self.artifact.source.morphology['name']+'-kernel' + name = self.artifact.source.morphology['name'] + '-kernel' a = self.new_artifact(name) with self.local_artifact_cache.put(a) as dest: for basename in ['zImage', 'vmlinuz']: @@ -570,7 +579,7 @@ class SystemKindBuilder(BuilderBase): # pragma: no cover break -class SystemKindBuilderFactory(object): # pragma: no cover +class SystemKindBuilderFactory(object): # pragma: no cover '''A factory class for SystemKindBuilder objects.''' @@ -579,16 +588,16 @@ class SystemKindBuilderFactory(object): # pragma: no cover def register(self, klass): self.system_kinds.append(klass) - + def new(self, system_kind, args, kwargs): for klass in self.system_kinds: if klass.system_kind == system_kind: return klass(*args, **kwargs) raise morphlib.Error("Don't know how to build system kind %s" % - system_kind) + system_kind) -class SystemBuilder(BuilderBase): # pragma: no cover +class SystemBuilder(BuilderBase): # pragma: no cover '''Build system image artifacts.''' @@ -596,21 +605,21 @@ class SystemBuilder(BuilderBase): # pragma: no cover BuilderBase.__init__(self, *args, **kwargs) self.args = args self.kwargs = kwargs - + def build_and_cache(self): system_kind = self.artifact.source.morphology['system-kind'] builder = self.app.system_kind_builder_factory.new( - system_kind, self.args, self.kwargs) + system_kind, self.args, self.kwargs) logging.debug('Building system with %s' % repr(builder)) self.app.status(msg='Building system %(system_name)s', system_name=self.artifact.name) return builder.build_and_cache() -class Builder(object): # pragma: no cover +class Builder(object): # pragma: no cover '''Helper class to build with the right BuilderBase subclass.''' - + classes = { 'chunk': ChunkBuilder, 'stratum': StratumBuilder, @@ -628,17 +637,16 @@ class Builder(object): # pragma: no cover self.build_env = build_env self.max_jobs = max_jobs self.setup_mounts = setup_mounts - + def build_and_cache(self, artifact): kind = artifact.source.morphology['kind'] o = self.classes[kind](self.app, self.staging_area, self.local_artifact_cache, self.remote_artifact_cache, artifact, - self.repo_cache, self.build_env, + self.repo_cache, self.build_env, self.max_jobs, self.setup_mounts) logging.debug('Builder.build: artifact %s with %s' % (artifact.name, repr(o))) built_artifacts = o.build_and_cache() logging.debug('Builder.build: done') return built_artifacts - |