diff options
-rw-r--r-- | morphlib/artifactresolver.py | 32 | ||||
-rw-r--r-- | morphlib/buildcommand.py | 159 | ||||
-rw-r--r-- | morphlib/builder2.py | 240 | ||||
-rw-r--r-- | morphlib/cachekeycomputer.py | 55 | ||||
-rw-r--r-- | morphlib/plugins/cross-bootstrap_plugin.py | 121 | ||||
-rw-r--r-- | morphlib/remoteartifactcache.py | 4 |
6 files changed, 314 insertions, 297 deletions
diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py index a60a8989..5deb25b7 100644 --- a/morphlib/artifactresolver.py +++ b/morphlib/artifactresolver.py @@ -149,7 +149,7 @@ class ArtifactResolver(object): if sta_name in stratum_source.artifacts: stratum_artifact = \ stratum_source.artifacts[sta_name] - system.add_dependency(stratum_artifact) + source.add_dependency(stratum_artifact) artifacts.append(stratum_artifact) queue.append(stratum_source) @@ -182,10 +182,10 @@ class ArtifactResolver(object): artifacts.append(other_stratum) for stratum in strata: - if other_stratum.depends_on(stratum): + if other_source.depends_on(stratum): raise MutualDependencyError(stratum, other_stratum) - stratum.add_dependency(other_stratum) + source.add_dependency(other_stratum) queue.append(other_source) @@ -208,12 +208,9 @@ class ArtifactResolver(object): build_depends = info.get('build-depends', None) - for ca_name in chunk_source.split_rules.artifacts: - chunk_artifact = chunk_source.artifacts[ca_name] - - # Add our stratum's build depends as dependencies of this chunk - for other_stratum in stratum_build_depends: - chunk_artifact.add_dependency(other_stratum) + # Add our stratum's build depends as dependencies of this chunk + for other_stratum in stratum_build_depends: + chunk_source.add_dependency(other_stratum) # Add dependencies between chunks mentioned in this stratum for name in build_depends: # pragma: no cover @@ -222,22 +219,19 @@ class ArtifactResolver(object): source, info['name'], name) other_artifacts = name_to_processed_artifacts[name] for other_artifact in other_artifacts: - for ca_name in chunk_source.split_rules.artifacts: - chunk_artifact = chunk_source.artifacts[ca_name] - chunk_artifact.add_dependency(other_artifact) + chunk_source.add_dependency(other_artifact) # Add build dependencies between our stratum's artifacts # and the chunk artifacts produced by this stratum. matches, overlaps, unmatched = source.split_rules.partition( ((chunk_name, ca_name) for ca_name in chunk_source.split_rules.artifacts)) - for stratum in strata: - for (chunk_name, ca_name) in matches[stratum.name]: - chunk_artifact = chunk_source.artifacts[ca_name] - stratum.add_dependency(chunk_artifact) - # Only return chunks required to build strata we need - if chunk_artifact not in artifacts: - artifacts.append(chunk_artifact) + for (chunk_name, ca_name) in matches[source.name]: + chunk_artifact = chunk_source.artifacts[ca_name] + source.add_dependency(chunk_artifact) + # Only return chunks required to build strata we need + if chunk_artifact not in artifacts: + artifacts.append(chunk_artifact) # Add these chunks to the processed artifacts, so other diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 436e23eb..c8d9930c 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -14,6 +14,7 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import itertools import os import shutil import logging @@ -175,9 +176,9 @@ class BuildCommand(object): self.app.status(msg='Computing cache keys', chatty=True) ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env) - for artifact in artifacts: - artifact.cache_key = ckc.compute_key(artifact) - artifact.cache_id = ckc.get_cache_id(artifact) + for source in set(a.source for a in artifacts): + source.cache_key = ckc.compute_key(source) + source.cache_id = ckc.get_cache_id(source) root_artifact.build_env = build_env return root_artifact @@ -210,7 +211,7 @@ class BuildCommand(object): if src.morphology['kind'] == 'stratum': name = src.name ref = src.sha1[:7] - self.app.status(msg='Stratum [%(name)s] version is %(ref)s', + self.app.status(msg='Stratum [%(name)s] version is %(ref)s', name=name, ref=ref) if name in stratum_names: raise morphlib.Error( @@ -251,78 +252,94 @@ class BuildCommand(object): def _find_root_artifacts(self, artifacts): '''Find all the root artifacts among a set of artifacts in a DAG. - + It would be nice if the ArtifactResolver would return its results in a more useful order to save us from needing to do this -- the root object is known already since that's the one the user asked us to build. - + ''' - return [a for a in artifacts if not a.dependents] + return [a for a in artifacts if not a.dependent_sources] + + @staticmethod + def get_ordered_sources(artifacts): + ordered_sources = [] + known_sources = set() + for artifact in artifacts: + if artifact.source not in known_sources: + known_sources.add(artifact.source) + yield artifact.source def build_in_order(self, root_artifact): '''Build everything specified in a build order.''' - self.app.status(msg='Building a set of artifacts', chatty=True) + self.app.status(msg='Building a set of sources', chatty=True) build_env = root_artifact.build_env - artifacts = root_artifact.walk() + ordered_sources = list(self.get_ordered_sources(root_artifact.walk())) old_prefix = self.app.status_prefix - for i, a in enumerate(artifacts): + for i, s in enumerate(ordered_sources): self.app.status_prefix = ( old_prefix + '[Build %(index)d/%(total)d] [%(name)s] ' % { 'index': (i+1), - 'total': len(artifacts), - 'name': a.name, + 'total': len(ordered_sources), + 'name': s.name, }) - self.cache_or_build_artifact(a, build_env) - - self.app.status(msg='%(kind)s %(name)s is cached at %(cachepath)s', - kind=a.source.morphology['kind'], name=a.name, - cachepath=self.lac.artifact_filename(a), - chatty=(a.source.morphology['kind'] != "system")) + self.cache_or_build_source(s, build_env) self.app.status_prefix = old_prefix - def cache_or_build_artifact(self, artifact, build_env): - '''Make the built artifact available in the local cache. + def cache_or_build_source(self, source, build_env): + '''Make artifacts of the built source available in the local cache. This can be done by retrieving from a remote artifact cache, or if - that doesn't work for some reason, by building the artifact locally. + that doesn't work for some reason, by building the source locally. ''' + artifacts = source.artifacts.values() if self.rac is not None: try: - self.cache_artifacts_locally([artifact]) + self.cache_artifacts_locally(artifacts) except morphlib.remoteartifactcache.GetError: # Error is logged by the RemoteArtifactCache object. pass - if not self.lac.has(artifact): - self.build_artifact(artifact, build_env) + if any(not self.lac.has(artifact) for artifact in artifacts): + self.build_source(source, build_env) + + for a in artifacts: + self.app.status(msg='%(kind)s %(name)s is cached at %(cachepath)s', + kind=source.morphology['kind'], name=a.name, + cachepath=self.lac.artifact_filename(a), + chatty=(source.morphology['kind'] != "system")) - def build_artifact(self, artifact, build_env): - '''Build one artifact. + def build_source(self, source, build_env): + '''Build all artifacts for one source. All the dependencies are assumed to be built and available in either the local or remote cache already. ''' self.app.status(msg='Building %(kind)s %(name)s', - name=artifact.name, - kind=artifact.source.morphology['kind']) - - self.fetch_sources(artifact) - deps = self.get_recursive_deps(artifact) + name=source.name, + kind=source.morphology['kind']) + + self.fetch_sources(source) + # TODO: Make an artifact.walk() that takes multiple root artifacts. + # as this does a walk for every artifact. This was the status + # quo before build logic was made to work per-source, but we can + # now do better. + deps = self.get_recursive_deps(source.artifacts.values()) self.cache_artifacts_locally(deps) use_chroot = False setup_mounts = False - if artifact.source.morphology['kind'] == 'chunk': - build_mode = artifact.source.build_mode - extra_env = {'PREFIX': artifact.source.prefix} + if source.morphology['kind'] == 'chunk': + build_mode = source.build_mode + extra_env = {'PREFIX': source.prefix} - dep_prefix_set = artifact.get_dependency_prefix_set() + dep_prefix_set = set(a.source.prefix for a in deps + if a.source.morphology['kind'] == 'chunk') extra_path = [os.path.join(d, 'bin') for d in dep_prefix_set] if build_mode not in ['bootstrap', 'staging', 'test']: @@ -340,37 +357,44 @@ class BuildCommand(object): extra_env=extra_env, extra_path=extra_path) try: - self.install_dependencies(staging_area, deps, artifact) + self.install_dependencies(staging_area, deps, source) except BaseException: staging_area.abort() raise else: staging_area = self.create_staging_area(build_env, False) - self.build_and_cache(staging_area, artifact, setup_mounts) + self.build_and_cache(staging_area, source, setup_mounts) self.remove_staging_area(staging_area) - def get_recursive_deps(self, artifact): - return artifact.walk()[:-1] + def get_recursive_deps(self, artifacts): + deps = set() + ordered_deps = [] + for artifact in artifacts: + for dep in artifact.walk(): + if dep not in deps and dep not in artifacts: + deps.add(dep) + ordered_deps.append(dep) + return ordered_deps - def fetch_sources(self, artifact): + def fetch_sources(self, source): '''Update the local git repository cache with the sources.''' - repo_name = artifact.source.repo_name + repo_name = source.repo_name if self.app.settings['no-git-update']: self.app.status(msg='Not updating existing git repository ' '%(repo_name)s ' 'because of no-git-update being set', chatty=True, repo_name=repo_name) - artifact.source.repo = self.lrc.get_repo(repo_name) + source.repo = self.lrc.get_repo(repo_name) return if self.lrc.has_repo(repo_name): - artifact.source.repo = self.lrc.get_repo(repo_name) + source.repo = self.lrc.get_repo(repo_name) try: - sha1 = artifact.source.sha1 - artifact.source.repo.resolve_ref(sha1) + sha1 = source.sha1 + source.repo.resolve_ref(sha1) self.app.status(msg='Not updating git repository ' '%(repo_name)s because it ' 'already contains sha1 %(sha1)s', @@ -379,17 +403,17 @@ class BuildCommand(object): except morphlib.cachedrepo.InvalidReferenceError: self.app.status(msg='Updating %(repo_name)s', repo_name=repo_name) - artifact.source.repo.update() + source.repo.update() else: self.app.status(msg='Cloning %(repo_name)s', repo_name=repo_name) - artifact.source.repo = self.lrc.cache_repo(repo_name) + source.repo = self.lrc.cache_repo(repo_name) # Update submodules. done = set() self.app.cache_repo_and_submodules( - self.lrc, artifact.source.repo.url, - artifact.source.sha1, done) + self.lrc, source.repo.url, + source.sha1, done) def cache_artifacts_locally(self, artifacts): '''Get artifacts missing from local cache from remote cache.''' @@ -455,14 +479,25 @@ class BuildCommand(object): # Nasty hack to avoid installing chunks built in 'bootstrap' mode in a # different stratum when constructing staging areas. - def is_stratum(self, a): - return a.source.morphology['kind'] == 'stratum' + # TODO: make nicer by having chunk morphs keep a reference to the + # stratum they were in + def in_same_stratum(self, s1, s2): + '''Checks whether two chunk sources are from the same stratum. - def in_same_stratum(self, a, b): - return len(filter(self.is_stratum, a.dependencies)) == \ - len(filter(self.is_stratum, b.dependencies)) + In the absence of morphologies tracking where they came from, + this checks whether both sources are depended on by artifacts + that belong to sources which have the same morphology. - def install_dependencies(self, staging_area, artifacts, target_artifact): + ''' + def dependent_stratum_morphs(source): + dependent_sources = set(itertools.chain.from_iterable( + a.dependent_sources for a in source.artifacts.itervalues())) + dependent_strata = set(s for s in dependent_sources + if s.morphology['kind'] == 'stratum') + return set(s.morphology for s in dependent_strata) + return dependent_stratum_morphs(s1) == dependent_stratum_morphs(s2) + + def install_dependencies(self, staging_area, artifacts, target_source): '''Install chunk artifacts into staging area. We only ever care about chunk artifacts as build dependencies, @@ -477,29 +512,29 @@ class BuildCommand(object): if artifact.source.morphology['kind'] != 'chunk': continue if artifact.source.build_mode == 'bootstrap': - if not self.in_same_stratum(artifact, target_artifact): + if not self.in_same_stratum(artifact.source, target_source): continue self.app.status( msg='Installing chunk %(chunk_name)s from cache %(cache)s', chunk_name=artifact.name, - cache=artifact.cache_key[:7], + cache=artifact.source.cache_key[:7], chatty=True) handle = self.lac.get(artifact) staging_area.install_artifact(handle) - if target_artifact.source.build_mode == 'staging': + if target_source.build_mode == 'staging': morphlib.builder2.ldconfig(self.app.runcmd, staging_area.dirname) - def build_and_cache(self, staging_area, artifact, setup_mounts): - '''Build an artifact and put it into the local artifact cache.''' + def build_and_cache(self, staging_area, source, setup_mounts): + '''Build a source and put its artifacts into the local cache.''' self.app.status(msg='Starting actual build: %(name)s ' '%(sha1)s', - name=artifact.name, sha1=artifact.source.sha1[:7]) + name=source.name, sha1=source.sha1[:7]) builder = morphlib.builder2.Builder( self.app, staging_area, self.lac, self.rac, self.lrc, self.app.settings['max-jobs'], setup_mounts) - return builder.build_and_cache(artifact) + return builder.build_and_cache(source) class InitiatorBuildCommand(BuildCommand): diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 681ad6be..20cae225 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -162,15 +162,15 @@ def get_stratum_files(f, lac): # pragma: no cover cf.close() -def get_overlaps(artifact, constituents, lac): # pragma: no cover +def get_overlaps(source, constituents, lac): # pragma: no cover # check whether strata overlap installed = defaultdict(set) for dep in constituents: handle = lac.get(dep) - if artifact.source.morphology['kind'] == 'stratum': + if source.morphology['kind'] == 'stratum': for filename in get_chunk_files(handle): installed[filename].add(dep) - elif artifact.source.morphology['kind'] == 'system': + elif source.morphology['kind'] == 'system': for filename in get_stratum_files(handle, lac): installed[filename].add(dep) handle.close() @@ -207,13 +207,13 @@ class BuilderBase(object): '''Base class for building artifacts.''' def __init__(self, app, staging_area, local_artifact_cache, - remote_artifact_cache, artifact, repo_cache, max_jobs, + remote_artifact_cache, source, repo_cache, max_jobs, setup_mounts): self.app = app self.staging_area = staging_area self.local_artifact_cache = local_artifact_cache self.remote_artifact_cache = remote_artifact_cache - self.artifact = artifact + self.source = source self.repo_cache = repo_cache self.max_jobs = max_jobs self.build_watch = morphlib.stopwatch.Stopwatch() @@ -233,13 +233,13 @@ class BuilderBase(object): logging.debug('Writing metadata to the cache') with self.local_artifact_cache.put_source_metadata( - self.artifact.source, self.artifact.cache_key, + self.source, self.source.cache_key, 'meta') as f: json.dump(meta, f, indent=4, sort_keys=True, encoding='unicode-escape') f.write('\n') - def create_metadata(self, artifact_name, contents=[]): + def create_metadata(self, artifact_name, contents=[]): # pragma: no cover '''Create metadata to artifact to allow it to be reproduced later. The metadata is represented as a dict, which later on will be @@ -247,20 +247,20 @@ class BuilderBase(object): ''' - assert isinstance(self.artifact.source.repo, + assert isinstance(self.source.repo, morphlib.cachedrepo.CachedRepo) meta = { 'artifact-name': artifact_name, - 'source-name': self.artifact.source.morphology['name'], - 'kind': self.artifact.source.morphology['kind'], - 'description': self.artifact.source.morphology['description'], - 'repo': self.artifact.source.repo.url, - 'repo-alias': self.artifact.source.repo_name, - 'original_ref': self.artifact.source.original_ref, - 'sha1': self.artifact.source.sha1, - 'morphology': self.artifact.source.filename, - 'cache-key': self.artifact.cache_key, - 'cache-id': self.artifact.cache_id, + 'source-name': self.source.name, + 'kind': self.source.morphology['kind'], + 'description': self.source.morphology['description'], + 'repo': self.source.repo.url, + 'repo-alias': self.source.repo_name, + 'original_ref': self.source.original_ref, + 'sha1': self.source.sha1, + 'morphology': self.source.filename, + 'cache-key': self.source.cache_key, + 'cache-id': self.source.cache_id, 'morph-version': { 'ref': morphlib.gitversion.ref, 'tree': morphlib.gitversion.tree, @@ -279,7 +279,8 @@ class BuilderBase(object): os.makedirs(dirname) return open(filename, mode) - def write_metadata(self, instdir, artifact_name, contents=[]): + def write_metadata(self, instdir, artifact_name, + contents=[]): # pragma: no cover '''Write the metadata for an artifact. The file will be located under the ``baserock`` directory under @@ -308,7 +309,7 @@ class ChunkBuilder(BuilderBase): def create_devices(self, destdir): # pragma: no cover '''Creates device nodes if the morphology specifies them''' - morphology = self.artifact.source.morphology + morphology = self.source.morphology perms_mask = stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO if 'devices' in morphology and morphology['devices'] is not None: for dev in morphology['devices']: @@ -332,14 +333,14 @@ class ChunkBuilder(BuilderBase): with self.build_watch('overall-build'): builddir, destdir = self.staging_area.chroot_open( - self.artifact.source, self.setup_mounts) + self.source, self.setup_mounts) stdout = (self.app.output if self.app.settings['build-log-on-stdout'] else None) cache = self.local_artifact_cache logpath = cache.get_source_metadata_filename( - self.artifact.source, self.artifact.cache_key, 'build-log') + self.source, self.source.cache_key, 'build-log') _, temppath = tempfile.mkstemp(dir=os.path.dirname(logpath)) @@ -375,7 +376,7 @@ class ChunkBuilder(BuilderBase): def run_commands(self, builddir, destdir, logfilepath, stdout=None): # pragma: no cover - m = self.artifact.source.morphology + m = self.source.morphology bs = morphlib.buildsystem.lookup_build_system(m['build-system']) relative_builddir = self.staging_area.relative(builddir) @@ -407,7 +408,7 @@ class ChunkBuilder(BuilderBase): for cmd in cmds: if in_parallel: - max_jobs = self.artifact.source.morphology['max-jobs'] + max_jobs = self.source.morphology['max-jobs'] if max_jobs is None: max_jobs = self.max_jobs extra_env['MAKEFLAGS'] = '-j%s' % max_jobs @@ -474,7 +475,7 @@ class ChunkBuilder(BuilderBase): def assemble_chunk_artifacts(self, destdir): # pragma: no cover built_artifacts = [] filenames = [] - source = self.artifact.source + source = self.source split_rules = source.split_rules morphology = source.morphology sys_tag = 'system-integration' @@ -533,7 +534,7 @@ class ChunkBuilder(BuilderBase): return built_artifacts def get_sources(self, srcdir): # pragma: no cover - s = self.artifact.source + s = self.source extract_sources(self.app, self.repo_cache, s.repo, s.sha1, srcdir) @@ -547,42 +548,42 @@ class StratumBuilder(BuilderBase): def build_and_cache(self): # pragma: no cover with self.build_watch('overall-build'): - constituents = [d for d in self.artifact.dependencies + constituents = [d for d in self.source.dependencies if self.is_constituent(d)] # the only reason the StratumBuilder has to download chunks is to # check for overlap now that strata are lists of chunks with self.build_watch('check-chunks'): - # download the chunk artifact if necessary - download_depends(constituents, - self.local_artifact_cache, - self.remote_artifact_cache) - # check for chunk overlaps - overlaps = get_overlaps(self.artifact, constituents, - self.local_artifact_cache) - if len(overlaps) > 0: - logging.warning('Overlaps in stratum artifact %s detected' - % self.artifact.name) - log_overlaps(overlaps) - self.app.status(msg='Overlaps in stratum artifact ' - '%(stratum_name)s detected', - stratum_name=self.artifact.name, - error=True) - write_overlap_metadata(self.artifact, overlaps, - self.local_artifact_cache) + for a_name, a in self.source.artifacts.iteritems(): + # download the chunk artifact if necessary + download_depends(constituents, + self.local_artifact_cache, + self.remote_artifact_cache) + # check for chunk overlaps + overlaps = get_overlaps(self.source, constituents, + self.local_artifact_cache) + if len(overlaps) > 0: + logging.warning( + 'Overlaps in stratum artifact %s detected' %a_name) + log_overlaps(overlaps) + self.app.status(msg='Overlaps in stratum artifact ' + '%(stratum_name)s detected', + stratum_name=a_name, error=True) + write_overlap_metadata(a, overlaps, + self.local_artifact_cache) with self.build_watch('create-chunk-list'): lac = self.local_artifact_cache - meta = self.create_metadata(self.artifact.name, - [x.name for x in constituents]) - with lac.put_artifact_metadata(self.artifact, 'meta') as f: - json.dump(meta, f, indent=4, sort_keys=True, - encoding='unicode-escape') - with self.local_artifact_cache.put(self.artifact) as f: - json.dump([c.basename() for c in constituents], f, - encoding='unicode-escape') + for a_name, a in self.source.artifacts.iteritems(): + meta = self.create_metadata( + a_name, + [x.name for x in constituents]) + with lac.put_artifact_metadata(a, 'meta') as f: + json.dump(meta, f, indent=4, sort_keys=True) + with self.local_artifact_cache.put(a) as f: + json.dump([c.basename() for c in constituents], f) self.save_build_times() - return [self.artifact] + return self.source.artifacts.values() class SystemBuilder(BuilderBase): # pragma: no cover @@ -596,43 +597,42 @@ class SystemBuilder(BuilderBase): # pragma: no cover def build_and_cache(self): self.app.status(msg='Building system %(system_name)s', - system_name=self.artifact.source.morphology['name']) + system_name=self.source.name) with self.build_watch('overall-build'): - arch = self.artifact.source.morphology['arch'] - - rootfs_name = self.artifact.source.morphology['name'] - handle = self.local_artifact_cache.put(self.artifact) - - try: - fs_root = self.staging_area.destdir(self.artifact.source) - self.unpack_strata(fs_root) - self.write_metadata(fs_root, rootfs_name) - self.run_system_integration_commands(fs_root) - unslashy_root = fs_root[1:] - def uproot_info(info): - info.name = relpath(info.name, unslashy_root) - if info.islnk(): - info.linkname = relpath(info.linkname, - unslashy_root) - return info - artiname = self.artifact.source.morphology['name'] - tar = tarfile.open(fileobj=handle, mode="w", name=artiname) - self.app.status(msg='Constructing tarball of root filesystem', - chatty=True) - tar.add(fs_root, recursive=True, filter=uproot_info) - tar.close() - except BaseException, e: - logging.error(traceback.format_exc()) - self.app.status(msg='Error while building system', - error=True) - handle.abort() - raise - - handle.close() + arch = self.source.morphology['arch'] + + for a_name, artifact in self.source.artifacts.iteritems(): + handle = self.local_artifact_cache.put(artifact) + + try: + fs_root = self.staging_area.destdir(self.source) + self.unpack_strata(fs_root) + self.write_metadata(fs_root, a_name) + self.run_system_integration_commands(fs_root) + unslashy_root = fs_root[1:] + def uproot_info(info): + info.name = relpath(info.name, unslashy_root) + if info.islnk(): + info.linkname = relpath(info.linkname, + unslashy_root) + return info + tar = tarfile.open(fileobj=handle, mode="w", name=a_name) + self.app.status(msg='Constructing tarball of rootfs', + chatty=True) + tar.add(fs_root, recursive=True, filter=uproot_info) + tar.close() + except BaseException as e: + logging.error(traceback.format_exc()) + self.app.status(msg='Error while building system', + error=True) + handle.abort() + raise + else: + handle.close() self.save_build_times() - return [self.artifact] + return self.source.artifacts.itervalues() def unpack_one_stratum(self, stratum_artifact, target): '''Unpack a single stratum into a target directory''' @@ -658,37 +658,37 @@ class SystemBuilder(BuilderBase): # pragma: no cover self.app.status(msg='Unpacking strata to %(path)s', path=path, chatty=True) with self.build_watch('unpack-strata'): - # download the stratum artifacts if necessary - download_depends(self.artifact.dependencies, - self.local_artifact_cache, - self.remote_artifact_cache, - ('meta',)) - - # download the chunk artifacts if necessary - for stratum_artifact in self.artifact.dependencies: - f = self.local_artifact_cache.get(stratum_artifact) - chunks = [ArtifactCacheReference(a) - for a in json.load(f, encoding='unicode-escape')] - download_depends(chunks, + for a_name, a in self.source.artifacts.iteritems(): + # download the stratum artifacts if necessary + download_depends(self.source.dependencies, self.local_artifact_cache, - self.remote_artifact_cache) - f.close() - - # check whether the strata overlap - overlaps = get_overlaps(self.artifact, self.artifact.dependencies, - self.local_artifact_cache) - if len(overlaps) > 0: - self.app.status(msg='Overlaps in system artifact ' - '%(artifact_name)s detected', - artifact_name=self.artifact.name, - error=True) - log_overlaps(overlaps) - write_overlap_metadata(self.artifact, overlaps, - self.local_artifact_cache) - - # unpack it from the local artifact cache - for stratum_artifact in self.artifact.dependencies: - self.unpack_one_stratum(stratum_artifact, path) + self.remote_artifact_cache, + ('meta',)) + + # download the chunk artifacts if necessary + for stratum_artifact in self.source.dependencies: + f = self.local_artifact_cache.get(stratum_artifact) + chunks = [ArtifactCacheReference(c) for c in json.load(f)] + download_depends(chunks, + self.local_artifact_cache, + self.remote_artifact_cache) + f.close() + + # check whether the strata overlap + overlaps = get_overlaps(self.source, self.source.dependencies, + self.local_artifact_cache) + if len(overlaps) > 0: + self.app.status(msg='Overlaps in system artifact ' + '%(artifact_name)s detected', + artifact_name=a_name, + error=True) + log_overlaps(overlaps) + write_overlap_metadata(a, overlaps, + self.local_artifact_cache) + + # unpack it from the local artifact cache + for stratum_artifact in self.source.dependencies: + self.unpack_one_stratum(stratum_artifact, path) ldconfig(self.app.runcmd, path) @@ -779,15 +779,15 @@ class Builder(object): # pragma: no cover self.max_jobs = max_jobs self.setup_mounts = setup_mounts - def build_and_cache(self, artifact): - kind = artifact.source.morphology['kind'] + def build_and_cache(self, source): + kind = source.morphology['kind'] o = self.classes[kind](self.app, self.staging_area, self.local_artifact_cache, - self.remote_artifact_cache, artifact, + self.remote_artifact_cache, source, self.repo_cache, self.max_jobs, self.setup_mounts) self.app.status(msg='Builder.build: artifact %s with %s' % - (artifact.name, repr(o)), + (source.name, repr(o)), chatty=True) built_artifacts = o.build_and_cache() self.app.status(msg='Builder.build: done', diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py index a9f5aabe..c3a01b9e 100644 --- a/morphlib/cachekeycomputer.py +++ b/morphlib/cachekeycomputer.py @@ -32,16 +32,15 @@ class CacheKeyComputer(object): "USER", "USERNAME"] return dict([(k, env[k]) for k in keys]) - def compute_key(self, artifact): + def compute_key(self, source): try: - ret = self._hashed[artifact] - return ret + return self._hashed[source] except KeyError: - ret = self._hash_id(self.get_cache_id(artifact)) - self._hashed[artifact] = ret - logging.debug('computed cache key %s for artifact %s from source ', - ret, (artifact.source.repo_name, - artifact.source.sha1, artifact.source.filename)) + ret = self._hash_id(self.get_cache_id(source)) + self._hashed[source] = ret + logging.debug( + 'computed cache key %s for artifact %s from source ', + ret, (source.repo_name, source.sha1, source.filename)) return ret def _hash_id(self, cache_id): @@ -71,47 +70,47 @@ class CacheKeyComputer(object): for item in tup: self._hash_thing(sha, item) - def get_cache_id(self, artifact): + def get_cache_id(self, source): try: - ret = self._calculated[artifact] + ret = self._calculated[source] return ret except KeyError: - cacheid = self._calculate(artifact) - self._calculated[artifact] = cacheid + cacheid = self._calculate(source) + self._calculated[source] = cacheid return cacheid - def _calculate(self, artifact): + def _calculate(self, source): keys = { 'env': self._filterenv(self._build_env.env), - 'kids': [{'artifact': a.name, 'cache-key': self.compute_key(a)} - for a in artifact.dependencies], - 'metadata-version': 1 # bump if /baserock metadata format changes + 'kids': [{'artifact': a.name, + 'cache-key': self.compute_key(a.source)} + for a in source.dependencies], + 'metadata-version': 1 } - kind = artifact.source.morphology['kind'] + morphology = source.morphology + kind = morphology['kind'] if kind == 'chunk': - keys['build-mode'] = artifact.source.build_mode - keys['prefix'] = artifact.source.prefix - keys['tree'] = artifact.source.tree + keys['build-mode'] = source.build_mode + keys['prefix'] = source.prefix + keys['tree'] = source.tree keys['split-rules'] = [(a, [rgx.pattern for rgx in r._regexes]) - for (a, r) in artifact.source.split_rules] + for (a, r) in source.split_rules] # Include morphology contents, since it doesn't always come # from the source tree - morphology = artifact.source.morphology + keys['devices'] = morphology.get('devices') + keys['max-jobs'] = morphology.get('max-jobs') + keys['system-integration'] = morphology.get('system-integration', + {}) + # products is omitted as they are part of the split-rules # include {pre-,,post-}{configure,build,test,install}-commands # in morphology key for prefix in ('pre-', '', 'post-'): for cmdtype in ('configure', 'build', 'test', 'install'): cmd_field = prefix + cmdtype + '-commands' keys[cmd_field] = morphology[cmd_field] - keys['devices'] = morphology.get('devices') - keys['max-jobs'] = morphology.get('max-jobs') - keys['system-integration'] = morphology.get('system-integration', - {}) - # products is omitted as they are part of the split-rules elif kind in ('system', 'stratum'): - morphology = artifact.source.morphology morph_dict = dict((k, morphology[k]) for k in morphology.keys()) # Disregard all fields of a morphology that aren't important diff --git a/morphlib/plugins/cross-bootstrap_plugin.py b/morphlib/plugins/cross-bootstrap_plugin.py index 95e991f4..7b53a4a5 100644 --- a/morphlib/plugins/cross-bootstrap_plugin.py +++ b/morphlib/plugins/cross-bootstrap_plugin.py @@ -58,38 +58,39 @@ class BootstrapSystemBuilder(morphlib.builder2.BuilderBase): def build_and_cache(self): with self.build_watch('overall-build'): - handle = self.local_artifact_cache.put(self.artifact) - fs_root = self.staging_area.destdir(self.artifact.source) - try: - self.unpack_binary_chunks(fs_root) - self.unpack_sources(fs_root) - self.write_build_script(fs_root) - system_name = self.artifact.source.morphology['name'] - self.create_tarball(handle, fs_root, system_name) - except BaseException, e: - logging.error(traceback.format_exc()) - self.app.status(msg='Error while building bootstrap image', - error=True) - handle.abort() - raise - - handle.close() + for system_name, artifact in self.source.artifacts.iteritems(): + handle = self.local_artifact_cache.put(artifact) + fs_root = self.staging_area.destdir(self.source) + try: + self.unpack_binary_chunks(fs_root) + self.unpack_sources(fs_root) + self.write_build_script(fs_root) + self.create_tarball(handle, fs_root, system_name) + except BaseException, e: + logging.error(traceback.format_exc()) + self.app.status(msg='Error while building bootstrap image', + error=True) + handle.abort() + raise + + handle.close() self.save_build_times() - return [self.artifact] + return self.source.artifacts.items() def unpack_binary_chunks(self, dest): cache = self.local_artifact_cache - for chunk_artifact in self.artifact.source.cross_chunks: - with cache.get(chunk_artifact) as chunk_file: - try: - morphlib.bins.unpack_binary_from_file(chunk_file, dest) - except BaseException, e: - self.app.status( - msg='Error unpacking binary chunk %(name)s', - name=chunk_artifact.name, - error=True) - raise + for chunk_source in self.source.cross_sources: + for chunk_artifact in chunk_source.artifacts.itervalues(): + with cache.get(chunk_artifact) as chunk_file: + try: + morphlib.bins.unpack_binary_from_file(chunk_file, dest) + except BaseException, e: + self.app.status( + msg='Error unpacking binary chunk %(name)s', + name=chunk_artifact.name, + error=True) + raise def unpack_sources(self, path): # Multiple chunks sources may be built from the same repo ('linux' @@ -98,24 +99,18 @@ class BootstrapSystemBuilder(morphlib.builder2.BuilderBase): # # It might be neater to build these as "source artifacts" individually, # but that would waste huge amounts of space in the artifact cache. - for a in self.artifact.walk(): - if a in self.artifact.source.cross_chunks: - continue - if a.source.morphology['kind'] != 'chunk': - continue - - escaped_source = escape_source_name(a.source) + for s in self.source.native_sources: + escaped_source = escape_source_name(s) source_dir = os.path.join(path, 'src', escaped_source) if not os.path.exists(source_dir): os.makedirs(source_dir) morphlib.builder2.extract_sources( - self.app, self.repo_cache, a.source.repo, a.source.sha1, - source_dir) + self.app, self.repo_cache, s.repo, s.sha1, source_dir) - name = a.source.morphology['name'] + name = s.name chunk_script = os.path.join(path, 'src', 'build-%s' % name) with morphlib.savefile.SaveFile(chunk_script, 'w') as f: - self.write_chunk_build_script(a, f) + self.write_chunk_build_script(s, f) os.chmod(chunk_script, 0777) def write_build_script(self, path): @@ -130,15 +125,8 @@ class BootstrapSystemBuilder(morphlib.builder2.BuilderBase): if k != 'PATH': f.write('export %s="%s"\n' % (k, v)) - # FIXME: really, of course, we need to iterate the sources not the - # artifacts ... this will break when we have chunk splitting! - for a in self.artifact.walk(): - if a in self.artifact.source.cross_chunks: - continue - if a.source.morphology['kind'] != 'chunk': - continue - - name = a.source.morphology['name'] + for s in self.source.native_sources: + name = s.name f.write('\necho Building %s\n' % name) f.write('mkdir /%s.inst\n' % name) f.write('env DESTDIR=/%s.inst $SRCDIR/build-%s\n' @@ -150,17 +138,17 @@ class BootstrapSystemBuilder(morphlib.builder2.BuilderBase): f.write(driver_footer) os.chmod(driver_script, 0777) - def write_chunk_build_script(self, chunk, f): - m = chunk.source.morphology + def write_chunk_build_script(self, source, f): + m = source.morphology f.write('#!/bin/sh\n') f.write('# Build script generated by morph\n') f.write('set -e\n') f.write('chunk_name=%s\n' % m['name']) - repo = escape_source_name(chunk.source) + repo = escape_source_name(source) f.write('cp -a $SRCDIR/%s $DESTDIR/$chunk_name.build\n' % repo) f.write('cd $DESTDIR/$chunk_name.build\n') - f.write('export PREFIX=%s\n' % chunk.source.prefix) + f.write('export PREFIX=%s\n' % source.prefix) bs = morphlib.buildsystem.lookup_build_system(m['build-system']) @@ -280,35 +268,36 @@ class CrossBootstrapPlugin(cliapp.Plugin): # Calculate build order # This is basically a hacked version of BuildCommand.build_in_order() - artifacts = system_artifact.walk() - cross_chunks = [] - native_chunks = [] - for a in artifacts: - if a.source.morphology['kind'] == 'chunk': - if a.source.build_mode == 'bootstrap': - cross_chunks.append(a) + sources = build_command.get_ordered_sources(system_artifact.walk()) + cross_sources = [] + native_sources = [] + for s in sources: + if s.morphology['kind'] == 'chunk': + if s.build_mode == 'bootstrap': + cross_sources.append(s) else: - native_chunks.append(a) + native_sources.append(s) - if len(cross_chunks) == 0: + if len(cross_sources) == 0: raise morphlib.Error( 'Nothing to cross-compile. Only chunks built in \'bootstrap\' ' 'mode can be cross-compiled.') - for i, a in enumerate(cross_chunks): - build_command.cache_or_build_artifact(a, build_env) + for s in cross_sources: + build_command.cache_or_build_source(s, build_env) - for i, a in enumerate(native_chunks): - build_command.fetch_sources(a) + for s in native_sources: + build_command.fetch_sources(s) # Install those to the output tarball ... self.app.status(msg='Building final bootstrap system image') - system_artifact.source.cross_chunks = cross_chunks + system_artifact.source.cross_sources = cross_sources + system_artifact.source.native_sources = native_sources staging_area = build_command.create_staging_area( build_env, use_chroot=False) builder = BootstrapSystemBuilder( self.app, staging_area, build_command.lac, build_command.rac, - system_artifact, build_command.lrc, 1, False) + system_artifact.source, build_command.lrc, 1, False) builder.build_and_cache() self.app.status( diff --git a/morphlib/remoteartifactcache.py b/morphlib/remoteartifactcache.py index 0f8edce8..4e09ce34 100644 --- a/morphlib/remoteartifactcache.py +++ b/morphlib/remoteartifactcache.py @@ -31,9 +31,9 @@ class GetError(cliapp.AppException): def __init__(self, cache, artifact): cliapp.AppException.__init__( - self, 'Failed to get the artifact %s with cache key %s ' + self, 'Failed to get the artifact %s ' 'from the artifact cache %s' % - (artifact.basename(), artifact.cache_key, cache)) + (artifact.basename(), cache)) class GetArtifactMetadataError(GetError): |