diff options
author | Richard Maw <richard.maw@gmail.com> | 2014-09-10 16:05:18 +0000 |
---|---|---|
committer | Richard Maw <richard.maw@gmail.com> | 2014-09-19 12:43:26 +0000 |
commit | 1202d2a452b3aefed351d08b1ffe2f68391b7416 (patch) | |
tree | e4daaac9e635370f6b93201ea37db08fcde1855f /morphlib/buildcommand.py | |
parent | a013626cf32e8f07be16590e3caa4d5f9839b426 (diff) | |
download | morph-1202d2a452b3aefed351d08b1ffe2f68391b7416.tar.gz |
Build per-source rather than per-artifact
Diffstat (limited to 'morphlib/buildcommand.py')
-rw-r--r-- | morphlib/buildcommand.py | 159 |
1 files changed, 97 insertions, 62 deletions
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 436e23eb..c8d9930c 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -14,6 +14,7 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import itertools import os import shutil import logging @@ -175,9 +176,9 @@ class BuildCommand(object): self.app.status(msg='Computing cache keys', chatty=True) ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env) - for artifact in artifacts: - artifact.cache_key = ckc.compute_key(artifact) - artifact.cache_id = ckc.get_cache_id(artifact) + for source in set(a.source for a in artifacts): + source.cache_key = ckc.compute_key(source) + source.cache_id = ckc.get_cache_id(source) root_artifact.build_env = build_env return root_artifact @@ -210,7 +211,7 @@ class BuildCommand(object): if src.morphology['kind'] == 'stratum': name = src.name ref = src.sha1[:7] - self.app.status(msg='Stratum [%(name)s] version is %(ref)s', + self.app.status(msg='Stratum [%(name)s] version is %(ref)s', name=name, ref=ref) if name in stratum_names: raise morphlib.Error( @@ -251,78 +252,94 @@ class BuildCommand(object): def _find_root_artifacts(self, artifacts): '''Find all the root artifacts among a set of artifacts in a DAG. - + It would be nice if the ArtifactResolver would return its results in a more useful order to save us from needing to do this -- the root object is known already since that's the one the user asked us to build. - + ''' - return [a for a in artifacts if not a.dependents] + return [a for a in artifacts if not a.dependent_sources] + + @staticmethod + def get_ordered_sources(artifacts): + ordered_sources = [] + known_sources = set() + for artifact in artifacts: + if artifact.source not in known_sources: + known_sources.add(artifact.source) + yield artifact.source def build_in_order(self, root_artifact): '''Build everything specified in a build order.''' - self.app.status(msg='Building a set of artifacts', chatty=True) + self.app.status(msg='Building a set of sources', chatty=True) build_env = root_artifact.build_env - artifacts = root_artifact.walk() + ordered_sources = list(self.get_ordered_sources(root_artifact.walk())) old_prefix = self.app.status_prefix - for i, a in enumerate(artifacts): + for i, s in enumerate(ordered_sources): self.app.status_prefix = ( old_prefix + '[Build %(index)d/%(total)d] [%(name)s] ' % { 'index': (i+1), - 'total': len(artifacts), - 'name': a.name, + 'total': len(ordered_sources), + 'name': s.name, }) - self.cache_or_build_artifact(a, build_env) - - self.app.status(msg='%(kind)s %(name)s is cached at %(cachepath)s', - kind=a.source.morphology['kind'], name=a.name, - cachepath=self.lac.artifact_filename(a), - chatty=(a.source.morphology['kind'] != "system")) + self.cache_or_build_source(s, build_env) self.app.status_prefix = old_prefix - def cache_or_build_artifact(self, artifact, build_env): - '''Make the built artifact available in the local cache. + def cache_or_build_source(self, source, build_env): + '''Make artifacts of the built source available in the local cache. This can be done by retrieving from a remote artifact cache, or if - that doesn't work for some reason, by building the artifact locally. + that doesn't work for some reason, by building the source locally. ''' + artifacts = source.artifacts.values() if self.rac is not None: try: - self.cache_artifacts_locally([artifact]) + self.cache_artifacts_locally(artifacts) except morphlib.remoteartifactcache.GetError: # Error is logged by the RemoteArtifactCache object. pass - if not self.lac.has(artifact): - self.build_artifact(artifact, build_env) + if any(not self.lac.has(artifact) for artifact in artifacts): + self.build_source(source, build_env) + + for a in artifacts: + self.app.status(msg='%(kind)s %(name)s is cached at %(cachepath)s', + kind=source.morphology['kind'], name=a.name, + cachepath=self.lac.artifact_filename(a), + chatty=(source.morphology['kind'] != "system")) - def build_artifact(self, artifact, build_env): - '''Build one artifact. + def build_source(self, source, build_env): + '''Build all artifacts for one source. All the dependencies are assumed to be built and available in either the local or remote cache already. ''' self.app.status(msg='Building %(kind)s %(name)s', - name=artifact.name, - kind=artifact.source.morphology['kind']) - - self.fetch_sources(artifact) - deps = self.get_recursive_deps(artifact) + name=source.name, + kind=source.morphology['kind']) + + self.fetch_sources(source) + # TODO: Make an artifact.walk() that takes multiple root artifacts. + # as this does a walk for every artifact. This was the status + # quo before build logic was made to work per-source, but we can + # now do better. + deps = self.get_recursive_deps(source.artifacts.values()) self.cache_artifacts_locally(deps) use_chroot = False setup_mounts = False - if artifact.source.morphology['kind'] == 'chunk': - build_mode = artifact.source.build_mode - extra_env = {'PREFIX': artifact.source.prefix} + if source.morphology['kind'] == 'chunk': + build_mode = source.build_mode + extra_env = {'PREFIX': source.prefix} - dep_prefix_set = artifact.get_dependency_prefix_set() + dep_prefix_set = set(a.source.prefix for a in deps + if a.source.morphology['kind'] == 'chunk') extra_path = [os.path.join(d, 'bin') for d in dep_prefix_set] if build_mode not in ['bootstrap', 'staging', 'test']: @@ -340,37 +357,44 @@ class BuildCommand(object): extra_env=extra_env, extra_path=extra_path) try: - self.install_dependencies(staging_area, deps, artifact) + self.install_dependencies(staging_area, deps, source) except BaseException: staging_area.abort() raise else: staging_area = self.create_staging_area(build_env, False) - self.build_and_cache(staging_area, artifact, setup_mounts) + self.build_and_cache(staging_area, source, setup_mounts) self.remove_staging_area(staging_area) - def get_recursive_deps(self, artifact): - return artifact.walk()[:-1] + def get_recursive_deps(self, artifacts): + deps = set() + ordered_deps = [] + for artifact in artifacts: + for dep in artifact.walk(): + if dep not in deps and dep not in artifacts: + deps.add(dep) + ordered_deps.append(dep) + return ordered_deps - def fetch_sources(self, artifact): + def fetch_sources(self, source): '''Update the local git repository cache with the sources.''' - repo_name = artifact.source.repo_name + repo_name = source.repo_name if self.app.settings['no-git-update']: self.app.status(msg='Not updating existing git repository ' '%(repo_name)s ' 'because of no-git-update being set', chatty=True, repo_name=repo_name) - artifact.source.repo = self.lrc.get_repo(repo_name) + source.repo = self.lrc.get_repo(repo_name) return if self.lrc.has_repo(repo_name): - artifact.source.repo = self.lrc.get_repo(repo_name) + source.repo = self.lrc.get_repo(repo_name) try: - sha1 = artifact.source.sha1 - artifact.source.repo.resolve_ref(sha1) + sha1 = source.sha1 + source.repo.resolve_ref(sha1) self.app.status(msg='Not updating git repository ' '%(repo_name)s because it ' 'already contains sha1 %(sha1)s', @@ -379,17 +403,17 @@ class BuildCommand(object): except morphlib.cachedrepo.InvalidReferenceError: self.app.status(msg='Updating %(repo_name)s', repo_name=repo_name) - artifact.source.repo.update() + source.repo.update() else: self.app.status(msg='Cloning %(repo_name)s', repo_name=repo_name) - artifact.source.repo = self.lrc.cache_repo(repo_name) + source.repo = self.lrc.cache_repo(repo_name) # Update submodules. done = set() self.app.cache_repo_and_submodules( - self.lrc, artifact.source.repo.url, - artifact.source.sha1, done) + self.lrc, source.repo.url, + source.sha1, done) def cache_artifacts_locally(self, artifacts): '''Get artifacts missing from local cache from remote cache.''' @@ -455,14 +479,25 @@ class BuildCommand(object): # Nasty hack to avoid installing chunks built in 'bootstrap' mode in a # different stratum when constructing staging areas. - def is_stratum(self, a): - return a.source.morphology['kind'] == 'stratum' + # TODO: make nicer by having chunk morphs keep a reference to the + # stratum they were in + def in_same_stratum(self, s1, s2): + '''Checks whether two chunk sources are from the same stratum. - def in_same_stratum(self, a, b): - return len(filter(self.is_stratum, a.dependencies)) == \ - len(filter(self.is_stratum, b.dependencies)) + In the absence of morphologies tracking where they came from, + this checks whether both sources are depended on by artifacts + that belong to sources which have the same morphology. - def install_dependencies(self, staging_area, artifacts, target_artifact): + ''' + def dependent_stratum_morphs(source): + dependent_sources = set(itertools.chain.from_iterable( + a.dependent_sources for a in source.artifacts.itervalues())) + dependent_strata = set(s for s in dependent_sources + if s.morphology['kind'] == 'stratum') + return set(s.morphology for s in dependent_strata) + return dependent_stratum_morphs(s1) == dependent_stratum_morphs(s2) + + def install_dependencies(self, staging_area, artifacts, target_source): '''Install chunk artifacts into staging area. We only ever care about chunk artifacts as build dependencies, @@ -477,29 +512,29 @@ class BuildCommand(object): if artifact.source.morphology['kind'] != 'chunk': continue if artifact.source.build_mode == 'bootstrap': - if not self.in_same_stratum(artifact, target_artifact): + if not self.in_same_stratum(artifact.source, target_source): continue self.app.status( msg='Installing chunk %(chunk_name)s from cache %(cache)s', chunk_name=artifact.name, - cache=artifact.cache_key[:7], + cache=artifact.source.cache_key[:7], chatty=True) handle = self.lac.get(artifact) staging_area.install_artifact(handle) - if target_artifact.source.build_mode == 'staging': + if target_source.build_mode == 'staging': morphlib.builder2.ldconfig(self.app.runcmd, staging_area.dirname) - def build_and_cache(self, staging_area, artifact, setup_mounts): - '''Build an artifact and put it into the local artifact cache.''' + def build_and_cache(self, staging_area, source, setup_mounts): + '''Build a source and put its artifacts into the local cache.''' self.app.status(msg='Starting actual build: %(name)s ' '%(sha1)s', - name=artifact.name, sha1=artifact.source.sha1[:7]) + name=source.name, sha1=source.sha1[:7]) builder = morphlib.builder2.Builder( self.app, staging_area, self.lac, self.rac, self.lrc, self.app.settings['max-jobs'], setup_mounts) - return builder.build_and_cache(artifact) + return builder.build_and_cache(source) class InitiatorBuildCommand(BuildCommand): |