diff options
| -rw-r--r-- | morphlib/__init__.py | 1 | ||||
| -rwxr-xr-x | morphlib/app.py | 280 | ||||
| -rw-r--r-- | morphlib/buildcommand.py | 298 | ||||
| -rw-r--r-- | morphlib/plugins/graphing_plugin.py | 2 | ||||
| -rw-r--r-- | without-test-modules | 1 |
5 files changed, 302 insertions, 280 deletions
diff --git a/morphlib/__init__.py b/morphlib/__init__.py index 60e289ea..f0d79910 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -29,6 +29,7 @@ import artifact import artifactcachereference import artifactresolver import bins +import buildcommand import buildenvironment import buildorder import buildsystem diff --git a/morphlib/app.py b/morphlib/app.py index de292674..6bda8bf8 100755 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -18,8 +18,6 @@ import cliapp import collections import logging import os -import shutil -import tempfile import time import warnings @@ -53,282 +51,6 @@ defaults = { } -class BuildCommand(object): - - '''High level logic for building. - - This controls how the whole build process goes. This is a separate - class to enable easy experimentation of different approaches to - the various parts of the process. - - ''' - - def __init__(self, app): - self.app = app - self.build_env = self.new_build_env() - self.ckc = self.new_cache_key_computer(self.build_env) - self.lac, self.rac = self.new_artifact_caches() - self.lrc, self.rrc = self.new_repo_caches() - - def build(self, args): - '''Build triplets specified on command line.''' - - self.app.status(msg='Build starts', chatty=True) - - for repo_name, ref, filename in self.app.itertriplets(args): - self.app.status(msg='Building %(repo_name)s %(ref)s %(filename)s', - repo_name=repo_name, ref=ref, filename=filename) - order = self.compute_build_order(repo_name, ref, filename) - self.build_in_order(order) - - self.app.status(msg='Build ends successfully', chatty=True) - - def new_build_env(self): - '''Create a new BuildEnvironment instance.''' - return morphlib.buildenvironment.BuildEnvironment(self.app.settings) - - def new_cache_key_computer(self, build_env): - '''Create a new cache key computer.''' - return morphlib.cachekeycomputer.CacheKeyComputer(build_env) - - def new_artifact_caches(self): - return morphlib.util.new_artifact_caches(self.app.settings) - - def create_artifact_cachedir(self): - return morphlib.util.create_artifact_cachedir(self.app.settings) - - def new_repo_caches(self): - return morphlib.util.new_repo_caches(self.app) - - def create_cachedir(self): - return morphlib.util.create_cachedir(self.app.settings) - - def compute_build_order(self, repo_name, ref, filename): - '''Compute build order for a triplet.''' - self.app.status(msg='Figuring out the right build order') - - self.app.status(msg='Creating source pool', chatty=True) - srcpool = self.app.create_source_pool( - self.lrc, self.rrc, (repo_name, ref, filename)) - - self.app.status(msg='Creating artifact resolver', chatty=True) - ar = morphlib.artifactresolver.ArtifactResolver() - - self.app.status(msg='Resolving artifacts', chatty=True) - artifacts = ar.resolve_artifacts(srcpool) - - self.app.status(msg='Computing cache keys', chatty=True) - for artifact in artifacts: - artifact.cache_key = self.ckc.compute_key(artifact) - artifact.cache_id = self.ckc.get_cache_id(artifact) - - self.app.status(msg='Computing build order', chatty=True) - order = morphlib.buildorder.BuildOrder(artifacts) - - return order - - def build_in_order(self, order): - '''Build everything specified in a build order.''' - self.app.status(msg='Building according to build ordering', - chatty=True) - for group in order.groups: - self.build_artifacts(group) - - def build_artifacts(self, artifacts): - '''Build a set of artifact. - - Typically, this would be a build group, but might be anything. - At this level of abstraction we don't care. - - ''' - - self.app.status(msg='Building a set of artifacts', chatty=True) - for artifact in artifacts: - self.build_artifact(artifact) - - def build_artifact(self, artifact): - '''Build one artifact. - - All the dependencies are assumed to be built and available - in either the local or remote cache already. - - ''' - - self.app.status(msg='Checking if %(kind)s %(name)s needs building', - kind=artifact.source.morphology['kind'], - name=artifact.name) - - if self.is_built(artifact): - self.app.status(msg='The %(kind)s %(name)s is already built', - kind=artifact.source.morphology['kind'], - name=artifact.name) - else: - self.app.status(msg='Building %(kind)s %(name)s', - kind=artifact.source.morphology['kind'], - name=artifact.name) - self.get_sources(artifact) - deps = self.get_recursive_deps(artifact) - self.cache_artifacts_locally(deps) - staging_area = self.create_staging_area(artifact) - if self.app.settings['staging-chroot']: - if artifact.source.morphology.needs_staging_area: - self.install_fillers(staging_area) - self.install_chunk_artifacts(staging_area, - deps) - self.build_and_cache(staging_area, artifact) - if self.app.settings['bootstrap']: - self.install_chunk_artifacts(staging_area, - (artifact,)) - self.remove_staging_area(staging_area) - - def is_built(self, artifact): - '''Does either cache already have the artifact?''' - return self.lac.has(artifact) or (self.rac and self.rac.has(artifact)) - - def get_recursive_deps(self, artifact): - done = set() - todo = set((artifact,)) - while todo: - for a in todo.pop().dependencies: - if a not in done: - done.add(a) - todo.add(a) - return done - - def get_sources(self, artifact): - '''Update the local git repository cache with the sources.''' - - repo_name = artifact.source.repo_name - if self.app.settings['no-git-update']: - self.app.status(msg='Not updating existing git repository ' - '%(repo_name)s ' - 'because of no-git-update being set', - chatty=True, - repo_name=repo_name) - artifact.source.repo = self.lrc.get_repo(repo_name) - return - - if self.lrc.has_repo(repo_name): - artifact.source.repo = self.lrc.get_repo(repo_name) - try: - sha1 = artifact.source.sha1 - artifact.source.repo.resolve_ref(sha1) - self.app.status(msg='Not updating git repository ' - '%(repo_name)s because it ' - 'already contains sha1 %(sha1)s', - chatty=True, repo_name=repo_name, - sha1=sha1) - except morphlib.cachedrepo.InvalidReferenceError: - self.app.status(msg='Updating %(repo_name)s', - repo_name=repo_name) - artifact.source.repo.update() - else: - self.app.status(msg='Cloning %(repo_name)s', - repo_name=repo_name) - artifact.source.repo = self.lrc.cache_repo(repo_name) - - # Update submodules. - done = set() - self.app.cache_repo_and_submodules( - self.lrc, artifact.source.repo.url, - artifact.source.sha1, done) - - def cache_artifacts_locally(self, artifacts): - '''Get artifacts missing from local cache from remote cache.''' - - def copy(remote, local): - shutil.copyfileobj(remote, local) - remote.close() - local.close() - - for artifact in artifacts: - if not self.lac.has(artifact): - self.app.status(msg='Fetching to local cache: ' - 'artifact %(name)s', - name=artifact.name) - copy(self.rac.get(artifact), self.lac.put(artifact)) - - if artifact.source.morphology.needs_artifact_metadata_cached: - if not self.lac.has_artifact_metadata(artifact, 'meta'): - self.app.status(msg='Fetching to local cache: ' - 'artifact metadata %(name)s', - name=artifact.name) - copy(self.rac.get_artifact_metadata(artifact, 'meta'), - self.lac.put_artifact_metadata(artifact, 'meta')) - - def create_staging_area(self, artifact): - '''Create the staging area for building a single artifact.''' - - if self.app.settings['staging-chroot']: - staging_root = tempfile.mkdtemp(dir=self.app.settings['tempdir']) - staging_temp = staging_root - else: - staging_root = '/' - staging_temp = tempfile.mkdtemp(dir=self.app.settings['tempdir']) - - self.app.status(msg='Creating staging area') - staging_area = morphlib.stagingarea.StagingArea(self.app, - staging_root, - staging_temp) - return staging_area - - def remove_staging_area(self, staging_area): - '''Remove the staging area.''' - - if staging_area.dirname != '/': - self.app.status(msg='Removing staging area') - staging_area.remove() - temp_path = staging_area.tempdir - if temp_path != '/' and os.path.exists(temp_path): - self.app.status(msg='Removing temporary staging directory') - shutil.rmtree(temp_path) - - def install_fillers(self, staging_area): - '''Install staging fillers into the staging area. - - This must not be called in bootstrap mode. - - ''' - - logging.debug('Pre-populating staging area %s' % staging_area.dirname) - logging.debug('Fillers: %s' % - repr(self.app.settings['staging-filler'])) - for filename in self.app.settings['staging-filler']: - with open(filename, 'rb') as f: - self.app.status(msg='Installing %(filename)s', - filename=filename) - staging_area.install_artifact(f) - - def install_chunk_artifacts(self, staging_area, artifacts): - '''Install chunk artifacts into staging area. - - We only ever care about chunk artifacts as build dependencies, - so this is not a generic artifact installer into staging area. - Any non-chunk artifacts are silently ignored. - - All artifacts MUST be in the local artifact cache already. - - ''' - - for artifact in artifacts: - if artifact.source.morphology['kind'] != 'chunk': - continue - self.app.status(msg='Installing chunk %(chunk_name)s', - chunk_name=artifact.name) - handle = self.lac.get(artifact) - staging_area.install_artifact(handle) - - def build_and_cache(self, staging_area, artifact): - '''Build an artifact and put it into the local artifact cache.''' - - self.app.status(msg='Starting actual build') - setup_mounts = self.app.settings['staging-chroot'] - builder = morphlib.builder2.Builder( - self.app, staging_area, self.lac, self.rac, self.lrc, - self.build_env, self.app.settings['max-jobs'], setup_mounts) - return builder.build_and_cache(artifact) - - class Morph(cliapp.Application): def add_settings(self): @@ -486,7 +208,7 @@ class Morph(cliapp.Application): ''' - build_command = BuildCommand(self) + build_command = morphlib.buildcommand.BuildCommand(self) build_command = self.hookmgr.call('new-build-command', build_command) build_command.build(args) diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py new file mode 100644 index 00000000..652bf303 --- /dev/null +++ b/morphlib/buildcommand.py @@ -0,0 +1,298 @@ +# Copyright (C) 2011-2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import os +import shutil +import logging +import tempfile + +import morphlib + + +class BuildCommand(object): + + '''High level logic for building. + + This controls how the whole build process goes. This is a separate + class to enable easy experimentation of different approaches to + the various parts of the process. + + ''' + + def __init__(self, app): + self.app = app + self.build_env = self.new_build_env() + self.ckc = self.new_cache_key_computer(self.build_env) + self.lac, self.rac = self.new_artifact_caches() + self.lrc, self.rrc = self.new_repo_caches() + + def build(self, args): + '''Build triplets specified on command line.''' + + self.app.status(msg='Build starts', chatty=True) + + for repo_name, ref, filename in self.app.itertriplets(args): + self.app.status(msg='Building %(repo_name)s %(ref)s %(filename)s', + repo_name=repo_name, ref=ref, filename=filename) + order = self.compute_build_order(repo_name, ref, filename) + self.build_in_order(order) + + self.app.status(msg='Build ends successfully', chatty=True) + + def new_build_env(self): + '''Create a new BuildEnvironment instance.''' + return morphlib.buildenvironment.BuildEnvironment(self.app.settings) + + def new_cache_key_computer(self, build_env): + '''Create a new cache key computer.''' + return morphlib.cachekeycomputer.CacheKeyComputer(build_env) + + def new_artifact_caches(self): + return morphlib.util.new_artifact_caches(self.app.settings) + + def create_artifact_cachedir(self): + return morphlib.util.create_artifact_cachedir(self.app.settings) + + def new_repo_caches(self): + return morphlib.util.new_repo_caches(self.app) + + def create_cachedir(self): + return morphlib.util.create_cachedir(self.app.settings) + + def compute_build_order(self, repo_name, ref, filename): + '''Compute build order for a triplet.''' + self.app.status(msg='Figuring out the right build order') + + self.app.status(msg='Creating source pool', chatty=True) + srcpool = self.app.create_source_pool( + self.lrc, self.rrc, (repo_name, ref, filename)) + + self.app.status(msg='Creating artifact resolver', chatty=True) + ar = morphlib.artifactresolver.ArtifactResolver() + + self.app.status(msg='Resolving artifacts', chatty=True) + artifacts = ar.resolve_artifacts(srcpool) + + self.app.status(msg='Computing cache keys', chatty=True) + for artifact in artifacts: + artifact.cache_key = self.ckc.compute_key(artifact) + artifact.cache_id = self.ckc.get_cache_id(artifact) + + self.app.status(msg='Computing build order', chatty=True) + order = morphlib.buildorder.BuildOrder(artifacts) + + return order + + def build_in_order(self, order): + '''Build everything specified in a build order.''' + self.app.status(msg='Building according to build ordering', + chatty=True) + for group in order.groups: + self.build_artifacts(group) + + def build_artifacts(self, artifacts): + '''Build a set of artifact. + + Typically, this would be a build group, but might be anything. + At this level of abstraction we don't care. + + ''' + + self.app.status(msg='Building a set of artifacts', chatty=True) + for artifact in artifacts: + self.build_artifact(artifact) + + def build_artifact(self, artifact): + '''Build one artifact. + + All the dependencies are assumed to be built and available + in either the local or remote cache already. + + ''' + + self.app.status(msg='Checking if %(kind)s %(name)s needs building', + kind=artifact.source.morphology['kind'], + name=artifact.name) + + if self.is_built(artifact): + self.app.status(msg='The %(kind)s %(name)s is already built', + kind=artifact.source.morphology['kind'], + name=artifact.name) + else: + self.app.status(msg='Building %(kind)s %(name)s', + kind=artifact.source.morphology['kind'], + name=artifact.name) + self.get_sources(artifact) + deps = self.get_recursive_deps(artifact) + self.cache_artifacts_locally(deps) + staging_area = self.create_staging_area(artifact) + if self.app.settings['staging-chroot']: + if artifact.source.morphology.needs_staging_area: + self.install_fillers(staging_area) + self.install_chunk_artifacts(staging_area, + deps) + self.build_and_cache(staging_area, artifact) + if self.app.settings['bootstrap']: + self.install_chunk_artifacts(staging_area, + (artifact,)) + self.remove_staging_area(staging_area) + + def is_built(self, artifact): + '''Does either cache already have the artifact?''' + return self.lac.has(artifact) or (self.rac and self.rac.has(artifact)) + + def get_recursive_deps(self, artifact): + done = set() + todo = set((artifact,)) + while todo: + for a in todo.pop().dependencies: + if a not in done: + done.add(a) + todo.add(a) + return done + + def get_sources(self, artifact): + '''Update the local git repository cache with the sources.''' + + repo_name = artifact.source.repo_name + if self.app.settings['no-git-update']: + self.app.status(msg='Not updating existing git repository ' + '%(repo_name)s ' + 'because of no-git-update being set', + chatty=True, + repo_name=repo_name) + artifact.source.repo = self.lrc.get_repo(repo_name) + return + + if self.lrc.has_repo(repo_name): + artifact.source.repo = self.lrc.get_repo(repo_name) + try: + sha1 = artifact.source.sha1 + artifact.source.repo.resolve_ref(sha1) + self.app.status(msg='Not updating git repository ' + '%(repo_name)s because it ' + 'already contains sha1 %(sha1)s', + chatty=True, repo_name=repo_name, + sha1=sha1) + except morphlib.cachedrepo.InvalidReferenceError: + self.app.status(msg='Updating %(repo_name)s', + repo_name=repo_name) + artifact.source.repo.update() + else: + self.app.status(msg='Cloning %(repo_name)s', + repo_name=repo_name) + artifact.source.repo = self.lrc.cache_repo(repo_name) + + # Update submodules. + done = set() + self.app.cache_repo_and_submodules( + self.lrc, artifact.source.repo.url, + artifact.source.sha1, done) + + def cache_artifacts_locally(self, artifacts): + '''Get artifacts missing from local cache from remote cache.''' + + def copy(remote, local): + shutil.copyfileobj(remote, local) + remote.close() + local.close() + + for artifact in artifacts: + if not self.lac.has(artifact): + self.app.status(msg='Fetching to local cache: ' + 'artifact %(name)s', + name=artifact.name) + copy(self.rac.get(artifact), self.lac.put(artifact)) + + if artifact.source.morphology.needs_artifact_metadata_cached: + if not self.lac.has_artifact_metadata(artifact, 'meta'): + self.app.status(msg='Fetching to local cache: ' + 'artifact metadata %(name)s', + name=artifact.name) + copy(self.rac.get_artifact_metadata(artifact, 'meta'), + self.lac.put_artifact_metadata(artifact, 'meta')) + + def create_staging_area(self, artifact): + '''Create the staging area for building a single artifact.''' + + if self.app.settings['staging-chroot']: + staging_root = tempfile.mkdtemp(dir=self.app.settings['tempdir']) + staging_temp = staging_root + else: + staging_root = '/' + staging_temp = tempfile.mkdtemp(dir=self.app.settings['tempdir']) + + self.app.status(msg='Creating staging area') + staging_area = morphlib.stagingarea.StagingArea(self.app, + staging_root, + staging_temp) + return staging_area + + def remove_staging_area(self, staging_area): + '''Remove the staging area.''' + + if staging_area.dirname != '/': + self.app.status(msg='Removing staging area') + staging_area.remove() + temp_path = staging_area.tempdir + if temp_path != '/' and os.path.exists(temp_path): + self.app.status(msg='Removing temporary staging directory') + shutil.rmtree(temp_path) + + def install_fillers(self, staging_area): + '''Install staging fillers into the staging area. + + This must not be called in bootstrap mode. + + ''' + + logging.debug('Pre-populating staging area %s' % staging_area.dirname) + logging.debug('Fillers: %s' % + repr(self.app.settings['staging-filler'])) + for filename in self.app.settings['staging-filler']: + with open(filename, 'rb') as f: + self.app.status(msg='Installing %(filename)s', + filename=filename) + staging_area.install_artifact(f) + + def install_chunk_artifacts(self, staging_area, artifacts): + '''Install chunk artifacts into staging area. + + We only ever care about chunk artifacts as build dependencies, + so this is not a generic artifact installer into staging area. + Any non-chunk artifacts are silently ignored. + + All artifacts MUST be in the local artifact cache already. + + ''' + + for artifact in artifacts: + if artifact.source.morphology['kind'] != 'chunk': + continue + self.app.status(msg='Installing chunk %(chunk_name)s', + chunk_name=artifact.name) + handle = self.lac.get(artifact) + staging_area.install_artifact(handle) + + def build_and_cache(self, staging_area, artifact): + '''Build an artifact and put it into the local artifact cache.''' + + self.app.status(msg='Starting actual build') + setup_mounts = self.app.settings['staging-chroot'] + builder = morphlib.builder2.Builder( + self.app, staging_area, self.lac, self.rac, self.lrc, + self.build_env, self.app.settings['max-jobs'], setup_mounts) + return builder.build_and_cache(artifact) diff --git a/morphlib/plugins/graphing_plugin.py b/morphlib/plugins/graphing_plugin.py index 5e25bd5c..3594e9b9 100644 --- a/morphlib/plugins/graphing_plugin.py +++ b/morphlib/plugins/graphing_plugin.py @@ -35,7 +35,7 @@ class GraphingPlugin(cliapp.Plugin): self.app.status(msg='Creating build order for ' '%(repo_name)s %(ref)s %(filename)s', repo_name=repo_name, ref=ref, filename=filename) - builder = morphlib.app.BuildCommand(self.app) + builder = morphlib.buildcommand.BuildCommand(self.app) order = builder.compute_build_order(repo_name, ref, filename) basename, ext = os.path.splitext(filename) diff --git a/without-test-modules b/without-test-modules index 67a1e143..91fcd117 100644 --- a/without-test-modules +++ b/without-test-modules @@ -13,4 +13,5 @@ morphlib/plugins/show_dependencies_plugin.py morphlib/plugins/update_gits_plugin.py morphlib/plugins/trebuchet_plugin.py morphlib/plugins/branch_and_merge_plugin.py +morphlib/buildcommand.py |