Create class for build logic and fix the logic

The build logic was spread all across the Morph class. This patch
gathers it into one class. Having it in one place is useful for
general code quality reasons, but also makes it feasible to experiment
with the build logic, and override only parts of it.

This patch also changes how the build logic works. Earlier we kept
one staging area for staging builds, and were unpacking everything in
a build group at a time. This meant that the staging area would contain
things that were not strictly build dependencies. This has a possiblity
of changing what gets built. The new logic creates a new, fresh, clean
staging area for each build.

This makes building a bit slower, but we'll fix that by making it really
fast to create a staging area and filling it with build dependencies.

1 files changed, 280 insertions, 206 deletions

diff --git a/morphlib/app.py b/morphlib/app.py
index 5fa268b5..922efbb3 100755
--- a/morphlib/app.py
+++ b/morphlib/app.py
@@ -51,6 +51,283 @@ defaults = {
 }
 
 
+class BuildCommand(object):
+
+    '''High level logic for building.
+    
+    This controls how the whole build process goes. This is a separate
+    class to enable easy experimentation of different approaches to
+    the various parts of the process.
+    
+    '''
+    
+    def __init__(self, app):
+        self.app = app
+        self.build_env = self.new_build_env()
+        self.ckc = self.new_cache_key_computer(self.build_env)
+        self.lac, self.rac = self.new_artifact_caches()
+        self.lrc, self.rrc = self.new_repo_caches()
+
+    def build(self, args):
+        '''Build triplets specified on command line.'''
+
+        self.app.status(msg='Build starts', chatty=True)
+        
+        for repo_name, ref, filename in self.app._itertriplets(args):
+            self.app.status(msg='Building %(repo_name)s %(ref)s %(filename)s',
+                            repo_name=repo_name, ref=ref, filename=filename)
+            order = self.compute_build_order(repo_name, ref, filename)
+            self.build_order(order)
+
+        self.app.status(msg='Build ends successfully', chatty=True)
+
+    def new_build_env(self):
+        '''Create a new BuildEnvironment instance.'''
+        return morphlib.buildenvironment.BuildEnvironment(self.app.settings)
+
+    def new_cache_key_computer(self, build_env):
+        '''Create a new cache key computer.'''
+        return morphlib.cachekeycomputer.CacheKeyComputer(build_env)
+
+    def new_artifact_caches(self):
+        '''Create new objects for local, remote artifact caches.'''
+
+        cachedir = self.create_cachedir()
+        artifact_cachedir = self.create_artifact_cachedir()
+
+        lac = morphlib.localartifactcache.LocalArtifactCache(artifact_cachedir)
+
+        rac_url = self.app.settings['cache-server']
+        if rac_url:
+            rac = morphlib.remoteartifactcache.RemoteArtifactCache(rac_url)
+        else:
+            rac = None
+        return lac, rac
+
+    def create_artifact_cachedir(self):
+        '''Create a new directory for the local artifact cache.'''
+
+        artifact_cachedir = os.path.join(
+                self.app.settings['cachedir'], 'artifacts')
+        if not os.path.exists(artifact_cachedir):
+            os.mkdir(artifact_cachedir)
+        return artifact_cachedir
+
+    def new_repo_caches(self):
+        '''Create new objects for local, remote git repository caches.'''
+
+        aliases = self.app.settings['repo-alias']
+        cachedir = self.create_cachedir()
+        gits_dir = os.path.join(cachedir, 'gits')
+        bundle_base_url = self.app.settings['bundle-server']
+        repo_resolver = morphlib.repoaliasresolver.RepoAliasResolver(aliases)
+        lrc = morphlib.localrepocache.LocalRepoCache(self.app,
+                gits_dir, repo_resolver, bundle_base_url=bundle_base_url)
+
+        url = self.app.settings['cache-server']
+        if url:
+            rrc = morphlib.remoterepocache.RemoteRepoCache(url, repo_resolver)
+        else:
+            rrc = None
+
+        return lrc, rrc
+
+    def create_cachedir(self):
+        '''Create a new cache directory.'''
+
+        cachedir = self.app.settings['cachedir']
+        if not os.path.exists(cachedir):
+            os.mkdir(cachedir)
+        return cachedir
+
+    def compute_build_order(self, repo_name, ref, filename):
+        '''Compute build order for a triplet.'''
+        self.app.status(msg='Figuring out the right build order')
+
+        self.app.status(msg='Creating source pool', chatty=True)
+        srcpool = self.app._create_source_pool(
+                        self.lrc, self.rrc, (repo_name, ref, filename))
+
+        self.app.status(msg='Creating artifact resolver', chatty=True)
+        ar = morphlib.artifactresolver.ArtifactResolver()
+
+        self.app.status(msg='Resolving artifacts', chatty=True)
+        artifacts = ar.resolve_artifacts(srcpool)
+
+        self.app.status(msg='Computing cache keys', chatty=True)
+        for artifact in artifacts:
+            artifact.cache_key = self.ckc.compute_key(artifact)
+            artifact.cache_id = self.ckc.get_cache_id(artifact)
+
+        self.app.status(msg='Computing build order', chatty=True)
+        order = morphlib.buildorder.BuildOrder(artifacts)
+        
+        return order
+
+    def build_order(self, order):
+        '''Build everything specified in a build order.'''
+        self.app.status(msg='Building according to build ordering',
+                        chatty=True)
+        for group in order.groups:
+            self.build_artifacts(group)
+            
+    def build_artifacts(self, artifacts):
+        '''Build a set of artifact.
+        
+        Typically, this would be a build group, but might be anything.
+        At this level of abstraction we don't care.
+        
+        '''
+
+        self.app.status(msg='Building a set of artifacts', chatty=True)
+        for artifact in artifacts:
+            self.build_artifact(artifact)
+
+    def build_artifact(self, artifact):
+        '''Build one artifact.
+        
+        All the dependencies are assumed to be built and available
+        in either the local or remote cache already.
+        
+        '''
+
+        self.app.status(msg='Checking if %(kind)s %(name)s needs building',
+                        kind=artifact.source.morphology['kind'],
+                        name=artifact.name)
+
+        if not self.is_built(artifact):
+            self.app.status(msg='Building %(kind)s %(name)s',
+                            kind=artifact.source.morphology['kind'],
+                            name=artifact.name)
+            self.get_sources(artifact)
+            self.cache_artifacts_locally(artifact.dependencies)
+            staging_area = self.create_staging_area(artifact)
+            if self.app.settings['staging-chroot']:
+                self.install_fillers(staging_area)
+                self.install_chunk_artifacts(staging_area, 
+                                             artifact.dependencies)
+            self.build_and_cache(staging_area, artifact)
+            self.remove_staging_area(staging_area)
+
+    def is_built(self, artifact):
+        '''Does either cache already have the artifact?'''
+        return self.lac.has(artifact) or (self.rac and self.rac.has(artifact))
+
+    def get_sources(self, artifact):
+        '''Update the local git repository cache with the sources.'''
+
+        if self.app.settings['no-git-update']:
+            self.app.status(msg='Not updating existing git repository',
+                            chatty=True)
+            artifact.source.repo = self.lrc.get_repo(artifact.source.repo_name)
+        else:
+            self.app.status(msg='Cloning/updating %(repo_name)s',
+                            repo_name=artifact.source.repo_name)
+            artifact.source.repo = self.lrc.cache_repo(
+                                        artifact.source.repo_name)
+            done = set()
+            self.app._cache_repo_and_submodules(
+                    self.lrc, artifact.source.repo.url,
+                    artifact.source.sha1, done)
+
+    def cache_artifacts_locally(self, artifacts):
+        '''Get artifacts missing from local cache from remote cache.'''
+
+        def copy(remote, local):
+            shutil.copyfileobj(remote, local)
+            remote.close()
+            local.close()
+    
+        for artifact in artifacts:
+            if not self.lac.has(artifact):
+                self.app.status(msg='Fetching to local cache: '
+                                    'artifact %(name)s',
+                                name=artifact.name)
+                copy(self.rac.get(artifact), self.lac.put(artifact))
+                
+            # For strata we also need the metadata in the local cache.
+            # FIXME: why do we need it and can we fix things so we don't?
+            if artifact.source.morphology['kind'] == 'stratum':
+                if not self.lac.has_artifact_metadata(artifact, 'meta'):
+                    self.app.status(msg='Fetching to local cache: '
+                                        'artifact metadata %(name)s',
+                                    name=artifact.name)
+                    copy(self.rac.get_artifact_metadata(artifact, 'meta'), 
+                         self.lac.put_artifact_metadata(artifact, 'meta'))
+                         
+    def create_staging_area(self, artifact):
+        '''Create the staging area for building a single artifact.'''
+
+        if self.app.settings['staging-chroot']:
+            staging_root = tempfile.mkdtemp(dir=self.app.settings['tempdir'])
+            staging_temp = staging_root
+        else:
+            staging_root = '/'
+            staging_temp = tempfile.mkdtemp(dir=self.app.settings['tempdir'])
+
+        self.app.status(msg='Creating staging area')
+        staging_area = morphlib.stagingarea.StagingArea(self.app,
+                                                        staging_root,
+                                                        staging_temp)
+        return staging_area
+
+    def remove_staging_area(self, staging_area):
+        '''Remove the staging area.'''
+
+        if staging_area.dirname != '/':
+            self.app.status(msg='Removing staging area')
+            staging_area.remove()
+        if (staging_area.tempdir != '/' and 
+            os.path.exists(staging_area.tempdir)):
+            self.app.status(msg='Removing temporary staging directory')
+            shutil.rmtree(staging_area.tempdir)
+
+    def install_fillers(self, staging_area):
+        '''Install staging fillers into the staging area.
+        
+        This must not be called in bootstrap mode.
+        
+        '''
+        
+        logging.debug('Pre-populating staging area %s' % staging_area.dirname)
+        logging.debug('Fillers: %s' % 
+                        repr(self.app.settings['staging-filler']))
+        for filename in self.app.settings['staging-filler']:
+            with open(filename, 'rb') as f:
+                self.app.status(msg='Installing %(filename)s', 
+                                filename=filename)
+                staging_area.install_artifact(f)
+
+    def install_chunk_artifacts(self, staging_area, artifacts):
+        '''Install chunk artifacts into staging area.
+        
+        We only ever care about chunk artifacts as build dependencies,
+        so this is not a generic artifact installer into staging area.
+        Any non-chunk artifacts are silently ignored.
+        
+        All artifacts MUST be in the local artifact cache already.
+        
+        '''
+
+        for artifact in artifacts:
+            if artifact.source.morphology['kind'] != 'chunk':
+                continue
+            self.app.status(msg='Installing chunk %(chunk_name)s',
+                            chunk_name=artifact.name)
+            handle = self.lac.get(artifact)
+            staging_area.install_artifact(handle)
+
+    def build_and_cache(self, staging_area, artifact):
+        '''Build an artifact and put it into the local artifact cache.'''
+
+        self.app.status(msg='Starting actual build')
+        setup_mounts = self.app.settings['staging-chroot']
+        builder = morphlib.builder2.Builder(self.app,
+                staging_area, self.lac, self.rac, self.lrc, self.build_env,
+                self.app.settings['max-jobs'], setup_mounts)
+        return builder.build_and_cache(artifact)
+
+
 class Morph(cliapp.Application):
 
     system_repo_base = 'morphs'
@@ -175,161 +452,6 @@ class Morph(cliapp.Application):
                               visit=add_to_pool)
         return pool
 
-    def compute_build_order(self, repo_name, ref, filename, ckc, lrc, rrc):
-        self.status(msg='Figuring out the right build order')
-
-        self.status(msg='Creating source pool', chatty=True)
-        srcpool = self._create_source_pool(
-                lrc, rrc, (repo_name, ref, filename))
-
-        self.status(msg='Creating artifact resolver', chatty=True)
-        ar = morphlib.artifactresolver.ArtifactResolver()
-
-        self.status(msg='Resolving artifacts', chatty=True)
-        artifacts = ar.resolve_artifacts(srcpool)
-
-        self.status(msg='Computing cache keys', chatty=True)
-        for artifact in artifacts:
-            artifact.cache_key = ckc.compute_key(artifact)
-            artifact.cache_id = ckc.get_cache_id(artifact)
-
-        self.status(msg='Computing build order', chatty=True)
-        order = morphlib.buildorder.BuildOrder(artifacts)
-        
-        return order
-
-    def find_what_needs_building(self, order, lac, rac):
-        self.status(msg='Finding out what needs to be built', chatty=True)
-        needed = []
-        for group in order.groups:
-            for artifact in group:
-                if not lac.has(artifact):
-                    if not rac or not rac.has(artifact):
-                        needed.append(artifact)
-        self.status(msg='There are %(count)s artifacts that need building',
-                    count=len(needed))
-        return needed
-
-    def get_source_repositories(self, needed, lrc):
-        done = set()
-        for artifact in needed:
-            if self.settings['no-git-update']:
-                artifact.source.repo = lrc.get_repo(artifact.source.repo_name)
-            else:
-                self.status(msg='Cloning/updating %(repo_name)s',
-                            repo_name=artifact.source.repo_name)
-                artifact.source.repo = lrc.cache_repo(
-                        artifact.source.repo_name)
-                self._cache_repo_and_submodules(
-                        lrc, artifact.source.repo.url,
-                        artifact.source.sha1, done)
-
-    def create_staging_area(self):
-        if self.settings['bootstrap']:
-            staging_root = '/'
-            staging_temp = tempfile.mkdtemp(dir=self.settings['tempdir'])
-            install_chunks = True
-            setup_mounts = False
-        elif self.settings['staging-chroot']:
-            staging_root = tempfile.mkdtemp(dir=self.settings['tempdir'])
-            staging_temp = staging_root
-            install_chunks = True
-            setup_mounts = True
-        else:
-            staging_root = '/'
-            staging_temp = tempfile.mkdtemp(dir=self.settings['tempdir'])
-            install_chunks = False
-            setup_mounts = False
-
-        self.status(msg='Creating staging area')
-        staging_area = morphlib.stagingarea.StagingArea(self,
-                                                        staging_root,
-                                                        staging_temp)
-        if self.settings['staging-chroot']:
-            self._install_initial_staging(staging_area)
-            
-        return staging_area, install_chunks, setup_mounts
-
-    def remove_staging_area(self, staging_area):
-        if staging_area.dirname != '/':
-            self.status(msg='Removing staging area')
-            staging_area.remove()
-        if (staging_area.tempdir != '/' and 
-            os.path.exists(staging_area.tempdir)):
-            self.status(msg='Removing temporary staging directory')
-            shutil.rmtree(staging_area.tempdir)
-
-    def install_artifacts(self, staging_area, lac, rac, chunk_artifacts):
-        for chunk_artifact in chunk_artifacts:
-            if not lac.has(chunk_artifact) and rac:
-                self.status(msg='Fetching artifact from remote server')
-                remote = rac.get(chunk_artifact)
-                local = lac.put(chunk_artifact)
-                shutil.copyfileobj(remote, local)
-                remote.close()
-                local.close()
-
-            self.status(msg='Installing chunk %(chunk_name)s',
-                        chunk_name=chunk_artifact.name)
-            handle = lac.get(chunk_artifact)
-            staging_area.install_artifact(handle)
-
-    def build_group(self, artifacts, builder, lac, staging_area):
-        for artifact in artifacts:
-            self.status(msg='Building %(kind)s %(artifact_name)s',
-                        artifact_name=artifact.name,
-                        kind=artifact.source.morphology['kind'])
-            builder.build_and_cache(artifact)
-
-    def new_build_env(self):
-        return morphlib.buildenvironment.BuildEnvironment(self.settings)
-
-    def new_cache_key_computer(self, build_env):
-        return morphlib.cachekeycomputer.CacheKeyComputer(build_env)
-
-    def create_cachedir(self):
-        cachedir = self.settings['cachedir']
-        if not os.path.exists(cachedir):
-            os.mkdir(cachedir)
-        return cachedir
-
-    def create_artifact_cachedir(self):
-        artifact_cachedir = os.path.join(
-                self.settings['cachedir'], 'artifacts')
-        if not os.path.exists(artifact_cachedir):
-            os.mkdir(artifact_cachedir)
-        return artifact_cachedir
-
-    def new_artifact_caches(self):
-        cachedir = self.create_cachedir()
-        artifact_cachedir = self.create_artifact_cachedir()
-
-        lac = morphlib.localartifactcache.LocalArtifactCache(artifact_cachedir)
-
-        rac_url = self.settings['cache-server']
-        if rac_url:
-            rac = morphlib.remoteartifactcache.RemoteArtifactCache(rac_url)
-        else:
-            rac = None
-        return lac, rac
-
-    def new_repo_caches(self):
-        aliases = self.settings['repo-alias']
-        cachedir = self.create_cachedir()
-        gits_dir = os.path.join(cachedir, 'gits')
-        bundle_base_url = self.settings['bundle-server']
-        repo_resolver = morphlib.repoaliasresolver.RepoAliasResolver(aliases)
-        lrc = morphlib.localrepocache.LocalRepoCache(self,
-                gits_dir, repo_resolver, bundle_base_url=bundle_base_url)
-
-        url = self.settings['cache-server']
-        if url:
-            rrc = morphlib.remoterepocache.RemoteRepoCache(url, repo_resolver)
-        else:
-            rrc = None
-
-        return lrc, rrc
-
     def cmd_build(self, args):
         '''Build a binary from a morphology.
         
@@ -342,57 +464,9 @@ class Morph(cliapp.Application):
         times as necessary.)
         
         '''
-
-        self.status(msg='Build starts')
-
-        build_env = self.new_build_env()
-        ckc = self.new_cache_key_computer(build_env)
-        lac, rac = self.new_artifact_caches()
-        lrc, rrc = self.new_repo_caches()
-
-        for repo_name, ref, filename in self._itertriplets(args):
-            self.status(msg='Building %(repo_name)s %(ref)s %(filename)s',
-                        repo_name=repo_name, ref=ref, filename=filename)
-            order = self.compute_build_order(repo_name, ref, filename,
-                                             ckc, lrc, rrc)
-            needed = self.find_what_needs_building(order, lac, rac)
-            self.get_source_repositories(needed, lrc)
-            staging_area, install_chunks, setup_mounts = \
-                self.create_staging_area()
-            builder = morphlib.builder2.Builder(self,
-                    staging_area, lac, rac, lrc, build_env,
-                    self.settings['max-jobs'], setup_mounts)
-
-            to_install = []
-            for group in order.groups:
-                if install_chunks:
-                    self.install_artifacts(staging_area, lac, rac, to_install)
-                    del to_install[:]
-                for artifact in set(group).difference(set(needed)):
-                    self.status(msg='Using cached %(artifact_name)s',
-                                artifact_name=artifact.name,
-                                chatty=True)
-                wanted = [x for x in group if x in needed]
-                self.build_group(wanted, builder, lac, staging_area)
-                to_install.extend(
-                        x for x in group
-                        if x.source.morphology['kind'] == 'chunk')
-
-            # If we are running bootstrap we probably also want the last
-            # build group to be installed as well
-            if self.settings['bootstrap']:
-                self.install_artifacts(staging_area, lac, rac, to_install)
-
-            self.remove_staging_area(staging_area)
-            self.status(msg='Build ends successfully')
-
-    def _install_initial_staging(self, staging_area):
-        logging.debug('Pre-populating staging area %s' % staging_area.dirname)
-        logging.debug('Fillers: %s' % repr(self.settings['staging-filler']))
-        for filename in self.settings['staging-filler']:
-            with open(filename, 'rb') as f:
-                self.status(msg='Installing %(filename)s', filename=filename)
-                staging_area.install_artifact(f)
+        
+        build_command = BuildCommand(self)
+        build_command.build(args)
 
     def cmd_show_dependencies(self, args):
         '''Dumps the dependency tree of all input morphologies.'''