Initial work on integrate the build order work into builder.

1 files changed, 185 insertions, 158 deletions

diff --git a/morphlib/builder.py b/morphlib/builder.py
index 95374e65..222e884c 100644
--- a/morphlib/builder.py
+++ b/morphlib/builder.py
@@ -17,10 +17,6 @@
 import json
 import logging
 import os
-import shutil
-import StringIO
-import tarfile
-import urlparse
 
 import morphlib
 
@@ -54,12 +50,11 @@ def ldconfig(ex, rootdir):
     else:
         logging.debug('No %s, not running ldconfig' % conf)
 
-class BinaryBlob(object):
 
-    def __init__(self, morph, repo, ref):
-        self.morph = morph
-        self.repo = repo
-        self.ref = ref
+class BlobBuilder(object):
+
+    def __init__(self, blob):
+        self.blob = blob
         
         # The following MUST get set by the caller.
         self.builddir = None
@@ -69,23 +64,66 @@ class BinaryBlob(object):
         self.msg = None
         self.cache_prefix = None
         self.tempdir = None
-        self.built = None
+        self.stage_items = []
         self.dump_memory_profile = lambda msg: None
 
         # Stopwatch to measure build times
         self.build_watch = morphlib.stopwatch.Stopwatch()
 
-    def needs_built(self):
-        return []
-
     def builds(self):
-        raise NotImplemented()
-    
+        ret = {}
+        for chunk_name in self.blob.chunks:
+            ret[chunk_name] = self.filename(chunk_name)
+        return ret
+
     def build(self):
-        raise NotImplemented()
+        # create the staging area on demand
+        if not os.path.exists(self.staging):
+            os.mkdir(self.staging)
+
+        # record all items built in the process
+        built_items = []
+
+        # get a list of all the items we have to build for this blob
+        builds = self.builds()
+
+        # if not all build items are in the cache, rebuild the blob
+        if not all(os.path.isfile(builds[name]) for name in builds):
+            with self.build_watch('overall-build'):
+                built_items += self.do_build()
+
+        # check again, fail if not all build items were actually built
+        if not all(os.path.isfile(builds[name]) for name in builds):
+            raise Exception('Not all builds results expected from %s were '
+                            'actually built' % self.blob)
+
+        # install all build items to the staging area
+        for name, filename in builds.items():
+            self.msg('Using cached %s %s at %s' % (self.blob.morph.kind,
+                                                   name, filename))
+            self.install_chunk(name, filename)
+            self.dump_memory_profile('after installing chunk')
+
+        return built_items
 
     def filename(self, name):
-        return '%s.%s.%s' % (self.cache_prefix, self.morph.kind, name)
+        return '%s.%s.%s' % (self.cache_prefix,
+                             self.blob.morph.kind,
+                             name)
+
+    def install_chunk(self, chunk_name, chunk_filename):
+        if self.blob.morph.kind != 'chunk':
+            return
+        if self.settings['bootstrap']:
+            self.msg('Unpacking item %s onto system' % chunk_name)
+            ex = morphlib.execute.Execute('/', self.msg)
+            morphlib.bins.unpack_binary(chunk_filename, '/', ex)
+            ldconfig(ex, '/')
+        else:
+            self.msg('Unpacking chunk %s into staging' % chunk_name)
+            ex = morphlib.execute.Execute(self.staging, self.msg)
+            morphlib.bins.unpack_binary(chunk_filename, self.staging, ex)
+            ldconfig(ex, self.staging)
 
     def prepare_binary_metadata(self, blob_name, **kwargs):
         '''Add metadata to a binary about to be built.'''
@@ -93,8 +131,8 @@ class BinaryBlob(object):
         self.msg('Adding metadata to %s' % blob_name)
         meta = {
             'name': blob_name,
-            'kind': self.morph.kind,
-            'description': self.morph.description,
+            'kind': self.blob.morph.kind,
+            'description': self.blob.morph.description,
         }
         for key, value in kwargs.iteritems():
             meta[key] = value
@@ -128,7 +166,7 @@ class BinaryBlob(object):
         self.write_cache_metadata(meta)
 
 
-class Chunk(BinaryBlob):
+class ChunkBuilder(BlobBuilder):
 
     build_system = {
         'autotools': {
@@ -147,21 +185,8 @@ class Chunk(BinaryBlob):
         },
     }
 
-    @property
-    def chunks(self):
-        if self.morph.chunks:
-            return self.morph.chunks
-        else:
-            return { self.morph.name: ['.'] }
-    
-    def builds(self):
-        ret = {}
-        for chunk_name in self.chunks:
-            ret[chunk_name] = self.filename(chunk_name)
-        return ret
-
-    def build(self):
-        logging.debug('Creating build tree at %s' % self.builddir)
+    def do_build(self):
+        self.msg('Creating build tree at %s' % self.builddir)
 
         self.ex = morphlib.execute.Execute(self.builddir, self.msg)
         self.setup_env()
@@ -169,23 +194,40 @@ class Chunk(BinaryBlob):
         self.create_source_and_tarball()
 
         os.mkdir(self.destdir)
-        if self.morph.build_system:
+        if self.blob.morph.build_system:
             self.build_using_buildsystem()
         else:
             self.build_using_commands()
         self.dump_memory_profile('after building chunk')
 
-        chunks = self.create_chunks(self.chunks)
-        self.dump_memory_profile('after creating chunk blobs')
+        chunks = self.create_chunks()
+        self.dump_memory_profile('after creating build chunks')
         return chunks
         
     def setup_env(self):
         path = self.ex.env['PATH']
-        tmpdir = self.ex.env.get('TMPDIR')
         tools = self.ex.env.get('BOOTSTRAP_TOOLS')
         distcc_hosts = self.ex.env.get('DISTCC_HOSTS')
+
+        # copy a set of white-listed variables from the original env
+        copied_vars = dict.fromkeys([
+            'TMPDIR',
+            'LD_PRELOAD',
+            'LD_LIBRARY_PATH',
+            'FAKEROOTKEY',
+            'FAKED_MODE',
+            'FAKEROOT_FD_BASE',
+        ], None)
+        for name in copied_vars:
+            copied_vars[name] = self.ex.env.get(name, None)
+
         self.ex.env.clear()
         
+        # apply the copied variables to the clean env
+        for name in copied_vars:
+            if copied_vars[name] is not None:
+                self.ex.env[name] = copied_vars[name]
+
         self.ex.env['TERM'] = 'dumb'
         self.ex.env['SHELL'] = '/bin/sh'
         self.ex.env['USER'] = \
@@ -193,8 +235,6 @@ class Chunk(BinaryBlob):
             self.ex.env['LOGNAME'] = 'tomjon'
         self.ex.env['LC_ALL'] = 'C'
         self.ex.env['HOME'] = os.path.join(self.tempdir.dirname)
-        if tmpdir is not None:
-            self.ex.env['TMPDIR'] = tmpdir
 
         if self.settings['keep-path'] or self.settings['bootstrap']:
             self.ex.env['PATH'] = path
@@ -215,8 +255,8 @@ class Chunk(BinaryBlob):
         if distcc_hosts is not None:
             self.ex.env['DISTCC_HOSTS'] = distcc_hosts
 
-        if self.morph.max_jobs:
-            max_jobs = int(self.morph.max_jobs)
+        if self.blob.morph.max_jobs:
+            max_jobs = int(self.blob.morph.max_jobs)
             logging.debug('max_jobs from morph: %s' % max_jobs)
         elif self.settings['max-jobs']:
             max_jobs = self.settings['max-jobs']
@@ -244,8 +284,8 @@ class Chunk(BinaryBlob):
                                         'for chunk')
             tarball = self.cache_prefix + '.src.tar'
             #FIXME Ugh use treeish everwhere
-            path = urlparse.urlparse(self.repo).path
-            t = morphlib.git.Treeish (path, self.ref)
+            path = urlparse.urlparse(self.blob.morph.repo).path
+            t = morphlib.git.Treeish(path, self.blob.morph.ref)
             morphlib.git.export_sources(t, tarball)
             self.dump_memory_profile('after exporting sources')
             os.mkdir(self.builddir)
@@ -254,7 +294,7 @@ class Chunk(BinaryBlob):
                                         'for chunk')
 
     def build_using_buildsystem(self):
-        bs_name = self.morph.build_system
+        bs_name = self.blob.morph.build_system
         self.msg('Building using well-known build system %s' % bs_name)
         bs = self.build_system[bs_name]
         self.run_sequentially('configure', bs['configure-commands'])
@@ -264,10 +304,10 @@ class Chunk(BinaryBlob):
 
     def build_using_commands(self):
         self.msg('Building using explicit commands')
-        self.run_sequentially('configure', self.morph.configure_commands)
-        self.run_in_parallel('build', self.morph.build_commands)
-        self.run_sequentially('test', self.morph.test_commands)
-        self.run_sequentially('install', self.morph.install_commands)
+        self.run_sequentially('configure', self.blob.morph.configure_commands)
+        self.run_in_parallel('build', self.blob.morph.build_commands)
+        self.run_sequentially('test', self.blob.morph.test_commands)
+        self.run_sequentially('install', self.blob.morph.install_commands)
 
     def run_in_parallel(self, what, commands):
         self.msg('commands: %s' % what)
@@ -284,58 +324,49 @@ class Chunk(BinaryBlob):
             self.ex.env['MAKEFLAGS'] = flags
             logging.debug('Restore MAKEFLAGS=%s' % self.ex.env['MAKEFLAGS'])
 
-    def create_chunks(self, chunks):
-        ret = {}
+    def create_chunks(self):
+        chunks = []
         with self.build_watch('create-chunks'):
-            for chunk_name in chunks:
+            for chunk_name in self.blob.chunks:
                 self.msg('Creating chunk %s' % chunk_name)
                 self.prepare_binary_metadata(chunk_name)
-                patterns = chunks[chunk_name]
+                patterns = self.blob.chunks[chunk_name]
                 patterns += [r'baserock/%s\.' % chunk_name]
                 filename = self.filename(chunk_name)
                 self.msg('Creating binary for %s' % chunk_name)
                 morphlib.bins.create_chunk(self.destdir, filename, patterns,
                                            self.ex, self.dump_memory_profile)
-                ret[chunk_name] = filename
+                chunks.append((chunk_name, filename))
 
         files = os.listdir(self.destdir)
         if files:
             raise Exception('DESTDIR %s is not empty: %s' %
                                 (self.destdir, files))
-        return ret
+        return chunks
 
 
-class Stratum(BinaryBlob):
+class StratumBuilder(BlobBuilder):
     
-    def needs_built(self):
-        for source in self.morph.sources:
-            project_name = source['name']
-            morph_name = source['morph'] if 'morph' in source else project_name
-            repo = source['repo']
-            ref = source['ref']
-            chunks = source['chunks'] if 'chunks' in source else [project_name]
-            yield repo, ref, morph_name, chunks
-
     def builds(self):
-        filename = self.filename(self.morph.name)
-        return { self.morph.name: filename }
+        filename = self.filename(self.blob.morph.name)
+        return { self.blob.morph.name: filename }
 
-    def build(self):
+    def do_build(self):
         os.mkdir(self.destdir)
         ex = morphlib.execute.Execute(self.destdir, self.msg)
         with self.build_watch('unpack-chunks'):
-            for chunk_name, filename in self.built:
+            for chunk_name, filename in self.stage_items:
                 self.msg('Unpacking chunk %s' % chunk_name)
                 morphlib.bins.unpack_binary(filename, self.destdir, ex)
         with self.build_watch('create-binary'):
-            self.prepare_binary_metadata(self.morph.name)
-            self.msg('Creating binary for %s' % self.morph.name)
-            filename = self.filename(self.morph.name)
+            self.prepare_binary_metadata(self.blob.morph.name)
+            self.msg('Creating binary for %s' % self.blob.morph.name)
+            filename = self.filename(self.blob.morph.name)
             morphlib.bins.create_stratum(self.destdir, filename, ex)
-        return { self.morph.name: filename }
+        return { self.blob.morph.name: filename }
 
 
-class System(BinaryBlob):
+class SystemBuilder(BlobBuilder):
 
     def needs_built(self):
         for stratum_name in self.morph.strata:
@@ -345,7 +376,7 @@ class System(BinaryBlob):
         filename = self.filename(self.morph.name)
         return { self.morph.name: filename }
 
-    def build(self):
+    def do_build(self):
         self.ex = morphlib.execute.Execute(self.tempdir.dirname, self.msg)
         
         # Create image.
@@ -391,7 +422,7 @@ class System(BinaryBlob):
                 for name, filename in self.built:
                     self.msg('unpack %s from %s' % (name, filename))
                     self.ex.runv(['tar', '-C', mount_point, '-xf', filename])
-                ldconfig(ex, mount_point)
+                ldconfig(self.ex, mount_point)
 
             # Create fstab.
             with self.build_watch('create-fstab'):
@@ -426,7 +457,7 @@ append root=/dev/sda1 init=/sbin/init quiet rw
             # Unmount.
             with self.build_watch('unmount-filesystem'):
                 self.ex.runv(['umount', mount_point])
-        except BaseException, e:
+        except BaseException:
             # Unmount.
             if mount_point is not None:
                 try:
@@ -458,15 +489,14 @@ class Builder(object):
     
     The objects may be chunks or strata.'''
     
-    def __init__(self, tempdir, app):
+    def __init__(self, tempdir, app, morph_loader):
         self.tempdir = tempdir
         self.real_msg = app.msg
         self.settings = app.settings
         self.dump_memory_profile = app.dump_memory_profile
         self.cachedir = morphlib.cachedir.CacheDir(self.settings['cachedir'])
         self.indent = 0
-        self.morph_loader = \
-            morphlib.morphologyloader.MorphologyLoader(self.settings)
+        self.morph_loader = morph_loader
 
     def msg(self, text):
         spaces = '  ' * self.indent
@@ -478,94 +508,91 @@ class Builder(object):
     def indent_less(self):
         self.indent -= 1
 
-    def build(self, repo, ref, filename):
-        '''Build a binary based on a morphology.'''
+    def build(self, blobs, build_order):
+        '''Build a list of groups of morphologies. Items in a group
+           can be built in parallel.'''
 
-        self.dump_memory_profile('at start of build method')
         self.indent_more()
-        self.msg('build %s|%s|%s' % (repo, ref, filename))
-        morph = self.morph_loader.load(repo, ref, filename)
-        repo = morph.repo
-        self.dump_memory_profile('after getting morph from git')
 
-        if morph.kind == 'chunk':
-            blob = Chunk(morph, repo, ref)
-        elif morph.kind == 'stratum':
-            blob = Stratum(morph, repo, ref)
-        elif morph.kind == 'system':
-            blob = System(morph, repo, ref)
-        else:
-            raise Exception('Unknown kind of morphology: %s' % morph.kind)
-        self.dump_memory_profile('after creating Chunk/Stratum/...')
+        # first pass: create builders for all blobs
+        builders = {}
+        for group in build_order:
+            for blob in group:
+                builder = self.create_blob_builder(blob)
+                builders[blob] = builder
+
+        # second pass: build group by group, item after item
+        ret = []
+        while len(build_order) > 0:
+            group = build_order.popleft()
+            while len(group) > 0:
+                blob = group.pop()
+
+                self.msg('Building %s' % blob)
+                self.indent_more()
 
-        cache_id = self.get_cache_id(repo, ref, filename)
-        logging.debug('cachae id: %s' % repr(cache_id))
-        self.dump_memory_profile('after computing cache id')
+                ## TODO this needs to be done recursively
+                ## make sure all dependencies are in the staging area
+                #for dependency in blob.dependencies:
+                #    depbuilder = builders[dependency]
+                #    depbuilder.stage()
 
-        blob.builddir = self.tempdir.join('%s.build' % morph.name)
-        blob.destdir = self.tempdir.join('%s.inst' % morph.name)
-        blob.staging = self.tempdir.join('staging')
-        if not os.path.exists(blob.staging):
-            os.mkdir(blob.staging)
-        blob.settings = self.settings
-        blob.msg = self.msg
-        blob.cache_prefix = self.cachedir.name(cache_id)
-        blob.tempdir = self.tempdir
-        blob.dump_memory_profile = self.dump_memory_profile
-
-        builds = blob.builds()
-        self.dump_memory_profile('after blob.builds()')
-        if all(os.path.exists(builds[x]) for x in builds):
-            for x in builds:
-                self.msg('using cached %s %s at %s' % 
-                            (morph.kind, x, builds[x]))
-                self.install_chunk(morph, x, builds[x], blob.staging)
-                self.dump_memory_profile('after installing chunk')
-            built = builds
-        else:
-            with blob.build_watch('overall-build'):
+                ## TODO this needs the set of recursively collected 
+                ## dependencies
+                ## make sure all non-dependencies are not staged
+                #for nondependency in (blobs - blob.dependencies):
+                #    depbuilder = builders[nondependency]
+                #    depbuilder.unstage()
 
-                with blob.build_watch('build-needed'):
-                    self.build_needed(blob)
-                    self.dump_memory_profile('after building needed')
+                built_items = builders[blob].build()
+                
+                if blob.parent:
+                    for item, filename in built_items:
+                        self.msg('Marking %s to be staged for %s' %
+                                 (item, blob.parent))
 
-                self.msg('Building %s %s' % (morph.kind, morph.name))
-                self.indent_more()
-                built = blob.build()
-                self.dump_memory_profile('after building blob')
-                self.indent_less()
-                for x in built:
-                    self.msg('%s %s cached at %s' % (morph.kind, x, built[x]))
-                    self.install_chunk(morph, x, built[x], blob.staging)
-                    self.dump_memory_profile('after installing chunks')
+                    parent_builder = builders[blob.parent]
+                    parent_builder.stage_items += built_items
 
-            blob.save_build_times()
+                self.indent_less()
 
         self.indent_less()
-        self.dump_memory_profile('at end of build method')
-        return morph, built
-
-    def build_needed(self, blob):
-        blob.built = []
-        for repo, ref, morph_name, blob_names in blob.needs_built():
-            morph_filename = '%s.morph' % morph_name
-            morph, cached = self.build(repo, ref, morph_filename)
-            for blob_name in blob_names:
-                blob.built.append((blob_name, cached[blob_name]))
-
-    def install_chunk(self, morph, chunk_name, chunk_filename, staging_dir):
-        if morph.kind != 'chunk':
-            return
-        if self.settings['bootstrap']:
-            self.msg('Unpacking chunk %s onto system' % chunk_name)
-            ex = morphlib.execute.Execute('/', self.msg)
-            morphlib.bins.unpack_binary(chunk_filename, '/', ex)
-            ldconfig(ex, '/')
+
+        return ret
+
+    def create_blob_builder(self, blob):
+        if isinstance(blob, morphlib.blobs.Stratum):
+            builder = StratumBuilder(blob)
+        elif isinstance(blob, morphlib.blobs.Chunk):
+            builder = ChunkBuilder(blob)
+        elif isinstance(blob, morphlib.blobs.System):
+            builder = SystemBuilder(blob)
         else:
-            self.msg('Unpacking chunk %s into staging' % chunk_name)
-            ex = morphlib.execute.Execute(staging_dir, self.msg)
-            morphlib.bins.unpack_binary(chunk_filename, staging_dir, ex)
-            ldconfig(ex, staging_dir)
+            raise TypeError('Blob %s has unknown type %s' %
+                            (str(blob), type(blob)))
+
+        cache_id = self.get_blob_cache_id(blob)
+        logging.debug('cache id: %s' % repr(cache_id))
+        self.dump_memory_profile('after computing cache id')
+
+        builder.builddir = self.tempdir.join('%s.build' % blob.morph.name)
+        builder.destdir = self.tempdir.join('%s.inst' % blob.morph.name)
+        builder.staging = self.tempdir.join('staging')
+        builder.settings = self.settings
+        builder.msg = self.msg
+        builder.cache_prefix = self.cachedir.name(cache_id)
+        builder.tempdir = self.tempdir
+        builder.dump_memory_profile = self.dump_memory_profile
+        
+        return builder
+
+    def get_blob_cache_id(self, blob):
+        # FIXME os.path.basename() only works if the .morph file is an
+        # immediate children of the repo location and is not located in
+        # a subfolder
+        return self.get_cache_id(blob.morph.repo,
+                                 blob.morph.ref,
+                                 os.path.basename(blob.morph.filename))
             
     def get_cache_id(self, repo, ref, morph_filename):
         logging.debug('get_cache_id(%s, %s, %s)' %