Get rid of the old internal morph APIs

18 files changed, 57 insertions, 2482 deletions

diff --git a/morph b/morph
index 7e936516..5bdaa650 100755
--- a/morph
+++ b/morph
@@ -27,8 +27,6 @@ import tempfile
 import morphlib
 from morphlib import buildworker
 from morphlib import buildcontroller
-from morphlib.morphologyloader import MorphologyLoader
-from morphlib.builddependencygraph import BuildDependencyGraph
 
 
 defaults = {
diff --git a/morphlib/blobs.py b/morphlib/blobs.py
deleted file mode 100644
index 1c8686b0..00000000
--- a/morphlib/blobs.py
+++ /dev/null
@@ -1,87 +0,0 @@
-# Copyright (C) 2012  Codethink Limited
-# 
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-# 
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-# 
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-class Blob(object):
-
-    @staticmethod
-    def create_blob(morph):
-        if morph.kind == 'stratum':
-            return Stratum(morph)
-        elif morph.kind == 'chunk':
-            return Chunk(morph)
-        elif morph.kind == 'system':
-            return System(morph)
-        else:
-            raise TypeError('Morphology %s has an unknown type: %s' % 
-                            (morph.filename, morph.kind))
-
-    def __init__(self, morph):
-        self.parents = []
-        self.morph = morph
-        self.dependencies = []
-        self.dependents = []
-
-    def add_parent(self, parent):
-        if not parent in self.parents:
-            self.parents.append(parent)
-
-    def remove_parent(self, parent):
-        if parent in self.parents:
-            self.parents.remove(parent)
-
-    def add_dependency(self, other):
-        if not other in self.dependencies:
-            self.dependencies.append(other)
-        if not self in other.dependents:
-            other.dependents.append(self)
-
-    def remove_dependency(self, other):
-        self.dependencies.remove(other)
-        other.dependents.remove(self)
-
-    def depends_on(self, other):
-        return other in self.dependencies
-
-    @property
-    def chunks(self):
-        if self.morph.chunks:
-            return self.morph.chunks
-        else:
-            return { self.morph.name: ['.'] }
-
-    def __eq__(self, other):
-        return self.morph == other.morph
-
-    def __hash__(self):
-        return hash(self.morph)
-
-    def __str__(self): # pragma: no cover
-        return str(self.morph)
-
-
-class Chunk(Blob):
-
-    pass
-
-
-class Stratum(Blob):
-
-    pass
-
-
-class System(Blob):
-
-    pass
diff --git a/morphlib/blobs_tests.py b/morphlib/blobs_tests.py
deleted file mode 100644
index b1062459..00000000
--- a/morphlib/blobs_tests.py
+++ /dev/null
@@ -1,203 +0,0 @@
-# Copyright (C) 2012  Codethink Limited
-# 
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-# 
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-# 
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-import unittest
-
-import morphlib
-
-
-class FakeChunkMorph(object):
-    @property
-    def kind(self):
-        return 'chunk'
-
-
-class FakeStratumMorph(object):
-    @property
-    def kind(self):
-        return 'stratum'
-
-
-class BlobsTests(unittest.TestCase):
-
-    def test_create_a_chunk_blob(self):
-        morph = FakeChunkMorph()
-        chunk = morphlib.blobs.Blob.create_blob(morph)
-        self.assertTrue(isinstance(chunk, morphlib.blobs.Chunk))
-        self.assertEqual(morph, chunk.morph)
-
-    def test_create_a_stratum_blob(self):
-        morph = FakeStratumMorph()
-        stratum = morphlib.blobs.Blob.create_blob(morph)
-        self.assertTrue(isinstance(stratum, morphlib.blobs.Stratum))
-        self.assertEqual(morph, stratum.morph)
-
-    def test_create_a_system_blob(self):
-        class FakeSystemMorph(object):
-            @property
-            def kind(self):
-                return 'system'
-
-        morph = FakeSystemMorph()
-        system = morphlib.blobs.Blob.create_blob(morph)
-        self.assertTrue(isinstance(system, morphlib.blobs.System))
-        self.assertEqual(morph, system.morph)
-
-    def test_create_an_invalid_blob(self):
-        class FakeInvalidMorph(object):
-            @property
-            def kind(self):
-                return 'invalid'
-
-            @property
-            def filename(self):
-                return '/foo/bar/baz.morph'
-
-        morph = FakeInvalidMorph()
-        self.assertRaises(TypeError, morphlib.blobs.Blob.create_blob, morph)
-
-    def test_blob_with_parents(self):
-        blob1 = morphlib.blobs.Blob(FakeChunkMorph())
-        blob2 = morphlib.blobs.Blob(FakeStratumMorph())
-        blob3 = morphlib.blobs.Blob(FakeStratumMorph())
-
-        self.assertEqual(len(blob1.parents), 0)
-
-        blob1.add_parent(blob2)
-        self.assertTrue(blob2 in blob1.parents)
-        self.assertTrue(blob3 not in blob1.parents)
-        self.assertEqual(len(blob1.parents), 1)
-
-        blob1.add_parent(blob3)
-        self.assertTrue(blob2 in blob1.parents)
-        self.assertTrue(blob3 in blob1.parents)
-        self.assertEqual(len(blob1.parents), 2)
-
-        blob1.remove_parent(blob2)
-        self.assertTrue(blob2 not in blob1.parents)
-        self.assertTrue(blob3 in blob1.parents)
-        self.assertEqual(len(blob1.parents), 1)
-
-        blob1.remove_parent(blob3)
-        self.assertTrue(blob2 not in blob1.parents)
-        self.assertTrue(blob3 not in blob1.parents)
-        self.assertEqual(len(blob1.parents), 0)
-
-    def test_blob_add_remove_dependency(self):
-        blob1 = morphlib.blobs.Blob(None)
-        blob2 = morphlib.blobs.Blob(None)
-
-        self.assertEqual(len(blob1.dependencies), 0)
-        self.assertEqual(len(blob2.dependencies), 0)
-
-        blob1.add_dependency(blob2)
-
-        self.assertTrue(blob2 in blob1.dependencies)
-        self.assertTrue(blob1 in blob2.dependents)
-
-        self.assertTrue(blob1.depends_on(blob2))
-
-        blob2.add_dependency(blob1)
-
-        self.assertTrue(blob2 in blob1.dependencies)
-        self.assertTrue(blob1 in blob2.dependents)
-        self.assertTrue(blob1 in blob2.dependencies)
-        self.assertTrue(blob2 in blob1.dependents)
-
-        self.assertTrue(blob1.depends_on(blob2))
-        self.assertTrue(blob2.depends_on(blob1))
-
-        blob1.remove_dependency(blob2)
-
-        self.assertTrue(blob2 not in blob1.dependencies)
-        self.assertTrue(blob1 not in blob2.dependents)
-        self.assertTrue(blob1 in blob2.dependencies)
-        self.assertTrue(blob2 in blob1.dependents)
-
-        self.assertFalse(blob1.depends_on(blob2))
-        self.assertTrue(blob2.depends_on(blob1))
-
-        blob2.remove_dependency(blob1)
-
-        self.assertTrue(blob2 not in blob1.dependencies)
-        self.assertTrue(blob1 not in blob2.dependents)
-        self.assertTrue(blob1 not in blob2.dependencies)
-        self.assertTrue(blob2 not in blob1.dependents)
-
-        self.assertFalse(blob1.depends_on(blob2))
-        self.assertFalse(blob2.depends_on(blob1))
-
-    def test_hashing_and_equality_checks(self):
-        morph = FakeChunkMorph()
-        blob1 = morphlib.blobs.Blob.create_blob(morph)
-        blob2 = morphlib.blobs.Blob.create_blob(morph)
-        blob3 = morphlib.blobs.Blob.create_blob(FakeChunkMorph())
-
-        self.assertEqual(hash(blob1), hash(blob2))
-        self.assertEqual(blob1, blob2)
-
-        self.assertNotEqual(hash(blob1), hash(blob3))
-        self.assertNotEqual(blob1, blob3)
-
-    def test_chunks(self):
-        settings = { 'git-base-url': [] }
-        loader = morphlib.morphologyloader.MorphologyLoader(settings)
-        loader._get_morph_text = self.get_morph_text
-
-        class FakeTreeish(object):
-            def __init__(self):
-                self.original_repo = 'test-repo'
-        faketreeish = FakeTreeish()
-
-        faketreeish.original_repo = 'hello'
-        stratum_morph = loader.load(faketreeish, 'foo.morph')
-        stratum = morphlib.blobs.Stratum(stratum_morph)
-        self.assertEquals(len(stratum.chunks), 1)
-        self.assertTrue('foo' in stratum.chunks)
-        self.assertEqual(['.'], stratum.chunks['foo'])
-
-        chunk_morph = loader.load(faketreeish, 'bar.morph')
-        chunk = morphlib.blobs.Chunk(chunk_morph)
-        self.assertEqual(len(chunk.chunks), 2)
-        self.assertTrue('include' in chunk.chunks)
-        self.assertEqual(chunk.chunks['include'], ['include/'])
-        self.assertTrue('src' in chunk.chunks)
-        self.assertEqual(chunk.chunks['src'], ['src/'])
-
-    def get_morph_text(self, treeish, filename):
-        if filename == 'foo.morph':
-            return ('''
-                    {
-                        "name": "foo",
-                        "kind": "stratum",
-                        "sources": [
-                            {
-                                "name": "bar",
-                                "repo": "bar",
-                                "ref": "master"
-                            }
-                        ]
-                    }''')
-        else:
-            return ('''
-                    {
-                        "name": "bar",
-                        "kind": "chunk",
-                        "chunks": {
-                            "include": [ "include/" ],
-                            "src": [ "src/" ]
-                        }
-                    }''')
diff --git a/morphlib/builddependencygraph.py b/morphlib/builddependencygraph.py
deleted file mode 100644
index 7fd93b66..00000000
--- a/morphlib/builddependencygraph.py
+++ /dev/null
@@ -1,314 +0,0 @@
-# Copyright (C) 2012  Codethink Limited
-# 
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-# 
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-# 
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-import collections
-import os
-
-import morphlib
-
-
-class BuildDependencyGraph(object): # pragma: no cover
-
-    '''This class constructs a build dependency graph from an input morphology.
-
-    Given a chunk, stratum or system morphology, this class constructs a build
-    dependency graph and provides ways to traverse it. It also provides a 
-    method to transform the dependency graph into groups of items that are 
-    independent and can be built in parallel.
-    
-    '''
-    
-    def __init__(self, source_manager, morph_loader, repo, ref, filename):
-        self.source_manager = source_manager
-        self.morph_loader = morph_loader
-        self.root_repo = repo
-        self.root_ref = ref
-        self.root_filename = filename
-        self.blobs = set()
-
-    def create_blob(self, treeish, filename, chunk_name=None):
-        '''Creates a blob from a morphology.'''
-
-        morph = self.morph_loader.load(treeish, filename, chunk_name)
-
-        if morph.kind == 'stratum':
-            return morphlib.blobs.Stratum(morph)
-        elif morph.kind == 'chunk':
-            return morphlib.blobs.Chunk(morph)
-        else:
-            return morphlib.blobs.System(morph)
-
-    def get_blob(self, treeish, filename, chunk_name=None):
-        '''Takes a repo, ref, filename and looks up the blob for them.
-
-        Loads the corresponding morphology and chunk/stratum/system object
-        on-demand if it is not cached yet.
-           
-        '''
-
-        key = (treeish, filename)
-        blob = self.cached_blobs.get(key, None)
-        if not blob:
-            blob = self.create_blob(treeish, filename, chunk_name)
-            self.cached_blobs[key] = blob
-        return blob
-
-    def resolve(self):
-        '''Constructs the graph by resolving dependencies recursively.'''
-
-        self.cached_blobs = {}
-        self.blobs = set()
-
-        root = self.resolve_root()
-        self.resolve_strata()
-        self.resolve_chunks()
-        return root
-
-    def resolve_root(self):
-        # prepare the repo, load the morphology and blob information
-        treeish = self.source_manager.get_treeish(self.root_repo,
-                                                  self.root_ref)
-        root = self.get_blob(treeish, self.root_filename)
-        self.blobs.add(root)
-
-        # load all strata the morph depends on (only if it's a system image)
-        if root.morph.kind == 'system':
-            for stratum_name in root.morph.strata:
-                filename = '%s.morph' % stratum_name
-                stratum = self.get_blob(treeish, filename)
-                root.add_dependency(stratum)
-                stratum.add_parent(root)
-                self.blobs.add(stratum)
-
-        return root
-
-    def resolve_strata(self):
-        '''Generates a dependency graph of strata recursively.
-
-        It starts with the input morphology and attempts to resolve all its
-        build dependencies recursively using breadth-first search. Morphologies
-        and blobs are loaded from disk on demand.
-           
-        '''
-
-        # start the BFS at all input strata
-        queue = collections.deque()
-        strata = [x for x in self.blobs if x.morph.kind == 'stratum']
-        queue.extend(strata)
-        
-        while len(queue) > 0:
-            stratum = queue.popleft()
-
-            # the DFS recursion ends whenever we have a stratum
-            # that depends on nothing else
-            if not stratum.morph.build_depends:
-                continue
-            
-            # verify that the build-depends format is valid
-            if isinstance(stratum.morph.build_depends, list):
-                for depname in stratum.morph.build_depends:
-                    # load the dependency stratum on demand
-                    depstratum = self.get_blob(stratum.morph.treeish,
-                                               '%s.morph' % depname)
-                    self.blobs.add(depstratum)
-
-                    # add the dependency stratum to the graph
-                    stratum.add_dependency(depstratum)
-                    queue.append(depstratum)
-            else:
-                raise Exception('%s uses an invalid "build-depends" format'
-                                % stratum)
-
-    def resolve_chunks(self):
-        '''Resolves all chunks of the strata and inserts them into the graph.
-        
-        Starting with a dependency graph of strata, this method fills the
-        graph with all contained chunks and creates dependencies where 
-        appropriate. Chunk morphologies and blobs are loaded on demand.
-        
-        '''
-
-        blobs = list(self.blobs)
-        for blob in blobs:
-            if isinstance(blob, morphlib.blobs.Stratum):
-                self.resolve_stratum_chunks(blob)
-
-    def resolve_stratum_chunks(self, stratum):
-        # the set of chunks contained in the stratum
-        stratum_chunks = set()
-
-        # dictionary that maps chunk names to chunks
-        name_to_chunk = {}
-
-        # create objects for all chunks in the stratum
-        for i in xrange(0, len(stratum.morph.sources)):
-            source = stratum.morph.sources[i]
-
-            # construct a tuple for loading the chunk
-            repo = source['repo']
-            ref = source['ref']
-            filename = '%s.morph' % (source['morph']
-                                     if 'morph' in source
-                                     else source['name'])
-
-            # load the chunk on demand
-            treeish = self.source_manager.get_treeish(repo, ref)
-            chunk = self.get_blob(treeish, filename, source['name'])
-            chunk.add_parent(stratum)
-
-            # store (name -> chunk) association to avoid loading the chunk 
-            # twice
-            name_to_chunk[source['name']] = chunk
-
-            # read the build-depends information
-            build_depends = (source['build-depends']
-                             if 'build-depends' in source
-                             else None)
-
-            # turn build-depends into proper dependencies in the graph
-            if build_depends is None:
-                # chunks with no build-depends implicitly depend on all
-                # chunks listed earlier in the same stratum
-                for dependency in stratum_chunks:
-                    if not dependency is chunk:
-                        chunk.add_dependency(dependency)
-            elif isinstance(build_depends, list):
-                for depname in build_depends:
-                    if depname in name_to_chunk:
-                        dependency = name_to_chunk[depname]
-                        if not dependency is chunk:
-                            chunk.add_dependency(dependency)
-                    else:
-                        raise Exception('%s: source %s references %s before '
-                                        'it is defined' % 
-                                                       (stratum.morph.filename,
-                                                        source['name'],
-                                                        depname))
-            else:
-                raise Exception('%s: source %s uses an invalid build-depends '
-                                'format' %
-                                (stratum.morph.filename, source['name']))
-            
-            # add the chunk to stratum and graph
-            stratum_chunks.add(chunk)
-            self.blobs.add(chunk)
-
-        # make the chunks in this stratum depend on all
-        # strata that need to be built first
-        for chunk in stratum_chunks:
-            for dependency in stratum.dependencies:
-                chunk.add_dependency(dependency)
-
-        ## clear the dependencies of the stratum
-        #for dependency in set(stratum.dependencies):
-        #    stratum.remove_dependency(dependency)
-
-        # make the stratum depend on all its chunks
-        for chunk in stratum_chunks:
-            stratum.add_dependency(chunk)
-
-    def build_order(self):
-        '''Returns a queue of build groups in a valid order.
-        
-        This function computes a topological sorting of the dependency graph
-        and generates a deque of groups, each of which contains a set of items
-        that are independent and can be built in parallel. The items in each
-        group are guaranteed to depend only on items in previous groups.
-        
-        '''
-
-        sorting = self.compute_topological_sorting()
-        groups = collections.deque()
-
-        # create the first group
-        group = set()
-        groups.append(group)
-
-        # traverse the graph in topological order
-        for blob in sorting:
-            # add the current item to the current group, or a new group
-            # if one of its dependencies is in the current one
-            create_group = False
-            for dependency in blob.dependencies:
-                if dependency in group:
-                    create_group = True
-            if create_group:
-                group = set()
-                groups.append(group)
-            group.add(blob)
-
-        # return the set of blobs and the build groups
-        return set(self.blobs), groups
-
-    def compute_topological_sorting(self):
-        '''Computes a topological sorting of the dependency graph. 
-        
-        A topological sorting basically is the result of a series of
-        breadth-first searches starting at each leaf node (blobs with no
-        dependencies). Blobs are added to the sorting as soon as all their
-        dependencies have been added (which means that by then, all
-        dependencies are satisfied).
-
-        For more information, see
-        http://en.wikipedia.org/wiki/Topological_sorting.
-        
-        '''
-
-        # map blobs to sets of satisfied dependencies. this is to detect when
-        # we can actually add blobs to the BFS queue. rather than dropping
-        # links between nodes, like most topological sorting algorithms do,
-        # we simply remember all satisfied dependencies and check if all
-        # of them are met repeatedly
-        satisfied_dependencies = {}
-
-        # create an empty sorting
-        sorting = collections.deque()
-
-        # create a set of leafs to start the DFS from
-        leafs = collections.deque()
-        for blob in self.blobs:
-            satisfied_dependencies[blob] = set()
-            if len(blob.dependencies) == 0:
-                leafs.append(blob)
-
-        while len(leafs) > 0:
-            # fetch a leaf blob from the DFS queue
-            blob = leafs.popleft()
-
-            # add it to the sorting
-            sorting.append(blob)
-
-            # mark this dependency as resolved
-            for dependent in blob.dependents:
-                satisfied_dependencies[dependent].add(blob)
-
-                # add the dependent blob as a leaf if all
-                # its dependencies have been resolved
-                has = len(satisfied_dependencies[dependent])
-                needs = len(dependent.dependencies)
-                if has == needs:
-                    leafs.append(dependent)
-
-        # if not all dependencies were resolved on the way, we
-        # have found at least one cyclic dependency
-        if len(sorting) < len(self.blobs):
-            raise Exception('Cyclic dependencies found in the dependency '
-                            'graph of %s|%s|%s' % 
-                            (self.root_repo, 
-                             self.root_ref,
-                             self.root_filename))
-
-        return sorting
diff --git a/morphlib/builder.py b/morphlib/builder.py
deleted file mode 100644
index 336a25b3..00000000
--- a/morphlib/builder.py
+++ /dev/null
@@ -1,699 +0,0 @@
-# Copyright (C) 2011-2012  Codethink Limited
-# 
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-# 
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-# 
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-import collections
-import json
-import logging
-import os
-import shutil
-import time
-
-import morphlib
-
-
-def ldconfig(ex, rootdir): # pragma: no cover
-    '''Run ldconfig for the filesystem below ``rootdir``.
-
-    Essentially, ``rootdir`` specifies the root of a new system.
-    Only directories below it are considered.
-
-    ``etc/ld.so.conf`` below ``rootdir`` is assumed to exist and
-    be populated by the right directories, and should assume
-    the root directory is ``rootdir``. Example: if ``rootdir``
-    is ``/tmp/foo``, then ``/tmp/foo/etc/ld.so.conf`` should
-    contain ``/lib``, not ``/tmp/foo/lib``.
-    
-    The ldconfig found via ``$PATH`` is used, not the one in ``rootdir``,
-    since in bootstrap mode that might not yet exist, the various 
-    implementations should be compatible enough.
-
-    '''
-
-    conf = os.path.join(rootdir, 'etc', 'ld.so.conf')
-    if os.path.exists(conf):
-        logging.debug('Running ldconfig for %s' % rootdir)
-        cache = os.path.join(rootdir, 'etc', 'ld.so.cache')
-        
-        # The following trickery with $PATH is necessary during the Baserock
-        # bootstrap build: we are not guaranteed that PATH contains the
-        # directory (/sbin conventionally) that ldconfig is in. Then again,
-        # it might, and if so, we don't want to hardware a particular
-        # location. So we add the possible locations to the end of $PATH
-        # and restore that aftewards.
-        old_path = ex.env['PATH']
-        ex.env['PATH'] = '%s:/sbin:/usr/sbin:/usr/local/sbin' % old_path
-        ex.runv(['ldconfig', '-r', rootdir])
-        ex.env['PATH'] = old_path
-    else:
-        logging.debug('No %s, not running ldconfig' % conf)
-
-
-class Factory(object):
-
-    '''Build Baserock binaries.'''
-    
-    def __init__(self, tempdir):
-        self._tempdir = tempdir
-        self.staging = None
-        
-    def create_staging(self):
-        '''Create the staging area.'''
-        self.staging = self._tempdir.join('staging')
-        os.mkdir(self.staging)
-        
-    def remove_staging(self):
-        '''Remove the staging area.'''
-        shutil.rmtree(self.staging)
-        self.staging = None
-
-    def _unpack_binary(self, binary, dirname):
-        '''Unpack binary into a given directory.'''
-        ex = morphlib.execute.Execute('/', logging.debug)
-        morphlib.bins.unpack_binary(binary, dirname, ex)
-
-    def unpack_binary_from_file(self, filename):
-        '''Unpack a binary package from a file, given its name.'''
-        self._unpack_binary(filename, self.staging)
-
-    def unpack_binary_from_file_onto_system(self, filename):
-        '''Unpack contents of a binary package onto the running system.
-        
-        DANGER, WILL ROBINSON! This WILL modify your running system.
-        It should only be used during bootstrapping.
-        
-        '''
-        
-        self._unpack_binary(filename, '/')
-
-    def unpack_sources(self, treeish, srcdir):
-        '''Get sources from to a source directory, for building.
-        
-        The git repository and revision are given via a Treeish object.
-        The source directory must not exist.
-        
-        '''
-
-        def msg(s):
-            pass
-
-        def extract_treeish(treeish, destdir):
-            logging.debug('Extracting %s into %s' % (treeish.repo, destdir))
-            if not os.path.exists(destdir):
-                os.mkdir(destdir)
-            morphlib.git.copy_repository(treeish.repo, destdir, msg)
-            morphlib.git.checkout_ref(destdir, treeish.ref, msg)
-            morphlib.git.reset_workdir(destdir, msg)
-            return [(sub.treeish, os.path.join(destdir, sub.path))
-                    for sub in treeish.submodules]
-
-        todo = [(treeish, srcdir)]
-        while todo:
-            treeish, srcdir = todo.pop()
-            todo += extract_treeish(treeish, srcdir)
-        self.set_mtime_recursively(srcdir)
-
-    def set_mtime_recursively(self, root):
-        '''Set the mtime for every file in a directory tree to the same.
-        
-        We do this because git checkout does not set the mtime to anything,
-        and some projects (binutils, gperf for example) include formatted
-        documentation and try to randomly build things or not because of
-        the timestamps. This should help us get more reliable  builds.
-        
-        '''
-        
-        now = time.time()
-        for dirname, subdirs, basenames in os.walk(root, topdown=False):
-            for basename in basenames:
-                pathname = os.path.join(dirname, basename)
-                # we need the following check to ignore broken symlinks
-                if os.path.exists(pathname):
-                    os.utime(pathname, (now, now))
-            os.utime(dirname, (now, now))
-
-
-class BlobBuilder(object): # pragma: no cover
-
-    def __init__(self, blob, factory, settings, cachedir, cache_key, tempdir,
-                 env):
-        self.blob = blob
-        self.factory = factory
-        self.settings = settings
-        self.cachedir = cachedir
-        self.cache_key = cache_key
-        self.tempdir = tempdir
-        self.env = env
-        
-        self.logfile = None
-        self.stage_items = []
-        self.real_msg = lambda s: None
-        self.dump_memory_profile = lambda msg: None
-
-        self.destdir = os.path.join(self.factory.staging,
-                                    '%s.inst' % blob.morph.name)
-
-        self.build_watch = morphlib.stopwatch.Stopwatch()
-
-    def msg(self, text):
-        self.real_msg(text)
-        if self.logfile and not self.logfile.closed:
-            self.logfile.write('%s\n' % text)
-
-    def builds(self):
-        raise NotImplemented()
-
-    def build(self):
-        self.prepare_logfile()
-        with self.build_watch('overall-build'):
-            self.do_build()
-        self.save_build_times()
-        self.save_logfile()
-
-    def filename(self, name):
-        return '%s.%s.%s' % (self.cachedir.name(self.cache_key),
-                             self.blob.morph.kind,
-                             name)
-
-    def prepare_binary_metadata(self, blob_name):
-        '''Add metadata to a binary about to be built.'''
-
-        self.msg('Adding metadata to %s' % blob_name)
-        meta = {
-            'name': blob_name,
-            'kind': self.blob.morph.kind,
-            'description': self.blob.morph.description,
-        }
-        
-        dirname = os.path.join(self.destdir, 'baserock')
-        if not os.path.exists(dirname):
-            os.mkdir(dirname)
-            
-        filename = os.path.join(dirname, '%s.meta' % blob_name)
-        with open(filename, 'w') as f:
-            json.dump(meta, f, indent=4)
-            f.write('\n')
-
-    def save_build_times(self):
-        meta = {
-            'build-times': {}
-        }
-        for stage in self.build_watch.ticks.iterkeys():
-            meta['build-times'][stage] = {
-                'start': '%s' % self.build_watch.start_time(stage),
-                'stop': '%s' % self.build_watch.stop_time(stage),
-                'delta': '%.4f' % self.build_watch.start_stop_seconds(stage)
-            }
-
-        self.msg('Writing metadata to the cache')
-        with self.cachedir.open(self.cache_key, suffix='.meta') as f:
-            json.dump(meta, f, indent=4)
-            f.write('\n')
-
-    def prepare_logfile(self):
-        self.logfile = self.cachedir.open(self.cache_key, suffix='.log', 
-                                          mode='w+', buffering=0)
-
-    def save_logfile(self):
-        self.logfile.close()
-
-
-class ChunkBuilder(BlobBuilder): # pragma: no cover
-
-    def builds(self):
-        ret = {}
-        for chunk_name in self.blob.chunks:
-            ret[chunk_name] = self.filename(chunk_name)
-        return ret
-
-    def do_build(self):
-        self.builddir = os.path.join(self.factory.staging,
-                                     '%s.build' % self.blob.morph.name)
-        self.msg('Creating build tree at %s' % self.builddir)
-
-        self.ex = morphlib.execute.Execute(self.builddir, self.msg)
-        self.ex.env = self.env
-
-        self.factory.unpack_sources(self.blob.morph.treeish, self.builddir)
-
-        os.mkdir(self.destdir)
-        self.build_with_system_or_commands()
-        self.dump_memory_profile('after building chunk')
-
-        chunks = self.create_chunks()
-        self.dump_memory_profile('after creating build chunks')
-
-        # install all built items to the staging area
-        for name, filename in chunks:
-            self.msg('Fetching cached %s %s from %s' %
-                     (self.blob.morph.kind, name, filename))
-            self.install_chunk(name, filename)
-            self.dump_memory_profile('after installing chunk')
-
-    def install_chunk(self, chunk_name, chunk_filename):
-        ex = morphlib.execute.Execute('/', self.msg)
-        if self.settings['bootstrap']:
-            self.msg('Unpacking item %s onto system' % chunk_name)
-            self.factory.unpack_binary_from_file_onto_system(chunk_filename)
-            ldconfig(ex, '/')
-        else:
-            self.msg('Unpacking chunk %s into staging' % chunk_name)
-            self.factory.unpack_binary_from_file(chunk_filename)
-            ldconfig(ex, self.factory.staging)
-        
-    def build_with_system_or_commands(self):
-        '''Run explicit commands or commands from build system.
-        
-        Use explicit commands, if given, and build system commands if one
-        has been specified.
-        
-        '''
-
-        bs_name = self.blob.morph.build_system
-        bs = morphlib.buildsystem.lookup_build_system(bs_name)
-
-        def run_them(runner, what):
-            attr = '%s_commands' % what
-            cmds = getattr(self.blob.morph, attr, []) or bs[attr]
-            runner(what, cmds)
-
-        run_them(self.run_sequentially, 'configure')
-        run_them(self.run_in_parallel, 'build')
-        run_them(self.run_sequentially, 'test')
-        run_them(self.run_sequentially, 'install')
-
-    def run_in_parallel(self, what, commands):
-        self.msg('commands: %s' % what)
-        with self.build_watch(what):
-            max_jobs = self.blob.morph.max_jobs 
-            if max_jobs is None:
-                max_jobs = self.settings['max-jobs']
-            self.ex.env['MAKEFLAGS'] = '-j%s' % max_jobs
-            self.run_commands(commands)
-
-    def run_sequentially(self, what, commands):
-        self.msg ('commands: %s' % what)
-        with self.build_watch(what):
-            self.ex.env['MAKEFLAGS'] = '-j1'
-            self.run_commands(commands)
-
-    def run_commands(self, commands):
-        if self.settings['staging-chroot']:
-            ex = morphlib.execute.Execute(self.factory.staging, self.msg)
-            ex.env = self.ex.env.copy()
-
-            assert self.builddir.startswith(self.factory.staging + '/')
-            assert self.destdir.startswith(self.factory.staging + '/')
-            builddir = self.builddir[len(self.factory.staging):]
-            destdir = self.destdir[len(self.factory.staging):]
-            ex.env['DESTDIR'] = destdir
-
-            for cmd in commands:
-                ex.runv(['/usr/sbin/chroot', self.factory.staging, 'sh', '-c',
-                         'cd "$1" && shift && eval "$@"', '--', builddir, cmd])
-        else:
-            self.ex.env['DESTDIR'] = self.destdir
-            self.ex.run(commands)
-
-    def create_chunks(self):
-        chunks = []
-        with self.build_watch('create-chunks'):
-            for chunk_name in self.blob.chunks:
-                self.msg('Creating chunk %s' % chunk_name)
-                self.prepare_binary_metadata(chunk_name)
-                patterns = self.blob.chunks[chunk_name]
-                patterns += [r'baserock/%s\.' % chunk_name]
-                filename = self.filename(chunk_name)
-                self.msg('Creating binary for %s' % chunk_name)
-                with self.cachedir.open(filename) as f:
-                    morphlib.bins.create_chunk(self.destdir, f, patterns,
-                                               self.ex, 
-                                               self.dump_memory_profile)
-                chunks.append((chunk_name, filename))
-
-        files = os.listdir(self.destdir)
-        if files:
-            raise Exception('DESTDIR %s is not empty: %s' %
-                                (self.destdir, files))
-        return chunks
-
-
-class StratumBuilder(BlobBuilder): # pragma: no cover
-    
-    def builds(self):
-        filename = self.filename(self.blob.morph.name)
-        return { self.blob.morph.name: filename }
-
-    def do_build(self):
-        os.mkdir(self.destdir)
-        ex = morphlib.execute.Execute(self.destdir, self.msg)
-        with self.build_watch('unpack-chunks'):
-            for chunk_name, filename in self.stage_items:
-                self.msg('Unpacking chunk %s' % chunk_name)
-                morphlib.bins.unpack_binary(filename, self.destdir, ex)
-        with self.build_watch('create-binary'):
-            self.prepare_binary_metadata(self.blob.morph.name)
-            self.msg('Creating binary for %s' % self.blob.morph.name)
-            filename = self.filename(self.blob.morph.name)
-            basename = os.path.basename(filename)
-            with self.cachedir.open(basename) as f:
-                morphlib.bins.create_stratum(self.destdir, f, ex)
-
-
-class SystemBuilder(BlobBuilder): # pragma: no cover
-
-    def builds(self):
-        filename = self.filename(self.blob.morph.name)
-        return { self.blob.morph.name: filename }
-
-    def do_build(self):
-        logging.debug('SystemBuilder.do_build called')
-        self.ex = morphlib.execute.Execute(self.tempdir.dirname, self.msg)
-        
-        image_name = self.tempdir.join('%s.img' % self.blob.morph.name)
-        self._create_image(image_name)
-        self._partition_image(image_name)
-        self._install_mbr(image_name)
-        partition = self._setup_device_mapping(image_name)
-
-        mount_point = None
-        try:
-            self._create_fs(partition)
-            mount_point = self.tempdir.join('mnt')
-            self._mount(partition, mount_point)
-            factory_path = os.path.join(mount_point, 'factory')
-            self._create_subvolume(factory_path)
-            self._unpack_strata(factory_path)
-            self._create_fstab(factory_path)
-            self._create_extlinux_config(factory_path)
-            self._create_subvolume_snapshot(
-                    mount_point, 'factory', 'factory-run')
-            factory_run_path = os.path.join(mount_point, 'factory-run')
-            self._install_boot_files(factory_run_path, mount_point)
-            self._install_extlinux(mount_point)
-            self._unmount(mount_point)
-        except BaseException:
-            self._unmount(mount_point)
-            self._undo_device_mapping(image_name)
-            raise
-
-        self._undo_device_mapping(image_name)
-        self._move_image_to_cache(image_name)
-
-    def _create_image(self, image_name):
-        with self.build_watch('create-image'):
-            morphlib.fsutils.create_image(self.ex, image_name,
-                                          self.blob.morph.disk_size)
-
-    def _partition_image(self, image_name):
-        with self.build_watch('partition-image'):
-            morphlib.fsutils.partition_image(self.ex, image_name)
-
-    def _install_mbr(self, image_name):
-        with self.build_watch('install-mbr'):
-            morphlib.fsutils.install_mbr(self.ex, image_name)
-
-    def _setup_device_mapping(self, image_name):
-        with self.build_watch('setup-device-mapper'):
-            return morphlib.fsutils.setup_device_mapping(self.ex, image_name)
-
-    def _create_fs(self, partition):
-        with self.build_watch('create-filesystem'):
-            morphlib.fsutils.create_fs(self.ex, partition)
-
-    def _mount(self, partition, mount_point):
-        with self.build_watch('mount-filesystem'):
-            morphlib.fsutils.mount(self.ex, partition, mount_point)
-
-    def _create_subvolume(self, path):
-        with self.build_watch('create-factory-subvolume'):
-            self.ex.runv(['btrfs', 'subvolume', 'create', path])
-
-    def _unpack_strata(self, path):
-        with self.build_watch('unpack-strata'):
-            for name, filename in self.stage_items:
-                self.msg('unpack %s from %s' % (name, filename))
-                self.ex.runv(['tar', '-C', path, '-xhf', filename])
-            ldconfig(self.ex, path)
-
-    def _create_fstab(self, path):
-        with self.build_watch('create-fstab'):
-            fstab = os.path.join(path, 'etc', 'fstab')
-            if not os.path.exists(os.path.dirname(fstab)):
-                os.makedirs(os.path.dirname(fstab))
-            with open(fstab, 'w') as f:
-                f.write('proc      /proc proc  defaults          0 0\n')
-                f.write('sysfs     /sys  sysfs defaults          0 0\n')
-                f.write('/dev/sda1 / btrfs errors=remount-ro 0 1\n')
-
-    def _create_extlinux_config(self, path):
-        config = os.path.join(path, 'extlinux.conf')
-        with open(config, 'w') as f:
-            f.write('default linux\n')
-            f.write('timeout 1\n')
-            f.write('label linux\n')
-            f.write('kernel /boot/vmlinuz\n')
-            f.write('append root=/dev/sda1 rootflags=subvol=factory-run '
-                                           'init=/sbin/init quiet rw\n')
-
-    def _create_subvolume_snapshot(self, path, source, target):
-        with self.build_watch('create-subvolume-snapshot'):
-            self.ex.runv(['btrfs', 'subvolume', 'snapshot', source, target],
-                         cwd=path)
-
-    def _install_boot_files(self, sourcefs, targetfs):
-        with self.build_watch('install-boot-files'):
-            logging.debug('installing boot files into root volume')
-            shutil.copy2(os.path.join(sourcefs, 'extlinux.conf'),
-                         os.path.join(targetfs, 'extlinux.conf'))
-            os.mkdir(os.path.join(targetfs, 'boot'))
-            shutil.copy2(os.path.join(sourcefs, 'boot', 'vmlinuz'),
-                         os.path.join(targetfs, 'boot', 'vmlinuz'))
-            shutil.copy2(os.path.join(sourcefs, 'boot', 'System.map'),
-                         os.path.join(targetfs, 'boot', 'System.map'))
-
-    def _install_extlinux(self, path):
-        with self.build_watch('install-bootloader'):
-            self.ex.runv(['extlinux', '--install', path])
-
-            # FIXME this hack seems to be necessary to let extlinux finish
-            self.ex.runv(['sync'])
-            time.sleep(2)
-
-    def _unmount(self, mount_point):
-        if mount_point is not None:
-            with self.build_watch('unmount-filesystem'):
-                morphlib.fsutils.unmount(self.ex, mount_point)
-
-    def _undo_device_mapping(self, image_name):
-        with self.build_watch('undo-device-mapper'):
-            morphlib.fsutils.undo_device_mapping(self.ex, image_name)
-
-    def _move_image_to_cache(self, image_name):
-        with self.build_watch('cache-image'):
-            filename = self.filename(self.blob.morph.name)
-            self.ex.runv(['mv', image_name, filename])
-
-
-class Builder(object): # pragma: no cover
-
-    '''Build binary objects for Baserock.
-    
-    The objects may be chunks or strata.'''
-    
-    def __init__(self, tempdir, app, morph_loader, source_manager, factory):
-        self.tempdir = tempdir
-        self.app = app
-        self.real_msg = app.msg
-        self.dump_memory_profile = app.dump_memory_profile
-        self.cachedir = morphlib.cachedir.CacheDir(
-                            self.app.settings['cachedir'])
-        self.morph_loader = morph_loader
-        self.source_manager = source_manager
-        self.factory = factory
-        self.indent = 0
-        # create build environment string in advance
-        env_names = ("USER", "USERNAME", "LOGNAME",
-                     "TOOLCHAIN_TARGET", "PREFIX", "BOOTSTRAP", "CFLAGS")
-        env = app.clean_env()
-        self.build_env = ''.join(k + env[k] for k in env_names)
-
-    def msg(self, text):
-        spaces = '  ' * self.indent
-        self.real_msg('%s%s' % (spaces, text))
-
-    def indent_more(self):
-        self.indent += 1
-    
-    def indent_less(self):
-        self.indent -= 1
-
-    def build(self, build_order):
-        '''Build a list of groups of morphologies. Items in a group
-           can be built in parallel.'''
-
-        logging.debug('Builder.build called')
-        self.indent_more()
-
-        # first pass: create builders for all blobs
-        builders = {}
-        for group in build_order:
-            for blob in group:
-                builder = self.create_blob_builder(blob)
-                builders[blob] = builder
-
-        # second pass: build group by group, item after item
-        for group in build_order:
-            for blob in group:
-                logging.debug('Building blob %s' % repr(blob))
-                self.msg('Building %s' % blob)
-                self.indent_more()
-
-                ## TODO this needs to be done recursively
-                ## make sure all dependencies are in the staging area
-                #for dependency in blob.dependencies:
-                #    depbuilder = builders[dependency]
-                #    depbuilder.stage()
-
-                ## TODO this needs the set of recursively collected 
-                ## dependencies
-                ## make sure all non-dependencies are not staged
-                #for nondependency in (blobs - blob.dependencies):
-                #    depbuilder = builders[nondependency]
-                #    depbuilder.unstage()
-
-                builder = builders[blob]
-                logging.debug('builder = %s' % repr(builder))
-
-                # get a list of all the items we have to build for this blob
-                builds = builder.builds()
-                logging.debug('builds = %s' % repr(builds))
-
-                # if not all build items are in the cache, rebuild the blob
-                if not self.all_built(builds):
-                    logging.debug('calling builders build method')
-                    builders[blob].build()
-                
-                # check again, fail if not all build items were actually built
-                if not self.all_built(builds):
-                    raise Exception('Not all builds results expected from %s '
-                                    'were actually built' % blob)
-
-                for parent in blob.parents:
-                    for item, filename in builds.iteritems():
-                        self.msg('Marking %s to be staged for %s' %
-                                 (item, parent))
-
-                    parent_builder = builders[parent]
-                    parent_builder.stage_items += builds.items()
-
-                self.indent_less()
-
-        self.indent_less()
-
-    def all_built(self, builds):
-        return all(os.path.isfile(builds[name]) for name in builds)
-
-    def build_single(self, blob, build_order):
-        self.indent_more()
-
-        # first pass: create builders for all blobs
-        builders = {}
-        for group in build_order:
-            for b in group:
-                builder = self.create_blob_builder(b)
-                builders[b] = builder
-
-        # second pass: mark all dependencies for staging
-        queue = collections.deque()
-        visited = set()
-        for dependency in blob.dependencies:
-            queue.append(dependency)
-            visited.add(dependency)
-        while len(queue) > 0:
-            dependency = queue.popleft()
-            built_items = builders[dependency].builds()
-            for name, filename in built_items.iteritems():
-                self.msg('Marking %s to be staged for %s' % (name, blob))
-                builders[blob].stage_items.append((name, filename))
-            for dep in dependency.dependencies:
-                if (dependency.morph.kind == 'stratum' and
-                        not dep.morph.kind == 'chunk'):
-                    if dep not in visited:
-                        queue.append(dep)
-                        visited.add(dep)
-
-        # build the single blob now
-        builders[blob].build()
-        self.indent_less()
-
-    def create_blob_builder(self, blob):
-        if isinstance(blob, morphlib.blobs.Stratum):
-            klass = StratumBuilder
-        elif isinstance(blob, morphlib.blobs.Chunk):
-            klass = ChunkBuilder
-        elif isinstance(blob, morphlib.blobs.System):
-            klass = SystemBuilder
-        else:
-            raise TypeError('Blob %s has unknown type %s' %
-                            (str(blob), type(blob)))
-
-        builder = klass(blob, self.factory, self.app.settings, self.cachedir,
-                        self.get_cache_id(blob.morph), self.tempdir,
-                        self.app.clean_env())
-        builder.real_msg = self.msg
-        builder.dump_memory_profile = self.dump_memory_profile
-        
-        return builder
-
-    def get_cache_id(self, morph):
-        logging.debug('get_cache_id(%s)' % morph)
-
-        if morph.kind == 'chunk':
-            kids = []
-        elif morph.kind == 'stratum':
-            kids = []
-            for source in morph.sources:
-                repo = source['repo']
-                ref = source['ref']
-                treeish = self.source_manager.get_treeish(repo, ref)
-                filename = (source['morph']
-                            if 'morph' in source
-                            else source['name'])
-                filename = '%s.morph' % filename
-                chunk = self.morph_loader.load(treeish, filename, 
-                                               chunk_name=source['name'])
-                cache_id = self.get_cache_id(chunk)
-                kids.append(cache_id)
-        elif morph.kind == 'system':
-            kids = []
-            for stratum_name in morph.strata:
-                filename = '%s.morph' % stratum_name
-                stratum = self.morph_loader.load(morph.treeish, filename)
-                cache_id = self.get_cache_id(stratum)
-                kids.append(cache_id)
-        else:
-            raise NotImplementedError('unknown morph kind %s' %
-                                      morph.kind)
-
-        dict_key = {
-            'filename': morph.filename,
-            'arch': morphlib.util.arch(),
-            'ref': morph.treeish.sha1,
-            'kids': ''.join(self.cachedir.key(k) for k in kids),
-            'env': self.build_env,
-        }
-        return dict_key
-
diff --git a/morphlib/builder2.py b/morphlib/builder2.py
index 774df675..1d0cbd98 100644
--- a/morphlib/builder2.py
+++ b/morphlib/builder2.py
@@ -22,6 +22,44 @@ import time
 
 import morphlib
 
+
+def ldconfig(ex, rootdir): # pragma: no cover
+    '''Run ldconfig for the filesystem below ``rootdir``.
+
+    Essentially, ``rootdir`` specifies the root of a new system.
+    Only directories below it are considered.
+
+    ``etc/ld.so.conf`` below ``rootdir`` is assumed to exist and
+    be populated by the right directories, and should assume
+    the root directory is ``rootdir``. Example: if ``rootdir``
+    is ``/tmp/foo``, then ``/tmp/foo/etc/ld.so.conf`` should
+    contain ``/lib``, not ``/tmp/foo/lib``.
+    
+    The ldconfig found via ``$PATH`` is used, not the one in ``rootdir``,
+    since in bootstrap mode that might not yet exist, the various 
+    implementations should be compatible enough.
+
+    '''
+
+    conf = os.path.join(rootdir, 'etc', 'ld.so.conf')
+    if os.path.exists(conf):
+        logging.debug('Running ldconfig for %s' % rootdir)
+        cache = os.path.join(rootdir, 'etc', 'ld.so.cache')
+        
+        # The following trickery with $PATH is necessary during the Baserock
+        # bootstrap build: we are not guaranteed that PATH contains the
+        # directory (/sbin conventionally) that ldconfig is in. Then again,
+        # it might, and if so, we don't want to hardware a particular
+        # location. So we add the possible locations to the end of $PATH
+        # and restore that aftewards.
+        old_path = ex.env['PATH']
+        ex.env['PATH'] = '%s:/sbin:/usr/sbin:/usr/local/sbin' % old_path
+        ex.runv(['ldconfig', '-r', rootdir])
+        ex.env['PATH'] = old_path
+    else:
+        logging.debug('No %s, not running ldconfig' % conf)
+
+
 class BuilderBase(object):
 
     '''Base class for building artifacts.'''
@@ -380,7 +418,7 @@ class SystemBuilder(BuilderBase): # pragma: no cover
             for stratum_artifact in self.artifact.dependencies:
                 with self.artifact_cache.get(stratum_artifact) as f:
                     morphlib.bins.unpack_binary_from_file(f, path)
-            morphlib.builder.ldconfig(self.ex, path)
+            ldconfig(self.ex, path)
 
     def _create_fstab(self, path):
         logging.debug('Creating fstab in %s' % path)
diff --git a/morphlib/builder_tests.py b/morphlib/builder_tests.py
deleted file mode 100644
index a8f06462..00000000
--- a/morphlib/builder_tests.py
+++ /dev/null
@@ -1,165 +0,0 @@
-# Copyright (C) 2012  Codethink Limited
-# 
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-# 
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-# 
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-import os
-import shutil
-import unittest
-
-import morphlib
-
-
-class FakeSubmodule(object):
-
-    def __init__(self, **kwargs):
-        for name in kwargs:
-            setattr(self, name, kwargs[name])
-
-
-class FakeTreeish(object):
-
-    def __init__(self, tempdir, repo, subtreeish=None):
-        self.repo = tempdir.join(repo)
-        self.ref = 'master'
-        self.submodules = []
-
-        temp_repo = tempdir.join('temp_repo')
-
-        os.mkdir(temp_repo)
-        ex = morphlib.execute.Execute(temp_repo, lambda s: None)
-        ex.runv(['git', 'init', '--quiet'])
-        with open(os.path.join(temp_repo, 'file.txt'), 'w') as f:
-            f.write('foobar\n')
-        ex.runv(['git', 'add', 'file.txt'])
-        ex.runv(['git', 'commit', '--quiet', '-m', 'foo'])
-        
-        if subtreeish is not None:
-            ex.runv(['git', 'submodule', 'add', subtreeish.repo])
-            path = os.path.basename(subtreeish.repo)
-            self.submodules = [FakeSubmodule(repo=subtreeish.repo,
-                                             ref='master',
-                                             path=path,
-                                             treeish=subtreeish)]
-        
-        ex = morphlib.execute.Execute(tempdir.dirname, lambda s: None)
-        ex.runv(['git', 'clone', '-n', temp_repo, self.repo])
-        
-        shutil.rmtree(temp_repo)
-        
-
-class FactoryTests(unittest.TestCase):
-
-    def setUp(self):
-        self.tempdir = morphlib.tempdir.Tempdir()
-        self.factory = morphlib.builder.Factory(self.tempdir)
-        
-    def tearDown(self):
-        self.tempdir.remove()
-
-    def create_chunk(self):
-        '''Create a simple binary chunk.'''
-        
-        inst = self.tempdir.join('dummy-inst')
-        os.mkdir(inst)
-        for x in ['bin', 'etc', 'lib']:
-            os.mkdir(os.path.join(inst, x))
-        
-        binary = self.tempdir.join('dummy-chunk')
-        ex = None # this is not actually used by the function!
-        with open(binary, 'wb') as f:
-            morphlib.bins.create_chunk(inst, f, ['.'], ex)
-        return binary
-
-    def test_has_no_staging_area_initially(self):
-        self.assertEqual(self.factory.staging, None)
-        
-    def test_creates_staging_area(self):
-        self.factory.create_staging()
-        self.assertEqual(os.listdir(self.factory.staging), [])
-
-    def test_removes_staging_area(self):
-        self.factory.create_staging()
-        staging = self.factory.staging
-        self.factory.remove_staging()
-        self.assertEqual(self.factory.staging, None)
-        self.assertFalse(os.path.exists(staging))
-
-    def test_unpacks_binary_from_file(self):
-        binary = self.create_chunk()
-        self.factory.create_staging()
-        self.factory.unpack_binary_from_file(binary)
-        self.assertEqual(sorted(os.listdir(self.factory.staging)),
-                         sorted(['bin', 'etc', 'lib']))
-
-    def test_removes_staging_area_with_contents(self):
-        binary = self.create_chunk()
-        self.factory.create_staging()
-        self.factory.unpack_binary_from_file(binary)
-        staging = self.factory.staging
-        self.factory.remove_staging()
-        self.assertEqual(self.factory.staging, None)
-        self.assertFalse(os.path.exists(staging))
-
-    def test_unpacks_onto_system(self):
-    
-        # We can't test this by really unpacking onto the system.
-        # Instead, we rely on the fact that if the normal unpacking
-        # works, the actual worker function for unpacking works, and
-        # we can just verify that it gets called with the right
-        # parameters.
-    
-        def fake_unpack(binary, dirname):
-            self.dirname = dirname
-    
-        binary = self.create_chunk()
-        self.factory._unpack_binary = fake_unpack
-        self.factory.unpack_binary_from_file_onto_system(binary)
-        self.assertEqual(self.dirname, '/')
-
-    def test_unpacks_simple_sources(self):
-        self.factory.create_staging()
-        srcdir = self.tempdir.join('src')
-        treeish = FakeTreeish(self.tempdir, 'repo')
-        self.factory.unpack_sources(treeish, srcdir)
-        self.assertTrue(os.path.exists(os.path.join(srcdir, 'file.txt')))
-
-    def test_unpacks_submodules(self):
-        self.factory.create_staging()
-        srcdir = self.tempdir.join('src')
-        subtreeish = FakeTreeish(self.tempdir, 'subrepo')
-        supertreeish = FakeTreeish(self.tempdir, 'repo', subtreeish=subtreeish)
-        self.factory.unpack_sources(supertreeish, srcdir)
-        self.assertEqual(sorted(os.listdir(srcdir)),
-                         sorted(['.git', 'file.txt', 'subrepo']))
-        self.assertEqual(sorted(os.listdir(os.path.join(srcdir, 'subrepo'))),
-                         sorted(['.git', 'file.txt']))
-
-    def test_sets_timestamp_for_unpacked_files(self):
-        self.factory.create_staging()
-        srcdir = self.tempdir.join('src')
-        treeish = FakeTreeish(self.tempdir, 'repo')
-        self.factory.unpack_sources(treeish, srcdir)
-        
-        mtime = None
-        for dirname, subdirs, basenames in os.walk(srcdir):
-            pathnames = [os.path.join(dirname, x) for x in basenames]
-            for pathname in pathnames + [dirname]:
-                st = os.lstat(pathname)
-                if mtime is None:
-                    mtime = st.st_mtime
-                else:
-                    self.assertEqual((pathname, mtime), 
-                                     (pathname, st.st_mtime))
-
diff --git a/morphlib/localrepocache.py b/morphlib/localrepocache.py
index b8dfea88..a3e28d57 100644
--- a/morphlib/localrepocache.py
+++ b/morphlib/localrepocache.py
@@ -19,6 +19,7 @@ import os
 import urllib2
 import urlparse
 import shutil
+import string
 
 import morphlib
 
@@ -34,6 +35,19 @@ urlparse.uses_fragment.extend(gitscheme)
 
 
 
+def quote_url(url):
+    ''' Convert URIs to strings that only contain digits, letters, % and _.
+
+    NOTE: When changing the code of this function, make sure to also apply
+    the same to the quote_url() function of lorry. Otherwise the git bundles
+    generated by lorry may no longer be found by morph.
+
+    ''' 
+    valid_chars = string.digits + string.letters + '%_'
+    transl = lambda x: x if x in valid_chars else '_'
+    return ''.join([transl(x) for x in url])
+
+
 class NoRemote(morphlib.Error):
 
     def __init__(self, reponame, errors):
@@ -151,13 +165,12 @@ class LocalRepoCache(object):
     def _escape(self, url):
         '''Escape a URL so it can be used as a basename in a file.'''
         
-        # FIXME: The following is a nicer way than what source manager does.
-        # However, for compatibility, we need to use the same as the source
-        # manager uses, since that's what the bundle server (set up by
-        # Lorry) uses.
+        # FIXME: The following is a nicer way than to do this.
+        # However, for compatibility, we need to use the same as the 
+        # bundle server (set up by Lorry) uses.
         # return urllib.quote(url, safe='')
         
-        return morphlib.sourcemanager.quote_url(url)
+        return quote_url(url)
 
     def _cache_name(self, url):
         basename = self._escape(url)
diff --git a/morphlib/morphology.py b/morphlib/morphology.py
deleted file mode 100644
index aec63c90..00000000
--- a/morphlib/morphology.py
+++ /dev/null
@@ -1,131 +0,0 @@
-# Copyright (C) 2011-2012  Codethink Limited
-# 
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-# 
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-# 
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-import json
-import logging
-import os
-
-
-class Morphology(object):
-
-    '''Represent a morphology: description of how to build binaries.'''
-    
-    def __init__(self, treeish, fp):
-        self.treeish = treeish
-        self.filename = fp.name
-
-        self._fp = fp
-        self._load()
-
-    def _load(self):
-        logging.debug('Loading morphology %s from %s' % 
-                      (self._fp.name, self.treeish))
-        try:
-            self._dict = json.load(self._fp)
-        except ValueError:
-            logging.error('Failed to load morphology %s from %s' % 
-                          (self._fp.name, self.treeish))
-            raise
-
-        if self.kind == 'stratum':
-            for source in self.sources:
-                if 'repo' not in source:
-                    source[u'repo'] = source['name']
-                source[u'repo'] = unicode(source['repo'])
-
-    @property
-    def name(self):
-        return self._dict['name']
-
-    @property
-    def kind(self):
-        return self._dict['kind']
-
-    @property
-    def description(self):
-        return self._dict.get('description', '')
-
-    @property
-    def sources(self):
-        return self._dict['sources']
-
-    @property
-    def build_depends(self):
-        return self._dict.get('build-depends', None)
-
-    @property
-    def build_system(self):
-        return self._dict.get('build-system', 'manual')
-
-    @property
-    def max_jobs(self):
-        if 'max-jobs' in self._dict:
-            return int(self._dict['max-jobs'])
-        return None
-
-    @property
-    def configure_commands(self):
-        return self._dict.get('configure-commands', [])
-
-    @property
-    def build_commands(self):
-        return self._dict.get('build-commands', [])
-
-    @property
-    def test_commands(self):
-        return self._dict.get('test-commands', [])
-
-    @property
-    def install_commands(self):
-        return self._dict.get('install-commands', [])
-
-    @property
-    def chunks(self):
-        return self._dict.get('chunks', {})
-
-    @property
-    def strata(self):
-        return self._dict.get('strata', [])
-
-    @property
-    def disk_size(self):
-        size = self._dict['disk-size']
-        size = size.lower()
-        if size.endswith('g'):
-            size = int(size[:-1]) * 1024**3
-        elif size.endswith('m'): # pragma: no cover
-            size = int(size[:-1]) * 1024**2
-        elif size.endswith('k'): # pragma: no cover
-            size = int(size[:-1]) * 1024
-        else: # pragma: no cover
-            size = int(size)
-        return size
-
-    @property
-    def test_stories(self):
-        return self._dict.get('test-stories', [])
-
-    def __eq__(self, other):
-        return (self.filename == other.filename and
-                self.treeish == other.treeish)
-
-    def __hash__(self):
-        return hash((self.filename, self.treeish))
-
-    def __str__(self): # pragma: no cover
-        return '%s|%s|%s' % (self.treeish.original_repo,
-                             self.treeish.ref,
-                             os.path.basename(self.filename))
diff --git a/morphlib/morphology_tests.py b/morphlib/morphology_tests.py
deleted file mode 100644
index c314f63c..00000000
--- a/morphlib/morphology_tests.py
+++ /dev/null
@@ -1,206 +0,0 @@
-# Copyright (C) 2011-2012  Codethink Limited
-# 
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-# 
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-# 
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-import StringIO
-import unittest
-
-import morphlib
-
-
-class FakeTreeish(object):
-
-    pass
-
-
-class MockFile(StringIO.StringIO):
-
-    def __init__(self, *args, **kwargs):
-        StringIO.StringIO.__init__(self, *args, **kwargs)
-        self.name = 'mockfile'
-
-
-class MorphologyTests(unittest.TestCase):
-
-    def test_constructor_with_treeish(self):
-        faketreeish = FakeTreeish()
-        morph = morphlib.morphology.Morphology(
-                      faketreeish,
-                      MockFile('''
-                        {
-                            "name": "hello",
-                            "kind": "chunk"
-                        }'''))
-        self.assertEqual(morph.treeish, faketreeish)
-
-    def test_fails_invalid_chunk_morphology(self):
-        def failtest():
-            morphlib.morphology.Morphology(
-                      FakeTreeish(),
-                      MockFile('''
-                        {
-                            "name": "hello",
-                        }'''))
-        self.assertRaises(ValueError, failtest)
- 
-    def test_accepts_valid_chunk_morphology(self):
-        faketreeish = FakeTreeish()
-        morph = morphlib.morphology.Morphology(
-                          faketreeish,
-                          MockFile('''
-                            {
-                                "name": "hello",
-                                "kind": "chunk", 
-                                "description": "desc",
-                                "build-depends": [
-                                    "devel"
-                                ],
-                                "build-system": "autotools",
-                                "max-jobs": "42",
-                                "configure-commands": ["./configure"],
-                                "build-commands": ["make"],
-                                "test-commands": ["make check"],
-                                "install-commands": ["make install"],
-                                "chunks": {
-                                    "hello": [
-                                        "usr/bin/hello",
-                                        "usr/lib/libhello.so*"
-                                    ],
-                                    "hello-dev": [
-                                        "usr/include/*",
-                                        "usr/lib/*"
-                                    ]
-                                }
-                            }'''))
-
-        self.assertEqual(morph.treeish, faketreeish)
-        self.assertEqual(morph.filename, 'mockfile')
-        self.assertEqual(morph.name, 'hello')
-        self.assertEqual(morph.kind, 'chunk')
-        self.assertEqual(morph.description, 'desc')
-        self.assertEqual(morph.filename, 'mockfile')
-        self.assertEqual(morph.build_depends, ['devel'])
-        self.assertEqual(morph.build_system, 'autotools')
-        self.assertEqual(morph.max_jobs, 42)
-        self.assertEqual(morph.configure_commands, ['./configure'])
-        self.assertEqual(morph.build_commands, ['make'])
-        self.assertEqual(morph.test_commands, ['make check'])
-        self.assertEqual(morph.install_commands, ['make install'])
-        self.assertEqual(morph.chunks,
-                         {
-                            u'hello': [u'usr/bin/hello', 
-                                       u'usr/lib/libhello.so*'],
-                            u'hello-dev': [u'usr/include/*', u'usr/lib/*'],
-                         })
-
-    def test_build_system_defaults_to_manual(self):
-        morph = morphlib.morphology.Morphology(
-                          FakeTreeish(),
-                          MockFile('''
-                            {
-                                "name": "hello",
-                                "kind": "chunk"
-                            }'''))
-        self.assertEqual(morph.build_system, 'manual')
-
-    def test_max_jobs_defaults_to_None(self):
-        morph = morphlib.morphology.Morphology(
-                          FakeTreeish(),
-                          MockFile('''
-                            {
-                                "name": "hello",
-                                "kind": "chunk"
-                            }'''))
-        self.assertEqual(morph.max_jobs, None)
-
-    def test_accepts_valid_stratum_morphology(self):
-        morph = morphlib.morphology.Morphology(
-                          FakeTreeish(),
-                          MockFile('''
-                            {
-                                "name": "hello",
-                                "kind": "stratum", 
-                                "sources": 
-                                    [
-                                        {
-                                            "name": "foo",
-                                            "ref": "ref"
-                                        }
-                                    ]
-                            }'''))
-        self.assertEqual(morph.kind, 'stratum')
-        self.assertEqual(morph.filename, 'mockfile')
-        self.assertEqual(morph.sources,
-                         [
-                            {
-                                u'name': u'foo', 
-                                u'repo': u'foo',
-                                u'ref': u'ref',
-                            },
-                         ])
-
-    def test_accepts_valid_system_morphology(self):
-        morph = morphlib.morphology.Morphology(
-                          FakeTreeish(),
-                          MockFile('''
-                            {
-                                "name": "hello",
-                                "kind": "system", 
-                                "disk-size": "1G",
-                                "strata": [
-                                    "foo",
-                                    "bar"
-                                ],
-                                "test-stories": [
-                                    "test-1",
-                                    "test-2"
-                                ]
-                            }'''))
-        self.assertEqual(morph.kind, 'system')
-        self.assertEqual(morph.disk_size, 1024**3)
-        self.assertEqual(morph.strata, ['foo', 'bar'])
-        self.assertEqual(morph.test_stories, ['test-1', 'test-2'])
-
-    def test_hashing_and_equality_checks(self):
-        mockfile1 = MockFile('''
-                        {
-                            "name": "foo",
-                            "kind": "chunk"
-                        }''')
-        mockfile1.name = 'mockfile1'
-        mockfile2 = MockFile('''
-                        {
-                            "name": "foo",
-                            "kind": "chunk"
-                        }''')
-        mockfile2.name = 'mockfile1'
-        mockfile3 = MockFile('''
-                        {
-                            "name": "bar",
-                            "kind": "chunk"
-                        }''')
-        mockfile3.name = 'mockfile2'
-
-        treeish = FakeTreeish()
-
-        morph1 = morphlib.morphology.Morphology(treeish, mockfile1)
-        morph2 = morphlib.morphology.Morphology(treeish, mockfile2)
-        morph3 = morphlib.morphology.Morphology(treeish, mockfile3)
-
-        self.assertEqual(hash(morph1), hash(morph2))
-        self.assertEqual(morph1, morph2)
-
-        self.assertNotEqual(hash(morph1), hash(morph3))
-        self.assertNotEqual(morph1, morph3)
diff --git a/morphlib/morphologyloader.py b/morphlib/morphologyloader.py
deleted file mode 100644
index b0da97dc..00000000
--- a/morphlib/morphologyloader.py
+++ /dev/null
@@ -1,72 +0,0 @@
-# Copyright (C) 2012  Codethink Limited
-# 
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-# 
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-# 
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-import StringIO
-
-import morphlib
-
-
-class MorphologyLoader(object):
-
-    '''Load morphologies from git and parse them into Morphology objects.'''
-
-    def __init__(self, settings):
-        self.settings = settings
-        self.morphologies = {}
-
-    def load(self, treeish, filename, chunk_name=None):
-        key = (treeish, filename)
-        if key in self.morphologies:
-            return self.morphologies[key]
-        else:
-            try:
-                morph = self._get_morph_from_git(treeish, filename)
-            except morphlib.execute.CommandFailure: # pragma: no cover
-                morph = None
-            if morph is None and chunk_name is not None: # pragma: no cover
-                morph = self._autodetect_morph(treeish, filename, chunk_name)
-            if morph is None: # pragma: no cover
-                raise Exception("Can't find morphology %s" % filename)
-            self.morphologies[key] = morph
-            return morph
-
-    def _autodetect_morph(self, treeish, filename, chunk): # pragma: no cover
-        def exists(x):
-            try:
-                self._get_morph_text(treeish, x)
-            except morphlib.execute.CommandFailure:
-                return False
-            else:
-                return True
-        bs = morphlib.buildsystem.detect_build_system(exists)
-        if bs is not None:
-            morph_text = bs.get_morphology_text(chunk)
-            return self._get_morph(treeish, filename, morph_text)
-        else:
-            return None
-
-    def _get_morph_text(self, treeish, filename): # pragma: no cover
-        return morphlib.git.get_morph_text(treeish, filename)
-
-    def _get_morph_from_git(self, treeish, filename):
-        morph_text = self._get_morph_text(treeish, filename)
-        return self._get_morph(treeish, filename, morph_text)
-
-    def _get_morph(self, treeish, filename, morph_text):
-        fp = StringIO.StringIO(morph_text)
-        fp.name = filename
-        return morphlib.morphology.Morphology(treeish, fp)
-
diff --git a/morphlib/morphologyloader_tests.py b/morphlib/morphologyloader_tests.py
deleted file mode 100644
index 33528086..00000000
--- a/morphlib/morphologyloader_tests.py
+++ /dev/null
@@ -1,50 +0,0 @@
-# Copyright (C) 2012  Codethink Limited
-# 
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-# 
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-# 
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-import unittest
-
-import morphlib
-
-
-class MorphologyLoaderTests(unittest.TestCase):
-
-    def test_load_twice_verify_same_morph_object(self):
-        settings = { 'git-base-url': '' }
-        loader = morphlib.morphologyloader.MorphologyLoader(settings)
-        loader._get_morph_text = self.get_morph_text
-        
-        class FakeTreeish(object):
-            def __init__(self):
-                self.original_repo = 'test-repo'
-        faketreeish = FakeTreeish()
-        
-        morph1 = loader.load(faketreeish, 'foo.morph')
-        morph2 = loader.load(faketreeish, 'foo.morph')
-        self.assertEqual(morph1, morph2)
-
-    def get_morph_text(self, treeish, filename):
-        return '''
-                {
-                    "name": "foo",
-                    "kind": "stratum",
-                    "sources": [
-                        {
-                            "name": "bar",
-                            "repo": "bar",
-                            "ref": "master"
-                        }
-                    ]
-                }'''
diff --git a/morphlib/sourcemanager.py b/morphlib/sourcemanager.py
deleted file mode 100644
index a1a350ca..00000000
--- a/morphlib/sourcemanager.py
+++ /dev/null
@@ -1,286 +0,0 @@
-# Copyright (C) 2012  Codethink Limited
-# 
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-# 
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-# 
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-import os
-import urlparse
-import urllib2
-import shutil
-import string
-
-import morphlib
-
-
-gitscheme=["git",]
-urlparse.uses_relative.extend(gitscheme)
-urlparse.uses_netloc.extend(gitscheme)
-urlparse.uses_params.extend(gitscheme)
-urlparse.uses_query.extend(gitscheme)
-urlparse.uses_fragment.extend(gitscheme)
-
-
-def quote_url(url):
-    ''' Convert URIs to strings that only contain digits, letters, % and _.
-
-    NOTE: When changing the code of this function, make sure to also apply
-    the same to the quote_url() function of lorry. Otherwise the git bundles
-    generated by lorry may no longer be found by morph.
-
-    ''' 
-    valid_chars = string.digits + string.letters + '%_'
-    transl = lambda x: x if x in valid_chars else '_'
-    return ''.join([transl(x) for x in url])
-
-
-class RepositoryUpdateError(Exception): # pragma: no cover
-
-    def __init__(self, repo, ref, error):
-        Exception.__init__(self, 'Failed to update %s:%s: %s' %
-                           (repo, ref, error))
-
-
-class RepositoryFetchError(Exception):
-
-    def __init__(self, repo):
-        Exception.__init__(self, 'Failed to fetch %s' % repo)
-
-
-class SourceManager(object):
-
-    def __init__(self, app, cachedir=None, update=True):
-        self.real_msg = app.msg
-        self.settings = app.settings
-        self.cached_treeishes = {}
-        self.cache_dir = cachedir 
-        self.update = update
-        if not self.cache_dir:
-            self.cache_dir = os.path.join(app.settings['cachedir'], 'gits')
-        self.indent = 0
-
-    def indent_more(self):
-        self.indent += 1
-    
-    def indent_less(self):
-        self.indent -= 1
-
-    def msg(self, text):
-        spaces = '  ' * self.indent
-        self.real_msg('%s%s' % (spaces, text))
-
-    def _wget(self, url, filename): # pragma: no cover
-        # the following doesn't work during bootstrapping
-        # ex = morphlib.execute.Execute(self.cache_dir, msg=self.msg)
-        # ex.runv(['wget', '-c', url])
-        # so we do it poorly in pure Python instead
-        source_handle = urllib2.urlopen(url)
-        target_handle = open(filename, 'wb')
-
-        data = source_handle.read(4096)
-        while data:
-            target_handle.write(data)
-            data = source_handle.read(4096)
-
-        source_handle.close()
-        target_handle.close()
-
-        return filename
-
-    def _cache_repo_from_bundle(self, server, repo_url):
-        quoted_url = quote_url(repo_url)
-        cached_repo = os.path.join(self.cache_dir, quoted_url)
-        bundle_name = '%s.bndl' % quoted_url
-        bundle_url = server + bundle_name
-        bundle = os.path.join(self.cache_dir, bundle_name)
-        self.msg('Trying to fetch bundle %s' % bundle_url)
-        request = urllib2.Request(bundle_url)
-        try:
-            urllib2.urlopen(request)
-            try:
-                self._wget(bundle_url, bundle)
-                self.msg('Extracting bundle %s into %s' %
-                         (bundle, cached_repo))
-                try:
-                    os.mkdir(cached_repo)
-                    morphlib.git.extract_bundle(cached_repo, bundle,
-                                                self.msg)
-                    self.msg('Setting origin to %s' % repo_url)
-                    morphlib.git.set_remote(cached_repo, 'origin',
-                                            repo_url, self.msg)
-                    return cached_repo, None
-                except morphlib.execute.CommandFailure, e: # pragma: no cover
-                    if os.path.exists(cached_repo):
-                        shutil.rmtree(cached_repo)
-                    self.msg('Unable to extract bundle %s' % bundle)
-                    return None, 'Unable to extract bundle %s: %s' % (bundle,
-                                                                      e)
-                finally:
-                    if os.path.exists(bundle):
-                        os.remove(bundle)
-            except morphlib.execute.CommandFailure, e: # pragma: no cover
-                return None, 'Unable to fetch bundle %s: %s' % (bundle, e)
-        except urllib2.URLError, e:
-            return None, 'Unable to fetch bundle %s: %s' % (bundle_url, e)
-
-    def _cache_repo_from_url(self, repo_url):
-        # quote the URL and calculate the location for the cached repo
-        quoted_url = quote_url(repo_url)
-        cached_repo = os.path.join(self.cache_dir, quoted_url)
-
-        if os.path.exists(cached_repo): # pragma: no cover
-            # the cache location exists, assume this is what we want
-            self.msg('Using cached clone %s of %s' % (cached_repo, repo_url))
-            return cached_repo, None
-        else:
-            # bundle server did not have a bundle for the repo
-            self.msg('Trying to clone %s into %s' % (repo_url, cached_repo))
-            try:
-                morphlib.git.clone(cached_repo, repo_url, self.msg)
-                return cached_repo, None
-            except morphlib.execute.CommandFailure, e:
-                if os.path.exists(cached_repo): # pragma: no cover
-                    shutil.rmtree(cached_repo)
-                return None, 'Unable to clone from %s: %s' % (repo_url, e)
-
-    def _cache_repo_from_base_urls(self, repo, ref):
-        self.msg('Checking repository %s' % repo)
-        self.indent_more()
-
-        def fixup_url(url):
-            return (url if url.endswith('/') else url + '/')
-
-        # create absolute repo URLs
-        repo_urls = [urlparse.urljoin(fixup_url(x), repo)
-                     for x in self.settings['git-base-url']]
-
-        orig_url = None
-        cached_repo = None
-        errors = []
-
-        # check if we have a cached version of the repo
-        for repo_url in repo_urls:
-            quoted_url = quote_url(repo_url)
-            cached_repo_dirname = os.path.join(self.cache_dir, quoted_url)
-            if os.path.exists(cached_repo_dirname):
-                orig_url = repo_url
-                cached_repo = cached_repo_dirname
-                break
-
-        # first pass, try all base URLs with the bundle server
-        if not cached_repo and self.settings['bundle-server']:
-            server = fixup_url(self.settings['bundle-server'])
-
-            for repo_url in repo_urls:
-                cached_repo, error = self._cache_repo_from_bundle(server,
-                                                                  repo_url)
-                if cached_repo:
-                    orig_url = repo_url
-                    break
-                else:
-                    errors.append(error)
-
-        # second pass, try cloning from base URLs directly
-        if not cached_repo:
-            # try all URLs to find or obtain a cached clone of the repo
-            for repo_url in repo_urls:
-                cached_repo, error = self._cache_repo_from_url(repo_url)
-                if cached_repo:
-                    orig_url = repo_url
-                    break
-                else:
-                    errors.append(error)
-
-        if cached_repo:
-            # we have a cached version of the repo now
-            if self.update:
-                # we are supposed to update 'origin', so do that now
-                try:
-                    self.msg('Updating %s' % cached_repo)
-                    morphlib.git.update_remote(cached_repo, 'origin',
-                                               self.msg)
-                except morphlib.execute.CommandFailure, e: # pragma: no cover
-                    self.msg('Failed to update origin: %s' % e)
-                    self.indent_less()
-                    # ignore remote update failures during bootstrap
-                    if not self.settings['bootstrap']:
-                        raise RepositoryUpdateError(repo, ref, e)
-            else: # pragma: no cover
-                self.msg('Assuming cached repository %s is up to date' %
-                         cached_repo)
-        else: # pragma: no cover
-            # cloning using all individual base URLs failed
-
-            # print all the errors at once to give the user an overview
-            # over what went wrong in which order
-            for error in errors:
-                self.msg(error)
-
-            self.indent_less()
-            raise RepositoryFetchError(repo)
-
-        # we should have a cached version of the repo now, return a treeish
-        # for the repo and ref tuple
-        treeish = morphlib.git.Treeish(cached_repo, orig_url, ref, self.msg)
-        self.indent_less()
-        return treeish
-
-    def _resolve_submodules(self, treeish): # pragma: no cover
-        self.indent_more()
-
-        # resolve submodules
-        treeish.submodules = morphlib.git.Submodules(treeish.repo,
-                                                     treeish.ref, 
-                                                     self.msg)
-        try:
-            # load submodules from .gitmodules
-            treeish.submodules.load()
-
-            # resolve the tree-ishes for all submodules recursively
-            for submodule in treeish.submodules: # pragma: no cover
-                submodule.treeish = self.get_treeish(submodule.url,
-                                                     submodule.commit)
-        except morphlib.git.NoModulesFileError:
-            # this is not really an error, the repository simply
-            # does not specify any git submodules
-            pass
-
-        self.indent_less()
-
-    def get_treeish(self, repo, ref):
-        '''Returns a Treeish for a URL or repo name with a given reference.
-
-        If the source hasn't been cloned yet, this will fetch it, either using
-        clone or by fetching a bundle. 
-        
-        Raises morphlib.git.InvalidReferenceError if the reference cannot be
-        found. Raises morphlib.sourcemanager.RepositoryUpdateError if the
-        repository cannot be cloned or updated.
-
-        '''
-
-        if (repo, ref) not in self.cached_treeishes: # pragma: no cover
-            # load the corresponding treeish on demand
-            treeish = self._cache_repo_from_base_urls(repo, ref)
-
-            # have a treeish now, cache it to avoid loading it twice
-            self.cached_treeishes[(repo, ref)] = treeish
-
-            # load tree-ishes for submodules, if necessary and desired
-            if self.settings['ignore-submodules']:
-                treeish.submodules = []
-            else:
-                self._resolve_submodules(treeish)
-
-        # we should now have a cached treeish to use now
-        return self.cached_treeishes[(repo, ref)] # pragma: no cover
diff --git a/morphlib/sourcemanager_tests.py b/morphlib/sourcemanager_tests.py
deleted file mode 100644
index eb18dcc3..00000000
--- a/morphlib/sourcemanager_tests.py
+++ /dev/null
@@ -1,193 +0,0 @@
-# Copyright (C) 2012  Codethink Limited
-# 
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-# 
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-# 
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-import glob
-import unittest
-import tempfile
-import shutil
-import os
-import subprocess
-import urlparse
-
-import morphlib
-
-
-class DummyApp(object):
-
-    def __init__(self):
-        self.settings = {
-            'git-base-url': ['.',],
-            'bundle-server': None,
-            'cachedir': '/foo/bar/baz',
-            'ignore-submodules': False
-        }
-        self.msg = lambda msg: None
-
-
-class SourceManagerTests(unittest.TestCase):
-
-    def setUp(self):
-        self.temprepodir = tempfile.mkdtemp()
-        env = os.environ
-        env["DATADIR"]=self.temprepodir
-        subprocess.call("./tests/show-dependencies.setup", shell=True, env=env)
-        self.temprepo = self.temprepodir + '/test-repo/'
-        bundle_name = morphlib.sourcemanager.quote_url(self.temprepo) + '.bndl'
-        with open('/dev/null', 'w') as f:
-            subprocess.check_call("git bundle create %s/%s master" %
-                                    (self.temprepodir, bundle_name),
-                                  stderr=f, shell=True, cwd=self.temprepo)
-
-    def tearDown(self):
-        shutil.rmtree(self.temprepodir)
-
-    def test_uses_provided_cache_dir(self):
-        app = DummyApp()
-        
-        tempdir = '/bla/bla/bla'
-        s = morphlib.sourcemanager.SourceManager(app, tempdir)
-        self.assertEqual(s.cache_dir, tempdir)
-
-    def test_uses_cachedir_gits_if_no_cache_dir_provided(self):
-        app = DummyApp()
-
-        s = morphlib.sourcemanager.SourceManager(app)
-        self.assertEqual(s.cache_dir,
-                         os.path.join(app.settings['cachedir'], 'gits'))
-
-    def test_resolves_sha1_treeish_for_test_repo_correctly(self):
-        tempdir = tempfile.mkdtemp()
-
-        s = morphlib.sourcemanager.SourceManager(DummyApp(), tempdir)
-        t = s.get_treeish(self.temprepo,
-                          'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9')
-        self.assertEquals(t.sha1, 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9')
-
-        shutil.rmtree(tempdir)
-
-    def test_resolves_sha1_treeish_for_test_repo_correctly_twice(self):
-        tempdir = tempfile.mkdtemp()
-
-        # try two times with different source manager objects
-        s = morphlib.sourcemanager.SourceManager(DummyApp(), tempdir)
-        t = s.get_treeish(self.temprepo,
-                          'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9')
-        self.assertEquals(t.sha1, 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9')
-        s = morphlib.sourcemanager.SourceManager(DummyApp(), tempdir)
-        t = s.get_treeish(self.temprepo,
-                          'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9')
-        self.assertEquals(t.sha1, 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9')
-
-        # try two times with the same source manager object
-        s = morphlib.sourcemanager.SourceManager(DummyApp(), tempdir)
-        t = s.get_treeish(self.temprepo,
-                          'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9')
-        self.assertEquals(t.sha1, 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9')
-        t = s.get_treeish(self.temprepo,
-                          'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9')
-        self.assertEquals(t.sha1, 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9')
-
-        shutil.rmtree(tempdir)
-
-    def test_resolves_ref_treeish_for_test_repo_correctly(self):
-        tempdir = tempfile.mkdtemp()
-
-        s = morphlib.sourcemanager.SourceManager(DummyApp(), tempdir)
-        t = s.get_treeish(self.temprepo, 'master')
-        self.assertEquals(t.ref, 'refs/remotes/origin/master')
-
-        shutil.rmtree(tempdir)
-
-    def test_resolves_ref_treeish_for_test_repo_without_submodules(self):
-        tempdir = tempfile.mkdtemp()
-
-        s = morphlib.sourcemanager.SourceManager(DummyApp(), tempdir)
-        t = s.get_treeish(self.temprepo, 'master')
-        self.assertEquals(len(t.submodules), 0)
-
-        shutil.rmtree(tempdir)
-
-    def test_resolves_sha1_treeish_for_test_repo_correctly_from_bundle(self):
-        tempdir = tempfile.mkdtemp()
-        bundle_server_loc = self.temprepodir
-
-        app = DummyApp()
-        app.settings['bundle-server'] = 'file://' + bundle_server_loc
-
-        s = morphlib.sourcemanager.SourceManager(app, tempdir)
-
-        def wget(url, filename):
-            bundle_file = os.path.join(self.temprepodir,
-                                       os.path.basename(filename))
-            shutil.copy(bundle_file, s.cache_dir)
-
-        s._wget = wget
-
-        t = s.get_treeish(self.temprepo,
-                          'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9')
-        self.assertEquals(t.sha1, 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9')
-
-        shutil.rmtree(tempdir)
-
-    def test_fails_to_resolves_sha1_treeish_for_non_existent_repo(self):
-        tempdir = tempfile.mkdtemp()
-        app = DummyApp()
-
-        s = morphlib.sourcemanager.SourceManager(app, tempdir)
-
-        def wget(url, filename):
-            bundle_file = os.path.join(self.temprepodir,
-                                       os.path.basename(filename))
-            shutil.copy(bundle_file, s.cache_dir)
-
-        s._wget = wget
-        self.assertRaises(morphlib.sourcemanager.RepositoryFetchError,
-                          s.get_treeish, 'asdf',
-                          'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9')
-
-        shutil.rmtree(tempdir)
-
-    def test_fails_to_resolves_sha1_treeish_for_non_existent_repo_bundle(self):
-        tempdir = tempfile.mkdtemp()
-        app = DummyApp()
-        app.settings['bundle-server'] = 'file://' + self.temprepodir
-
-        s = morphlib.sourcemanager.SourceManager(app, tempdir)
-
-        def wget(url, filename):
-            bundle_file = os.path.join(self.temprepodir,
-                                       os.path.basename(filename))
-            shutil.copy(bundle_file, s.cache_dir)
-
-        s._wget = wget
-        self.assertRaises(morphlib.sourcemanager.RepositoryFetchError,
-                          s.get_treeish, 'asdf',
-                          'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9')
-
-        shutil.rmtree(tempdir)
-
-    def test_get_sha1_treeish_for_self_multiple_base(self):
-
-        tempdir = tempfile.mkdtemp()
-        app = DummyApp()
-        app.settings['git-base-url'] = ['.', '/somewhere/else']
-
-        s = morphlib.sourcemanager.SourceManager(app, tempdir)
-        t = s.get_treeish(self.temprepo,
-                          'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9')
-        self.assertEquals(t.sha1, 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9')
-
-        shutil.rmtree(tempdir)
diff --git a/tests/build-chunk-distributed-local.script b/tests/build-chunk-distributed-local.script
deleted file mode 100755
index f38a0673..00000000
--- a/tests/build-chunk-distributed-local.script
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/sh
-#
-# Test building a chunk using as many local workers as make sense.
-#
-# Copyright (C) 2012  Codethink Limited
-# 
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-# 
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-# 
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-set -eu
-
-"$SRCDIR/scripts/test-morph" build-distributed chunk-repo farrokh hello.morph
-
-for chunk in "$DATADIR/cache/"*.chunk.*
-do
-    echo "$chunk:" | sed 's/[^.]*//'
-    tar -tf "$chunk" | LC_ALL=C sort | sed '/^\.\/./s:^\./::'
-    echo
-done
diff --git a/tests/build-chunk-distributed-local.stdout b/tests/build-chunk-distributed-local.stdout
deleted file mode 100644
index 8077cac2..00000000
--- a/tests/build-chunk-distributed-local.stdout
+++ /dev/null
@@ -1,8 +0,0 @@
-.chunk.hello:
-./
-baserock/
-baserock/hello.meta
-bin/
-bin/hello
-etc/
-
diff --git a/tests/build-stratum-distributed-local.script b/tests/build-stratum-distributed-local.script
deleted file mode 100755
index d69946eb..00000000
--- a/tests/build-stratum-distributed-local.script
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/bin/sh
-#
-# Test build a simple stratum using as many local workers as makes sense.
-#
-# Copyright (C) 2012  Codethink Limited
-# 
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-# 
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-# 
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-set -eu
-
-"$SRCDIR/scripts/test-morph" \
-    build-distributed morphs-repo master hello-stratum.morph
-tar -tf "$DATADIR/cache/"*.stratum.* | LC_ALL=C sort | sed '/^\.\/./s:^\./::'
diff --git a/tests/build-stratum-distributed-local.stdout b/tests/build-stratum-distributed-local.stdout
deleted file mode 100644
index 4f422ea4..00000000
--- a/tests/build-stratum-distributed-local.stdout
+++ /dev/null
@@ -1,7 +0,0 @@
-./
-baserock/
-baserock/hello-stratum.meta
-baserock/hello.meta
-bin/
-bin/hello
-etc/