WIPbaserock/tiagogomes/submodules

Change-Id: I379a85dc54d48bca0f41b18d6e56eb8f2b1be68b

17 files changed, 336 insertions, 61 deletions

diff --git a/check b/check
index 1d56c514..122ad0e2 100755
--- a/check
+++ b/check
@@ -145,9 +145,9 @@ fi
 # Clean up artifacts from previous (possibly failed) runs, build,
 # and run the tests.
 
-if "$run_unit_tests"; then
-    python setup.py clean check
-fi
+#if "$run_unit_tests"; then
+#    python setup.py clean check
+#fi
 
 # Run scenario tests with yarn, if yarn is available.
 #
@@ -166,7 +166,7 @@ if "$run_yarns" && command -v yarn > /dev/null
 then
     yarn --env "PYTHONPATH=$PYTHONPATH" --env "TMPDIR=$TMPDIR" $snapshot \
     --tempdir "$TMPDIR" -s yarns/morph.shell-lib \
-        yarns/*.yarn
+        yarns/*.yarn --run 'test recursive sources2' --verbose --snapshot
 fi
 
 # cmdtest tests.
diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py
index f3936df1..2b40752d 100644
--- a/morphlib/artifactresolver.py
+++ b/morphlib/artifactresolver.py
@@ -189,6 +189,7 @@ class ArtifactResolver(object):
             # Resolve now to avoid a search for the parent morphology later
             chunk_source.build_mode = info['build-mode']
             chunk_source.prefix = info['prefix']
+            chunk_source.extra_sources = info['extra-sources']
 
             # Add these chunks to the processed artifacts, so other
             # chunks may refer to them.
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py
index 222d229a..49a23b45 100644
--- a/morphlib/buildcommand.py
+++ b/morphlib/buildcommand.py
@@ -59,10 +59,10 @@ class BuildCommand(object):
             repo_name=repo_name, ref=ref, filename=filename)
 
         self.app.status(msg='Traversing morphologies for %s' % filename)
-        srcpool = self.create_source_pool(
+        self.source_pool = self.create_source_pool(
             repo_name, ref, [filename], original_ref)
-        self.validate_sources(srcpool)
-        root_artifact = self.resolve_artifacts(srcpool)
+        self.validate_sources(self.source_pool)
+        root_artifact = self.resolve_artifacts(self.source_pool)
         self.build_in_order(root_artifact)
 
         self.app.status(
@@ -394,9 +394,25 @@ class BuildCommand(object):
     def fetch_sources(self, source):
         '''Update the local git repository cache with the sources.'''
 
+        def fetch_extra_sources(lrc, parent_repo, parent_ref, extra_sources):
+            for extra_source in extra_sources:
+                ref = extra_source.get('ref')
+                if not ref:
+                    ref = parent_repo.get_submodule_commit(
+                            parent_ref, extra_source['path'])
+                repo = self.lrc.get_updated_repo(extra_source['repo'],
+                                                 ref=ref)
+                fetch_extra_sources(lrc, repo, ref,
+                                    extra_source.get('extra-sources', []))
+
         repo_name = source.repo_name
         source.repo = self.lrc.get_updated_repo(repo_name, ref=source.sha1)
-        self.lrc.ensure_submodules(source.repo, source.sha1)
+        if source.morphology['kind'] == 'chunk':
+            if self.source_pool.definitions_version >= 8:
+                fetch_extra_sources(self.lrc, source.repo, source.sha1,
+                                    source.extra_sources)
+            else:
+                self.lrc.ensure_submodules(source.repo, source.sha1)
 
     def cache_artifacts_locally(self, artifacts):
         '''Get artifacts missing from local cache from remote cache.'''
@@ -542,7 +558,8 @@ class BuildCommand(object):
                         name=source.name, sha1=source.sha1[:7])
         builder = morphlib.builder.Builder(
             self.app, staging_area, self.lac, self.rac, self.lrc,
-            self.app.settings['max-jobs'], setup_mounts)
+            self.app.settings['max-jobs'], setup_mounts,
+            self.source_pool.definitions_version)
         return builder.build_and_cache(source)
 
 class InitiatorBuildCommand(BuildCommand):
diff --git a/morphlib/builder.py b/morphlib/builder.py
index cfd5510c..accf435e 100644
--- a/morphlib/builder.py
+++ b/morphlib/builder.py
@@ -35,7 +35,8 @@ import morphlib.gitversion
 
 SYSTEM_INTEGRATION_PATH = os.path.join('baserock', 'system-integration')
 
-def extract_sources(app, repo_cache, repo, sha1, srcdir): #pragma: no cover
+def extract_sources(app, definitions_version, repo_cache, repo, sha1,
+                    destdir, source): #pragma: no cover
     '''Get sources from git to a source directory, including submodules'''
 
     def extract_repo(repo, sha1, destdir):
@@ -57,12 +58,48 @@ def extract_sources(app, repo_cache, repo, sha1, srcdir): #pragma: no cover
                 sub_dir = os.path.join(destdir, sub.path)
                 tuples.append((cached_repo, sub.commit, sub_dir))
             return tuples
+        return []
 
-    todo = [(repo, sha1, srcdir)]
-    while todo:
-        repo, sha1, srcdir = todo.pop()
-        todo += extract_repo(repo, sha1, srcdir)
-    set_mtime_recursively(srcdir)
+    def extract_repo_version_8(repo, ref, extra_sources,
+                               rootdir, destdir):
+
+        app.status(msg='Extracting %(source)s into %(path)s',
+                   source=repo.original_name,
+                   path=destdir)
+        repo.checkout(ref, destdir)
+        morphlib.git.reset_workdir(app.runcmd, destdir)
+
+        for extra_source in extra_sources:
+            subrepo = repo_cache.get_repo(extra_source['repo'])
+            path = os.path.normpath(extra_source['path'])
+            checkout_dir = os.path.join(destdir, path)
+            if os.path.exists(checkout_dir):
+                if os.listdir(checkout_dir):
+                    raise cliapp.AppException(
+                            "Failed to clone '%s': the directory '%s' "
+                            "is not empty" %
+                            (subrepo.original_name, os.path.join(
+                                    rootdir, path)))
+            else:
+                os.makedirs(checkout_dir)
+            subref = extra_source.get('ref')
+            if not subref:
+                subref = repo.get_submodule_commit(ref, path)
+
+            extract_repo_version_8(subrepo, subref,
+                                   extra_source.get('extra-sources', []),
+                                   rootdir,
+                                   os.path.join(destdir, checkout_dir))
+    if definitions_version >= 8:
+        extract_repo_version_8(repo, sha1, source.extra_sources,
+                               os.path.basename(destdir), destdir)
+    else:
+        todo = [(repo, sha1, destdir)]
+        while todo:
+            repo, sha1, destdir = todo.pop()
+            todo += extract_repo(repo, sha1, destdir)
+    print "DESTDIR=", destdir
+    set_mtime_recursively(destdir)
 
 def set_mtime_recursively(root):  # pragma: no cover
     '''Set the mtime for every file in a directory tree to the same.
@@ -147,7 +184,7 @@ class BuilderBase(object):
 
     def __init__(self, app, staging_area, local_artifact_cache,
                  remote_artifact_cache, source, repo_cache, max_jobs,
-                 setup_mounts):
+                 setup_mounts, definitions_version):
         self.app = app
         self.staging_area = staging_area
         self.local_artifact_cache = local_artifact_cache
@@ -157,6 +194,7 @@ class BuilderBase(object):
         self.max_jobs = max_jobs
         self.build_watch = morphlib.stopwatch.Stopwatch()
         self.setup_mounts = setup_mounts
+        self.definitions_version = definitions_version
 
     def save_build_times(self):
         '''Write the times captured by the stopwatch'''
@@ -312,7 +350,6 @@ class ChunkBuilder(BuilderBase):
     def run_commands(self, logfilepath, stdout=None):  # pragma: no cover
         m = self.source.morphology
         bs = morphlib.buildsystem.lookup_build_system(m['build-system'])
-
         relative_builddir = self.staging_area.relative_builddir()
         relative_destdir = self.staging_area.relative_destdir()
         ccache_dir = self.staging_area.ccache_dir()
@@ -375,7 +412,6 @@ class ChunkBuilder(BuilderBase):
                                                 stderr=subprocess.STDOUT,
                                                 logfile=logfilepath,
                                                 ccache_dir=ccache_dir)
-
                     if stdout:
                         stdout.flush()
 
@@ -491,7 +527,8 @@ class ChunkBuilder(BuilderBase):
 
     def get_sources(self, srcdir):  # pragma: no cover
         s = self.source
-        extract_sources(self.app, self.repo_cache, s.repo, s.sha1, srcdir)
+        extract_sources(self.app, self.definitions_version, self.repo_cache,
+                        s.repo, s.sha1, srcdir, s)
 
 
 class StratumBuilder(BuilderBase):
@@ -726,7 +763,8 @@ class Builder(object):  # pragma: no cover
     }
 
     def __init__(self, app, staging_area, local_artifact_cache,
-                 remote_artifact_cache, repo_cache, max_jobs, setup_mounts):
+                 remote_artifact_cache, repo_cache, max_jobs, setup_mounts,
+                 definitions_version):
         self.app = app
         self.staging_area = staging_area
         self.local_artifact_cache = local_artifact_cache
@@ -734,6 +772,7 @@ class Builder(object):  # pragma: no cover
         self.repo_cache = repo_cache
         self.max_jobs = max_jobs
         self.setup_mounts = setup_mounts
+        self.definitions_version = definitions_version
 
     def build_and_cache(self, source):
         kind = source.morphology['kind']
@@ -741,7 +780,8 @@ class Builder(object):  # pragma: no cover
                                self.local_artifact_cache,
                                self.remote_artifact_cache, source,
                                self.repo_cache, self.max_jobs,
-                               self.setup_mounts)
+                               self.setup_mounts,
+                               self.definitions_version)
         self.app.status(msg='Builder.build: artifact %s with %s' %
                        (source.name, repr(o)),
                        chatty=True)
diff --git a/morphlib/builder_tests.py b/morphlib/builder_tests.py
index a571e3d0..e9629e1e 100644
--- a/morphlib/builder_tests.py
+++ b/morphlib/builder_tests.py
@@ -162,7 +162,8 @@ class BuilderBaseTests(unittest.TestCase):
                                                      self.artifact,
                                                      self.repo_cache,
                                                      self.max_jobs,
-                                                     False)
+                                                     False,
+                                                     7)
 
     def test_runs_desired_command(self):
         self.builder.runcmd(['foo', 'bar'])
diff --git a/morphlib/cachedrepo.py b/morphlib/cachedrepo.py
index 76cdaa86..0ca60fa2 100644
--- a/morphlib/cachedrepo.py
+++ b/morphlib/cachedrepo.py
@@ -245,6 +245,10 @@ class CachedRepo(object):
         else:
             return False
 
+    def get_submodule_commit(self, parent_ref, submodule_path): 
+        return self.gitdir.get_submodule_commit(parent_ref, # pragma: no cover
+                submodule_path)
+
     def update(self):
         '''Updates the cached repository using its origin remote.
 
diff --git a/morphlib/definitions_version.py b/morphlib/definitions_version.py
index 2fb7785a..c39fe422 100644
--- a/morphlib/definitions_version.py
+++ b/morphlib/definitions_version.py
@@ -24,7 +24,7 @@ import yaml
 import morphlib
 
 
-SUPPORTED_VERSIONS = [6, 7]
+SUPPORTED_VERSIONS = [6, 7, 8]
 
 
 class DefinitionsVersionError(cliapp.AppException):
diff --git a/morphlib/git.py b/morphlib/git.py
index b6f54d02..b43b335e 100644
--- a/morphlib/git.py
+++ b/morphlib/git.py
@@ -34,11 +34,17 @@ class NoModulesFileError(cliapp.AppException):
 
 class Submodule(object):
 
-    def __init__(self, name, url, path):
+    def __init__(self, name, url, sha1, path):
         self.name = name
         self.url = url
+        self.commit = sha1
         self.path = path
 
+    def __str__(self):
+        return "{name}|{url}|{path}".format(name=self.name,
+                                            url=self.url,
+                                            path=self.path)
+
 
 class InvalidSectionError(cliapp.AppException):
 
@@ -48,14 +54,6 @@ class InvalidSectionError(cliapp.AppException):
                            'title: [%s]' % (repo, ref, section))
 
 
-class MissingSubmoduleCommitError(cliapp.AppException):
-
-    def __init__(self, repo, ref, submodule):
-        Exception.__init__(self,
-                           '%s:%s:.gitmodules: No commit object found for '
-                           'submodule "%s"' % (repo, ref, submodule))
-
-
 class Submodules(object):
 
     def __init__(self, app, repo, ref):
@@ -86,6 +84,7 @@ class Submodules(object):
             raise NoModulesFileError(self.repo, self.ref)
 
     def _validate_and_read_entries(self, parser):
+        gd = morphlib.gitdir.GitDirectory(self.repo)
         for section in parser.sections():
             # validate section name against the 'section "foo"' pattern
             section_pattern = r'submodule "(.*)"'
@@ -96,33 +95,9 @@ class Submodules(object):
                 path = parser.get(section, 'path')
 
                 # create a submodule object
-                submodule = Submodule(name, url, path)
-                try:
-                    # list objects in the parent repo tree to find the commit
-                    # object that corresponds to the submodule
-                    commit = gitcmd(self.app.runcmd, 'ls-tree', self.ref,
-                                    submodule.path, cwd=self.repo)
-
-                    # read the commit hash from the output
-                    fields = commit.split()
-                    if len(fields) >= 2 and fields[1] == 'commit':
-                        submodule.commit = commit.split()[2]
-
-                        # fail if the commit hash is invalid
-                        if len(submodule.commit) != 40:
-                            raise MissingSubmoduleCommitError(self.repo,
-                                                              self.ref,
-                                                              submodule.name)
-
-                        # add a submodule object to the list
-                        self.submodules.append(submodule)
-                    else:
-                        logging.warning('Skipping submodule "%s" as %s:%s has '
-                                        'a non-commit object for it' %
-                                        (submodule.name, self.repo, self.ref))
-                except cliapp.AppException:
-                    raise MissingSubmoduleCommitError(self.repo, self.ref,
-                                                      submodule.name)
+                sha1 = gd.get_submodule_commit(self.ref, path)
+                submodule = Submodule(name, url, sha1, path)
+                self.submodules.append(submodule)
             else:
                 raise InvalidSectionError(self.repo, self.ref, section)
 
diff --git a/morphlib/gitdir.py b/morphlib/gitdir.py
index d1770275..8417176e 100644
--- a/morphlib/gitdir.py
+++ b/morphlib/gitdir.py
@@ -53,6 +53,14 @@ class ExpectedSha1Error(cliapp.AppException):
             self, 'SHA1 expected, got %s' % ref)
 
 
+class MissingSubmoduleCommitError(cliapp.AppException):
+
+    def __init__(self, repo, ref, submodule):
+        cliapp.AppException.__init__(self, # pragma
+                           '%s:%s:.gitmodules: No commit object found for '
+                           'submodule "%s"' % (repo, ref, submodule))
+
+
 class RefChangeError(cliapp.AppException):
     pass
 
@@ -861,6 +869,20 @@ class GitDirectory(object):
         except Exception as e:
             raise RefDeleteError(self, ref, old_sha1, e)
 
+    def get_submodule_commit(self, parent_ref, 
+                             submodule_path): # pragma: no cover
+        try:
+            lstree_output = morphlib.git.gitcmd(self._runcmd, 'ls-tree',
+                                                parent_ref, submodule_path)
+        except cliapp.AppException:
+            raise MissingSubmoduleCommitError(self.dirname, parent_ref,
+                                              submodule_path)
+        m = re.match("160000 commit (?P<sha1>\w{40})", lstree_output)
+        if not m:
+            raise MissingSubmoduleCommitError(self.dirname, parent_ref,
+                                              submodule_path)
+        return m.group('sha1')
+
     def describe(self):
         version = morphlib.git.gitcmd(self._runcmd, 'describe',
                                       '--always', '--dirty=-unreproducible')
diff --git a/morphlib/gitdir_tests.py b/morphlib/gitdir_tests.py
index 0ec7b0f1..e75ef483 100644
--- a/morphlib/gitdir_tests.py
+++ b/morphlib/gitdir_tests.py
@@ -102,6 +102,12 @@ class GitDirectoryTests(unittest.TestCase):
         gitdir = self.empty_git_directory()
         self.assertIsInstance(gitdir.get_index(), morphlib.gitindex.GitIndex)
 
+    def test_non_existent_submodule_path_raises_error(self):
+        gitdir = self.empty_git_directory()
+        self.assertRaises(
+                morphlib.gitdir.MissingSubmoduleCommitError,
+                gitdir.get_submodule_commit, 'master', 'somepath')
+
 
 class GitDirectoryAnchoredRefTests(unittest.TestCase):
 
diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py
index 7bd5c37a..79755b76 100644
--- a/morphlib/morphloader.py
+++ b/morphlib/morphloader.py
@@ -15,6 +15,7 @@
 # =*= License: GPL-2 =*=
 
 
+import os
 import collections
 import warnings
 import yaml
@@ -90,6 +91,13 @@ class InvalidTypeError(MorphologyValidationError):
             (field, expected, actual, morphology_name))
 
 
+class InvalidPathError(MorphologyValidationError):
+
+    def __init__(self, path, spec, morph_filename):
+        self.msg = ("Invalid path '%s' in %s from morphology %s"
+                    % (path, spec, morph_filename))
+
+
 class UnknownArchitectureError(MorphologyValidationError):
 
     def __init__(self, arch, morph_filename):
@@ -236,6 +244,7 @@ class MorphologyDumper(yaml.SafeDumper):
         'build-mode',
         'artifacts',
         'max-jobs',
+        'extra-sources',
         'products',
         'chunks',
         'build-system',
@@ -347,6 +356,7 @@ class MorphologyLoader(object):
             'strip-commands': None,
             'post-strip-commands': None,
             'devices': [],
+            'extra-sources': [],
             'products': [],
             'max-jobs': None,
             'build-system': 'manual',
@@ -554,6 +564,27 @@ class MorphologyLoader(object):
                 raise ChunkSpecNoBuildInstructionsError(
                     chunk_name, morph.filename)
 
+            def validate_extra_sources(extra_sources, morph_filename):
+                for extra_source in extra_sources:
+                    validate_chunk_str_field('repo', extra_source,
+                                             morph.filename)
+                    path = extra_source.get('path')
+                    if not path:
+                        raise MissingFieldError("'path' in %s" % extra_source,
+                                                morph.filename)
+                    path = os.path.normpath(path)
+                    if os.path.isabs(path) or path.startswith('..') or (
+                            path == '.') or not path.strip():
+                        raise InvalidPathError(extra_source['path'],
+                                               extra_source,
+                                               morph_filename)
+                    if 'extra-sources' in extra_source:
+                        validate_extra_sources(extra_source['extra-sources'],
+                                               morph.filename)
+
+            if 'extra-sources' in spec:
+                validate_extra_sources(spec['extra-sources'], morph.filename)
+
     @classmethod
     def _validate_chunk(cls, morphology):
         errors = []
@@ -706,6 +737,9 @@ class MorphologyLoader(object):
             if 'prefix' not in spec:
                 spec['prefix'] = \
                     self._static_defaults['chunk']['prefix']
+            if 'extra-sources' not in spec:
+                spec['extra-sources'] = \
+                    self._static_defaults['chunk']['extra-sources']
 
     def _unset_stratum_defaults(self, morph):
         for spec in morph['chunks']:
@@ -717,6 +751,9 @@ class MorphologyLoader(object):
             if 'prefix' in spec and spec['prefix'] == \
                     self._static_defaults['chunk']['prefix']:
                 del spec['prefix']
+            if 'extra-sources' in spec and spec['extra-sources'] == \
+                    self._static_defaults['chunk']['extra-sources']:
+                del spec['extra-sources']
 
     def _set_chunk_defaults(self, morph):
         if morph['max-jobs'] is not None:
diff --git a/morphlib/morphloader_tests.py b/morphlib/morphloader_tests.py
index 8039dfc9..64f9f4d4 100644
--- a/morphlib/morphloader_tests.py
+++ b/morphlib/morphloader_tests.py
@@ -278,6 +278,63 @@ build-system: manual
                 morphlib.morphloader.InvalidStringError):
             self.loader.validate(m)
 
+    def test_fails_to_validate_stratum_with_a_missing_path(self):
+        m = morphlib.morphology.Morphology({
+            'kind': 'stratum',
+            'name': 'foo',
+            'build-depends': [],
+            'chunks': [
+                {
+                    'name': 'chunk',
+                    'repo': 'test:repo',
+                    'ref': 'master',
+                    'build-system': 'manual',
+                    'build-depends': [],
+                    'extra-sources':
+                        [
+                            {
+                                'repo': 'foo',
+                                'path': 'somepath',
+                                'extra-sources':
+                                    [
+                                        {
+                                            'repo': 'bar',
+                                            'ref': 'master'
+                                        }
+                                    ]
+                            }
+                        ]
+                }
+            ]
+        })
+        self.assertRaises(
+            morphlib.morphloader.MissingFieldError, self.loader.validate, m)
+
+    def test_fails_to_validate_stratum_with_invalid_path(self):
+        m = morphlib.morphology.Morphology({
+            'kind': 'stratum',
+            'name': 'foo',
+            'build-depends': [],
+            'chunks': [
+                {
+                    'name': 'chunk',
+                    'repo': 'test:repo',
+                    'ref': 'master',
+                    'build-system': 'manual',
+                    'build-depends': [],
+                    'extra-sources':
+                        [
+                            {
+                                'repo': 'foo',
+                                'path': '../foo'
+                            }
+                        ]
+                }
+            ]
+        })
+        self.assertRaises(
+            morphlib.morphloader.InvalidPathError, self.loader.validate, m)
+
     def test_fails_to_validate_stratum_which_build_depends_on_self(self):
         text = '''\
 name: bad-stratum
@@ -572,6 +629,8 @@ build-system: manual
         })
         self.loader.set_defaults(m)
         self.loader.validate(m)
+        print m
+        self.maxDiff = 1000000000
         self.assertEqual(
             dict(m),
             {
@@ -601,6 +660,7 @@ build-system: manual
                 'pre-strip-commands': None,
                 'post-strip-commands': None,
 
+                'extra-sources': [],
                 'products': [],
                 'system-integration': [],
                 'devices': [],
@@ -654,6 +714,7 @@ build-system: manual
                         "morph": "bar",
                         'build-mode': 'bootstrap',
                         'build-depends': [],
+                        'extra-sources': [],
                         'prefix': '/usr',
                     },
                 ],
@@ -670,6 +731,7 @@ build-system: manual
                     "ref": "bar",
                     'build-mode': 'staging',
                     'build-depends': [],
+                    'extra-sources': [],
                     'prefix': '/usr',
                 },
             ],
diff --git a/morphlib/plugins/build_plugin.py b/morphlib/plugins/build_plugin.py
index 226a2c85..baa72756 100644
--- a/morphlib/plugins/build_plugin.py
+++ b/morphlib/plugins/build_plugin.py
@@ -310,6 +310,7 @@ class BuildPlugin(cliapp.Plugin):
         '''
         bc = morphlib.buildcommand.BuildCommand(self.app)
         bc.validate_sources(source_pool)
+        bc.source_pool = source_pool
         root = bc.resolve_artifacts(source_pool)
         if not component_names:
             component_names = [root.source.name]
diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py
index 108adc14..86f71c7d 100644
--- a/morphlib/sourceresolver.py
+++ b/morphlib/sourceresolver.py
@@ -406,7 +406,7 @@ class SourceResolver(object):
               predefined_split_rules)
 
     def traverse_morphs(self, definitions_repo, definitions_ref,
-                        system_filenames,
+                        system_filenames, pool,
                         visit=lambda rn, rf, fn, arf, m: None,
                         definitions_original_ref=None):
 
@@ -433,6 +433,7 @@ class SourceResolver(object):
 
             definitions_version = self._check_version_file(
                 definitions_checkout_dir)
+            pool.definitions_version = definitions_version
 
             predefined_build_systems, predefined_split_rules = \
                 self._get_defaults(
@@ -457,6 +458,7 @@ class SourceResolver(object):
                                    repo, ref, filename, buildsystem, visit,
                                    predefined_split_rules)
 
+
 class DuplicateChunkError(morphlib.Error):
 
     def _make_msg(self, (name, sources)): # pragma: no cover
@@ -517,7 +519,7 @@ def create_source_pool(lrc, rrc, repo, ref, filenames, cachedir,
     resolver = SourceResolver(lrc, rrc, tree_cache_manager, update_repos,
                               status_cb)
     resolver.traverse_morphs(repo, ref, filenames,
-                             visit=add_to_pool,
+                             pool, visit=add_to_pool,
                              definitions_original_ref=original_ref)
 
     # No two chunks may have the same name
diff --git a/scripts/test-shell.c b/scripts/test-shell.c
index 963d128c..9dadedcb 100644
--- a/scripts/test-shell.c
+++ b/scripts/test-shell.c
@@ -163,6 +163,12 @@ int run_commands(FILE *cmdstream){
                 ret = 1;
                 break;
             }
+        } else if (strstr(line, "file exists ") == line) {
+            char const *filename = line + sizeof("file exists ") -1;
+            fprintf(stderr, "FILENAME: %s\n", filename);
+            struct stat st;
+            int result = stat(filename, &st);
+            return result != 0;
         } else if (strstr(line, "create file ") == line) {
             char const *filename = line + sizeof("create file ") -1;
             FILE *outfile = fopen(filename, "w");
diff --git a/yarns/building.yarn b/yarns/building.yarn
index 8a98e5d9..32818cb1 100644
--- a/yarns/building.yarn
+++ b/yarns/building.yarn
@@ -37,6 +37,32 @@ Morph Building Tests
     THEN morph succeeded
     FINALLY the git server is shut down
 
+    SCENARIO build chunk with recursive sources
+    GIVEN a workspace
+    AND a git server
+    WHEN the user checks out the system branch called master
+    AND the user creates an uncommitted system morphology called systems/base-system.morph for our architecture in system branch master
+    AND from the directory workspace/master/test/morphs/systems the user attempts to morph build the system using the absolute path to base-system.morph
+    THEN morph succeeded
+    FINALLY the git server is shut down
+
+    SCENARIO  test recursive sources
+    GIVEN     a workspace
+    AND       a git server
+    GIVEN     a chunk with dependencies
+    WHEN      the user checks out the system branch called master
+    WHEN      the user attempts to build the system systems/test-system.morph in branch master
+    THEN      morph succeeded
+
+    SCENARIO  test recursive sources2
+    GIVEN     a workspace
+    AND       a git server
+    GIVEN     a chunk with recursive sources with non-empty paths
+    WHEN      the user checks out the system branch called master
+    WHEN      the user attempts to build the system systems/test-system.morph in branch master
+    THEN      morph failed
+
+
 System integrations
 -------------------
 
diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn
index 9f23107b..afa5df9d 100644
--- a/yarns/implementations.yarn
+++ b/yarns/implementations.yarn
@@ -78,6 +78,81 @@ locally, which we'll tell Morph to access using `file:` URLs. Specifically,
 we'll create a repository to hold system and stratum morphologies, and
 another to hold a chunk.
 
+    IMPLEMENTS GIVEN a chunk with recursive sources with non-empty paths
+    cd "$DATADIR/gits/morphs"
+    echo "version: 8" > VERSION
+    cat << EOF >> strata/core.morph
+    -   name: invalid-chunk
+        repo: test:chunk-with-submodules
+        ref: master
+        extra-sources:
+        - repo: file://$DATADIR/gits/test-chunk
+          path: somepath
+          ref: master
+        - repo: file://$DATADIR/gits/child-chunk
+          path: somepath
+    EOF
+    git add .
+    git commit -m "More stuff"
+
+
+    IMPLEMENTS GIVEN a chunk with dependencies
+    mkdir "$DATADIR/gits/grandchild-chunk"
+    cd "$DATADIR/gits/grandchild-chunk"
+    git init .
+    touch grandchild-file
+    git add .
+    git commit -m "Initial commit"
+
+    mkdir "$DATADIR/gits/child-chunk"
+    cd "$DATADIR/gits/child-chunk"
+    git init .
+    touch child-file
+    git add .
+    git commit -m "Initial commit"
+    git submodule add -b master file://$DATADIR/gits/grandchild-chunk
+    git commit -m "Initial submodule"
+    git checkout -b new-work
+    git mv child-file child-file-renamed
+    git commit -m "Moar work"
+
+    mkdir "$DATADIR/gits/chunk-with-submodules"
+    cd "$DATADIR/gits/chunk-with-submodules"
+    git init .
+    git add .
+    git commit --allow-empty -m "Initial commit"
+    git submodule add -b master file://$DATADIR/gits/child-chunk
+    #( cd child-chunk && git checkout master)
+    git add .
+    git commit -m "Add submodule"
+
+    cd "$DATADIR/gits/morphs"
+    echo "version: 8" > VERSION
+    cat << EOF >> strata/core.morph
+    -   name: chunk-with-submodules
+        morph: chunk-with-submodules.morph
+        repo: test:chunk-with-submodules
+        ref: master
+        extra-sources:
+        - repo: file://$DATADIR/gits/child-chunk
+          path: child-chunk
+          ref: new-work
+          extra-sources:
+          - repo: file://$DATADIR/gits/grandchild-chunk
+            path: grandchild-chunk
+    EOF
+
+    cat << EOF >> chunk-with-submodules.morph
+    name: chunk-with-submodules
+    kind: chunk
+    build-system: manual
+    build-commands:
+    - file exists child-chunk/child-file-renamed
+    - file exists child-chunk/grandchild-chunk/grandchild-file
+    EOF
+    git add .
+    git commit -m "Add moar stuff"
+
     IMPLEMENTS GIVEN a git server
 
     # Create a directory for all the git repositories.