Change how MorphologyLoader instances are created

Loading .morph files is becoming a bit more complicated, as we need to
deal with the VERSION file, and possibly soon with a DEFAULTS file as
well.

The logic of loading and parsing .morph files is done either in the
sourceresolver module, or the morphloader module. This change means that
all users of the latter module can use the get hold of a
MorphologyLoader instance with VERSION already parsed. If DEFAULTS is
added then it is also simple to parse DEFAULTS.

Change-Id: Ib33756e9dbd078e38f12dd7f776c89584b178959

20 files changed, 82 insertions, 41 deletions

diff --git a/morphlib/buildbranch.py b/morphlib/buildbranch.py
index e16bf2b4..6cec97fb 100644
--- a/morphlib/buildbranch.py
+++ b/morphlib/buildbranch.py
@@ -159,27 +159,31 @@ class BuildBranch(object):
             sha1 = gd.store_blob(loader.save_to_string(morphology))
             yield 0o100644, sha1, morphology.filename
 
+    def get_morphology_loader(self):
+        if self._sb:
+            return self._sb.get_morphology_loader()
+        else:
+            return self._root.get_morphology_loader()
+
     def load_all_morphologies(self, loader):
         if self._sb:
-            return self._sb.load_all_morphologies(loader)
+            return self._sb.load_all_morphologies()
         else:
-            return self._root.load_all_morphologies(loader)
+            return self._root.load_all_morphologies()
 
-    def inject_build_refs(self, loader, use_local_repos,
-                          inject_cb=lambda **kwargs: None):
+    def inject_build_refs(self, use_local_repos, inject_cb=lambda **kwargs:
+                          None):
         '''Update system and stratum morphologies to point to our branch.
 
         For all edited repositories, this alter the temporary GitIndex
         of the morphs repositories to point their temporary build branch
         versions.
 
-        `loader` is a MorphologyLoader that is used to convert morphology
-        files into their in-memory representations and back again.
-
         '''
         root_repo = self._root.remote_url
         root_ref = self._root.HEAD
         morphs = morphlib.morphset.MorphologySet()
+        loader = self.get_morphology_loader()
 
         for morph in self.load_all_morphologies(loader):
             morphs.add_morphology(morph)
@@ -362,7 +366,7 @@ class BuildBranch(object):
 
 
 @contextlib.contextmanager
-def pushed_build_branch(bb, loader, changes_need_pushing, name, email,
+def pushed_build_branch(bb, changes_need_pushing, name, email,
                         build_uuid, status):
     with contextlib.closing(bb) as bb:
         def report_add(gd, build_ref, changed):
@@ -385,8 +389,7 @@ def pushed_build_branch(bb, loader, changes_need_pushing, name, email,
             status(msg='Injecting temporary build refs '\
                            'into morphologies in %(dirname)s',
                        dirname=gd.dirname, chatty=True)
-        bb.inject_build_refs(loader=loader,
-                             use_local_repos=not changes_need_pushing,
+        bb.inject_build_refs(use_local_repos=not changes_need_pushing,
                              inject_cb=report_inject)
 
         def report_commit(gd, build_ref):
diff --git a/morphlib/definitions_repo.py b/morphlib/definitions_repo.py
index c1381af6..53328e85 100644
--- a/morphlib/definitions_repo.py
+++ b/morphlib/definitions_repo.py
@@ -124,10 +124,8 @@ class DefinitionsRepo(gitdir.GitDirectory):
             bb = morphlib.buildbranch.BuildBranch(
                 build_ref_prefix, uuid, definitions_repo=self)
 
-        loader = morphlib.morphloader.MorphologyLoader()
         pbb = morphlib.buildbranch.pushed_build_branch(
-            bb, loader=loader,
-            changes_need_pushing=push, name=git_user_name,
+            bb, changes_need_pushing=push, name=git_user_name,
             email=git_user_email, build_uuid=uuid,
             status=status_cb)
         return pbb   # (repo_url, commit, original_ref)
@@ -228,11 +226,30 @@ class DefinitionsRepo(gitdir.GitDirectory):
                     'to "origin", or use the --local-changes=include feature '
                     'of Morph.' % (e.ref, e.repo_url, ref))
 
-    def load_all_morphologies(self, loader):
+    def get_morphology_loader(self):
+        '''Return a MorphologyLoader instance.
+
+        This may read the VERSION and DEFAULTS file and pass appropriate
+        information to the MorphologyLoader constructor.
+
+        '''
+        mf = morphlib.morphologyfinder.MorphologyFinder(self)
+
+        version_text = mf.read_file('VERSION')
+        version = morphlib.definitions_version.check_version_file(version_text)
+
+        loader = morphlib.morphloader.MorphologyLoader(
+            definitions_version=version)
+
+        return loader
+
+    def load_all_morphologies(self, loader=None):
+        loader = loader or self.get_morphology_loader()
+
         mf = morphlib.morphologyfinder.MorphologyFinder(self)
         for filename in (f for f in mf.list_morphologies()
                          if not self.is_symlink(f)):
-            text = mf.read_morphology(filename)
+            text = mf.read_file(filename)
             m = loader.load_from_string(text, filename=filename)
             m.repo_url = self.remote_url
             m.ref = self.HEAD
diff --git a/morphlib/definitions_version.py b/morphlib/definitions_version.py
index 44bed178..56cef164 100644
--- a/morphlib/definitions_version.py
+++ b/morphlib/definitions_version.py
@@ -76,3 +76,5 @@ def check_version_file(version_text):  # pragma: no cover
 
     if version not in SUPPORTED_VERSIONS:
         raise UnknownVersionError(version)
+
+    return version
diff --git a/morphlib/morphologyfinder.py b/morphlib/morphologyfinder.py
index 708c86a9..335c01b6 100644
--- a/morphlib/morphologyfinder.py
+++ b/morphlib/morphologyfinder.py
@@ -33,7 +33,7 @@ class MorphologyFinder(object):
         self.gitdir = gitdir
         self.ref = ref
 
-    def read_morphology(self, filename):
+    def read_file(self, filename):
         '''Return the un-parsed text of a morphology.
         
         For the given morphology name, locate and return the contents
diff --git a/morphlib/morphologyfinder_tests.py b/morphlib/morphologyfinder_tests.py
index a83f89a5..59772e23 100644
--- a/morphlib/morphologyfinder_tests.py
+++ b/morphlib/morphologyfinder_tests.py
@@ -83,29 +83,29 @@ class MorphologyFinderTests(unittest.TestCase):
     def test_read_morph_in_HEAD(self):
         gd = morphlib.gitdir.GitDirectory(self.dirname)
         mf = morphlib.morphologyfinder.MorphologyFinder(gd, 'HEAD')
-        self.assertEqual(mf.read_morphology('bar.morph'),
+        self.assertEqual(mf.read_file('bar.morph'),
                          "dummy morphology text")
 
     def test_read_morph_in_master(self):
         gd = morphlib.gitdir.GitDirectory(self.dirname)
         mf = morphlib.morphologyfinder.MorphologyFinder(gd, 'master')
-        self.assertEqual(mf.read_morphology('bar.morph'),
+        self.assertEqual(mf.read_file('bar.morph'),
                          "dummy morphology text")
 
     def test_read_morph_raises_with_invalid_ref(self):
         gd = morphlib.gitdir.GitDirectory(self.dirname)
         mf = morphlib.morphologyfinder.MorphologyFinder(gd, 'invalid_ref')
         self.assertRaises(morphlib.gitdir.InvalidRefError,
-                          mf.read_morphology, 'bar')
+                          mf.read_file, 'bar')
 
     def test_read_morph_in_work_tree(self):
         gd = morphlib.gitdir.GitDirectory(self.dirname)
         mf = morphlib.morphologyfinder.MorphologyFinder(gd)
-        self.assertEqual(mf.read_morphology('foo.morph'),
+        self.assertEqual(mf.read_file('foo.morph'),
                          "altered morphology text")
 
     def test_read_morph_raises_no_worktree_no_ref(self):
         gd = morphlib.gitdir.GitDirectory(self.mirror)
         mf = morphlib.morphologyfinder.MorphologyFinder(gd)
         self.assertRaises(morphlib.gitdir.NoWorkingTreeError,
-                          mf.read_morphology, 'bar.morph')
+                          mf.read_file, 'bar.morph')
diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py
index 08589ea6..f206ef73 100644
--- a/morphlib/plugins/branch_and_merge_plugin.py
+++ b/morphlib/plugins/branch_and_merge_plugin.py
@@ -294,7 +294,7 @@ class BranchAndMergePlugin(cliapp.Plugin):
 
         ws = morphlib.workspace.open('.')
         sb = morphlib.sysbranchdir.open_from_within('.')
-        loader = morphlib.morphloader.MorphologyLoader()
+        loader = sb.get_morphology_loader()
         morphs = self._load_all_sysbranch_morphologies(sb, loader)
 
         def edit_chunk(morph, chunk_name):
@@ -486,7 +486,7 @@ class BranchAndMergePlugin(cliapp.Plugin):
         '''Read in all the morphologies in the root repository.'''
         self.app.status(msg='Loading in all morphologies')
         morphs = morphlib.morphset.MorphologySet()
-        for morph in sb.load_all_morphologies(loader):
+        for morph in sb.load_all_morphologies():
             morphs.add_morphology(morph)
         return morphs
 
@@ -606,7 +606,7 @@ class BranchAndMergePlugin(cliapp.Plugin):
             gd.branch(system_branch, base_ref)
             gd.checkout(system_branch)
 
-            loader = morphlib.morphloader.MorphologyLoader()
+            loader = sb.get_morphology_loader()
             morphs = self._load_all_sysbranch_morphologies(sb, loader)
 
             morphs.repoint_refs(sb.root_repository_url,
diff --git a/morphlib/plugins/diff_plugin.py b/morphlib/plugins/diff_plugin.py
index 06566438..9855c39f 100644
--- a/morphlib/plugins/diff_plugin.py
+++ b/morphlib/plugins/diff_plugin.py
@@ -108,7 +108,7 @@ class DiffPlugin(cliapp.Plugin):
                 definitions = mf.list_morphologies()
             system_paths = set()
             for definition in definitions:
-                m = ml.parse_morphology_text(mf.read_morphology(definition),
+                m = ml.parse_morphology_text(mf.read_file(definition),
                                              definition)
                 if m.get('kind') == 'system' or 'strata' in m:
                     system_paths.add(definition)
diff --git a/morphlib/plugins/get_chunk_details_plugin.py b/morphlib/plugins/get_chunk_details_plugin.py
index e58ed6c5..90e71c52 100644
--- a/morphlib/plugins/get_chunk_details_plugin.py
+++ b/morphlib/plugins/get_chunk_details_plugin.py
@@ -49,13 +49,12 @@ class GetChunkDetailsPlugin(cliapp.Plugin):
 
         definitions_repo = morphlib.definitions_repo.open(
             '.', search_for_root=True, search_workspace=True, app=self.app)
-        loader = morphlib.morphloader.MorphologyLoader()
 
         aliases = self.app.settings['repo-alias']
         self.resolver = morphlib.repoaliasresolver.RepoAliasResolver(aliases)
 
         found = 0
-        for morph in definitions_repo.load_all_morphologies(loader):
+        for morph in definitions_repo.load_all_morphologies():
             if morph['kind'] == 'stratum':
                 if (stratum_name == None or
                         morph['name'] == stratum_name):
diff --git a/morphlib/plugins/get_repo_plugin.py b/morphlib/plugins/get_repo_plugin.py
index 38039406..9c1c19f1 100644
--- a/morphlib/plugins/get_repo_plugin.py
+++ b/morphlib/plugins/get_repo_plugin.py
@@ -122,10 +122,9 @@ class GetRepoPlugin(cliapp.Plugin):
 
         definitions_repo = morphlib.definitions_repo.open(
             '.', search_for_root=True, search_workspace=True, app=self.app)
-        loader = morphlib.morphloader.MorphologyLoader()
 
         self.app.status(msg='Loading in all morphologies')
-        for morph in definitions_repo.load_all_morphologies(loader):
+        for morph in definitions_repo.load_all_morphologies():
             if morph['kind'] == 'stratum':
                 for chunk in morph['chunks']:
                     if chunk['name'] == chunk_name:
diff --git a/morphlib/sysbranchdir.py b/morphlib/sysbranchdir.py
index d23d9eea..25fc7f67 100644
--- a/morphlib/sysbranchdir.py
+++ b/morphlib/sysbranchdir.py
@@ -207,8 +207,11 @@ class SystemBranchDirectory(object):
                 for dirname in
                 morphlib.util.find_leaves(self.root_directory, '.git'))
 
-    def load_all_morphologies(self, loader): # pragma: no cover
-        return self.definitions_repo.load_all_morphologies(loader)
+    def get_morphology_loader(self):  # pragma: no cover
+        return self.definitions_repo.get_morphology_loader()
+
+    def load_all_morphologies(self):  # pragma: no cover
+        return self.definitions_repo.load_all_morphologies()
 
 
 def create(root_directory, root_repository_url, system_branch_name):
diff --git a/tests.branching/add-then-edit.script b/tests.branching/add-then-edit.script
index bd1094e5..59b7e60f 100755
--- a/tests.branching/add-then-edit.script
+++ b/tests.branching/add-then-edit.script
@@ -36,6 +36,7 @@ stratum["chunks"].append({
     "name": "goodbye",
     "ref": "master",
     "repo": "test:goodbye",
+    "build-system": "manual",
 })
 with open("hello-stratum.morph", "w") as f:
     yaml.dump(stratum, f)
diff --git a/tests.branching/branch-creates-new-system-branch-not-from-master.stdout b/tests.branching/branch-creates-new-system-branch-not-from-master.stdout
index c61624b4..78b3ca33 100644
--- a/tests.branching/branch-creates-new-system-branch-not-from-master.stdout
+++ b/tests.branching/branch-creates-new-system-branch-not-from-master.stdout
@@ -7,6 +7,7 @@ d ./newbranch/test
 d ./newbranch/test/morphs
 d ./newbranch/test/morphs/.git
 f ./newbranch/.morph-system-branch/config
+f ./newbranch/test/morphs/VERSION
 f ./newbranch/test/morphs/hello-stratum.morph
 f ./newbranch/test/morphs/hello-system.morph
 f ./newbranch/test/morphs/this.is.alfred
diff --git a/tests.branching/branch-creates-new-system-branch.stdout b/tests.branching/branch-creates-new-system-branch.stdout
index a7318378..882d5401 100644
--- a/tests.branching/branch-creates-new-system-branch.stdout
+++ b/tests.branching/branch-creates-new-system-branch.stdout
@@ -7,6 +7,7 @@ d ./newbranch/test
 d ./newbranch/test/morphs
 d ./newbranch/test/morphs/.git
 f ./newbranch/.morph-system-branch/config
+f ./newbranch/test/morphs/VERSION
 f ./newbranch/test/morphs/hello-stratum.morph
 f ./newbranch/test/morphs/hello-system.morph
 Current branches:
diff --git a/tests.branching/branch-works-anywhere.stdout b/tests.branching/branch-works-anywhere.stdout
index 4e317902..4e6c3e27 100644
--- a/tests.branching/branch-works-anywhere.stdout
+++ b/tests.branching/branch-works-anywhere.stdout
@@ -7,6 +7,7 @@ d ./branch1/test
 d ./branch1/test/morphs
 d ./branch1/test/morphs/.git
 f ./branch1/.morph-system-branch/config
+f ./branch1/test/morphs/VERSION
 f ./branch1/test/morphs/hello-stratum.morph
 f ./branch1/test/morphs/hello-system.morph
 Workspace after creating the second branch:
@@ -23,9 +24,11 @@ d ./branch2/test
 d ./branch2/test/morphs
 d ./branch2/test/morphs/.git
 f ./branch1/.morph-system-branch/config
+f ./branch1/test/morphs/VERSION
 f ./branch1/test/morphs/hello-stratum.morph
 f ./branch1/test/morphs/hello-system.morph
 f ./branch2/.morph-system-branch/config
+f ./branch2/test/morphs/VERSION
 f ./branch2/test/morphs/hello-stratum.morph
 f ./branch2/test/morphs/hello-system.morph
 Workspace after creating the third branch:
@@ -47,12 +50,15 @@ d ./branch3/test
 d ./branch3/test/morphs
 d ./branch3/test/morphs/.git
 f ./branch1/.morph-system-branch/config
+f ./branch1/test/morphs/VERSION
 f ./branch1/test/morphs/hello-stratum.morph
 f ./branch1/test/morphs/hello-system.morph
 f ./branch2/.morph-system-branch/config
+f ./branch2/test/morphs/VERSION
 f ./branch2/test/morphs/hello-stratum.morph
 f ./branch2/test/morphs/hello-system.morph
 f ./branch3/.morph-system-branch/config
+f ./branch3/test/morphs/VERSION
 f ./branch3/test/morphs/hello-stratum.morph
 f ./branch3/test/morphs/hello-system.morph
 Workspace after creating the fourth branch:
@@ -79,14 +85,18 @@ d ./branch4/test
 d ./branch4/test/morphs
 d ./branch4/test/morphs/.git
 f ./branch1/.morph-system-branch/config
+f ./branch1/test/morphs/VERSION
 f ./branch1/test/morphs/hello-stratum.morph
 f ./branch1/test/morphs/hello-system.morph
 f ./branch2/.morph-system-branch/config
+f ./branch2/test/morphs/VERSION
 f ./branch2/test/morphs/hello-stratum.morph
 f ./branch2/test/morphs/hello-system.morph
 f ./branch3/.morph-system-branch/config
+f ./branch3/test/morphs/VERSION
 f ./branch3/test/morphs/hello-stratum.morph
 f ./branch3/test/morphs/hello-system.morph
 f ./branch4/.morph-system-branch/config
+f ./branch4/test/morphs/VERSION
 f ./branch4/test/morphs/hello-stratum.morph
 f ./branch4/test/morphs/hello-system.morph
diff --git a/tests.branching/checkout-existing-branch.stdout b/tests.branching/checkout-existing-branch.stdout
index a6026269..f2b704e9 100644
--- a/tests.branching/checkout-existing-branch.stdout
+++ b/tests.branching/checkout-existing-branch.stdout
@@ -7,6 +7,7 @@ d ./master/test
 d ./master/test/morphs
 d ./master/test/morphs/.git
 f ./master/.morph-system-branch/config
+f ./master/test/morphs/VERSION
 f ./master/test/morphs/hello-stratum.morph
 f ./master/test/morphs/hello-system.morph
 Current branches:
diff --git a/tests.branching/checkout-works-anywhere.stdout b/tests.branching/checkout-works-anywhere.stdout
index ed8b1567..d7f6903a 100644
--- a/tests.branching/checkout-works-anywhere.stdout
+++ b/tests.branching/checkout-works-anywhere.stdout
@@ -7,6 +7,7 @@ d ./master/test
 d ./master/test/morphs
 d ./master/test/morphs/.git
 f ./master/.morph-system-branch/config
+f ./master/test/morphs/VERSION
 f ./master/test/morphs/hello-stratum.morph
 f ./master/test/morphs/hello-system.morph
 Workspace after checking out master from within a new branch:
@@ -23,8 +24,10 @@ d ./newbranch/test
 d ./newbranch/test/morphs
 d ./newbranch/test/morphs/.git
 f ./master/.morph-system-branch/config
+f ./master/test/morphs/VERSION
 f ./master/test/morphs/hello-stratum.morph
 f ./master/test/morphs/hello-system.morph
 f ./newbranch/.morph-system-branch/config
+f ./newbranch/test/morphs/VERSION
 f ./newbranch/test/morphs/hello-stratum.morph
 f ./newbranch/test/morphs/hello-system.morph
diff --git a/tests.branching/edit-checkouts-existing-chunk.stdout b/tests.branching/edit-checkouts-existing-chunk.stdout
index f6ac79c2..a859587b 100644
--- a/tests.branching/edit-checkouts-existing-chunk.stdout
+++ b/tests.branching/edit-checkouts-existing-chunk.stdout
@@ -8,4 +8,4 @@ test:morphs
 
 
 Files in hello:
-hello.morph
+dummy
diff --git a/tests.branching/edit-updates-stratum.stdout b/tests.branching/edit-updates-stratum.stdout
index ee9510b5..2372745e 100644
--- a/tests.branching/edit-updates-stratum.stdout
+++ b/tests.branching/edit-updates-stratum.stdout
@@ -1,8 +1,8 @@
 diff --git a/hello-stratum.morph b/hello-stratum.morph
-index f335879..7bf9d37 100644
+index 159854f..0036ee3 100644
 --- a/hello-stratum.morph
 +++ b/hello-stratum.morph
-@@ -3,6 +3,7 @@ kind: stratum
+@@ -3,7 +3,8 @@ kind: stratum
  chunks:
  - name: hello
    repo: test:hello
@@ -11,3 +11,4 @@ index f335879..7bf9d37 100644
 +  unpetrify-ref: master
    build-depends: []
    build-mode: bootstrap
+   build-system: manual
diff --git a/tests.branching/setup b/tests.branching/setup
index 59979ca8..8e82f32c 100755
--- a/tests.branching/setup
+++ b/tests.branching/setup
@@ -46,6 +46,8 @@ mkdir "$DATADIR/morphs"
 ## Create a link to this repo that has a .git suffix
 ln -s "$DATADIR/morphs" "$DATADIR/morphs.git"
 
+echo 'version: 6' > "$DATADIR/morphs/VERSION"
+
 cat <<EOF > "$DATADIR/morphs/hello-system.morph"
 name: hello-system
 kind: system
@@ -63,8 +65,10 @@ chunks:
   ref: master
   build-depends: []
   build-mode: bootstrap
+  build-system: manual
 EOF
 
+
 scripts/run-git-in "$DATADIR/morphs" init
 scripts/run-git-in "$DATADIR/morphs" add .
 scripts/run-git-in "$DATADIR/morphs" commit -m initial
@@ -77,15 +81,10 @@ scripts/run-git-in "$DATADIR/morphs" add this.is.alfred
 scripts/run-git-in "$DATADIR/morphs" commit --quiet -m 'mark as alfred'
 scripts/run-git-in "$DATADIR/morphs" checkout master
 
-
 # Create a dummy chunk repository
 mkdir "$DATADIR/hello"
 
-cat <<EOF > "$DATADIR/hello/hello.morph"
-name: hello
-kind: chunk
-build-system: dummy
-EOF
+echo 'Chunk repo is empty' > "$DATADIR"/hello/dummy
 
 scripts/run-git-in "$DATADIR/hello" init
 scripts/run-git-in "$DATADIR/hello" add .
diff --git a/tests.branching/setup-second-chunk b/tests.branching/setup-second-chunk
index c8baa946..416a773a 100755
--- a/tests.branching/setup-second-chunk
+++ b/tests.branching/setup-second-chunk
@@ -27,7 +27,6 @@ create_chunk() {
     cd "$1"
 
     cat <<EOF > "$1/$2.morph"
-build-system: dummy
 kind: chunk
 name: $2
 EOF
@@ -49,11 +48,13 @@ chunks:
   ref: master
   build-depends: []
   build-mode: bootstrap
+  build-system: manual
 - name: goodbye
   repo: test:goodbye
   ref: master
   build-depends: []
   build-mode: bootstrap
+  build-system: manual
 EOF
 
 git commit -q --all -m "Add goodbye to hello-stratum"