morph merge: Use a special git merge driver for morphology files

We now have a two-stage merge process. Stage one only runs if there
are changes in both branches on the same file. At the end of stage
one we assume that all the components that were edited in the FROM
branch still have their 'ref' field set to the FROM branch. Morph
then iterates through these repositories, performing a merge in each
one, and then updates the refs in the morphologies again with the
correct target branches.

4 files changed, 127 insertions, 12 deletions

diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py
index 30f02fde..279a9948 100644
--- a/morphlib/plugins/branch_and_merge_plugin.py
+++ b/morphlib/plugins/branch_and_merge_plugin.py
@@ -29,7 +29,6 @@ import uuid
 
 import morphlib
 
-
 class BranchAndMergePlugin(cliapp.Plugin):
 
     def __init__(self):
@@ -58,6 +57,8 @@ class BranchAndMergePlugin(cliapp.Plugin):
                                 arg_synopsis='SYSTEM')
         self.app.add_subcommand('foreach', self.foreach,
                                 arg_synopsis='COMMAND')
+        # This command should be hidden, once cliapp supports such a thing
+        self.app.add_subcommand('merge-morphology', self.merge_morphology)
 
     def disable(self):
         pass
@@ -710,10 +711,51 @@ class BranchAndMergePlugin(cliapp.Plugin):
 
         return old, new
 
+    def configure_merge_driver(self, repo_dir):
+        self.set_repo_config(repo_dir, 'merge.morph.name',
+            'Morphology merge driver')
+        self.set_repo_config(repo_dir, 'merge.morph.driver',
+            '%s merge-morphology $MORPH_FROM_BRANCH $MORPH_TO_BRANCH '
+            '%%O %%A %%B' % self.app.__file__)
+
+    MERGE_ATTRIBUTE = '*.morph\tmerge=morph\n'
+
+    def enable_merge_driver(self, repo_dir):
+        attributes_file = os.path.join(repo_dir, ".git", "info", "attributes")
+        with open(attributes_file, 'a') as f:
+            f.write(self.MERGE_ATTRIBUTE)
+
+    def disable_merge_driver(self, repo_dir):
+        attributes_file = os.path.join(repo_dir, ".git", "info", "attributes")
+        with open(attributes_file, 'r') as f:
+            attributes = f.read()
+        if attributes == self.MERGE_ATTRIBUTE:
+            os.unlink(attributes_file)
+        elif attributes.endswith(self.MERGE_ATTRIBUTE):
+            with morphlib.savefile.SaveFile(attributes_file, 'w') as f:
+                f.write(attributes[:-len(self.MERGE_ATTRIBUTE)])
+
     def merge_repo(self, dirty_repos, failed_repos,
                    from_branch_dir, from_repo, from_ref,
                    to_branch_dir, to_repo, to_ref):
-        '''Merge changes for a system branch in a specific repository'''
+        '''Merge changes for a system branch in a specific repository
+
+        This done using a standard 'git pull', with a custom merge driver to
+        handle the .morph files.
+
+        The merge driver is only invoked when there is a three-way merge
+        conflict on a file. If the file has changed in the FROM branch but not
+        in the TO branch (relative to the base version) git will choose the
+        FROM version without considering its contents.
+
+        Since git may automatically update morphologies in TO to use the refs
+        out of FROM, the merge driver does the same, and we then reset them
+        back and write the morphologies out again after recursively merging
+        the components. It's important that we disable the merge driver again
+        after the pull, or we risk confusing later users of the same repo who
+        do a manual git merge.
+
+        '''
 
         if to_repo in failed_repos:
             return None
@@ -732,16 +774,25 @@ class BranchAndMergePlugin(cliapp.Plugin):
             raise cliapp.AppException('repository %s has uncommitted '
                                       'changes' % to_repo)
 
-        # repo must be made into a URL to avoid ':' in pathnames confusing git
+        self.configure_merge_driver(to_repo_dir)
+        self.enable_merge_driver(to_repo_dir)
+        env = dict(os.environ)
+        env['MORPH_FROM_BRANCH'] = from_ref
+        env['MORPH_TO_BRANCH'] = to_ref
+        # ':' in pathnames confuses git, so we have to pass it a URL
         from_url = urlparse.urljoin('file://', from_repo_dir)
-
         status, output, error = self.app.runcmd_unchecked(
             ['git', 'pull', '--quiet', '--no-commit', '--no-ff',
-             from_url, from_ref], cwd=to_repo_dir)
+             from_url, from_ref], cwd=to_repo_dir, env=env)
+        self.disable_merge_driver(to_repo_dir)
+
         if status != 0:
             self.app.output.write(
-                'Merge errors encountered merging into %s in repo %s:\n%s%s\n'
-                % (to_ref, to_repo, error, output))
+                'Merge errors encountered in %s, branch %s:\n%s'
+                % (to_repo, to_ref, output))
+
+        if status != 0:
+            self.app.output.write('\n')
             failed_repos.add(to_repo)
             return None
         return to_repo_dir
@@ -875,6 +926,69 @@ class BranchAndMergePlugin(cliapp.Plugin):
                 self.reset_work_tree_safe(repo_dir)
             raise
 
+    def merge_morphology_contents(self, from_branch, from_morph, to_morph):
+        if from_morph['kind'] != to_morph['kind'] or \
+           from_morph['name'] != to_morph['name']:
+            raise cliapp.AppException('mismatch in name or kind')
+
+        pairs = []
+        if to_morph['kind'] == 'system':
+            pairs = zip(from_morph['strata'], to_morph['strata'])
+        elif to_morph['kind'] == 'stratum':
+            pairs = zip(from_morph['chunks'], to_morph['chunks'])
+
+        for from_child, to_child in pairs:
+            if from_child['morph'] != to_child['morph'] or \
+               from_child['repo'] != to_child['repo']:
+                continue
+
+            # 'morph merge' has two stages, of which this is the first. In the
+            # second stage we iterate each component whose 'ref' points to
+            # 'from_branch', merge inside its repository, and then update the
+            # morphology again to point at the merged ref in that repository.
+            # That is why we apparently merge the 'ref' field backwards here.
+            if from_child['ref'] == from_branch:
+                to_child['ref'] = from_branch
+
+    def merge_morphology(self, args):
+        '''Automatically merge changes between two conflicting morphologies.
+
+        Normally executed as a git merge driver.
+
+        In the future, this function can be expanded to resolve changes in
+        field ordering.
+
+        '''
+
+        if len(args) != 5:
+            raise cliapp.AppException('this command is not meant to be run '
+                                      'manually.')
+
+        from_branch = args[0]
+        to_branch = args[1]
+        base_file = args[2]
+        to_file = args[3]     # local
+        from_file = args[4]   # remote
+
+        with open(from_file) as f:
+            from_morph = morphlib.morph2.Morphology(f.read())
+        with open(to_file) as f:
+            to_morph = morphlib.morph2.Morphology(f.read())
+
+        self.merge_morphology_contents(from_branch, from_morph, to_morph)
+
+        # git merge gives us temporary files which we can overwrite with our
+        # resolution
+        with morphlib.savefile.SaveFile(to_file, 'w') as f:
+            to_morph.write_to_file(f)
+
+        # Leave the rest of the merging to git for now.
+        status, output, error = self.app.runcmd_unchecked(
+            ['git', 'merge-file', '-LHEAD', '-LBASE', '-L%s' % from_branch,
+             to_file, base_file, from_file])
+        if status != 0:
+            raise cliapp.AppException('Morph did not resolve all conflicts')
+
     def build(self, args):
         if len(args) != 1:
             raise cliapp.AppException('morph build expects exactly one '
diff --git a/tests.branching/merge-conflict-chunks.stdout b/tests.branching/merge-conflict-chunks.stdout
index cfb9949d..75d50c7f 100644
--- a/tests.branching/merge-conflict-chunks.stdout
+++ b/tests.branching/merge-conflict-chunks.stdout
@@ -1,9 +1,9 @@
-Merge errors encountered merging into test/stable in repo baserock:stratum2-hello:
+Merge errors encountered in baserock:stratum2-hello, branch test/stable:
 Auto-merging conflict.txt
 CONFLICT (add/add): Merge conflict in conflict.txt
 Automatic merge failed; fix conflicts and then commit the result.
 
-Merge errors encountered merging into test/stable in repo baserock:stratum3-hello:
+Merge errors encountered in baserock:stratum3-hello, branch test/stable:
 Auto-merging conflict.txt
 CONFLICT (add/add): Merge conflict in conflict.txt
 Automatic merge failed; fix conflicts and then commit the result.
diff --git a/tests.branching/merge-conflict-stratum.stdout b/tests.branching/merge-conflict-stratum.stdout
index a3bb4cc0..20064406 100644
--- a/tests.branching/merge-conflict-stratum.stdout
+++ b/tests.branching/merge-conflict-stratum.stdout
@@ -1,6 +1,5 @@
-Merge errors encountered merging into test/stable in repo baserock:morphs:
+Merge errors encountered in baserock:morphs, branch test/stable:
 Auto-merging hello-system.morph
-CONFLICT (content): Merge conflict in hello-system.morph
 Auto-merging hello-stratum.morph
 CONFLICT (content): Merge conflict in hello-stratum.morph
 Automatic merge failed; fix conflicts and then commit the result.
diff --git a/tests.branching/merge.script b/tests.branching/merge.script
index 9dbc2d84..d73e4835 100755
--- a/tests.branching/merge.script
+++ b/tests.branching/merge.script
@@ -31,11 +31,13 @@ cd "$DATADIR/workspace"
 "$SRCDIR/scripts/test-morph" branch baserock:morphs baserock/newbranch
 
 # Make a change to a chunk.
+cd baserock/newbranch
 "$SRCDIR/scripts/test-morph" edit hello-system hello-stratum hello
-cd baserock/newbranch/baserock:hello
+cd baserock:hello
 touch newfile.txt
 git add newfile.txt
 git commit -m foo --quiet
+git push --quiet origin baserock/newbranch
 
 # Commit in morphs repo
 cd ../baserock:morphs