Merge remote-tracking branch 'origin/baserock/richardmaw/S8938/null-repo-ref'

Reviewed-by: Lars Wirzenius
Reviewed-by: Jonathan Maw

13 files changed, 204 insertions, 13 deletions

diff --git a/morphlib/app.py b/morphlib/app.py
index 08b020ff..a0833d45 100644
--- a/morphlib/app.py
+++ b/morphlib/app.py
@@ -318,11 +318,15 @@ class Morph(cliapp.Application):
 
             visit(reponame, ref, filename, absref, tree, morphology)
             if morphology['kind'] == 'system':
-                queue.extend((s['repo'], s['ref'], '%s.morph' % s['morph'])
+                queue.extend((s['repo'] or reponame,
+                              s['ref'] or ref,
+                              '%s.morph' % s['morph'])
                              for s in morphology['strata'])
             elif morphology['kind'] == 'stratum':
                 if morphology['build-depends']:
-                    queue.extend((s['repo'], s['ref'], '%s.morph' % s['morph'])
+                    queue.extend((s['repo'] or reponame,
+                                  s['ref'] or ref,
+                                  '%s.morph' % s['morph'])
                                  for s in morphology['build-depends'])
                 queue.extend((c['repo'], c['ref'], '%s.morph' % c['morph'])
                              for c in morphology['chunks'])
diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py
index 186d5357..17f038a2 100644
--- a/morphlib/artifactresolver.py
+++ b/morphlib/artifactresolver.py
@@ -155,8 +155,8 @@ class ArtifactResolver(object):
 
         for info in source.morphology['strata']:
             stratum_source = self._source_pool.lookup(
-                info['repo'],
-                info['ref'],
+                info['repo'] or source.repo_name,
+                info['ref'] or source.original_ref,
                 '%s.morph' % info['morph'])
 
             stratum_name = stratum_source.morphology.builds_artifacts[0]
@@ -178,8 +178,8 @@ class ArtifactResolver(object):
         if stratum.source.morphology['build-depends']:
             for stratum_info in stratum.source.morphology['build-depends']:
                 other_source = self._source_pool.lookup(
-                    stratum_info['repo'],
-                    stratum_info['ref'],
+                    stratum_info['repo'] or stratum.source.repo_name,
+                    stratum_info['ref'] or stratum.source.original_ref,
                     '%s.morph' % stratum_info['morph'])
 
                 other_stratum = self._get_artifact(
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py
index d7007233..e76b7a14 100644
--- a/morphlib/buildcommand.py
+++ b/morphlib/buildcommand.py
@@ -163,8 +163,8 @@ class BuildCommand(object):
 
     def _validate_cross_refs_for_xxx(self, src, srcpool, specs, wanted):
         for spec in specs:
-            repo_name = spec['repo']
-            ref = spec['ref']
+            repo_name = spec['repo'] or src.repo_name
+            ref = spec['ref'] or src.original_ref
             filename = '%s.morph' % spec['morph']
             logging.debug(
                 'Validating cross ref to %s:%s:%s' %
diff --git a/morphlib/morph2.py b/morphlib/morph2.py
index a733ce77..6975e699 100644
--- a/morphlib/morph2.py
+++ b/morphlib/morph2.py
@@ -252,7 +252,8 @@ class Morphology(object):
                 continue
 
             value = self._apply_changes_for_key(key, live_dict, original_dict)
-            if value is not None:
+            # VILE HACK to preserve nulls in repo/ref fields
+            if value is not None or key in ('repo', 'ref'):
                 output_dict[key] = value
         return output_dict
 
diff --git a/morphlib/morphset.py b/morphlib/morphset.py
index 3c07d58e..9ef1e804 100644
--- a/morphlib/morphset.py
+++ b/morphlib/morphset.py
@@ -95,9 +95,11 @@ class MorphologySet(object):
 
         repo_url, ref, morph = self._find_spec(
             system_morph['strata'], stratum_name)
-        if repo_url is None:
+        if (repo_url, ref, morph) == (None, None, None):
             raise StratumNotInSystemError(system_morph['name'], stratum_name)
-        m = self._get_morphology(repo_url, ref, '%s.morph' % morph)
+        m = self._get_morphology(repo_url or system_morph.repo_url,
+                                 ref or system_morph.ref,
+                                 '%s.morph' % morph)
         if m is None:
             raise StratumNotInSetError(stratum_name)
         return m
@@ -118,7 +120,7 @@ class MorphologySet(object):
 
         repo_url, ref, morph = self._find_spec(
             stratum_morph['chunks'], chunk_name)
-        if repo_url is None:
+        if (repo_url, ref, morph) == (None, None, None):
             raise ChunkNotInStratumError(stratum_morph['name'], chunk_name)
         return repo_url, ref, morph
 
diff --git a/morphlib/plugins/branch_and_merge_new_plugin.py b/morphlib/plugins/branch_and_merge_new_plugin.py
index 39552ef0..edda7d9c 100644
--- a/morphlib/plugins/branch_and_merge_new_plugin.py
+++ b/morphlib/plugins/branch_and_merge_new_plugin.py
@@ -266,7 +266,9 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin):
         # of triplets (repo url, ref, filename).
 
         return [
-            (spec['repo'], spec['ref'], '%s.morph' % spec['morph'])
+            (spec['repo'] or morph.repo_url,
+             spec['ref'] or morph.ref,
+             '%s.morph' % spec['morph'])
             for spec in specs
         ]
 
@@ -679,6 +681,9 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin):
         #TODO: Stop using app.resolve_ref
         def resolve_refs(morphs):
             for repo, ref in morphs.list_refs():
+                # You can't resolve null refs, so don't attempt to.
+                if repo is None or ref is None:
+                    continue
                 # TODO: Handle refs that are only in workspace in general
                 if (repo == sb.root_repository_url
                     and ref == sb.system_branch_name):
diff --git a/scripts/nullify-local-refs b/scripts/nullify-local-refs
new file mode 100755
index 00000000..5db5c587
--- /dev/null
+++ b/scripts/nullify-local-refs
@@ -0,0 +1,18 @@
+#!/usr/bin/python
+
+import yaml, sys
+repo = sys.argv[1]
+ref = sys.argv[2]
+for filename in sys.argv[3:]:
+    with open(filename, "r") as f:
+        d = yaml.load(f)
+    if "strata" in d:
+        for spec in d["strata"]:
+            if spec["repo"] == repo and spec["ref"] == ref:
+                spec["repo"] = spec["ref"] = None
+    if "build-depends" in d:
+        for spec in d["build-depends"]:
+            if spec["repo"] == repo and spec["ref"] == ref:
+                spec["repo"] = spec["ref"] = None
+    with open(filename, "w") as f:
+        yaml.dump(d, f)
diff --git a/tests.build/build-system-with-null-refs.script b/tests.build/build-system-with-null-refs.script
new file mode 100755
index 00000000..e23dcafa
--- /dev/null
+++ b/tests.build/build-system-with-null-refs.script
@@ -0,0 +1,24 @@
+#!/bin/sh
+#
+# Copyright (C) 2013  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+## Test building a system with null refs
+
+set -eu
+
+"$SRCDIR/scripts/test-morph" build-morphology \
+    test:morphs-repo master hello-system
diff --git a/tests.build/build-system-with-null-refs.setup b/tests.build/build-system-with-null-refs.setup
new file mode 100755
index 00000000..cbf53076
--- /dev/null
+++ b/tests.build/build-system-with-null-refs.setup
@@ -0,0 +1,23 @@
+#!/bin/sh
+#
+# Copyright (C) 2013  Codethink Limited
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+set -eu
+
+cd "$DATADIR/morphs-repo"
+"$SRCDIR/scripts/nullify-local-refs" test:morphs master *.morph
+git add *.morph
+git commit --quiet -m "Nullify all refs"
diff --git a/tests.deploy/deploy-with-null-refs.script b/tests.deploy/deploy-with-null-refs.script
new file mode 100755
index 00000000..c283debf
--- /dev/null
+++ b/tests.deploy/deploy-with-null-refs.script
@@ -0,0 +1,35 @@
+#!/bin/bash
+#
+# Copyright (C) 2013  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+
+set -eu
+
+
+. "$SRCDIR/tests.deploy/setup-build"
+
+cd "$DATADIR/workspace/branch1"
+"$SRCDIR/scripts/nullify-local-refs" test:morphs master test:morphs/*.morph
+
+"$SRCDIR/scripts/test-morph" build hello-system
+
+"$SRCDIR/scripts/test-morph" build linux-system
+
+"$SRCDIR/scripts/test-morph" --log "$DATADIR/deploy.log" \
+    deploy test_cluster \
+    linux-system-2.HOSTNAME="baserock-rocks-even-more" \
+    > /dev/null
diff --git a/yarns/branches-workspaces.yarn b/yarns/branches-workspaces.yarn
index 5273f396..f523ebcd 100644
--- a/yarns/branches-workspaces.yarn
+++ b/yarns/branches-workspaces.yarn
@@ -290,3 +290,65 @@ Creating a tag twice should fail.
     WHEN        attempting to tag system branch foo as test123
     THEN        morph failed
 
+Working with null repositories and refs
+---------------------------------------
+
+It is convenient to not explicitly name the repository and branch of
+a stratum morphology, instead assuming it is the same as the current
+morphology.
+
+These can be checked out like normal system branches.
+
+    SCENARIO    check out an existing system branch with null refs
+    GIVEN       a workspace
+    AND         a git server
+    AND         null refs for local strata
+    WHEN        checking out the master system branch
+    THEN        the system branch master is checked out
+
+Likewise we can also create new system branches from these, and we
+wouldn't need to worry about changing the system branch ref.
+
+
+    SCENARIO    branch off a system branch with null refs
+    GIVEN       a workspace
+    AND         a git server
+    AND         null refs for local strata
+    WHEN        creating system branch foo
+    THEN        the system branch foo is checked out
+
+When we edit a morphology with null refs, they stay null.
+
+    SCENARIO    editing with null refs
+    GIVEN       a workspace
+    AND         a git server
+    AND         null refs for local strata
+
+When creating the branch, the refs remain null.
+
+    WHEN        creating system branch foo
+    THEN        in branch foo, system test-system refs test-stratum in None
+
+After editing the stratum they remain null.
+
+    WHEN    editing stratum test-stratum in system test-system in branch foo
+    THEN    in branch foo, system test-system refs test-stratum in None
+
+Refs to chunks are still altered as usual
+
+    WHEN editing chunk test-chunk in test-stratum in test-system in branch foo
+    THEN in branch foo, system test-system refs test-stratum in None
+    AND  in branch foo, stratum test-stratum refs test-chunk in foo
+    AND  edited chunk test:test-chunk has git branch foo
+
+Petrifying also leaves null refs unmolested
+
+    SCENARIO    morph petrifies null refs
+    GIVEN       a workspace
+    AND         a git server
+    AND         null refs for local strata
+    WHEN        creating system branch foo
+    AND         pushing system branch foo to git server
+    AND         remembering all refs in foo
+    AND         petrifying foo
+    THEN        in branch foo, system test-system refs test-stratum in None
diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn
index cfb744f7..e35e4219 100644
--- a/yarns/implementations.yarn
+++ b/yarns/implementations.yarn
@@ -114,6 +114,18 @@ another to hold a chunk.
     repo-alias = test=file://$DATADIR/gits/%s#file://$DATADIR/gits/%s
     EOF
 
+Morphologies need to support having a null ref, which means look for the
+stratum in the same repository and ref. Testing this requires different
+morphologies.
+
+    IMPLEMENTS GIVEN null refs for local strata
+    nullify_local_refs test:morphs master \
+        "$DATADIR/gits/morphs/test-system.morph" \
+        "$DATADIR/gits/morphs/test-stratum.morph"
+    run_in "$DATADIR/gits/morphs" git add .
+    run_in "$DATADIR/gits/morphs" git commit -m "Use null refs."
+
+
 Implementation sections for system branch operations
 ----------------------------------------------------
 
diff --git a/yarns/morph.shell-lib b/yarns/morph.shell-lib
index 4fb1eb10..448c60ce 100644
--- a/yarns/morph.shell-lib
+++ b/yarns/morph.shell-lib
@@ -144,6 +144,11 @@ assert_morphologies_are_petrified()
 }
 
 
+nullify_local_refs()
+{
+    "$SRCDIR/scripts/nullify-local-refs" "$@"
+}
+
 # Currently, yarn isn't setting $SRCDIR to point at the project source
 # directory. We simulate this here.