morph build: Use the user's own repos when building

This means that Morph no longer requires changes to be pushed in order
to build them.

The repos from the system branch are currently cached in the local
repo cache as part of the build process, which is far from ideal.

Tests for 'morph build' now test build without push. The build
metadata now includes a repo path that is inside the TMPDIR, so the
tests have been rewritten to avoid having any hardcoded cache keys
because the cache keys are no longer static.

7 files changed, 28 insertions, 75 deletions

diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py
index 97f49eb4..1e2f381d 100644
--- a/morphlib/plugins/branch_and_merge_plugin.py
+++ b/morphlib/plugins/branch_and_merge_plugin.py
@@ -1116,6 +1116,7 @@ class BranchAndMergePlugin(cliapp.Plugin):
                                       'parameter: the system to build')
 
         system_name = args[0]
+        push = False
 
         # Deduce workspace and system branch and branch root repository.
         workspace = self.deduce_workspace()
@@ -1143,21 +1144,25 @@ class BranchAndMergePlugin(cliapp.Plugin):
         # Create the build refs for all these repositories and commit
         # all uncommitted changes to them, updating all references
         # to system branch refs to point to the build refs instead.
-        self.update_build_refs(build_repos, branch, build_uuid)
+        self.update_build_refs(build_repos, branch, build_uuid, push)
 
-        # Push the temporary build refs.
-        self.push_build_refs(build_repos)
+        if push:
+            self.push_build_refs(build_repos)
+            build_branch_root = branch_root
+        else:
+            dirname = build_repos[branch_root]['dirname']
+            build_branch_root = urlparse.urljoin('file://', dirname)
 
         # Run the build.
         build_command = morphlib.buildcommand.BuildCommand(self.app)
         build_command = self.app.hookmgr.call('new-build-command',
                                               build_command)
-        build_command.build([branch_root,
+        build_command.build([build_branch_root,
                              build_repos[branch_root]['build-ref'],
                              system_name])
 
-        # Delete the temporary refs on the server.
-        self.delete_remote_build_refs(build_repos)
+        if push:
+            self.delete_remote_build_refs(build_repos)
 
         self.app.status(msg='Finished build %(uuid)s', uuid=build_uuid)
 
@@ -1212,7 +1217,7 @@ class BranchAndMergePlugin(cliapp.Plugin):
 
         return build_repos
 
-    def inject_build_refs(self, morphology, build_repos):
+    def inject_build_refs(self, morphology, build_repos, will_push):
         # Starting from a system or stratum morphology, update all ref
         # pointers of strata or chunks involved in a system build (represented
         # by build_repos) to point to temporary build refs of the repos
@@ -1222,6 +1227,9 @@ class BranchAndMergePlugin(cliapp.Plugin):
                     info['morph'] in build_repos[info['repo']]['strata'] or
                     info['morph'] in build_repos[info['repo']]['chunks']):
                 info['ref'] = build_repos[info['repo']]['build-ref']
+                if not will_push:
+                    dirname = build_repos[info['repo']]['dirname']
+                    info['repo'] = urlparse.urljoin('file://', dirname)
         if morphology['kind'] == 'system':
             for info in morphology['strata']:
                 inject_build_ref(info)
@@ -1237,7 +1245,8 @@ class BranchAndMergePlugin(cliapp.Plugin):
                                      branch_uuid, repo_uuid)
             info['build-ref'] = build_ref
 
-    def update_build_refs(self, build_repos, system_branch, build_uuid):
+    def update_build_refs(self, build_repos, system_branch, build_uuid,
+                          will_push):
         '''Update build branches for each repository with any local changes '''
 
         # Define the committer.
@@ -1291,7 +1300,7 @@ class BranchAndMergePlugin(cliapp.Plugin):
             for filename in filenames:
                 # Inject temporary refs in the right places in each morphology.
                 morphology = self.load_morphology(repo_dir, filename)
-                self.inject_build_refs(morphology, build_repos)
+                self.inject_build_refs(morphology, build_repos, will_push)
                 handle, tmpfile = tempfile.mkstemp(suffix='.morph')
                 self.save_morphology(repo_dir, tmpfile, morphology)
 
diff --git a/tests.as-root/building-a-system-branch-multiple-times-doesnt-generate-new-artifacts.script b/tests.as-root/building-a-system-branch-multiple-times-doesnt-generate-new-artifacts.script
index ab84c431..9ccc3dee 100755
--- a/tests.as-root/building-a-system-branch-multiple-times-doesnt-generate-new-artifacts.script
+++ b/tests.as-root/building-a-system-branch-multiple-times-doesnt-generate-new-artifacts.script
@@ -32,14 +32,14 @@ source "$SRCDIR/tests.as-root/setup-build"
 # Build once.
 cd "$DATADIR/workspace"
 "$SRCDIR/scripts/test-morph" build linux-system
-"$SRCDIR/scripts/list-tree" "$DATADIR/cache/artifacts"
+ARTIFACT_COUNT="$(ls "$DATADIR/cache/artifacts" | wc -l)"
 
 # Build twice.
 cd "$DATADIR/workspace/branch1"
 "$SRCDIR/scripts/test-morph" build linux-system
-"$SRCDIR/scripts/list-tree" "$DATADIR/cache/artifacts"
+[ "$ARTIFACT_COUNT" -eq $(ls "$DATADIR/cache/artifacts" | wc -l) ]
 
 # Build thrice, and that should be enough.
 cd "$DATADIR/workspace/branch1/test:morphs"
 "$SRCDIR/scripts/test-morph" build linux-system
-"$SRCDIR/scripts/list-tree" "$DATADIR/cache/artifacts"
+[ "$ARTIFACT_COUNT" -eq $(ls "$DATADIR/cache/artifacts" | wc -l) ]
diff --git a/tests.as-root/building-a-system-branch-multiple-times-doesnt-generate-new-artifacts.stdout b/tests.as-root/building-a-system-branch-multiple-times-doesnt-generate-new-artifacts.stdout
deleted file mode 100644
index a02b977c..00000000
--- a/tests.as-root/building-a-system-branch-multiple-times-doesnt-generate-new-artifacts.stdout
+++ /dev/null
@@ -1,30 +0,0 @@
-d .
-f ./2311765afd12c233a772a33d7f7e31e2450c191379a430790ea03b07938b9e14.meta
-f ./2311765afd12c233a772a33d7f7e31e2450c191379a430790ea03b07938b9e14.stratum.linux-stratum
-f ./2311765afd12c233a772a33d7f7e31e2450c191379a430790ea03b07938b9e14.stratum.linux-stratum.meta
-f ./e8d3ecb31babcb58516f1298ccc5f63b167186e6527d831af5a788309a648cd6.build-log
-f ./e8d3ecb31babcb58516f1298ccc5f63b167186e6527d831af5a788309a648cd6.chunk.linux
-f ./e8d3ecb31babcb58516f1298ccc5f63b167186e6527d831af5a788309a648cd6.meta
-f ./f59cea90b0ea8c780c00adee730f6de993be0b744c7c92fb06b037b7879fc668.meta
-f ./f59cea90b0ea8c780c00adee730f6de993be0b744c7c92fb06b037b7879fc668.system.linux-system-kernel
-f ./f59cea90b0ea8c780c00adee730f6de993be0b744c7c92fb06b037b7879fc668.system.linux-system-rootfs
-d .
-f ./2311765afd12c233a772a33d7f7e31e2450c191379a430790ea03b07938b9e14.meta
-f ./2311765afd12c233a772a33d7f7e31e2450c191379a430790ea03b07938b9e14.stratum.linux-stratum
-f ./2311765afd12c233a772a33d7f7e31e2450c191379a430790ea03b07938b9e14.stratum.linux-stratum.meta
-f ./e8d3ecb31babcb58516f1298ccc5f63b167186e6527d831af5a788309a648cd6.build-log
-f ./e8d3ecb31babcb58516f1298ccc5f63b167186e6527d831af5a788309a648cd6.chunk.linux
-f ./e8d3ecb31babcb58516f1298ccc5f63b167186e6527d831af5a788309a648cd6.meta
-f ./f59cea90b0ea8c780c00adee730f6de993be0b744c7c92fb06b037b7879fc668.meta
-f ./f59cea90b0ea8c780c00adee730f6de993be0b744c7c92fb06b037b7879fc668.system.linux-system-kernel
-f ./f59cea90b0ea8c780c00adee730f6de993be0b744c7c92fb06b037b7879fc668.system.linux-system-rootfs
-d .
-f ./2311765afd12c233a772a33d7f7e31e2450c191379a430790ea03b07938b9e14.meta
-f ./2311765afd12c233a772a33d7f7e31e2450c191379a430790ea03b07938b9e14.stratum.linux-stratum
-f ./2311765afd12c233a772a33d7f7e31e2450c191379a430790ea03b07938b9e14.stratum.linux-stratum.meta
-f ./e8d3ecb31babcb58516f1298ccc5f63b167186e6527d831af5a788309a648cd6.build-log
-f ./e8d3ecb31babcb58516f1298ccc5f63b167186e6527d831af5a788309a648cd6.chunk.linux
-f ./e8d3ecb31babcb58516f1298ccc5f63b167186e6527d831af5a788309a648cd6.meta
-f ./f59cea90b0ea8c780c00adee730f6de993be0b744c7c92fb06b037b7879fc668.meta
-f ./f59cea90b0ea8c780c00adee730f6de993be0b744c7c92fb06b037b7879fc668.system.linux-system-kernel
-f ./f59cea90b0ea8c780c00adee730f6de993be0b744c7c92fb06b037b7879fc668.system.linux-system-rootfs
diff --git a/tests.as-root/building-a-system-branch-picks-up-uncommitted-changes.script b/tests.as-root/building-a-system-branch-picks-up-uncommitted-changes.script
index f959bb43..642093dc 100755
--- a/tests.as-root/building-a-system-branch-picks-up-uncommitted-changes.script
+++ b/tests.as-root/building-a-system-branch-picks-up-uncommitted-changes.script
@@ -34,14 +34,10 @@ cd "$DATADIR/workspace"
 "$SRCDIR/scripts/test-morph" build linux-system
 
 # Print tree SHA1s of the build ref of morphs and kernel.
-echo "Tree of morphs repo build branch after first build:"
 cd "$DATADIR/workspace/branch1/test:morphs"
-git log -1 --format=%T baserock/builds/123456789/987654321
-echo "Tree of kernel repo build branch after first build:"
+MORPHS_SHA1="$(git rev-parse baserock/builds/123456789/987654321)"
 cd "$DATADIR/workspace/branch1/test:kernel-repo"
-git log -1 --format=%T baserock/builds/123456789/AABBCCDDE
-
-echo
+KERNEL_SHA1="$(git rev-parse baserock/builds/123456789/AABBCCDDE)"
 
 # Make an uncommitted change to the linux morphology.
 cd "$DATADIR/workspace/branch1/test:kernel-repo"
@@ -55,9 +51,7 @@ cd "$DATADIR/workspace"
 # This time the tree SHA1 of morphs should be the same
 # but that of the kernel repo should be different because we
 # made a change.
-echo "Tree of morphs repo build branch after second build:"
 cd "$DATADIR/workspace/branch1/test:morphs"
-git log -1 --format=%T baserock/builds/123456789/987654321
-echo "Tree of kernel repo build branch after second build:"
+[ "$(git rev-parse baserock/builds/123456789/987654321)" != "$MORPHS_SHA1" ]
 cd "$DATADIR/workspace/branch1/test:kernel-repo"
-git log -1 --format=%T baserock/builds/123456789/AABBCCDDE
+[ "$(git rev-parse baserock/builds/123456789/AABBCCDDE)" != "$KERNEL_SHA1" ]
diff --git a/tests.as-root/building-a-system-branch-picks-up-uncommitted-changes.stdout b/tests.as-root/building-a-system-branch-picks-up-uncommitted-changes.stdout
deleted file mode 100644
index d5758d78..00000000
--- a/tests.as-root/building-a-system-branch-picks-up-uncommitted-changes.stdout
+++ /dev/null
@@ -1,9 +0,0 @@
-Tree of morphs repo build branch after first build:
-73b00844bfe028b01d2464727d703f1fcc1f8ef9
-Tree of kernel repo build branch after first build:
-83d99190fa36d14f1bd24d88ba0fbe2ce674dd7c
-
-Tree of morphs repo build branch after second build:
-73b00844bfe028b01d2464727d703f1fcc1f8ef9
-Tree of kernel repo build branch after second build:
-5b015689415c96cdd290834ba105a64be28a3cfb
diff --git a/tests.as-root/building-a-system-branch-works-anywhere.script b/tests.as-root/building-a-system-branch-works-anywhere.script
index 1ce59b0f..2bba83dc 100755
--- a/tests.as-root/building-a-system-branch-works-anywhere.script
+++ b/tests.as-root/building-a-system-branch-works-anywhere.script
@@ -52,11 +52,10 @@ cd "$DATADIR/workspace/branch1/test:kernel-repo"
 "$SRCDIR/scripts/list-tree" "$DATADIR/cache/artifacts" > "$DATADIR/output4"
 rm -rf "$DATADIR/cache/artifacts"/*
 
-# Verify that we're always building the same and that we're building
-# the right things after all.
-cat "$DATADIR/output1"
+# Verify that we build the right number of artifacts
+[ $(wc < "$DATADIR/output1" -l) -eq 10 ]
 
-# Print diffs of the build results, all of which should be empty.
+# List of files in the artifact cache should be identical after each build
 diff "$DATADIR/output1" "$DATADIR/output2"
 diff "$DATADIR/output2" "$DATADIR/output3"
 diff "$DATADIR/output3" "$DATADIR/output4"
diff --git a/tests.as-root/building-a-system-branch-works-anywhere.stdout b/tests.as-root/building-a-system-branch-works-anywhere.stdout
deleted file mode 100644
index 3bbb820f..00000000
--- a/tests.as-root/building-a-system-branch-works-anywhere.stdout
+++ /dev/null
@@ -1,10 +0,0 @@
-d .
-f ./2311765afd12c233a772a33d7f7e31e2450c191379a430790ea03b07938b9e14.meta
-f ./2311765afd12c233a772a33d7f7e31e2450c191379a430790ea03b07938b9e14.stratum.linux-stratum
-f ./2311765afd12c233a772a33d7f7e31e2450c191379a430790ea03b07938b9e14.stratum.linux-stratum.meta
-f ./e8d3ecb31babcb58516f1298ccc5f63b167186e6527d831af5a788309a648cd6.build-log
-f ./e8d3ecb31babcb58516f1298ccc5f63b167186e6527d831af5a788309a648cd6.chunk.linux
-f ./e8d3ecb31babcb58516f1298ccc5f63b167186e6527d831af5a788309a648cd6.meta
-f ./f59cea90b0ea8c780c00adee730f6de993be0b744c7c92fb06b037b7879fc668.meta
-f ./f59cea90b0ea8c780c00adee730f6de993be0b744c7c92fb06b037b7879fc668.system.linux-system-kernel
-f ./f59cea90b0ea8c780c00adee730f6de993be0b744c7c92fb06b037b7879fc668.system.linux-system-rootfs