Merge branch 'samthursfield/write-metadata-for-system' of git://git.baserock.org/baserock/morph

This includes a fixup of shell quoting in tests.as-root/lib.

Tests needs to be run with a larget TMPDIR than systemd's default,
since system images can be large.

12 files changed, 165 insertions, 41 deletions

diff --git a/morphlib/builder2.py b/morphlib/builder2.py
index 565fdf4f..d3768054 100644
--- a/morphlib/builder2.py
+++ b/morphlib/builder2.py
@@ -662,8 +662,8 @@ class DiskImageBuilder(SystemKindBuilder):  # pragma: no cover
         with self.build_watch('overall-build'):
             arch = self.artifact.source.morphology['arch']
 
-            rootfs_artifact = self.new_artifact(
-                self.artifact.source.morphology['name'] + '-rootfs')
+            rootfs_name = self.artifact.source.morphology['name'] + '-rootfs'
+            rootfs_artifact = self.new_artifact(rootfs_name)
             handle = self.local_artifact_cache.put(rootfs_artifact)
             (image_file_fd, image_name) = \
                 tempfile.mkstemp(dir=self.app.settings['tempdir'])
@@ -682,6 +682,7 @@ class DiskImageBuilder(SystemKindBuilder):  # pragma: no cover
                     factory_path = os.path.join(mount_point, 'factory')
                     self._create_subvolume(factory_path)
                     self.unpack_strata(factory_path)
+                    self.write_metadata(factory_path, rootfs_name)
                     self.create_fstab(factory_path)
                     self._create_bootloader_config(factory_path)
                     self._create_subvolume_snapshot(
diff --git a/morphlib/plugins/tarball-systembuilder_plugin.py b/morphlib/plugins/tarball-systembuilder_plugin.py
index 95cae4cd..14807924 100644
--- a/morphlib/plugins/tarball-systembuilder_plugin.py
+++ b/morphlib/plugins/tarball-systembuilder_plugin.py
@@ -49,6 +49,7 @@ class RootfsTarballBuilder(SystemKindBuilder):  # pragma: no cover
             try:
                 fs_root = self.staging_area.destdir(self.artifact.source)
                 self.unpack_strata(fs_root)
+                self.write_metadata(fs_root, rootfs_name)
                 self.create_fstab(fs_root)
                 self.copy_kernel_into_artifact_cache(fs_root)
                 unslashy_root = fs_root[1:]
diff --git a/scripts/test-morph b/scripts/test-morph
index 2b7fb57d..fb3e2cd4 100755
--- a/scripts/test-morph
+++ b/scripts/test-morph
@@ -27,7 +27,25 @@ set -eu
 # distributed builds, which invoke morph as a sub-process.
 export PATH="$SRCDIR:$PATH"
 
-"$SRCDIR/scripts/cmd-filter" "$SRCDIR/morph" --no-default-config \
-    --tarball-server= --cache-server= \
-    --config="$DATADIR/morph.conf" "$@"
+if [ "$1" = "--find-system-artifact" ]; then
+    shift
 
+    "$SRCDIR/morph" --no-default-config \
+        --tarball-server= --cache-server= \
+        --config="$DATADIR/morph.conf" --verbose "$@" > $DATADIR/stdout
+
+    ARTIFACT=$(grep "system \S\+-rootfs is cached at" "$DATADIR/stdout" | \
+        sed -nre "s/^.*system \S+-rootfs is cached at (\S+)$/\1/p")
+    rm "$DATADIR/stdout"
+
+    if [ ! -e "$ARTIFACT" ]; then
+        echo "Unable to find rootfs artifact: $ARTIFACT" 1>&2
+        exit 1
+    fi
+
+    echo $ARTIFACT
+else
+    "$SRCDIR/scripts/cmd-filter" "$SRCDIR/morph" --no-default-config \
+        --tarball-server= --cache-server= \
+        --config="$DATADIR/morph.conf" "$@"
+fi
diff --git a/tests.as-root/disk-builds-rootfs-and-kernel.script b/tests.as-root/disk-builds-rootfs-and-kernel.script
index f9748ca9..3b9174c2 100755
--- a/tests.as-root/disk-builds-rootfs-and-kernel.script
+++ b/tests.as-root/disk-builds-rootfs-and-kernel.script
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 #
 # A system kind of 'disk' should create the disk image, plus a kernel file.
 #
@@ -19,6 +19,8 @@
 
 set -eu
 
+. "$SRCDIR/tests.as-root/lib"
+
 cache="$DATADIR/cache/artifacts"
 kernelrepo="$DATADIR/kernel-repo"
 morphsrepo="$DATADIR/morphs-repo"
@@ -83,8 +85,8 @@ git add linux.morph
 
 git commit --quiet -m 'Make the kernel create a dummy zImage'
 
-"$SRCDIR/scripts/test-morph" \
-    build-morphology test:morphs-repo custom system > /dev/null
+rootfs=$("$SRCDIR/scripts/test-morph" --find-system-artifact \
+    build-morphology test:morphs-repo custom system)
 
 for suffix in kernel rootfs
 do
@@ -96,3 +98,12 @@ do
     fi
 done
 
+mkdir -p "$DATADIR/mnt"
+device=$(loopback_rootfs "$rootfs")
+mount "$device" "$DATADIR/mnt"
+trap 'cd /; umount -d "$device"' INT TERM ERR
+
+cd "$DATADIR/mnt"
+find | LC_ALL=C sort -u
+cd "$DATADIR"
+umount -d "$device"
diff --git a/tests.as-root/disk-builds-rootfs-and-kernel.stdout b/tests.as-root/disk-builds-rootfs-and-kernel.stdout
new file mode 100644
index 00000000..929f284d
--- /dev/null
+++ b/tests.as-root/disk-builds-rootfs-and-kernel.stdout
@@ -0,0 +1,30 @@
+.
+./boot
+./boot/System.map
+./boot/vmlinuz
+./factory
+./factory-run
+./factory-run/baserock
+./factory-run/baserock/hello.meta
+./factory-run/baserock/linux.meta
+./factory-run/baserock/stratum.meta
+./factory-run/baserock/system-rootfs.meta
+./factory-run/bin
+./factory-run/bin/hello
+./factory-run/boot
+./factory-run/boot/System.map
+./factory-run/boot/vmlinuz
+./factory-run/etc
+./factory-run/etc/fstab
+./factory/baserock
+./factory/baserock/hello.meta
+./factory/baserock/linux.meta
+./factory/baserock/stratum.meta
+./factory/baserock/system-rootfs.meta
+./factory/bin
+./factory/bin/hello
+./factory/boot
+./factory/boot/System.map
+./factory/boot/vmlinuz
+./factory/etc
+./factory/etc/fstab
diff --git a/tests.as-root/lib b/tests.as-root/lib
new file mode 100644
index 00000000..6818abde
--- /dev/null
+++ b/tests.as-root/lib
@@ -0,0 +1,35 @@
+#!/bin/sh
+#
+# Copyright (C) 2012  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# Helper functions for as-root tests
+
+loopback_rootfs() {
+    # Find offset partition offset in a rootfs and mount it
+    ROOTFS="$1"
+
+    mv "$ROOTFS" "$ROOTFS-unzipped.gz"
+    gunzip "$ROOTFS-unzipped.gz"
+
+    OFFSET=$(sfdisk -d "$ROOTFS-unzipped" | \
+        grep -m 1 -o 'start=\s\+\([0-9]\+\)' | awk '{ print $2 }')
+    OFFSET=$(expr "$OFFSET" '*' 512)
+
+    DEVICE=$(losetup --show -o "$OFFSET" -f "$ROOTFS-unzipped")
+    udevadm settle
+
+    echo "$DEVICE"
+}
diff --git a/tests.as-root/make-patch.script b/tests.as-root/make-patch.script
index 031ecb69..7f033c20 100755
--- a/tests.as-root/make-patch.script
+++ b/tests.as-root/make-patch.script
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 #
 # Test making a patch between two different system images
 #
@@ -19,6 +19,8 @@
 
 set -eu
 
+. "$SRCDIR/tests.as-root/lib"
+
 # Make a stratum that include hello and kernel chunks.
 cat <<EOF > "$DATADIR/morphs-repo/hello-stratum.morph"
 {
@@ -89,26 +91,10 @@ Done.
 EOF
 exit 0
 
-# find out the offset of the partition in the disk image
-# udevadm settle is used to wait until devices have appeared
-loop=$(losetup -f --show "$img1")
-udevadm settle
-offset=$(sfdisk -d "$loop" |
-             sed -n '/start=/s/^.*start=\s*\([0-9]\+\)[^0-9].*$/\1/p' |
-             head -n 1)
-ssz=`cat "/sys/devices/virtual/block/$(basename "$loop")/queue/hw_sector_size"`
-losetup -d "$loop"
-udevadm settle
-
-# mount the disk
-mkdir "$DATADIR/unpacked"
-# this has to be done by losetup then mount, because busybox mount doesn't
-# accept the -o offset= option
-loop=$(losetup -o "$(expr "$offset" '*' "$ssz")" -f --show "$img1")
-udevadm settle
-trap 'losetup -d "$loop"' INT TERM EXIT
-mount -o subvol=factory-run -t btrfs "$loop" "$DATADIR/unpacked"
-trap 'umount -d "$DATADIR/unpacked"' INT TERM EXIT
+mkdir -p "$DATADIR/unpacked"
+device=$(loopback_rootfs "$img1")
+mount -o subvol=factory-run "$device" "$DATADIR/unpacked"
+trap 'cd /; umount -d "$device"' INT TERM ERR
 
 cd "$DATADIR/unpacked"
 echo "old version:"
@@ -119,7 +105,7 @@ tbdiff-deploy "$patch" 2>/dev/null
 echo "new version:"
 ./bin/hello
 
-# Done.
 echo "Done."
-# chdir out of mounted folder, so it can be unmounted
+
 cd /
+umount -d "$device"
diff --git a/tests.as-root/rootfs-tarball-builds-rootfs-and-kernel.script b/tests.as-root/rootfs-tarball-builds-rootfs-and-kernel.script
index ab0186c1..99b5b73c 100755
--- a/tests.as-root/rootfs-tarball-builds-rootfs-and-kernel.script
+++ b/tests.as-root/rootfs-tarball-builds-rootfs-and-kernel.script
@@ -100,4 +100,3 @@ do
         exit 1
     fi
 done
-
diff --git a/tests.as-root/syslinux-disk-builds-rootfs-and-kernel.script b/tests.as-root/syslinux-disk-builds-rootfs-and-kernel.script
index f996aaa4..746ca3c5 100755
--- a/tests.as-root/syslinux-disk-builds-rootfs-and-kernel.script
+++ b/tests.as-root/syslinux-disk-builds-rootfs-and-kernel.script
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 #
 # A system kind of syslinux-disk should create the disk image, plus a kernel
 # file.
@@ -20,6 +20,8 @@
 
 set -eu
 
+. "$SRCDIR/tests.as-root/lib"
+
 # We only support x86 systems with syslinux-disk.
 arch=$(uname -m)
 case "$arch" in
@@ -90,8 +92,9 @@ git add linux.morph
 
 git commit --quiet -m 'Make the kernel create a dummy zImage'
 
-"$SRCDIR/scripts/test-morph" \
-    build-morphology test:morphs-repo custom system > /dev/null
+
+rootfs=$("$SRCDIR/scripts/test-morph" --find-system-artifact \
+    build-morphology test:morphs-repo custom system)
 
 for suffix in kernel rootfs
 do
@@ -103,3 +106,12 @@ do
     fi
 done
 
+mkdir -p "$DATADIR/mnt"
+device=$(loopback_rootfs "$rootfs")
+mount "$device" "$DATADIR/mnt"
+trap 'cd /; umount -d "$device"' INT TERM ERR
+
+cd "$DATADIR/mnt"
+find | LC_ALL=C sort -u
+cd "$DATADIR"
+umount -d "$device"
diff --git a/tests.as-root/syslinux-disk-builds-rootfs-and-kernel.stdout b/tests.as-root/syslinux-disk-builds-rootfs-and-kernel.stdout
new file mode 100644
index 00000000..1b643875
--- /dev/null
+++ b/tests.as-root/syslinux-disk-builds-rootfs-and-kernel.stdout
@@ -0,0 +1,33 @@
+.
+./boot
+./boot/System.map
+./boot/vmlinuz
+./extlinux.conf
+./factory
+./factory-run
+./factory-run/baserock
+./factory-run/baserock/hello.meta
+./factory-run/baserock/linux.meta
+./factory-run/baserock/stratum.meta
+./factory-run/baserock/system-rootfs.meta
+./factory-run/bin
+./factory-run/bin/hello
+./factory-run/boot
+./factory-run/boot/System.map
+./factory-run/boot/vmlinuz
+./factory-run/etc
+./factory-run/etc/fstab
+./factory-run/extlinux.conf
+./factory/baserock
+./factory/baserock/hello.meta
+./factory/baserock/linux.meta
+./factory/baserock/stratum.meta
+./factory/baserock/system-rootfs.meta
+./factory/bin
+./factory/bin/hello
+./factory/boot
+./factory/boot/System.map
+./factory/boot/vmlinuz
+./factory/etc
+./factory/etc/fstab
+./factory/extlinux.conf
diff --git a/tests.as-root/tarball-image-is-sensible.script b/tests.as-root/tarball-image-is-sensible.script
index 717d8e36..6e4af67b 100755
--- a/tests.as-root/tarball-image-is-sensible.script
+++ b/tests.as-root/tarball-image-is-sensible.script
@@ -26,14 +26,11 @@ esac
 
 . "$SRCDIR/scripts/fix-committer-info"
 
-tar=$("$SRCDIR/morph" \
-        --no-default-config --config="$DATADIR/morph.conf" --verbose \
-    build-morphology test:morphs-repo tarball-links hello-tarball 2>&1 |
-    grep "system hello-tarball-rootfs is cached at" |
-    sed -nre "s/^.*system hello-tarball-rootfs is cached at (\S+)$/\1/p")
+tar=$("$SRCDIR/scripts/test-morph" --find-system-artifact \
+    build-morphology test:morphs-repo tarball-links hello-tarball)
 
 extracted="$DATADIR/extracted"
 mkdir -p "$extracted"
 cd "$extracted"
 tar -xf "$tar"
-find . -mindepth 1 | LC_ALL=C sort -u | xargs ls -dF
+find . -mindepth 1 | xargs ls -dF | LC_ALL=C sort -u
diff --git a/tests.as-root/tarball-image-is-sensible.stdout b/tests.as-root/tarball-image-is-sensible.stdout
index 34eb6752..46a9409e 100644
--- a/tests.as-root/tarball-image-is-sensible.stdout
+++ b/tests.as-root/tarball-image-is-sensible.stdout
@@ -1,5 +1,6 @@
 ./baserock/
 ./baserock/hello-stratum.meta
+./baserock/hello-tarball-rootfs.meta
 ./baserock/hello.meta
 ./baserock/link-stratum.meta
 ./baserock/links.meta