Merge branch 'baserock/richardmaw/S10721-img-pkg-v2'

Reviewed-by: Sam Thursfield
Reviewed-by: Lars Wirzenius

7 files changed, 349 insertions, 1 deletions

diff --git a/image-package-example.morph b/image-package-example.morph
new file mode 100644
index 00000000..eeb5b3f2
--- /dev/null
+++ b/image-package-example.morph
@@ -0,0 +1,12 @@
+name: image-package-example
+kind: cluster
+description: |
+  Packaged system and script for installing it, for deferred instantiation.
+systems:
+- morph: base-system-x86_32-generic
+  deploy:
+    imgpkg:
+      type: image-package
+      location: image-package-example.tar
+      BOOTLOADER_BLOBS: "/usr/share/syslinux/mbr.bin"
+      INCLUDE_SCRIPTS: "image-package-example/make-disk-image.sh.in:image-package-example/disk-install.sh.in:image-package-example/common.sh.in"
diff --git a/image-package-example/README b/image-package-example/README
new file mode 100644
index 00000000..c1322f25
--- /dev/null
+++ b/image-package-example/README
@@ -0,0 +1,9 @@
+Image package example scripts
+=============================
+
+These are scripts used to create disk images or install the system onto
+an existing disk.
+
+This is also implemented independently for the rawdisk.write write
+extension; see morphlib.writeexts.WriteExtension.create_local_system()
+for a similar, python implementation.
diff --git a/image-package-example/common.sh.in b/image-package-example/common.sh.in
new file mode 100644
index 00000000..9a7389a7
--- /dev/null
+++ b/image-package-example/common.sh.in
@@ -0,0 +1,72 @@
+#!/bin/false
+# Script library to be used by disk-install.sh and make-disk-image.sh
+
+status(){
+    echo "$@"
+}
+
+info(){
+    echo "$@" >&2
+}
+
+warn(){
+    echo "$@" >&2
+}
+
+extract_rootfs(){
+    tar -C "$1" -xf @@ROOTFS_TAR_PATH@@ .
+}
+
+make_disk_image(){
+    truncate --size "$1" "$2"
+}
+
+format_disk(){
+    local disk="$1"
+    mkfs.ext4 -F -L rootfs "$disk"
+}
+
+install_fs_config(){
+    local mountpoint="$1"
+    local rootdisk="${2-/dev/vda}"
+    cat >>"$mountpoint/etc/fstab" <<EOF
+$rootdisk / ext4 rw,errors=remount-ro 0 0
+EOF
+    install -D -m 644 /proc/self/fd/0 "$mountpoint/boot/extlinux.conf" <<EOF
+DEFAULT baserock
+LABEL   baserock
+SAY     Booting Baserock
+LINUX   /boot/vmlinuz
+APPEND  root=$rootdisk
+EOF
+}
+
+install_bootloader(){
+    local disk="$1"
+    local mountpoint="$2"
+    dd if=@@IMAGE_DIR@@/mbr.bin conv=notrunc bs=440 count=1 of="$disk"
+    extlinux --install "$mountpoint/boot"
+}
+
+loop_file(){
+    losetup --show --find "$1"
+}
+unloop_file(){
+    #losetup --detach "$1"
+    # unlooping handled by umount -d, for busybox compatibility
+    true
+}
+
+temp_mount(){
+    local mp="$(mktemp -d)"
+    if ! mount "$@" "$mp"; then
+        rmdir "$mp"
+        return 1
+    fi
+    echo "$mp"
+}
+untemp_mount(){
+    # Unmount and detach in one step for busybox compatibility
+    umount -d "$1"
+    rmdir "$1"
+}
diff --git a/image-package-example/disk-install.sh.in b/image-package-example/disk-install.sh.in
new file mode 100644
index 00000000..bc8e0e67
--- /dev/null
+++ b/image-package-example/disk-install.sh.in
@@ -0,0 +1,51 @@
+#!/bin/sh
+# Script for writing the system to an existing disk.
+# This formats the disk, extracts the rootfs to it, installs the
+# bootloader, and ensures there's appropriate configuration for the
+# bootloader, kernel and userland to agree what the rootfs is.
+
+set -eu
+
+usage(){
+    cat <<EOF
+usage: $0 DISK [TARGET_DISK]
+
+DISK:        Where the disk appears on your development machine
+TARGET_DISK: What the disk will appear as on the target machine
+EOF
+}
+
+. @@SCRIPT_DIR@@/common.sh
+
+if [ "$#" -lt 1 -o "$#" -gt 2 ]; then
+    usage
+    exit 1
+fi
+
+DISK="$1"
+TARGET_DISK="${1-/dev/sda}"
+
+status Formatting "$DISK" as ext4
+format_disk "$DISK"
+(
+    info Mounting "$DISK"
+    MP="$(temp_mount -t ext4 "$DISK")"
+    info Mounted "$DISK" to "$MP"
+    set +e
+    (
+        set -e
+        info Copying rootfs onto disk
+        extract_rootfs "$MP"
+        info Configuring disk paths
+        install_fs_config "$MP" "$TARGET_DISK"
+        info Installing bootloader
+        install_bootloader "$DISK" "$MP"
+    )
+    ret="$?"
+    if [ "$ret" != 0 ]; then
+        warn Filling rootfs failed with "$ret"
+    fi
+    info Unmounting "$DISK" from "$MP" and removing "$MP"
+    untemp_mount "$MP"
+    exit "$ret"
+)
diff --git a/image-package-example/make-disk-image.sh.in b/image-package-example/make-disk-image.sh.in
new file mode 100644
index 00000000..61264fa0
--- /dev/null
+++ b/image-package-example/make-disk-image.sh.in
@@ -0,0 +1,36 @@
+#!/bin/sh
+# Script for writing the system to a disk image file.
+# This creates a file of the right size, attaches it to a loop device,
+# then hands the rest of the work off to disk-install.sh
+
+usage(){
+    cat <<EOF
+usage: $0 FILENAME SIZE [TARGET_DISK]
+
+FILENAME:    Location to write the disk image to
+SIZE:        Size to create the disk image with
+TARGET_DISK: What the disk will appear as on the target machine
+EOF
+}
+
+. @@SCRIPT_DIR@@/common.sh
+
+if [ "$#" -lt 2 -o "$#" -gt 3 ]; then
+    usage
+    exit 1
+fi
+
+DISK_IMAGE="$1"
+DISK_SIZE="$2"
+TARGET_DISK="${3-/dev/vda}"
+
+make_disk_image "$DISK_SIZE" "$DISK_IMAGE"
+
+(
+    LOOP="$(loop_file "$DISK_IMAGE")"
+    set +e
+    @@SCRIPT_DIR@@/disk-install.sh "$DISK_IMAGE" "$TARGET_DISK"
+    ret="$?"
+    unloop_file "$LOOP"
+    exit "$ret"
+)
diff --git a/image-package.write b/image-package.write
new file mode 100755
index 00000000..15ceadcf
--- /dev/null
+++ b/image-package.write
@@ -0,0 +1,168 @@
+#!/bin/sh
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# =*= License: GPL-2 =*=
+#
+#
+# This is a write extension for making a package that can be used to
+# install the produced system. Ideally we'd instead have Baserock
+# everywhere to do the deployment, but we need to support this workflow
+# until that is possible.
+#
+# This write extension produces a tarball, which contains:
+#    - a tarball of the configured system root file system
+#    - any supporting files listed in BOOTLOADER_BLOBS
+#    - any supporting scripts, generated from templates listed in
+#      INCLUDE_SCRIPTS
+#
+# The extension requires the following environment variables to be set:
+#
+#   * BOOTLOADER_BLOBS: files to include besides rootfs tarball,
+#                       paths are relative to the root of the built rootfs
+#                       works on any kind of file in the rootfs, named
+#                       BOOTLOADER_BLOBS since that's the common use-case
+#                       :-separated by default
+#   * INCLUDE_SCRIPTS:  script templates that are included in the package
+#                       after being filled out
+#                       file paths are relative to the definitions repository
+#                       :-separated by default
+#
+# The script templates may contain any of the following strings, which
+# will be replaced with a string which will expand to the appropriate
+# value as a shell word:
+#    - @@SCRIPT_DIR@@:      the path the script files are installed to
+#    - @@IMAGE_DIR@@:       the path BOOTLOADER_BLOBS are installed to
+#    - @@ROOTFS_TAR_PATH@@: path to the rootfs tarball
+#
+# The interpolated strings may run commands dependant on the current
+# working directory, so if `cd` is required, bind these values to a
+# variable beforehand.
+#
+# The following optional variables can be set as well:
+#
+#   * INCLUDE_SCRIPTS_SEPARATOR: character to separate INCLUDE_SCRIPTS with (default: :)
+#   * BOOTLOADER_BLOBS_SEPARATOR: character to separate BOOTLOADER_BLOBS with (default: :)
+#   * SCRIPT_SUBDIR: where in the package processed scripts are installed to (default: tools)
+#   * IMAGE_SUBDIR:  where in the package BOOTLOADER_BLOBS are copied to (default: image_files)
+#   * ROOTFS_TAR:    name to call the rootfs tarball inside IMAGE_SUBDIR (default: rootfs.tar)
+#   * OUTPUT_COMPRESS: compression used for output tarball (default: none)
+#   * ROOTFS_COMPRESS: compression used for rootfs (default: none)
+
+set -eu
+
+die(){
+    echo "$@" >&2
+    exit 1
+}
+
+warn(){
+    echo "$@" >&2
+}
+
+info(){
+    echo "$@" >&2
+}
+
+shellescape(){
+    echo "'$(echo "$1" | sed -e "s/'/'\\''/g")'"
+}
+
+sedescape(){
+    # Escape the passed in string so it can be safely interpolated into
+    # a sed expression as a literal value.
+    echo "$1" | sed -e 's/[\/&]/\\&/g'
+}
+
+ROOTDIR="$1"
+OUTPUT_TAR="$2"
+td="$(mktemp -d)"
+IMAGE_SUBDIR="${IMAGE_SUBDIR-image_files}"
+SCRIPT_SUBDIR="${SCRIPT_SUBDIR-tools}"
+ROOTFS_TAR="${ROOTFS_TAR-rootfs.tar}"
+
+# Generate shell snippets that will expand to paths to various resources
+# needed by the scripts.
+# They expand to a single shell word, so constructs like the following work
+#     SCRIPT_DIR=@@SCRIPT_DIR@@
+#     dd if="$SCRIPT_DIR/mbr" of="$disk" count=1
+#     tar -C "$mountpoint" -xf @@ROOTFS_TAR_PATH@@ .
+find_script_dir='"$(readlink -f "$(dirname "$0")")"'
+image_dir="$find_script_dir/../$(shellescape "$IMAGE_SUBDIR")"
+rootfs_tar_path="$image_dir/$(shellescape "$ROOTFS_TAR")"
+
+install_script(){
+    local source_file="$1"
+    local output_dir="$2"
+    local target_file="$output_dir/$SCRIPT_SUBDIR/$(basename "$source_file" .in)"
+    sed -e "s/@@SCRIPT_DIR@@/$(sedescape "$find_script_dir")/g" \
+        -e "s/@@IMAGE_DIR@@/$(sedescape "$image_dir")/g" \
+        -e "s/@@ROOTFS_TAR_PATH@@/$(sedescape "$rootfs_tar_path")/g" \
+        "$source_file" \
+    |   install -D -m 755 /proc/self/fd/0 "$target_file"
+}
+
+install_scripts(){
+    local output_dir="$1"
+    (
+        IFS="${INCLUDE_SCRIPTS_SEPARATOR-:}"
+        for script in $INCLUDE_SCRIPTS; do
+            local script_path="$(pwd)/$script"
+            if [ ! -e "$script_path" ]; then
+                warn Script "$script" not found, ignoring
+                continue
+            fi
+            install_script "$script" "$output_dir"
+        done
+    )
+}
+
+install_bootloader_blobs(){
+    local output_dir="$1"
+    local image_dir="$output_dir/$IMAGE_SUBDIR"
+    (
+        IFS="${BOOTLOADER_BLOBS_SEPARATOR-:}"
+        for blob in $BOOTLOADER_BLOBS; do
+            local blob_path="$ROOTDIR/$blob"
+            if [ ! -e "$blob_path" ]; then
+                warn Bootloader blob "$blob" not found, ignoring
+                continue
+            fi
+            install -D -m644 "$blob_path" "$image_dir/$(basename "$blob_path")"
+        done
+    )
+}
+
+# Determine a basename for our directory as the same as our tarball with
+# extensions removed. This is needed, since tarball packages usually
+# have a base directory of its contents, rather then extracting into the
+# current directory.
+output_dir="$(basename "$OUTPUT_TAR")"
+for ext in .xz .bz2 .gzip .gz .tgz .tar; do
+    output_dir="${output_dir%$ext}"
+done
+
+info Installing scripts
+install_scripts "$td/$output_dir"
+
+info Installing bootloader blobs
+install_bootloader_blobs "$td/$output_dir"
+
+info Writing rootfs tar to "$IMAGE_SUBDIR/$ROOTFS_TAR"
+tar -C "$ROOTDIR" -c . \
+| sh -c "${ROOTFS_COMPRESS-cat}" >"$td/$output_dir/$IMAGE_SUBDIR/$ROOTFS_TAR"
+
+info Writing image package tar to "$OUTPUT_TAR"
+tar -C "$td" -c "$output_dir" | sh -c "${OUTPUT_COMPRESS-cat}" >"$OUTPUT_TAR"
diff --git a/sdk.write b/sdk.write
index 83ed54d4..3b07ea9d 100755
--- a/sdk.write
+++ b/sdk.write
@@ -1,5 +1,5 @@
 #!/bin/sh
-# Copyright (C) 2013  Codethink Limited
+# Copyright (C) 2014  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by