Add Mason scripts on top of distbuild system

The distbuild system can be configured to act as a CI controller.

Providing appropriate config makes it copy all the scripts and systemd
units out of the mason directory onto the target, such that it will
start building and testing the configured cluster morphology on boot.

9 files changed, 553 insertions, 0 deletions

diff --git a/distbuild-system-x86_64.morph b/distbuild-system-x86_64.morph
index 52d91626..7f234e6e 100644
--- a/distbuild-system-x86_64.morph
+++ b/distbuild-system-x86_64.morph
@@ -7,6 +7,7 @@ configuration-extensions:
 - install-files
 - distbuild
 - fstab
+- mason
 description: Morph distributed build node for x86_64
 kind: system
 name: distbuild-system-x86_64
diff --git a/mason.configure b/mason.configure
new file mode 100644
index 00000000..8528ad89
--- /dev/null
+++ b/mason.configure
@@ -0,0 +1,92 @@
+#!/bin/sh
+#
+# Copyright (C) 2014 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# This is a "morph deploy" configuration extension to fully configure
+# a Mason instance at deployment time. It uses the following variables
+# from the environment:
+#
+#   * MASON_CLUSTER_MORPHOLOGY
+#   * MASON_UPSTREAM_TROVE_ADDRESS
+#   * MASON_DEFINITIONS_REF
+#   * MASON_DISTBUILD_ARCH
+#   * MASON_TEST_HOST
+#   * TROVE_HOST
+#   * TROVE_ID
+#   * CONTROLLERHOST
+
+set -e
+
+ROOT="$1"
+
+if [ "$MASON_CLUSTER_MORPHOLOGY" = "" \
+     -o "$MASON_UPSTREAM_TROVE_ADDRESS" = "" \
+     -o "$MASON_DEFINITIONS_REF" = "" \
+     -o "$MASON_DISTBUILD_ARCH" = "" \
+     -o "$MASON_TEST_HOST" = "" ]; then
+    echo Not configuring as Mason, some options not defined
+    exit 0
+fi
+
+
+##########################################################################
+# Functions
+##########################################################################
+
+shellescape() {
+    echo "'$(echo "$1" | sed -e "s/'/'\\''/g")'"
+}
+
+
+##########################################################################
+# Generate config variable shell snippet
+##########################################################################
+
+MASON_CONFIG="$ROOT"/root/mason.conf
+
+echo "Creating $MASON_CONFIG"
+cat >>"$MASON_CONFIG" <<EOF
+#################### START OF DEPLOY TIME CONFIGURATION #######################
+
+UPSTREAM_TROVE_ADDRESS=$(shellescape "$MASON_UPSTREAM_TROVE_ADDRESS")
+DEFINITIONS_REF=$(shellescape "$MASON_DEFINITIONS_REF")
+DISTBUILD_ARCH=$(shellescape "$MASON_DISTBUILD_ARCH")
+DISTBUILD_CONTROLLER_ADDRESS=$(shellescape "$CONTROLLERHOST")
+DISTBUILD_TROVE_ADDRESS=$(shellescape "$TROVE_HOST")
+TROVE_ID=$(shellescape "$TROVE_ID")
+BUILD_CLUSTER_MORPHOLOGY=$(shellescape "$MASON_CLUSTER_MORPHOLOGY")
+TEST_VM_HOST_SSH_URL=$(shellescape "$MASON_TEST_HOST")
+
+##################### END OF DEPLOY TIME CONFIGURATION ########################
+EOF
+
+
+##########################################################################
+# Copy Mason files into root filesystem
+##########################################################################
+
+cp mason/mason.sh "$ROOT"/root/mason.sh
+cp mason/mason-report.sh "$ROOT"/root/mason-report.sh
+
+cp mason/mason.timer "$ROOT"/etc/systemd/system/mason.timer
+ln -s ../mason.timer "$ROOT"/etc/systemd/system/multi-user.target.wants/mason.timer
+
+cp mason/mason.service "$ROOT"/etc/systemd/system/mason.service
+ln -s ../mason.service "$ROOT"/etc/systemd/system/multi-user.target.wants/mason.service
+
+cp mason/httpd.service "$ROOT"/etc/systemd/system/httpd.service
+ln -s ../httpd.service "$ROOT"/etc/systemd/system/multi-user.target.wants/httpd.service
+mkdir -p /srv/mason
diff --git a/mason.morph b/mason.morph
new file mode 100644
index 00000000..d6052146
--- /dev/null
+++ b/mason.morph
@@ -0,0 +1,58 @@
+name: example-mason-cluster
+kind: cluster
+description: |
+  This is a template cluster morphology that can be adapted to set up a
+  Mason.  Masons are composed of a trove and a distbuild system.
+
+  It is suggested that you use mason/mason-generator.sh to adapt this
+  template to suit your needs.  It also handles the generation of
+  keys to let the systems communicate.
+systems:
+- morph: trove-system-x86_64
+  deploy:
+    red-box-v1-trove:
+      type: kvm
+      location: kvm+ssh://vm-user@vm-host/red-box-v1-trove/vm-path/red-box-v1-trove.img
+      VERSION_LABEL: 45
+      DISK_SIZE: 100G
+      RAM_SIZE: 8G
+      VCPUS: 2
+      HOSTNAME: red-box-v1-trove
+      TROVE_ID: red-box-v1-trove
+      TROVE_HOST: red-box-v1
+      TROVE_COMPANY: Company name goes here
+      LORRY_SSH_KEY: ssh_keys/lorry.key
+      UPSTREAM_TROVE: upstream-trove
+      UPSTREAM_TROVE_PROTOCOL: http
+      TROVE_ADMIN_USER: adminuser
+      TROVE_ADMIN_EMAIL: adminuser@example.com
+      TROVE_ADMIN_NAME: Nobody
+      TROVE_ADMIN_SSH_PUBKEY: ssh_keys/id_rsa.pub
+      WORKER_SSH_PUBKEY: ssh_keys/worker.key.pub
+      MASON_SSH_PUBKEY: ssh_keys/mason.key.pub
+      AUTOSTART: yes
+- morph: distbuild-system-x86_64
+  deploy-defaults:
+    TROVE_ID: red-box-v1-trove
+    TROVE_HOST: red-box-v1-trove.example.com
+    CONTROLLERHOST: red-box-v1-controller
+    DISTBUILD_CONTROLLER: no
+    DISTBUILD_WORKER: yes
+    VCPUS: 2
+    RAM_SIZE: 8G
+    #FSTAB_SRC: LABEL=src /srv/distbuild auto defaults,rw,noatime 0 2
+    INSTALL_FILES: distbuild/manifest
+    WORKER_SSH_KEY: ssh_keys/worker.key
+  deploy:
+    red-box-v1-controller:
+      type: kvm
+      location: kvm+ssh://vm-user@vm-host/red-box-v1-controller/vm-path/red-box-v1-controller.img
+      DISK_SIZE: 60G
+      DISTBUILD_CONTROLLER: yes
+      HOSTNAME: red-box-v1-controller
+      WORKERS: red-box-v1-controller
+      MASON_CLUSTER_MORPHOLOGY: ci.morph
+      MASON_DEFINITIONS_REF: master
+      MASON_DISTBUILD_ARCH: x86_64
+      MASON_UPSTREAM_TROVE_ADDRESS: upstream-trove
+      MASON_TEST_HOST: vm-user@vm-host:/vm-path/
diff --git a/mason/httpd.service b/mason/httpd.service
new file mode 100644
index 00000000..7572b732
--- /dev/null
+++ b/mason/httpd.service
@@ -0,0 +1,10 @@
+[Unit]
+Description=HTTP server for Mason
+After=network.target
+
+[Service]
+User=root
+ExecStart=/usr/sbin/httpd -f -p 80 -h /srv/mason
+
+[Install]
+WantedBy=multi-user.target
diff --git a/mason/mason-generator.sh b/mason/mason-generator.sh
new file mode 100755
index 00000000..a0b2fb0a
--- /dev/null
+++ b/mason/mason-generator.sh
@@ -0,0 +1,94 @@
+#!/bin/sh
+
+set -e
+
+if [ "$1" == "-h" -o "$1" == "--help" ]; then
+  echo "Usage:"
+  echo "  `basename $0` HOST_PREFIX UPSTREAM_TROVE_HOSTNAME VM_USER VM_HOST VM_PATH [HOST_POSTFIX]"
+  echo ""
+  echo "Where:"
+  echo "  HOST_PREFIX              -- Name of your Mason instance"
+  echo "                              e.g. \"my-mason\" to produce hostnames:"
+  echo "                              my-mason-trove and my-mason-controller"
+  echo "  UPSTREAM_TROVE_HOSTNAME  -- Upstream trove's hostname"
+  echo "  VM_USER                  -- User on VM host for VM deployment"
+  echo "  VM_HOST                  -- VM host for VM deployment"
+  echo "  VM_PATH                  -- Path to store VM images in on VM host"
+  echo "  HOST_POSTFIX             -- e.g. \".example.com\" to get"
+  echo "                              my-mason-trove.example.com"
+  echo ""
+  echo "This script makes deploying a Mason system simpler by automating"
+  echo "the generation of keys for the systems to use, building of the"
+  echo "systems, filling out the mason deployment cluster morphology"
+  echo "template with useful values, and finally deploying the systems."
+  echo ""
+  echo "To ensure that the deployed system can deploy test systems, you"
+  echo "must supply an ssh key to the VM host.  Do so with the following"
+  echo "command:"
+  echo "  ssh-copy-id -i ssh_keys-HOST_PREFIX/mason.key.pub VM_USER@VM_HOST"
+  echo ""
+  exit 0
+fi
+
+
+HOST_PREFIX=$1
+UPSTREAM_TROVE=$2
+VM_USER=$3
+VM_HOST=$4
+VM_PATH=$5
+HOST_POSTFIX=$6
+
+sedescape() {
+    # Escape all non-alphanumeric characters
+    printf "%s\n" "$1" | sed -e 's/\W/\\&/g'
+}
+
+
+##############################################################################
+# Key generation
+##############################################################################
+
+mkdir "ssh_keys-${HOST_PREFIX}"
+cd "ssh_keys-${HOST_PREFIX}"
+ssh-keygen -t rsa -b 2048 -f mason.key -C mason@TROVE_HOST -N ''
+ssh-keygen -t rsa -b 2048 -f lorry.key -C lorry@TROVE_HOST -N ''
+ssh-keygen -t rsa -b 2048 -f worker.key -C worker@TROVE_HOST -N ''
+ssh-keygen -t rsa -b 2048 -f id_rsa -C trove-admin@TROVE_HOST -N ''
+cd ../
+
+
+##############################################################################
+# Mason setup
+##############################################################################
+
+cp mason.morph mason-${HOST_PREFIX}.morph
+
+sed -i "s/red-box-v1/$(sedescape "$HOST_PREFIX")/g" "mason-$HOST_PREFIX.morph"
+sed -i "s/ssh_keys/ssh_keys-$(sedescape "$HOST_PREFIX")/g" "mason-$HOST_PREFIX.morph"
+sed -i "s/upstream-trove/$(sedescape "$UPSTREAM_TROVE")/" "mason-$HOST_PREFIX.morph"
+sed -i "s/vm-user/$(sedescape "$VM_USER")/g" "mason-$HOST_PREFIX.morph"
+sed -i "s/vm-host/$(sedescape "$VM_HOST")/g" "mason-$HOST_PREFIX.morph"
+sed -i "s/vm-path/$(sedescape "$VM_PATH")/g" "mason-$HOST_PREFIX.morph"
+sed -i "s/\.example\.com/$(sedescape "$HOST_POSTFIX")/g" "mason-$HOST_PREFIX.morph"
+
+
+##############################################################################
+# System building
+##############################################################################
+
+morph build trove-system-x86_64
+morph build distbuild-system-x86_64
+
+
+##############################################################################
+# System deployment
+##############################################################################
+
+morph deploy mason-${HOST_PREFIX}.morph
+
+
+##############################################################################
+# Cleanup
+##############################################################################
+
+rm mason-${HOST_PREFIX}.morph
diff --git a/mason/mason-report.sh b/mason/mason-report.sh
new file mode 100755
index 00000000..d6cf0c19
--- /dev/null
+++ b/mason/mason-report.sh
@@ -0,0 +1,209 @@
+#!/bin/bash
+
+set -x
+
+. /root/mason.conf
+
+REPORT_PATH=/root/report.html
+SERVER_PATH=/srv/mason
+
+sed_escape() {
+    printf "%s\n" "$1" | sed -e 's/\W/\\&/g'
+}
+
+create_report() {
+cat > $REPORT_PATH <<'EOF'
+<html>
+<head>
+<meta charset="UTF-8">
+<meta http-equiv="refresh" content="10">
+<style>
+html, body {
+	margin: 0;
+	padding: 0;
+}
+p.branding {
+	background: black;
+	color: #fff;
+	padding: 0.4em;
+	margin: 0;
+	font-weight: bold;
+}
+h1 {
+	background: #225588;
+	color: white;
+	margin: 0;
+	padding: 0.6em;
+}
+table {
+	width: 90%;
+	margin: 1em auto 6em auto;
+	border: 1px solid black;
+	border-spacing: 0;
+}
+table tr.headings {
+	background: #555;
+	color: white;
+}
+table tr.pass {
+	background: #aaffaa;
+}
+table tr.pass:hover {
+	background: #bbffbb;
+}
+table tr.fail {
+	background: #ffaaaa;
+}
+table tr.fail:hover {
+	background: #ffbbbb;
+}
+table tr.headings th {
+	font-weight: bold;
+	text-align: left;
+	padding: 3px 2px;
+}
+table td {
+	padding: 2px;
+}
+td.result {
+	font-weight: bold;
+	text-transform: uppercase;
+}
+td.result a {
+	text-decoration: none;
+}
+td.result a:before {
+	content: "➫ ";
+}
+tr.pass td.result a {
+	color: #252;
+}
+tr.pass td.result a:hover {
+	color: #373;
+}
+tr.fail td.result a {
+	color: #622;
+}
+tr.fail td.result a:hover {
+	color: #933;
+}
+td.ref {
+	font-family: monospace;
+}
+td.ref a {
+	color: #333;
+}
+td.ref a:hover {
+	color: #555;
+}
+table tr.pass td, table tr.fail td {
+	border-top: solid white 1px;
+}
+p {
+	margin: 1.3em;
+}
+code {
+	padding: 0.3em 0.5em;
+	background: #eee;
+	border: 1px solid #bbb;
+	border-radius: 1em;
+}
+#footer {
+	margin: 0;
+	background: #aaa;
+	color: #222;
+	border-top: #888 1px solid;
+	font-size: 80%;
+	padding: 0;
+	text-align: right;
+	position: fixed;
+	bottom: 0;
+	width: 100%;
+}
+</style>
+</head>
+<body>
+<p class="branding">Mason</p>
+<h1>Baserock: Continuous Delivery</h1>
+<p>Build log of changes to <code>BRANCH</code> from <code>TROVE</code>.  Most recent first.</p>
+<table>
+<tr class="headings">
+	<th>Started</th>
+	<th>Ref</th>
+	<th>Duration</th>
+	<th>Result</th>
+</tr>
+<!--INSERTION POINT-->
+</table>
+<div id="footer">
+<p>Generated by Mason</p>
+</div>
+</body>
+</html>
+EOF
+
+    sed -i 's/BRANCH/'"$(sed_escape "$1")"'/' $REPORT_PATH
+    sed -i 's/TROVE/'"$(sed_escape "$2")"'/' $REPORT_PATH
+}
+
+update_report() {
+    # Give function params sensible names
+    build_start_time="$1"
+    build_trove_host="$2"
+    build_ref="$3"
+    build_sha1="$4"
+    build_duration="$5"
+    build_result="$6"
+
+    # Generate template if report file is not there
+    if [ ! -f $REPORT_PATH ]; then
+        create_report $build_ref $build_trove_host
+    fi
+
+    # Build table row for insertion into report file
+    msg='<tr class="'"${build_result}"'"><td>'"${build_start_time}"'</td><td class="ref"><a href="http://'"${build_trove_host}"'/cgi-bin/cgit.cgi/baserock/baserock/definitions.git/commit/?h='"${build_ref}"'&id='"${build_sha1}"'">'"${build_sha1}"'</a></td><td>'"${build_duration}s"'</td><td class="result"><a href="log/'"${build_sha1}"'--'"${build_start_time}"'.log">'"${build_result}"'</a></td></tr>'
+
+    # Insert report line, newest at top
+    sed -i 's/<!--INSERTION POINT-->/<!--INSERTION POINT-->\n'"$(sed_escape "$msg")"'/' $REPORT_PATH
+}
+
+START_TIME=`date +%Y-%m-%d\ %T`
+
+logfile="$(mktemp)"
+/root/mason.sh 2>&1 | tee "$logfile"
+case "${PIPESTATUS[0]}" in
+0)
+    RESULT=pass
+    ;;
+33)
+    RESULT=skip
+    ;;
+*)
+    RESULT=fail
+    ;;
+esac
+
+# TODO: Update page with last executed time
+if [ "$RESULT" = skip ]; then
+    rm "$logfile"
+    exit 0
+fi
+
+DURATION=$(( $(date +%s) - $(date --date="$START_TIME" +%s) ))
+SHA1="$(cd "ws/$DEFINITIONS_REF/$UPSTREAM_TROVE_ADDRESS/baserock/baserock/definitions" && git rev-parse HEAD)"
+
+update_report "$START_TIME" \
+              "$DISTBUILD_TROVE_ADDRESS" \
+              "$DEFINITIONS_REF" \
+              "$SHA1" \
+              "$DURATION" \
+              "$RESULT"
+
+
+#
+# Copy report into server directory
+#
+
+cp "$REPORT_PATH" "$SERVER_PATH/index.html"
+mkdir /srv/mason/log
+mv "$logfile" /srv/mason/log/"$SHA1--$START_TIME.log"
diff --git a/mason/mason.service b/mason/mason.service
new file mode 100644
index 00000000..16b5dc3f
--- /dev/null
+++ b/mason/mason.service
@@ -0,0 +1,9 @@
+[Unit]
+Description=Mason: Continuous Delivery Service
+
+[Service]
+User=root
+ExecStart=/root/mason-report.sh
+
+[Install]
+WantedBy=multi-user.target
diff --git a/mason/mason.sh b/mason/mason.sh
new file mode 100755
index 00000000..dfed71f7
--- /dev/null
+++ b/mason/mason.sh
@@ -0,0 +1,70 @@
+#!/bin/sh
+
+set -e
+set -x
+
+# Load our deployment config
+. /root/mason.conf
+
+if [ ! -e ws ]; then
+    morph init ws
+fi
+cd ws
+
+definitions_repo="$DEFINITIONS_REF"/"$UPSTREAM_TROVE_ADDRESS"/baserock/baserock/definitions
+if [ ! -e "$definitions_repo" ]; then
+    morph checkout git://"$UPSTREAM_TROVE_ADDRESS"/baserock/baserock/definitions.git "$DEFINITIONS_REF"
+    cd "$definitions_repo"
+    git config user.name "$TROVE_ID"-mason
+    git config user.email "$TROVE_ID"-mason@$(hostname)
+else
+    cd "$definitions_repo"
+    SHA1_PREV="$(git rev-parse HEAD)"
+fi
+
+git remote update origin
+git clean -fxd
+git reset --hard origin/"$DEFINITIONS_REF"
+
+SHA1="$(git rev-parse HEAD)"
+
+if [ -f "$HOME/success" ] && [ "$SHA1" = "$SHA1_PREV" ]; then
+    echo INFO: No changes to "$DEFINITIONS_REF", nothing to do
+    exit 33
+fi
+
+rm -f "$HOME/success"
+
+echo INFO: Mason building: $DEFINITIONS_REF at $SHA1
+
+"scripts/release-build" --no-default-configs \
+        --trove-host "$UPSTREAM_TROVE_ADDRESS" \
+	--controllers "$DISTBUILD_ARCH:$DISTBUILD_CONTROLLER_ADDRESS" \
+	"$BUILD_CLUSTER_MORPHOLOGY"
+
+releases_made="$(cd release && ls | wc -l)"
+if [ "$releases_made" = 0 ]; then
+    echo ERROR: No release images created
+    exit 1
+else
+    echo INFO: Created "$releases_made" release images
+fi
+
+"scripts/release-test" \
+	--deployment-host "$DISTBUILD_ARCH":"$TEST_VM_HOST_SSH_URL" \
+	--trove-host "$DISTBUILD_TROVE_ADDRESS" \
+	--trove-id "$TROVE_ID" \
+	"$BUILD_CLUSTER_MORPHOLOGY"
+
+"scripts/release-upload" --build-trove-host "$DISTBUILD_TROVE_ADDRESS" \
+	--arch "$DISTBUILD_ARCH" \
+	--log-level=debug --log="$HOME"/release-upload.log \
+	--public-trove-host "$UPSTREAM_TROVE_ADDRESS" \
+	--public-trove-username root \
+	--public-trove-artifact-dir /home/cache/artifacts \
+	--no-upload-release-artifacts \
+	"$BUILD_CLUSTER_MORPHOLOGY"
+
+echo INFO: Artifact upload complete for $DEFINITIONS_REF at $SHA1
+
+touch "$HOME/success"
diff --git a/mason/mason.timer b/mason/mason.timer
new file mode 100644
index 00000000..107dff97
--- /dev/null
+++ b/mason/mason.timer
@@ -0,0 +1,10 @@
+[Unit]
+Description=Runs Mason continually with 1 min between calls
+
+[Timer]
+#Time between Mason finishing and calling it again
+OnUnitActiveSec=1min
+Unit=mason.service
+
+[Install]
+WantedBy=multi-user.target