From 8867889800efcb321d0061893fe6092f8932c4d3 Mon Sep 17 00:00:00 2001 From: Michael Drake Date: Tue, 5 Aug 2014 14:52:57 +0000 Subject: Add Mason scripts on top of distbuild system The distbuild system can be configured to act as a CI controller. Providing appropriate config makes it copy all the scripts and systemd units out of the mason directory onto the target, such that it will start building and testing the configured cluster morphology on boot. --- distbuild-system-x86_64.morph | 1 + mason.configure | 92 +++++++++++++++++++ mason.morph | 58 ++++++++++++ mason/httpd.service | 10 ++ mason/mason-generator.sh | 94 +++++++++++++++++++ mason/mason-report.sh | 209 ++++++++++++++++++++++++++++++++++++++++++ mason/mason.service | 9 ++ mason/mason.sh | 70 ++++++++++++++ mason/mason.timer | 10 ++ 9 files changed, 553 insertions(+) create mode 100644 mason.configure create mode 100644 mason.morph create mode 100644 mason/httpd.service create mode 100755 mason/mason-generator.sh create mode 100755 mason/mason-report.sh create mode 100644 mason/mason.service create mode 100755 mason/mason.sh create mode 100644 mason/mason.timer diff --git a/distbuild-system-x86_64.morph b/distbuild-system-x86_64.morph index 52d91626..7f234e6e 100644 --- a/distbuild-system-x86_64.morph +++ b/distbuild-system-x86_64.morph @@ -7,6 +7,7 @@ configuration-extensions: - install-files - distbuild - fstab +- mason description: Morph distributed build node for x86_64 kind: system name: distbuild-system-x86_64 diff --git a/mason.configure b/mason.configure new file mode 100644 index 00000000..8528ad89 --- /dev/null +++ b/mason.configure @@ -0,0 +1,92 @@ +#!/bin/sh +# +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# This is a "morph deploy" configuration extension to fully configure +# a Mason instance at deployment time. It uses the following variables +# from the environment: +# +# * MASON_CLUSTER_MORPHOLOGY +# * MASON_UPSTREAM_TROVE_ADDRESS +# * MASON_DEFINITIONS_REF +# * MASON_DISTBUILD_ARCH +# * MASON_TEST_HOST +# * TROVE_HOST +# * TROVE_ID +# * CONTROLLERHOST + +set -e + +ROOT="$1" + +if [ "$MASON_CLUSTER_MORPHOLOGY" = "" \ + -o "$MASON_UPSTREAM_TROVE_ADDRESS" = "" \ + -o "$MASON_DEFINITIONS_REF" = "" \ + -o "$MASON_DISTBUILD_ARCH" = "" \ + -o "$MASON_TEST_HOST" = "" ]; then + echo Not configuring as Mason, some options not defined + exit 0 +fi + + +########################################################################## +# Functions +########################################################################## + +shellescape() { + echo "'$(echo "$1" | sed -e "s/'/'\\''/g")'" +} + + +########################################################################## +# Generate config variable shell snippet +########################################################################## + +MASON_CONFIG="$ROOT"/root/mason.conf + +echo "Creating $MASON_CONFIG" +cat >>"$MASON_CONFIG" < $REPORT_PATH <<'EOF' + + + + + + + +

Mason

+

Baserock: Continuous Delivery

+

Build log of changes to BRANCH from TROVE. Most recent first.

+ + + + + + + + +
StartedRefDurationResult
+ + + +EOF + + sed -i 's/BRANCH/'"$(sed_escape "$1")"'/' $REPORT_PATH + sed -i 's/TROVE/'"$(sed_escape "$2")"'/' $REPORT_PATH +} + +update_report() { + # Give function params sensible names + build_start_time="$1" + build_trove_host="$2" + build_ref="$3" + build_sha1="$4" + build_duration="$5" + build_result="$6" + + # Generate template if report file is not there + if [ ! -f $REPORT_PATH ]; then + create_report $build_ref $build_trove_host + fi + + # Build table row for insertion into report file + msg=''"${build_start_time}"''"${build_sha1}"''"${build_duration}s"''"${build_result}"'' + + # Insert report line, newest at top + sed -i 's//\n'"$(sed_escape "$msg")"'/' $REPORT_PATH +} + +START_TIME=`date +%Y-%m-%d\ %T` + +logfile="$(mktemp)" +/root/mason.sh 2>&1 | tee "$logfile" +case "${PIPESTATUS[0]}" in +0) + RESULT=pass + ;; +33) + RESULT=skip + ;; +*) + RESULT=fail + ;; +esac + +# TODO: Update page with last executed time +if [ "$RESULT" = skip ]; then + rm "$logfile" + exit 0 +fi + +DURATION=$(( $(date +%s) - $(date --date="$START_TIME" +%s) )) +SHA1="$(cd "ws/$DEFINITIONS_REF/$UPSTREAM_TROVE_ADDRESS/baserock/baserock/definitions" && git rev-parse HEAD)" + +update_report "$START_TIME" \ + "$DISTBUILD_TROVE_ADDRESS" \ + "$DEFINITIONS_REF" \ + "$SHA1" \ + "$DURATION" \ + "$RESULT" + + +# +# Copy report into server directory +# + +cp "$REPORT_PATH" "$SERVER_PATH/index.html" +mkdir /srv/mason/log +mv "$logfile" /srv/mason/log/"$SHA1--$START_TIME.log" diff --git a/mason/mason.service b/mason/mason.service new file mode 100644 index 00000000..16b5dc3f --- /dev/null +++ b/mason/mason.service @@ -0,0 +1,9 @@ +[Unit] +Description=Mason: Continuous Delivery Service + +[Service] +User=root +ExecStart=/root/mason-report.sh + +[Install] +WantedBy=multi-user.target diff --git a/mason/mason.sh b/mason/mason.sh new file mode 100755 index 00000000..dfed71f7 --- /dev/null +++ b/mason/mason.sh @@ -0,0 +1,70 @@ +#!/bin/sh + +set -e +set -x + +# Load our deployment config +. /root/mason.conf + +if [ ! -e ws ]; then + morph init ws +fi +cd ws + +definitions_repo="$DEFINITIONS_REF"/"$UPSTREAM_TROVE_ADDRESS"/baserock/baserock/definitions +if [ ! -e "$definitions_repo" ]; then + morph checkout git://"$UPSTREAM_TROVE_ADDRESS"/baserock/baserock/definitions.git "$DEFINITIONS_REF" + cd "$definitions_repo" + git config user.name "$TROVE_ID"-mason + git config user.email "$TROVE_ID"-mason@$(hostname) +else + cd "$definitions_repo" + SHA1_PREV="$(git rev-parse HEAD)" +fi + +git remote update origin +git clean -fxd +git reset --hard origin/"$DEFINITIONS_REF" + +SHA1="$(git rev-parse HEAD)" + +if [ -f "$HOME/success" ] && [ "$SHA1" = "$SHA1_PREV" ]; then + echo INFO: No changes to "$DEFINITIONS_REF", nothing to do + exit 33 +fi + +rm -f "$HOME/success" + +echo INFO: Mason building: $DEFINITIONS_REF at $SHA1 + +"scripts/release-build" --no-default-configs \ + --trove-host "$UPSTREAM_TROVE_ADDRESS" \ + --controllers "$DISTBUILD_ARCH:$DISTBUILD_CONTROLLER_ADDRESS" \ + "$BUILD_CLUSTER_MORPHOLOGY" + +releases_made="$(cd release && ls | wc -l)" +if [ "$releases_made" = 0 ]; then + echo ERROR: No release images created + exit 1 +else + echo INFO: Created "$releases_made" release images +fi + +"scripts/release-test" \ + --deployment-host "$DISTBUILD_ARCH":"$TEST_VM_HOST_SSH_URL" \ + --trove-host "$DISTBUILD_TROVE_ADDRESS" \ + --trove-id "$TROVE_ID" \ + "$BUILD_CLUSTER_MORPHOLOGY" + +"scripts/release-upload" --build-trove-host "$DISTBUILD_TROVE_ADDRESS" \ + --arch "$DISTBUILD_ARCH" \ + --log-level=debug --log="$HOME"/release-upload.log \ + --public-trove-host "$UPSTREAM_TROVE_ADDRESS" \ + --public-trove-username root \ + --public-trove-artifact-dir /home/cache/artifacts \ + --no-upload-release-artifacts \ + "$BUILD_CLUSTER_MORPHOLOGY" + +echo INFO: Artifact upload complete for $DEFINITIONS_REF at $SHA1 + +touch "$HOME/success" diff --git a/mason/mason.timer b/mason/mason.timer new file mode 100644 index 00000000..107dff97 --- /dev/null +++ b/mason/mason.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Runs Mason continually with 1 min between calls + +[Timer] +#Time between Mason finishing and calling it again +OnUnitActiveSec=1min +Unit=mason.service + +[Install] +WantedBy=multi-user.target -- cgit v1.2.1