summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMichael Drake <michael.drake@codethink.co.uk>2014-08-07 10:19:47 +0000
committerMichael Drake <michael.drake@codethink.co.uk>2014-08-07 10:19:47 +0000
commite752380753b9657c60bae790c10f2a0867e56009 (patch)
treed7d7663723495e61da0bf2a2b4c95a3296cc0233
parentb782018ff9f04190574b1d482655b1945521daf0 (diff)
parent2bae2433c5dc35097bca0165ab50a433ef95d738 (diff)
downloaddefinitions-e752380753b9657c60bae790c10f2a0867e56009.tar.gz
Merge branch 'baserock/michaeldrake/mason-devel'
Reviewed-By: Lars Wirzenius <lars.wirzenius@codethink.co.uk>
-rw-r--r--distbuild-system-x86_64.morph1
-rw-r--r--mason.configure101
-rw-r--r--mason.morph58
-rw-r--r--mason/httpd.service10
-rwxr-xr-xmason/mason-generator.sh94
-rwxr-xr-xmason/mason-report.sh209
-rw-r--r--mason/mason.service9
-rwxr-xr-xmason/mason.sh70
-rw-r--r--mason/mason.timer10
-rwxr-xr-xscripts/release-test402
-rwxr-xr-xtrove.configure4
11 files changed, 967 insertions, 1 deletions
diff --git a/distbuild-system-x86_64.morph b/distbuild-system-x86_64.morph
index 52d91626..7f234e6e 100644
--- a/distbuild-system-x86_64.morph
+++ b/distbuild-system-x86_64.morph
@@ -7,6 +7,7 @@ configuration-extensions:
- install-files
- distbuild
- fstab
+- mason
description: Morph distributed build node for x86_64
kind: system
name: distbuild-system-x86_64
diff --git a/mason.configure b/mason.configure
new file mode 100644
index 00000000..fb73b01c
--- /dev/null
+++ b/mason.configure
@@ -0,0 +1,101 @@
+#!/bin/sh
+#
+# Copyright (C) 2014 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# This is a "morph deploy" configuration extension to fully configure
+# a Mason instance at deployment time. It uses the following variables
+# from the environment:
+#
+# * MASON_CLUSTER_MORPHOLOGY
+# * MASON_UPSTREAM_TROVE_ADDRESS
+# * MASON_DEFINITIONS_REF
+# * MASON_DISTBUILD_ARCH
+# * MASON_TEST_HOST
+# * TROVE_HOST
+# * TROVE_ID
+# * CONTROLLERHOST
+
+set -e
+
+ROOT="$1"
+
+if [ "$MASON_CLUSTER_MORPHOLOGY" = "" \
+ -o "$MASON_UPSTREAM_TROVE_ADDRESS" = "" \
+ -o "$MASON_DEFINITIONS_REF" = "" \
+ -o "$MASON_DISTBUILD_ARCH" = "" \
+ -o "$MASON_TEST_HOST" = "" ]; then
+ echo Not configuring as Mason, some options not defined
+ exit 0
+fi
+
+
+##########################################################################
+# Functions
+##########################################################################
+
+shellescape() {
+ echo "'$(echo "$1" | sed -e "s/'/'\\''/g")'"
+}
+
+
+##########################################################################
+# Generate config variable shell snippet
+##########################################################################
+
+MASON_CONFIG="$ROOT"/root/mason.conf
+
+echo "Creating $MASON_CONFIG"
+cat >>"$MASON_CONFIG" <<EOF
+#################### START OF DEPLOY TIME CONFIGURATION #######################
+
+UPSTREAM_TROVE_ADDRESS=$(shellescape "$MASON_UPSTREAM_TROVE_ADDRESS")
+DEFINITIONS_REF=$(shellescape "$MASON_DEFINITIONS_REF")
+DISTBUILD_ARCH=$(shellescape "$MASON_DISTBUILD_ARCH")
+DISTBUILD_CONTROLLER_ADDRESS=$(shellescape "$CONTROLLERHOST")
+DISTBUILD_TROVE_ADDRESS=$(shellescape "$TROVE_HOST")
+TROVE_ID=$(shellescape "$TROVE_ID")
+BUILD_CLUSTER_MORPHOLOGY=$(shellescape "$MASON_CLUSTER_MORPHOLOGY")
+TEST_VM_HOST_SSH_URL=$(shellescape "$MASON_TEST_HOST")
+
+##################### END OF DEPLOY TIME CONFIGURATION ########################
+EOF
+
+
+##########################################################################
+# Copy Mason files into root filesystem
+##########################################################################
+
+cp mason/mason.sh "$ROOT"/root/mason.sh
+cp mason/mason-report.sh "$ROOT"/root/mason-report.sh
+
+cp mason/mason.timer "$ROOT"/etc/systemd/system/mason.timer
+ln -s ../mason.timer "$ROOT"/etc/systemd/system/multi-user.target.wants/mason.timer
+
+cp mason/mason.service "$ROOT"/etc/systemd/system/mason.service
+ln -s ../mason.service "$ROOT"/etc/systemd/system/multi-user.target.wants/mason.service
+
+##########################################################################
+# Set up httpd web server
+##########################################################################
+
+cp mason/httpd.service "$ROOT"/etc/systemd/system/httpd.service
+ln -s ../httpd.service "$ROOT"/etc/systemd/system/multi-user.target.wants/httpd.service
+
+mkdir -p "$ROOT"/srv/mason
+
+cat >>"$ROOT"/etc/httpd.conf <<EOF
+.log:text/plain
+EOF
diff --git a/mason.morph b/mason.morph
new file mode 100644
index 00000000..d6052146
--- /dev/null
+++ b/mason.morph
@@ -0,0 +1,58 @@
+name: example-mason-cluster
+kind: cluster
+description: |
+ This is a template cluster morphology that can be adapted to set up a
+ Mason. Masons are composed of a trove and a distbuild system.
+
+ It is suggested that you use mason/mason-generator.sh to adapt this
+ template to suit your needs. It also handles the generation of
+ keys to let the systems communicate.
+systems:
+- morph: trove-system-x86_64
+ deploy:
+ red-box-v1-trove:
+ type: kvm
+ location: kvm+ssh://vm-user@vm-host/red-box-v1-trove/vm-path/red-box-v1-trove.img
+ VERSION_LABEL: 45
+ DISK_SIZE: 100G
+ RAM_SIZE: 8G
+ VCPUS: 2
+ HOSTNAME: red-box-v1-trove
+ TROVE_ID: red-box-v1-trove
+ TROVE_HOST: red-box-v1
+ TROVE_COMPANY: Company name goes here
+ LORRY_SSH_KEY: ssh_keys/lorry.key
+ UPSTREAM_TROVE: upstream-trove
+ UPSTREAM_TROVE_PROTOCOL: http
+ TROVE_ADMIN_USER: adminuser
+ TROVE_ADMIN_EMAIL: adminuser@example.com
+ TROVE_ADMIN_NAME: Nobody
+ TROVE_ADMIN_SSH_PUBKEY: ssh_keys/id_rsa.pub
+ WORKER_SSH_PUBKEY: ssh_keys/worker.key.pub
+ MASON_SSH_PUBKEY: ssh_keys/mason.key.pub
+ AUTOSTART: yes
+- morph: distbuild-system-x86_64
+ deploy-defaults:
+ TROVE_ID: red-box-v1-trove
+ TROVE_HOST: red-box-v1-trove.example.com
+ CONTROLLERHOST: red-box-v1-controller
+ DISTBUILD_CONTROLLER: no
+ DISTBUILD_WORKER: yes
+ VCPUS: 2
+ RAM_SIZE: 8G
+ #FSTAB_SRC: LABEL=src /srv/distbuild auto defaults,rw,noatime 0 2
+ INSTALL_FILES: distbuild/manifest
+ WORKER_SSH_KEY: ssh_keys/worker.key
+ deploy:
+ red-box-v1-controller:
+ type: kvm
+ location: kvm+ssh://vm-user@vm-host/red-box-v1-controller/vm-path/red-box-v1-controller.img
+ DISK_SIZE: 60G
+ DISTBUILD_CONTROLLER: yes
+ HOSTNAME: red-box-v1-controller
+ WORKERS: red-box-v1-controller
+ MASON_CLUSTER_MORPHOLOGY: ci.morph
+ MASON_DEFINITIONS_REF: master
+ MASON_DISTBUILD_ARCH: x86_64
+ MASON_UPSTREAM_TROVE_ADDRESS: upstream-trove
+ MASON_TEST_HOST: vm-user@vm-host:/vm-path/
diff --git a/mason/httpd.service b/mason/httpd.service
new file mode 100644
index 00000000..7572b732
--- /dev/null
+++ b/mason/httpd.service
@@ -0,0 +1,10 @@
+[Unit]
+Description=HTTP server for Mason
+After=network.target
+
+[Service]
+User=root
+ExecStart=/usr/sbin/httpd -f -p 80 -h /srv/mason
+
+[Install]
+WantedBy=multi-user.target
diff --git a/mason/mason-generator.sh b/mason/mason-generator.sh
new file mode 100755
index 00000000..a0b2fb0a
--- /dev/null
+++ b/mason/mason-generator.sh
@@ -0,0 +1,94 @@
+#!/bin/sh
+
+set -e
+
+if [ "$1" == "-h" -o "$1" == "--help" ]; then
+ echo "Usage:"
+ echo " `basename $0` HOST_PREFIX UPSTREAM_TROVE_HOSTNAME VM_USER VM_HOST VM_PATH [HOST_POSTFIX]"
+ echo ""
+ echo "Where:"
+ echo " HOST_PREFIX -- Name of your Mason instance"
+ echo " e.g. \"my-mason\" to produce hostnames:"
+ echo " my-mason-trove and my-mason-controller"
+ echo " UPSTREAM_TROVE_HOSTNAME -- Upstream trove's hostname"
+ echo " VM_USER -- User on VM host for VM deployment"
+ echo " VM_HOST -- VM host for VM deployment"
+ echo " VM_PATH -- Path to store VM images in on VM host"
+ echo " HOST_POSTFIX -- e.g. \".example.com\" to get"
+ echo " my-mason-trove.example.com"
+ echo ""
+ echo "This script makes deploying a Mason system simpler by automating"
+ echo "the generation of keys for the systems to use, building of the"
+ echo "systems, filling out the mason deployment cluster morphology"
+ echo "template with useful values, and finally deploying the systems."
+ echo ""
+ echo "To ensure that the deployed system can deploy test systems, you"
+ echo "must supply an ssh key to the VM host. Do so with the following"
+ echo "command:"
+ echo " ssh-copy-id -i ssh_keys-HOST_PREFIX/mason.key.pub VM_USER@VM_HOST"
+ echo ""
+ exit 0
+fi
+
+
+HOST_PREFIX=$1
+UPSTREAM_TROVE=$2
+VM_USER=$3
+VM_HOST=$4
+VM_PATH=$5
+HOST_POSTFIX=$6
+
+sedescape() {
+ # Escape all non-alphanumeric characters
+ printf "%s\n" "$1" | sed -e 's/\W/\\&/g'
+}
+
+
+##############################################################################
+# Key generation
+##############################################################################
+
+mkdir "ssh_keys-${HOST_PREFIX}"
+cd "ssh_keys-${HOST_PREFIX}"
+ssh-keygen -t rsa -b 2048 -f mason.key -C mason@TROVE_HOST -N ''
+ssh-keygen -t rsa -b 2048 -f lorry.key -C lorry@TROVE_HOST -N ''
+ssh-keygen -t rsa -b 2048 -f worker.key -C worker@TROVE_HOST -N ''
+ssh-keygen -t rsa -b 2048 -f id_rsa -C trove-admin@TROVE_HOST -N ''
+cd ../
+
+
+##############################################################################
+# Mason setup
+##############################################################################
+
+cp mason.morph mason-${HOST_PREFIX}.morph
+
+sed -i "s/red-box-v1/$(sedescape "$HOST_PREFIX")/g" "mason-$HOST_PREFIX.morph"
+sed -i "s/ssh_keys/ssh_keys-$(sedescape "$HOST_PREFIX")/g" "mason-$HOST_PREFIX.morph"
+sed -i "s/upstream-trove/$(sedescape "$UPSTREAM_TROVE")/" "mason-$HOST_PREFIX.morph"
+sed -i "s/vm-user/$(sedescape "$VM_USER")/g" "mason-$HOST_PREFIX.morph"
+sed -i "s/vm-host/$(sedescape "$VM_HOST")/g" "mason-$HOST_PREFIX.morph"
+sed -i "s/vm-path/$(sedescape "$VM_PATH")/g" "mason-$HOST_PREFIX.morph"
+sed -i "s/\.example\.com/$(sedescape "$HOST_POSTFIX")/g" "mason-$HOST_PREFIX.morph"
+
+
+##############################################################################
+# System building
+##############################################################################
+
+morph build trove-system-x86_64
+morph build distbuild-system-x86_64
+
+
+##############################################################################
+# System deployment
+##############################################################################
+
+morph deploy mason-${HOST_PREFIX}.morph
+
+
+##############################################################################
+# Cleanup
+##############################################################################
+
+rm mason-${HOST_PREFIX}.morph
diff --git a/mason/mason-report.sh b/mason/mason-report.sh
new file mode 100755
index 00000000..d6cf0c19
--- /dev/null
+++ b/mason/mason-report.sh
@@ -0,0 +1,209 @@
+#!/bin/bash
+
+set -x
+
+. /root/mason.conf
+
+REPORT_PATH=/root/report.html
+SERVER_PATH=/srv/mason
+
+sed_escape() {
+ printf "%s\n" "$1" | sed -e 's/\W/\\&/g'
+}
+
+create_report() {
+cat > $REPORT_PATH <<'EOF'
+<html>
+<head>
+<meta charset="UTF-8">
+<meta http-equiv="refresh" content="10">
+<style>
+html, body {
+ margin: 0;
+ padding: 0;
+}
+p.branding {
+ background: black;
+ color: #fff;
+ padding: 0.4em;
+ margin: 0;
+ font-weight: bold;
+}
+h1 {
+ background: #225588;
+ color: white;
+ margin: 0;
+ padding: 0.6em;
+}
+table {
+ width: 90%;
+ margin: 1em auto 6em auto;
+ border: 1px solid black;
+ border-spacing: 0;
+}
+table tr.headings {
+ background: #555;
+ color: white;
+}
+table tr.pass {
+ background: #aaffaa;
+}
+table tr.pass:hover {
+ background: #bbffbb;
+}
+table tr.fail {
+ background: #ffaaaa;
+}
+table tr.fail:hover {
+ background: #ffbbbb;
+}
+table tr.headings th {
+ font-weight: bold;
+ text-align: left;
+ padding: 3px 2px;
+}
+table td {
+ padding: 2px;
+}
+td.result {
+ font-weight: bold;
+ text-transform: uppercase;
+}
+td.result a {
+ text-decoration: none;
+}
+td.result a:before {
+ content: "➫ ";
+}
+tr.pass td.result a {
+ color: #252;
+}
+tr.pass td.result a:hover {
+ color: #373;
+}
+tr.fail td.result a {
+ color: #622;
+}
+tr.fail td.result a:hover {
+ color: #933;
+}
+td.ref {
+ font-family: monospace;
+}
+td.ref a {
+ color: #333;
+}
+td.ref a:hover {
+ color: #555;
+}
+table tr.pass td, table tr.fail td {
+ border-top: solid white 1px;
+}
+p {
+ margin: 1.3em;
+}
+code {
+ padding: 0.3em 0.5em;
+ background: #eee;
+ border: 1px solid #bbb;
+ border-radius: 1em;
+}
+#footer {
+ margin: 0;
+ background: #aaa;
+ color: #222;
+ border-top: #888 1px solid;
+ font-size: 80%;
+ padding: 0;
+ text-align: right;
+ position: fixed;
+ bottom: 0;
+ width: 100%;
+}
+</style>
+</head>
+<body>
+<p class="branding">Mason</p>
+<h1>Baserock: Continuous Delivery</h1>
+<p>Build log of changes to <code>BRANCH</code> from <code>TROVE</code>. Most recent first.</p>
+<table>
+<tr class="headings">
+ <th>Started</th>
+ <th>Ref</th>
+ <th>Duration</th>
+ <th>Result</th>
+</tr>
+<!--INSERTION POINT-->
+</table>
+<div id="footer">
+<p>Generated by Mason</p>
+</div>
+</body>
+</html>
+EOF
+
+ sed -i 's/BRANCH/'"$(sed_escape "$1")"'/' $REPORT_PATH
+ sed -i 's/TROVE/'"$(sed_escape "$2")"'/' $REPORT_PATH
+}
+
+update_report() {
+ # Give function params sensible names
+ build_start_time="$1"
+ build_trove_host="$2"
+ build_ref="$3"
+ build_sha1="$4"
+ build_duration="$5"
+ build_result="$6"
+
+ # Generate template if report file is not there
+ if [ ! -f $REPORT_PATH ]; then
+ create_report $build_ref $build_trove_host
+ fi
+
+ # Build table row for insertion into report file
+ msg='<tr class="'"${build_result}"'"><td>'"${build_start_time}"'</td><td class="ref"><a href="http://'"${build_trove_host}"'/cgi-bin/cgit.cgi/baserock/baserock/definitions.git/commit/?h='"${build_ref}"'&id='"${build_sha1}"'">'"${build_sha1}"'</a></td><td>'"${build_duration}s"'</td><td class="result"><a href="log/'"${build_sha1}"'--'"${build_start_time}"'.log">'"${build_result}"'</a></td></tr>'
+
+ # Insert report line, newest at top
+ sed -i 's/<!--INSERTION POINT-->/<!--INSERTION POINT-->\n'"$(sed_escape "$msg")"'/' $REPORT_PATH
+}
+
+START_TIME=`date +%Y-%m-%d\ %T`
+
+logfile="$(mktemp)"
+/root/mason.sh 2>&1 | tee "$logfile"
+case "${PIPESTATUS[0]}" in
+0)
+ RESULT=pass
+ ;;
+33)
+ RESULT=skip
+ ;;
+*)
+ RESULT=fail
+ ;;
+esac
+
+# TODO: Update page with last executed time
+if [ "$RESULT" = skip ]; then
+ rm "$logfile"
+ exit 0
+fi
+
+DURATION=$(( $(date +%s) - $(date --date="$START_TIME" +%s) ))
+SHA1="$(cd "ws/$DEFINITIONS_REF/$UPSTREAM_TROVE_ADDRESS/baserock/baserock/definitions" && git rev-parse HEAD)"
+
+update_report "$START_TIME" \
+ "$DISTBUILD_TROVE_ADDRESS" \
+ "$DEFINITIONS_REF" \
+ "$SHA1" \
+ "$DURATION" \
+ "$RESULT"
+
+
+#
+# Copy report into server directory
+#
+
+cp "$REPORT_PATH" "$SERVER_PATH/index.html"
+mkdir /srv/mason/log
+mv "$logfile" /srv/mason/log/"$SHA1--$START_TIME.log"
diff --git a/mason/mason.service b/mason/mason.service
new file mode 100644
index 00000000..16b5dc3f
--- /dev/null
+++ b/mason/mason.service
@@ -0,0 +1,9 @@
+[Unit]
+Description=Mason: Continuous Delivery Service
+
+[Service]
+User=root
+ExecStart=/root/mason-report.sh
+
+[Install]
+WantedBy=multi-user.target
diff --git a/mason/mason.sh b/mason/mason.sh
new file mode 100755
index 00000000..dfed71f7
--- /dev/null
+++ b/mason/mason.sh
@@ -0,0 +1,70 @@
+#!/bin/sh
+
+set -e
+set -x
+
+# Load our deployment config
+. /root/mason.conf
+
+if [ ! -e ws ]; then
+ morph init ws
+fi
+cd ws
+
+definitions_repo="$DEFINITIONS_REF"/"$UPSTREAM_TROVE_ADDRESS"/baserock/baserock/definitions
+if [ ! -e "$definitions_repo" ]; then
+ morph checkout git://"$UPSTREAM_TROVE_ADDRESS"/baserock/baserock/definitions.git "$DEFINITIONS_REF"
+ cd "$definitions_repo"
+ git config user.name "$TROVE_ID"-mason
+ git config user.email "$TROVE_ID"-mason@$(hostname)
+else
+ cd "$definitions_repo"
+ SHA1_PREV="$(git rev-parse HEAD)"
+fi
+
+git remote update origin
+git clean -fxd
+git reset --hard origin/"$DEFINITIONS_REF"
+
+SHA1="$(git rev-parse HEAD)"
+
+if [ -f "$HOME/success" ] && [ "$SHA1" = "$SHA1_PREV" ]; then
+ echo INFO: No changes to "$DEFINITIONS_REF", nothing to do
+ exit 33
+fi
+
+rm -f "$HOME/success"
+
+echo INFO: Mason building: $DEFINITIONS_REF at $SHA1
+
+"scripts/release-build" --no-default-configs \
+ --trove-host "$UPSTREAM_TROVE_ADDRESS" \
+ --controllers "$DISTBUILD_ARCH:$DISTBUILD_CONTROLLER_ADDRESS" \
+ "$BUILD_CLUSTER_MORPHOLOGY"
+
+releases_made="$(cd release && ls | wc -l)"
+if [ "$releases_made" = 0 ]; then
+ echo ERROR: No release images created
+ exit 1
+else
+ echo INFO: Created "$releases_made" release images
+fi
+
+"scripts/release-test" \
+ --deployment-host "$DISTBUILD_ARCH":"$TEST_VM_HOST_SSH_URL" \
+ --trove-host "$DISTBUILD_TROVE_ADDRESS" \
+ --trove-id "$TROVE_ID" \
+ "$BUILD_CLUSTER_MORPHOLOGY"
+
+"scripts/release-upload" --build-trove-host "$DISTBUILD_TROVE_ADDRESS" \
+ --arch "$DISTBUILD_ARCH" \
+ --log-level=debug --log="$HOME"/release-upload.log \
+ --public-trove-host "$UPSTREAM_TROVE_ADDRESS" \
+ --public-trove-username root \
+ --public-trove-artifact-dir /home/cache/artifacts \
+ --no-upload-release-artifacts \
+ "$BUILD_CLUSTER_MORPHOLOGY"
+
+echo INFO: Artifact upload complete for $DEFINITIONS_REF at $SHA1
+
+touch "$HOME/success"
diff --git a/mason/mason.timer b/mason/mason.timer
new file mode 100644
index 00000000..107dff97
--- /dev/null
+++ b/mason/mason.timer
@@ -0,0 +1,10 @@
+[Unit]
+Description=Runs Mason continually with 1 min between calls
+
+[Timer]
+#Time between Mason finishing and calling it again
+OnUnitActiveSec=1min
+Unit=mason.service
+
+[Install]
+WantedBy=multi-user.target
diff --git a/scripts/release-test b/scripts/release-test
new file mode 100755
index 00000000..ddfe6bbd
--- /dev/null
+++ b/scripts/release-test
@@ -0,0 +1,402 @@
+#!/usr/bin/env python
+#
+# Copyright 2014 Codethink Ltd
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+'''release-test
+
+This script deploys the set of systems in the cluster morphology it is
+instructed to read, to test that they work correctly.
+
+'''
+
+import cliapp
+import os
+import pipes
+import shlex
+import shutil
+import socket
+import tempfile
+import time
+import uuid
+
+import morphlib
+
+
+class MorphologyFrobber(object):
+ def __init__(self):
+ self.sb = sb = morphlib.sysbranchdir.open_from_within('.')
+ defs_repo_path = sb.get_git_directory_name(sb.root_repository_url)
+ self.defs_repo = morphlib.gitdir.GitDirectory(defs_repo_path)
+ self.loader = morphlib.morphloader.MorphologyLoader()
+ self.finder = morphlib.morphologyfinder.MorphologyFinder(self.defs_repo)
+
+ def load_morphology(self, path):
+ text = self.finder.read_morphology(path)
+ return self.loader.load_from_string(text)
+
+ @classmethod
+ def iterate_systems(cls, systems_list):
+ for system in systems_list:
+ yield morphlib.util.sanitise_morphology_path(system['morph'])
+ if 'subsystems' in system:
+ for subsystem in cls.iterate_systems(system['subsystems']):
+ yield subsystem
+
+ def iterate_cluster_deployments(cls, cluster_morph):
+ for system in cluster_morph['systems']:
+ path = morphlib.util.sanitise_morphology_path(system['morph'])
+ defaults = system.get('deploy-defaults', {})
+ for name, options in system['deploy'].iteritems():
+ config = dict(defaults)
+ config.update(options)
+ yield path, name, config
+
+ def load_cluster_systems(self, cluster_morph):
+ for system_path in set(self.iterate_systems(cluster_morph['systems'])):
+ system_text = self.finder.read_morphology(system_path)
+ system_morph = self.loader.load_from_string(system_text)
+ yield system_path, system_morph
+
+
+class TimeoutError(cliapp.AppException):
+
+ """Error to be raised when a connection waits too long"""
+
+ def __init__(self, msg):
+ super(TimeoutError, self).__init__(msg)
+
+
+class VMHost(object):
+ def __init__(self, user, address, disk_path):
+ self.user = user
+ self.address = address
+ self.disk_path = disk_path
+
+ @property
+ def ssh_host(self):
+ return '{user}@{address}'.format(user=self.user, address=self.address)
+
+ def runcmd(self, *args, **kwargs):
+ cliapp.ssh_runcmd(self.ssh_host, *args, **kwargs)
+
+ def virsh(self, *args, **kwargs):
+ self.runcmd(['virsh', '-c', 'qemu:///system'] + list(args), **kwargs)
+
+
+class Instance(object):
+ def __init__(self, deployment, config, host_machine, vm_id, rootfs_path):
+ self.deployment = deployment
+ self.config = config
+ # TODO: Stop assuming test machine can DHCP and be assigned its
+ # hostname in the deployer's resolve search path.
+ self.ip_address = self.config['HOSTNAME']
+ self.host_machine = host_machine
+ self.vm_id = vm_id
+ self.rootfs_path = rootfs_path
+
+ @property
+ def ssh_host(self):
+ # TODO: Stop assuming we ssh into test instances as root
+ return 'root@{host}'.format(host=self.ip_address)
+
+ def runcmd(self, argv, chdir='.', **kwargs):
+ ssh_cmd = ['ssh', '-o', 'StrictHostKeyChecking=no', self.ssh_host]
+ cmd = ['sh', '-c', 'cd "$1" && shift && exec "$@"', '-', chdir]
+ cmd += argv
+ ssh_cmd.append(' '.join(map(pipes.quote, cmd)))
+ return cliapp.runcmd(ssh_cmd, **kwargs)
+
+ def _wait_for_dhcp(self, timeout):
+ '''
+ Block until given hostname resolves successfully.
+
+ Raises TimeoutError if the hostname has not appeared in 'timeout'
+ seconds.
+ '''
+ start_time = time.time()
+ while True:
+ try:
+ socket.gethostbyname(self.ip_address)
+ return
+ except socket.gaierror:
+ pass
+ if time.time() > start_time + timeout:
+ raise TimeoutError("Host %s did not appear after %i seconds" %
+ (self.ip_address, timeout))
+ time.sleep(0.5)
+
+ def _wait_for_ssh(self, timeout):
+ """Wait until the deployed VM is responding via SSH"""
+ start_time = time.time()
+ while True:
+ try:
+ self.runcmd(['true'], stdin=None, stdout=None, stderr=None)
+ return
+ except cliapp.AppException:
+ # TODO: Stop assuming the ssh part of the command is what failed
+ if time.time() > start_time + timeout:
+ raise TimeoutError("%s sshd did not start after %i seconds"
+ % (self.ip_address, timeout))
+ time.sleep(0.5)
+
+ def wait_online(self, timeout=10):
+ self._wait_for_dhcp(timeout)
+ self._wait_for_ssh(timeout)
+
+ def delete(self):
+ # Forget config of deployed system as if it ever reappears it
+ # will have different host keys
+ cliapp.runcmd(['ssh-keygen', '-R', self.config['HOSTNAME']])
+ # Stop and remove VM
+ try:
+ self.host_machine.virsh('destroy', self.vm_id)
+ except cliapp.AppException as e:
+ # TODO: Stop assuming that destroy failed because it wasn't running
+ pass
+ try:
+ self.host_machine.virsh('undefine', self.vm_id)
+ except cliapp.AppException as e:
+ # TODO: Stop assuming that undefine failed because it was
+ # already removed
+ pass
+ # TODO: Remove all attached disks, rather than just assuming it
+ # only has one
+ self.host_machine.runcmd(['rm', self.rootfs_path])
+
+
+class Deployment(object):
+ def __init__(self, cluster_path, name,
+ deployment_config, host_machine):
+ self.cluster_path = cluster_path
+ self.name = name
+ self.deployment_config = deployment_config
+ self.host_machine = host_machine
+
+ @staticmethod
+ def _ssh_host_key_exists(hostname):
+ """Check if an ssh host key exists in known_hosts"""
+ with open('/root/.ssh/known_hosts', 'r') as known_hosts:
+ return any(line.startswith(hostname) for line in known_hosts)
+
+ def _update_known_hosts(self):
+ if not self._ssh_host_key_exists(self.host_machine.address):
+ with open('/root/.ssh/known_hosts', 'a') as known_hosts:
+ cliapp.runcmd(['ssh-keyscan', self.host_machine.address],
+ stdout=known_hosts)
+
+ @staticmethod
+ def _generate_sshkey_config(tempdir, config):
+ manifest = os.path.join(tempdir, 'manifest')
+ with open(manifest, 'w') as f:
+ f.write('0040700 0 0 /root/.ssh\n')
+ f.write('overwrite 0100600 0 0 /root/.ssh/authorized_keys\n')
+ authkeys = os.path.join(tempdir, 'root', '.ssh', 'authorized_keys')
+ os.makedirs(os.path.dirname(authkeys))
+ with open(authkeys, 'w') as auth_f:
+ with open('/root/.ssh/id_rsa.pub', 'r') as key_f:
+ shutil.copyfileobj(key_f, auth_f)
+
+ install_files = shlex.split(config.get('INSTALL_FILES', ''))
+ install_files.append(manifest)
+ yield 'INSTALL_FILES', ' '.join(pipes.quote(f) for f in install_files)
+
+ def deploy(self):
+ self._update_known_hosts()
+
+ hostname = str(uuid.uuid4())
+ vm_id = hostname
+ image_base = self.host_machine.disk_path
+ rootpath = '{image_base}/{hostname}.img'.format(image_base=image_base,
+ hostname=hostname)
+ loc = 'kvm+ssh://{ssh_host}/{id}/{path}'.format(
+ ssh_host=self.host_machine.ssh_host, id=vm_id, path=rootpath)
+
+ options = {
+ 'type': 'kvm',
+ 'location': loc,
+ 'AUTOSTART': 'True',
+ 'HOSTNAME': hostname,
+ 'DISK_SIZE': '20G',
+ 'RAM_SIZE': '2G',
+ 'VERSION_LABEL': 'release-test',
+ }
+
+ tempdir = tempfile.mkdtemp()
+ try:
+ options.update(
+ self._generate_sshkey_config(tempdir,
+ self.deployment_config))
+
+ args = ['morph', 'deploy', self.cluster_path, self.name]
+ for k, v in options.iteritems():
+ args.append('%s.%s=%s' % (self.name, k, v))
+ cliapp.runcmd(args, stdin=None, stdout=None, stderr=None)
+
+ config = dict(self.deployment_config)
+ config.update(options)
+
+ return Instance(self, config, self.host_machine, vm_id, rootpath)
+ finally:
+ shutil.rmtree(tempdir)
+
+
+class ReleaseApp(cliapp.Application):
+
+ """Cliapp application which handles automatic builds and tests"""
+
+ def add_settings(self):
+ """Add the command line options needed"""
+ group_main = 'Program Options'
+ self.settings.string_list(['deployment-host'],
+ 'ARCH:HOST:PATH that VMs can be deployed to',
+ default=None,
+ group=group_main)
+ self.settings.string(['trove-host'],
+ 'Address of Trove for test systems to build from',
+ default=None,
+ group=group_main)
+ self.settings.string(['trove-id'],
+ 'ID of Trove for test systems to build from',
+ default=None,
+ group=group_main)
+ self.settings.string(['build-ref-prefix'],
+ 'Prefix of build branches for test systems',
+ default=None,
+ group=group_main)
+
+ @staticmethod
+ def _run_tests(instance, system_path, system_morph,
+ (trove_host, trove_id, build_ref_prefix),
+ morph_frobber, systems):
+ instance.wait_online()
+
+ tests = []
+ def baserock_build_test(instance):
+ instance.runcmd(['git', 'config', '--global', 'user.name',
+ 'Test Instance of %s' % instance.deployment.name])
+ instance.runcmd(['git', 'config', '--global', 'user.email',
+ 'ci-test@%s' % instance.config['HOSTNAME']])
+ instance.runcmd(['mkdir', '-p', '/src/ws', '/src/cache',
+ '/src/tmp'])
+ def morph_cmd(*args, **kwargs):
+ # TODO: decide whether to use cached artifacts or not by
+ # adding --artifact-cache-server= --cache-server=
+ argv = ['morph', '--log=/src/morph.log', '--cachedir=/src/cache',
+ '--tempdir=/src/tmp', '--log-max=100M',
+ '--trove-host', trove_host, '--trove-id', trove_id,
+ '--build-ref-prefix', build_ref_prefix]
+ argv.extend(args)
+ instance.runcmd(argv, **kwargs)
+
+ repo = morph_frobber.sb.root_repository_url
+ ref = morph_frobber.defs_repo.HEAD
+ sha1 = morph_frobber.defs_repo.resolve_ref_to_commit(ref)
+ morph_cmd('init', '/src/ws')
+ chdir = '/src/ws'
+
+ morph_cmd('checkout', repo, ref, chdir=chdir)
+ # TODO: Add a morph subcommand that gives the path to the root repository.
+ repo_path = os.path.relpath(
+ morph_frobber.sb.get_git_directory_name(repo),
+ morph_frobber.sb.root_directory)
+ chdir = os.path.join(chdir, ref, repo_path)
+
+ instance.runcmd(['git', 'reset', '--hard', sha1], chdir=chdir)
+ print 'Building test systems for {sys}'.format(sys=system_path)
+ for to_build_path, to_build_morph in systems.iteritems():
+ if to_build_morph['arch'] == system_morph['arch']:
+ print 'Test building {path}'.format(path=to_build_path)
+ morph_cmd('build', to_build_path, chdir=chdir,
+ stdin=None, stdout=None, stderr=None)
+ print 'Finished Building test systems'
+
+ def python_smoke_test(instance):
+ instance.runcmd(['python', '-c', 'print "Hello World"'])
+
+ # TODO: Come up with a better way of determining which tests to run
+ if 'devel' in system_path:
+ tests.append(baserock_build_test)
+ else:
+ tests.append(python_smoke_test)
+
+ for test in tests:
+ test(instance)
+
+ def deploy_and_test_systems(self, cluster_path,
+ deployment_hosts, build_test_config):
+ """Run the deployments and tests"""
+
+ image_path = '/opt/ci/'
+ version = 'release-test'
+
+ morph_frobber = MorphologyFrobber()
+ cluster_morph = morph_frobber.load_morphology(cluster_path)
+ systems = dict(morph_frobber.load_cluster_systems(cluster_morph))
+
+ for system_path, deployment_name, deployment_config in \
+ morph_frobber.iterate_cluster_deployments(cluster_morph):
+
+ system_morph = systems[system_path]
+ # We can only test systems in KVM that have a BSP
+ if not any('bsp' in si['morph'] for si in system_morph['strata']):
+ continue
+
+ # We can only test systems in KVM that we have a host for
+ if system_morph['arch'] not in deployment_hosts:
+ continue
+ host_machine = deployment_hosts[system_morph['arch']]
+ deployment = Deployment(cluster_path, deployment_name,
+ deployment_config, host_machine)
+
+ instance = deployment.deploy()
+ try:
+ self._run_tests(instance, system_path, system_morph,
+ build_test_config, morph_frobber, systems)
+ finally:
+ instance.delete()
+
+ def process_args(self, args):
+ """Process the command line args and kick off the builds/tests"""
+ if self.settings['build-ref-prefix'] is None:
+ self.settings['build-ref-prefix'] = (
+ os.path.join(self.settings['trove-id'], 'builds'))
+ for setting in ('deployment-host', 'trove-host',
+ 'trove-id', 'build-ref-prefix'):
+ self.settings.require(setting)
+
+ deployment_hosts = {}
+ for host_config in self.settings['deployment-host']:
+ arch, address = host_config.split(':', 1)
+ user, address = address.split('@', 1)
+ address, disk_path = address.split(':', 1)
+ if user == '':
+ user = 'root'
+ # TODO: Don't assume root is the user with deploy access
+ deployment_hosts[arch] = VMHost(user, address, disk_path)
+
+ build_test_config = (self.settings['trove-host'],
+ self.settings['trove-id'],
+ self.settings['build-ref-prefix'])
+
+ if len(args) != 1:
+ raise cliapp.AppException('Usage: release-test CLUSTER')
+ cluster_path = morphlib.util.sanitise_morphology_path(args[0])
+ self.deploy_and_test_systems(cluster_path, deployment_hosts,
+ build_test_config)
+
+
+if __name__ == '__main__':
+ ReleaseApp().run()
diff --git a/trove.configure b/trove.configure
index c7a4f3af..4cc9720a 100755
--- a/trove.configure
+++ b/trove.configure
@@ -25,6 +25,7 @@
# * TROVE_COMPANY
# * LORRY_SSH_KEY
# * UPSTREAM_TROVE
+# * UPSTREAM_TROVE_PROTOCOL
# * TROVE_ADMIN_USER
# * TROVE_ADMIN_EMAIL
# * TROVE_ADMIN_NAME
@@ -136,7 +137,8 @@ trove_configuration={
optional_keys = ('MASON_ID', 'HOSTNAME', 'TROVE_HOSTNAME',
- 'LORRY_CONTROLLER_MINIONS', 'TROVE_BACKUP_KEYS')
+ 'LORRY_CONTROLLER_MINIONS', 'TROVE_BACKUP_KEYS',
+ 'UPSTREAM_TROVE_PROTOCOL')
for key in optional_keys:
if key in os.environ: