diff options
author | Michael Drake <michael.drake@codethink.co.uk> | 2014-08-07 10:19:47 +0000 |
---|---|---|
committer | Michael Drake <michael.drake@codethink.co.uk> | 2014-08-07 10:19:47 +0000 |
commit | e752380753b9657c60bae790c10f2a0867e56009 (patch) | |
tree | d7d7663723495e61da0bf2a2b4c95a3296cc0233 | |
parent | b782018ff9f04190574b1d482655b1945521daf0 (diff) | |
parent | 2bae2433c5dc35097bca0165ab50a433ef95d738 (diff) | |
download | definitions-e752380753b9657c60bae790c10f2a0867e56009.tar.gz |
Merge branch 'baserock/michaeldrake/mason-devel'
Reviewed-By: Lars Wirzenius <lars.wirzenius@codethink.co.uk>
-rw-r--r-- | distbuild-system-x86_64.morph | 1 | ||||
-rw-r--r-- | mason.configure | 101 | ||||
-rw-r--r-- | mason.morph | 58 | ||||
-rw-r--r-- | mason/httpd.service | 10 | ||||
-rwxr-xr-x | mason/mason-generator.sh | 94 | ||||
-rwxr-xr-x | mason/mason-report.sh | 209 | ||||
-rw-r--r-- | mason/mason.service | 9 | ||||
-rwxr-xr-x | mason/mason.sh | 70 | ||||
-rw-r--r-- | mason/mason.timer | 10 | ||||
-rwxr-xr-x | scripts/release-test | 402 | ||||
-rwxr-xr-x | trove.configure | 4 |
11 files changed, 967 insertions, 1 deletions
diff --git a/distbuild-system-x86_64.morph b/distbuild-system-x86_64.morph index 52d91626..7f234e6e 100644 --- a/distbuild-system-x86_64.morph +++ b/distbuild-system-x86_64.morph @@ -7,6 +7,7 @@ configuration-extensions: - install-files - distbuild - fstab +- mason description: Morph distributed build node for x86_64 kind: system name: distbuild-system-x86_64 diff --git a/mason.configure b/mason.configure new file mode 100644 index 00000000..fb73b01c --- /dev/null +++ b/mason.configure @@ -0,0 +1,101 @@ +#!/bin/sh +# +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# This is a "morph deploy" configuration extension to fully configure +# a Mason instance at deployment time. It uses the following variables +# from the environment: +# +# * MASON_CLUSTER_MORPHOLOGY +# * MASON_UPSTREAM_TROVE_ADDRESS +# * MASON_DEFINITIONS_REF +# * MASON_DISTBUILD_ARCH +# * MASON_TEST_HOST +# * TROVE_HOST +# * TROVE_ID +# * CONTROLLERHOST + +set -e + +ROOT="$1" + +if [ "$MASON_CLUSTER_MORPHOLOGY" = "" \ + -o "$MASON_UPSTREAM_TROVE_ADDRESS" = "" \ + -o "$MASON_DEFINITIONS_REF" = "" \ + -o "$MASON_DISTBUILD_ARCH" = "" \ + -o "$MASON_TEST_HOST" = "" ]; then + echo Not configuring as Mason, some options not defined + exit 0 +fi + + +########################################################################## +# Functions +########################################################################## + +shellescape() { + echo "'$(echo "$1" | sed -e "s/'/'\\''/g")'" +} + + +########################################################################## +# Generate config variable shell snippet +########################################################################## + +MASON_CONFIG="$ROOT"/root/mason.conf + +echo "Creating $MASON_CONFIG" +cat >>"$MASON_CONFIG" <<EOF +#################### START OF DEPLOY TIME CONFIGURATION ####################### + +UPSTREAM_TROVE_ADDRESS=$(shellescape "$MASON_UPSTREAM_TROVE_ADDRESS") +DEFINITIONS_REF=$(shellescape "$MASON_DEFINITIONS_REF") +DISTBUILD_ARCH=$(shellescape "$MASON_DISTBUILD_ARCH") +DISTBUILD_CONTROLLER_ADDRESS=$(shellescape "$CONTROLLERHOST") +DISTBUILD_TROVE_ADDRESS=$(shellescape "$TROVE_HOST") +TROVE_ID=$(shellescape "$TROVE_ID") +BUILD_CLUSTER_MORPHOLOGY=$(shellescape "$MASON_CLUSTER_MORPHOLOGY") +TEST_VM_HOST_SSH_URL=$(shellescape "$MASON_TEST_HOST") + +##################### END OF DEPLOY TIME CONFIGURATION ######################## +EOF + + +########################################################################## +# Copy Mason files into root filesystem +########################################################################## + +cp mason/mason.sh "$ROOT"/root/mason.sh +cp mason/mason-report.sh "$ROOT"/root/mason-report.sh + +cp mason/mason.timer "$ROOT"/etc/systemd/system/mason.timer +ln -s ../mason.timer "$ROOT"/etc/systemd/system/multi-user.target.wants/mason.timer + +cp mason/mason.service "$ROOT"/etc/systemd/system/mason.service +ln -s ../mason.service "$ROOT"/etc/systemd/system/multi-user.target.wants/mason.service + +########################################################################## +# Set up httpd web server +########################################################################## + +cp mason/httpd.service "$ROOT"/etc/systemd/system/httpd.service +ln -s ../httpd.service "$ROOT"/etc/systemd/system/multi-user.target.wants/httpd.service + +mkdir -p "$ROOT"/srv/mason + +cat >>"$ROOT"/etc/httpd.conf <<EOF +.log:text/plain +EOF diff --git a/mason.morph b/mason.morph new file mode 100644 index 00000000..d6052146 --- /dev/null +++ b/mason.morph @@ -0,0 +1,58 @@ +name: example-mason-cluster +kind: cluster +description: | + This is a template cluster morphology that can be adapted to set up a + Mason. Masons are composed of a trove and a distbuild system. + + It is suggested that you use mason/mason-generator.sh to adapt this + template to suit your needs. It also handles the generation of + keys to let the systems communicate. +systems: +- morph: trove-system-x86_64 + deploy: + red-box-v1-trove: + type: kvm + location: kvm+ssh://vm-user@vm-host/red-box-v1-trove/vm-path/red-box-v1-trove.img + VERSION_LABEL: 45 + DISK_SIZE: 100G + RAM_SIZE: 8G + VCPUS: 2 + HOSTNAME: red-box-v1-trove + TROVE_ID: red-box-v1-trove + TROVE_HOST: red-box-v1 + TROVE_COMPANY: Company name goes here + LORRY_SSH_KEY: ssh_keys/lorry.key + UPSTREAM_TROVE: upstream-trove + UPSTREAM_TROVE_PROTOCOL: http + TROVE_ADMIN_USER: adminuser + TROVE_ADMIN_EMAIL: adminuser@example.com + TROVE_ADMIN_NAME: Nobody + TROVE_ADMIN_SSH_PUBKEY: ssh_keys/id_rsa.pub + WORKER_SSH_PUBKEY: ssh_keys/worker.key.pub + MASON_SSH_PUBKEY: ssh_keys/mason.key.pub + AUTOSTART: yes +- morph: distbuild-system-x86_64 + deploy-defaults: + TROVE_ID: red-box-v1-trove + TROVE_HOST: red-box-v1-trove.example.com + CONTROLLERHOST: red-box-v1-controller + DISTBUILD_CONTROLLER: no + DISTBUILD_WORKER: yes + VCPUS: 2 + RAM_SIZE: 8G + #FSTAB_SRC: LABEL=src /srv/distbuild auto defaults,rw,noatime 0 2 + INSTALL_FILES: distbuild/manifest + WORKER_SSH_KEY: ssh_keys/worker.key + deploy: + red-box-v1-controller: + type: kvm + location: kvm+ssh://vm-user@vm-host/red-box-v1-controller/vm-path/red-box-v1-controller.img + DISK_SIZE: 60G + DISTBUILD_CONTROLLER: yes + HOSTNAME: red-box-v1-controller + WORKERS: red-box-v1-controller + MASON_CLUSTER_MORPHOLOGY: ci.morph + MASON_DEFINITIONS_REF: master + MASON_DISTBUILD_ARCH: x86_64 + MASON_UPSTREAM_TROVE_ADDRESS: upstream-trove + MASON_TEST_HOST: vm-user@vm-host:/vm-path/ diff --git a/mason/httpd.service b/mason/httpd.service new file mode 100644 index 00000000..7572b732 --- /dev/null +++ b/mason/httpd.service @@ -0,0 +1,10 @@ +[Unit] +Description=HTTP server for Mason +After=network.target + +[Service] +User=root +ExecStart=/usr/sbin/httpd -f -p 80 -h /srv/mason + +[Install] +WantedBy=multi-user.target diff --git a/mason/mason-generator.sh b/mason/mason-generator.sh new file mode 100755 index 00000000..a0b2fb0a --- /dev/null +++ b/mason/mason-generator.sh @@ -0,0 +1,94 @@ +#!/bin/sh + +set -e + +if [ "$1" == "-h" -o "$1" == "--help" ]; then + echo "Usage:" + echo " `basename $0` HOST_PREFIX UPSTREAM_TROVE_HOSTNAME VM_USER VM_HOST VM_PATH [HOST_POSTFIX]" + echo "" + echo "Where:" + echo " HOST_PREFIX -- Name of your Mason instance" + echo " e.g. \"my-mason\" to produce hostnames:" + echo " my-mason-trove and my-mason-controller" + echo " UPSTREAM_TROVE_HOSTNAME -- Upstream trove's hostname" + echo " VM_USER -- User on VM host for VM deployment" + echo " VM_HOST -- VM host for VM deployment" + echo " VM_PATH -- Path to store VM images in on VM host" + echo " HOST_POSTFIX -- e.g. \".example.com\" to get" + echo " my-mason-trove.example.com" + echo "" + echo "This script makes deploying a Mason system simpler by automating" + echo "the generation of keys for the systems to use, building of the" + echo "systems, filling out the mason deployment cluster morphology" + echo "template with useful values, and finally deploying the systems." + echo "" + echo "To ensure that the deployed system can deploy test systems, you" + echo "must supply an ssh key to the VM host. Do so with the following" + echo "command:" + echo " ssh-copy-id -i ssh_keys-HOST_PREFIX/mason.key.pub VM_USER@VM_HOST" + echo "" + exit 0 +fi + + +HOST_PREFIX=$1 +UPSTREAM_TROVE=$2 +VM_USER=$3 +VM_HOST=$4 +VM_PATH=$5 +HOST_POSTFIX=$6 + +sedescape() { + # Escape all non-alphanumeric characters + printf "%s\n" "$1" | sed -e 's/\W/\\&/g' +} + + +############################################################################## +# Key generation +############################################################################## + +mkdir "ssh_keys-${HOST_PREFIX}" +cd "ssh_keys-${HOST_PREFIX}" +ssh-keygen -t rsa -b 2048 -f mason.key -C mason@TROVE_HOST -N '' +ssh-keygen -t rsa -b 2048 -f lorry.key -C lorry@TROVE_HOST -N '' +ssh-keygen -t rsa -b 2048 -f worker.key -C worker@TROVE_HOST -N '' +ssh-keygen -t rsa -b 2048 -f id_rsa -C trove-admin@TROVE_HOST -N '' +cd ../ + + +############################################################################## +# Mason setup +############################################################################## + +cp mason.morph mason-${HOST_PREFIX}.morph + +sed -i "s/red-box-v1/$(sedescape "$HOST_PREFIX")/g" "mason-$HOST_PREFIX.morph" +sed -i "s/ssh_keys/ssh_keys-$(sedescape "$HOST_PREFIX")/g" "mason-$HOST_PREFIX.morph" +sed -i "s/upstream-trove/$(sedescape "$UPSTREAM_TROVE")/" "mason-$HOST_PREFIX.morph" +sed -i "s/vm-user/$(sedescape "$VM_USER")/g" "mason-$HOST_PREFIX.morph" +sed -i "s/vm-host/$(sedescape "$VM_HOST")/g" "mason-$HOST_PREFIX.morph" +sed -i "s/vm-path/$(sedescape "$VM_PATH")/g" "mason-$HOST_PREFIX.morph" +sed -i "s/\.example\.com/$(sedescape "$HOST_POSTFIX")/g" "mason-$HOST_PREFIX.morph" + + +############################################################################## +# System building +############################################################################## + +morph build trove-system-x86_64 +morph build distbuild-system-x86_64 + + +############################################################################## +# System deployment +############################################################################## + +morph deploy mason-${HOST_PREFIX}.morph + + +############################################################################## +# Cleanup +############################################################################## + +rm mason-${HOST_PREFIX}.morph diff --git a/mason/mason-report.sh b/mason/mason-report.sh new file mode 100755 index 00000000..d6cf0c19 --- /dev/null +++ b/mason/mason-report.sh @@ -0,0 +1,209 @@ +#!/bin/bash + +set -x + +. /root/mason.conf + +REPORT_PATH=/root/report.html +SERVER_PATH=/srv/mason + +sed_escape() { + printf "%s\n" "$1" | sed -e 's/\W/\\&/g' +} + +create_report() { +cat > $REPORT_PATH <<'EOF' +<html> +<head> +<meta charset="UTF-8"> +<meta http-equiv="refresh" content="10"> +<style> +html, body { + margin: 0; + padding: 0; +} +p.branding { + background: black; + color: #fff; + padding: 0.4em; + margin: 0; + font-weight: bold; +} +h1 { + background: #225588; + color: white; + margin: 0; + padding: 0.6em; +} +table { + width: 90%; + margin: 1em auto 6em auto; + border: 1px solid black; + border-spacing: 0; +} +table tr.headings { + background: #555; + color: white; +} +table tr.pass { + background: #aaffaa; +} +table tr.pass:hover { + background: #bbffbb; +} +table tr.fail { + background: #ffaaaa; +} +table tr.fail:hover { + background: #ffbbbb; +} +table tr.headings th { + font-weight: bold; + text-align: left; + padding: 3px 2px; +} +table td { + padding: 2px; +} +td.result { + font-weight: bold; + text-transform: uppercase; +} +td.result a { + text-decoration: none; +} +td.result a:before { + content: "➫ "; +} +tr.pass td.result a { + color: #252; +} +tr.pass td.result a:hover { + color: #373; +} +tr.fail td.result a { + color: #622; +} +tr.fail td.result a:hover { + color: #933; +} +td.ref { + font-family: monospace; +} +td.ref a { + color: #333; +} +td.ref a:hover { + color: #555; +} +table tr.pass td, table tr.fail td { + border-top: solid white 1px; +} +p { + margin: 1.3em; +} +code { + padding: 0.3em 0.5em; + background: #eee; + border: 1px solid #bbb; + border-radius: 1em; +} +#footer { + margin: 0; + background: #aaa; + color: #222; + border-top: #888 1px solid; + font-size: 80%; + padding: 0; + text-align: right; + position: fixed; + bottom: 0; + width: 100%; +} +</style> +</head> +<body> +<p class="branding">Mason</p> +<h1>Baserock: Continuous Delivery</h1> +<p>Build log of changes to <code>BRANCH</code> from <code>TROVE</code>. Most recent first.</p> +<table> +<tr class="headings"> + <th>Started</th> + <th>Ref</th> + <th>Duration</th> + <th>Result</th> +</tr> +<!--INSERTION POINT--> +</table> +<div id="footer"> +<p>Generated by Mason</p> +</div> +</body> +</html> +EOF + + sed -i 's/BRANCH/'"$(sed_escape "$1")"'/' $REPORT_PATH + sed -i 's/TROVE/'"$(sed_escape "$2")"'/' $REPORT_PATH +} + +update_report() { + # Give function params sensible names + build_start_time="$1" + build_trove_host="$2" + build_ref="$3" + build_sha1="$4" + build_duration="$5" + build_result="$6" + + # Generate template if report file is not there + if [ ! -f $REPORT_PATH ]; then + create_report $build_ref $build_trove_host + fi + + # Build table row for insertion into report file + msg='<tr class="'"${build_result}"'"><td>'"${build_start_time}"'</td><td class="ref"><a href="http://'"${build_trove_host}"'/cgi-bin/cgit.cgi/baserock/baserock/definitions.git/commit/?h='"${build_ref}"'&id='"${build_sha1}"'">'"${build_sha1}"'</a></td><td>'"${build_duration}s"'</td><td class="result"><a href="log/'"${build_sha1}"'--'"${build_start_time}"'.log">'"${build_result}"'</a></td></tr>' + + # Insert report line, newest at top + sed -i 's/<!--INSERTION POINT-->/<!--INSERTION POINT-->\n'"$(sed_escape "$msg")"'/' $REPORT_PATH +} + +START_TIME=`date +%Y-%m-%d\ %T` + +logfile="$(mktemp)" +/root/mason.sh 2>&1 | tee "$logfile" +case "${PIPESTATUS[0]}" in +0) + RESULT=pass + ;; +33) + RESULT=skip + ;; +*) + RESULT=fail + ;; +esac + +# TODO: Update page with last executed time +if [ "$RESULT" = skip ]; then + rm "$logfile" + exit 0 +fi + +DURATION=$(( $(date +%s) - $(date --date="$START_TIME" +%s) )) +SHA1="$(cd "ws/$DEFINITIONS_REF/$UPSTREAM_TROVE_ADDRESS/baserock/baserock/definitions" && git rev-parse HEAD)" + +update_report "$START_TIME" \ + "$DISTBUILD_TROVE_ADDRESS" \ + "$DEFINITIONS_REF" \ + "$SHA1" \ + "$DURATION" \ + "$RESULT" + + +# +# Copy report into server directory +# + +cp "$REPORT_PATH" "$SERVER_PATH/index.html" +mkdir /srv/mason/log +mv "$logfile" /srv/mason/log/"$SHA1--$START_TIME.log" diff --git a/mason/mason.service b/mason/mason.service new file mode 100644 index 00000000..16b5dc3f --- /dev/null +++ b/mason/mason.service @@ -0,0 +1,9 @@ +[Unit] +Description=Mason: Continuous Delivery Service + +[Service] +User=root +ExecStart=/root/mason-report.sh + +[Install] +WantedBy=multi-user.target diff --git a/mason/mason.sh b/mason/mason.sh new file mode 100755 index 00000000..dfed71f7 --- /dev/null +++ b/mason/mason.sh @@ -0,0 +1,70 @@ +#!/bin/sh + +set -e +set -x + +# Load our deployment config +. /root/mason.conf + +if [ ! -e ws ]; then + morph init ws +fi +cd ws + +definitions_repo="$DEFINITIONS_REF"/"$UPSTREAM_TROVE_ADDRESS"/baserock/baserock/definitions +if [ ! -e "$definitions_repo" ]; then + morph checkout git://"$UPSTREAM_TROVE_ADDRESS"/baserock/baserock/definitions.git "$DEFINITIONS_REF" + cd "$definitions_repo" + git config user.name "$TROVE_ID"-mason + git config user.email "$TROVE_ID"-mason@$(hostname) +else + cd "$definitions_repo" + SHA1_PREV="$(git rev-parse HEAD)" +fi + +git remote update origin +git clean -fxd +git reset --hard origin/"$DEFINITIONS_REF" + +SHA1="$(git rev-parse HEAD)" + +if [ -f "$HOME/success" ] && [ "$SHA1" = "$SHA1_PREV" ]; then + echo INFO: No changes to "$DEFINITIONS_REF", nothing to do + exit 33 +fi + +rm -f "$HOME/success" + +echo INFO: Mason building: $DEFINITIONS_REF at $SHA1 + +"scripts/release-build" --no-default-configs \ + --trove-host "$UPSTREAM_TROVE_ADDRESS" \ + --controllers "$DISTBUILD_ARCH:$DISTBUILD_CONTROLLER_ADDRESS" \ + "$BUILD_CLUSTER_MORPHOLOGY" + +releases_made="$(cd release && ls | wc -l)" +if [ "$releases_made" = 0 ]; then + echo ERROR: No release images created + exit 1 +else + echo INFO: Created "$releases_made" release images +fi + +"scripts/release-test" \ + --deployment-host "$DISTBUILD_ARCH":"$TEST_VM_HOST_SSH_URL" \ + --trove-host "$DISTBUILD_TROVE_ADDRESS" \ + --trove-id "$TROVE_ID" \ + "$BUILD_CLUSTER_MORPHOLOGY" + +"scripts/release-upload" --build-trove-host "$DISTBUILD_TROVE_ADDRESS" \ + --arch "$DISTBUILD_ARCH" \ + --log-level=debug --log="$HOME"/release-upload.log \ + --public-trove-host "$UPSTREAM_TROVE_ADDRESS" \ + --public-trove-username root \ + --public-trove-artifact-dir /home/cache/artifacts \ + --no-upload-release-artifacts \ + "$BUILD_CLUSTER_MORPHOLOGY" + +echo INFO: Artifact upload complete for $DEFINITIONS_REF at $SHA1 + +touch "$HOME/success" diff --git a/mason/mason.timer b/mason/mason.timer new file mode 100644 index 00000000..107dff97 --- /dev/null +++ b/mason/mason.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Runs Mason continually with 1 min between calls + +[Timer] +#Time between Mason finishing and calling it again +OnUnitActiveSec=1min +Unit=mason.service + +[Install] +WantedBy=multi-user.target diff --git a/scripts/release-test b/scripts/release-test new file mode 100755 index 00000000..ddfe6bbd --- /dev/null +++ b/scripts/release-test @@ -0,0 +1,402 @@ +#!/usr/bin/env python +# +# Copyright 2014 Codethink Ltd +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +'''release-test + +This script deploys the set of systems in the cluster morphology it is +instructed to read, to test that they work correctly. + +''' + +import cliapp +import os +import pipes +import shlex +import shutil +import socket +import tempfile +import time +import uuid + +import morphlib + + +class MorphologyFrobber(object): + def __init__(self): + self.sb = sb = morphlib.sysbranchdir.open_from_within('.') + defs_repo_path = sb.get_git_directory_name(sb.root_repository_url) + self.defs_repo = morphlib.gitdir.GitDirectory(defs_repo_path) + self.loader = morphlib.morphloader.MorphologyLoader() + self.finder = morphlib.morphologyfinder.MorphologyFinder(self.defs_repo) + + def load_morphology(self, path): + text = self.finder.read_morphology(path) + return self.loader.load_from_string(text) + + @classmethod + def iterate_systems(cls, systems_list): + for system in systems_list: + yield morphlib.util.sanitise_morphology_path(system['morph']) + if 'subsystems' in system: + for subsystem in cls.iterate_systems(system['subsystems']): + yield subsystem + + def iterate_cluster_deployments(cls, cluster_morph): + for system in cluster_morph['systems']: + path = morphlib.util.sanitise_morphology_path(system['morph']) + defaults = system.get('deploy-defaults', {}) + for name, options in system['deploy'].iteritems(): + config = dict(defaults) + config.update(options) + yield path, name, config + + def load_cluster_systems(self, cluster_morph): + for system_path in set(self.iterate_systems(cluster_morph['systems'])): + system_text = self.finder.read_morphology(system_path) + system_morph = self.loader.load_from_string(system_text) + yield system_path, system_morph + + +class TimeoutError(cliapp.AppException): + + """Error to be raised when a connection waits too long""" + + def __init__(self, msg): + super(TimeoutError, self).__init__(msg) + + +class VMHost(object): + def __init__(self, user, address, disk_path): + self.user = user + self.address = address + self.disk_path = disk_path + + @property + def ssh_host(self): + return '{user}@{address}'.format(user=self.user, address=self.address) + + def runcmd(self, *args, **kwargs): + cliapp.ssh_runcmd(self.ssh_host, *args, **kwargs) + + def virsh(self, *args, **kwargs): + self.runcmd(['virsh', '-c', 'qemu:///system'] + list(args), **kwargs) + + +class Instance(object): + def __init__(self, deployment, config, host_machine, vm_id, rootfs_path): + self.deployment = deployment + self.config = config + # TODO: Stop assuming test machine can DHCP and be assigned its + # hostname in the deployer's resolve search path. + self.ip_address = self.config['HOSTNAME'] + self.host_machine = host_machine + self.vm_id = vm_id + self.rootfs_path = rootfs_path + + @property + def ssh_host(self): + # TODO: Stop assuming we ssh into test instances as root + return 'root@{host}'.format(host=self.ip_address) + + def runcmd(self, argv, chdir='.', **kwargs): + ssh_cmd = ['ssh', '-o', 'StrictHostKeyChecking=no', self.ssh_host] + cmd = ['sh', '-c', 'cd "$1" && shift && exec "$@"', '-', chdir] + cmd += argv + ssh_cmd.append(' '.join(map(pipes.quote, cmd))) + return cliapp.runcmd(ssh_cmd, **kwargs) + + def _wait_for_dhcp(self, timeout): + ''' + Block until given hostname resolves successfully. + + Raises TimeoutError if the hostname has not appeared in 'timeout' + seconds. + ''' + start_time = time.time() + while True: + try: + socket.gethostbyname(self.ip_address) + return + except socket.gaierror: + pass + if time.time() > start_time + timeout: + raise TimeoutError("Host %s did not appear after %i seconds" % + (self.ip_address, timeout)) + time.sleep(0.5) + + def _wait_for_ssh(self, timeout): + """Wait until the deployed VM is responding via SSH""" + start_time = time.time() + while True: + try: + self.runcmd(['true'], stdin=None, stdout=None, stderr=None) + return + except cliapp.AppException: + # TODO: Stop assuming the ssh part of the command is what failed + if time.time() > start_time + timeout: + raise TimeoutError("%s sshd did not start after %i seconds" + % (self.ip_address, timeout)) + time.sleep(0.5) + + def wait_online(self, timeout=10): + self._wait_for_dhcp(timeout) + self._wait_for_ssh(timeout) + + def delete(self): + # Forget config of deployed system as if it ever reappears it + # will have different host keys + cliapp.runcmd(['ssh-keygen', '-R', self.config['HOSTNAME']]) + # Stop and remove VM + try: + self.host_machine.virsh('destroy', self.vm_id) + except cliapp.AppException as e: + # TODO: Stop assuming that destroy failed because it wasn't running + pass + try: + self.host_machine.virsh('undefine', self.vm_id) + except cliapp.AppException as e: + # TODO: Stop assuming that undefine failed because it was + # already removed + pass + # TODO: Remove all attached disks, rather than just assuming it + # only has one + self.host_machine.runcmd(['rm', self.rootfs_path]) + + +class Deployment(object): + def __init__(self, cluster_path, name, + deployment_config, host_machine): + self.cluster_path = cluster_path + self.name = name + self.deployment_config = deployment_config + self.host_machine = host_machine + + @staticmethod + def _ssh_host_key_exists(hostname): + """Check if an ssh host key exists in known_hosts""" + with open('/root/.ssh/known_hosts', 'r') as known_hosts: + return any(line.startswith(hostname) for line in known_hosts) + + def _update_known_hosts(self): + if not self._ssh_host_key_exists(self.host_machine.address): + with open('/root/.ssh/known_hosts', 'a') as known_hosts: + cliapp.runcmd(['ssh-keyscan', self.host_machine.address], + stdout=known_hosts) + + @staticmethod + def _generate_sshkey_config(tempdir, config): + manifest = os.path.join(tempdir, 'manifest') + with open(manifest, 'w') as f: + f.write('0040700 0 0 /root/.ssh\n') + f.write('overwrite 0100600 0 0 /root/.ssh/authorized_keys\n') + authkeys = os.path.join(tempdir, 'root', '.ssh', 'authorized_keys') + os.makedirs(os.path.dirname(authkeys)) + with open(authkeys, 'w') as auth_f: + with open('/root/.ssh/id_rsa.pub', 'r') as key_f: + shutil.copyfileobj(key_f, auth_f) + + install_files = shlex.split(config.get('INSTALL_FILES', '')) + install_files.append(manifest) + yield 'INSTALL_FILES', ' '.join(pipes.quote(f) for f in install_files) + + def deploy(self): + self._update_known_hosts() + + hostname = str(uuid.uuid4()) + vm_id = hostname + image_base = self.host_machine.disk_path + rootpath = '{image_base}/{hostname}.img'.format(image_base=image_base, + hostname=hostname) + loc = 'kvm+ssh://{ssh_host}/{id}/{path}'.format( + ssh_host=self.host_machine.ssh_host, id=vm_id, path=rootpath) + + options = { + 'type': 'kvm', + 'location': loc, + 'AUTOSTART': 'True', + 'HOSTNAME': hostname, + 'DISK_SIZE': '20G', + 'RAM_SIZE': '2G', + 'VERSION_LABEL': 'release-test', + } + + tempdir = tempfile.mkdtemp() + try: + options.update( + self._generate_sshkey_config(tempdir, + self.deployment_config)) + + args = ['morph', 'deploy', self.cluster_path, self.name] + for k, v in options.iteritems(): + args.append('%s.%s=%s' % (self.name, k, v)) + cliapp.runcmd(args, stdin=None, stdout=None, stderr=None) + + config = dict(self.deployment_config) + config.update(options) + + return Instance(self, config, self.host_machine, vm_id, rootpath) + finally: + shutil.rmtree(tempdir) + + +class ReleaseApp(cliapp.Application): + + """Cliapp application which handles automatic builds and tests""" + + def add_settings(self): + """Add the command line options needed""" + group_main = 'Program Options' + self.settings.string_list(['deployment-host'], + 'ARCH:HOST:PATH that VMs can be deployed to', + default=None, + group=group_main) + self.settings.string(['trove-host'], + 'Address of Trove for test systems to build from', + default=None, + group=group_main) + self.settings.string(['trove-id'], + 'ID of Trove for test systems to build from', + default=None, + group=group_main) + self.settings.string(['build-ref-prefix'], + 'Prefix of build branches for test systems', + default=None, + group=group_main) + + @staticmethod + def _run_tests(instance, system_path, system_morph, + (trove_host, trove_id, build_ref_prefix), + morph_frobber, systems): + instance.wait_online() + + tests = [] + def baserock_build_test(instance): + instance.runcmd(['git', 'config', '--global', 'user.name', + 'Test Instance of %s' % instance.deployment.name]) + instance.runcmd(['git', 'config', '--global', 'user.email', + 'ci-test@%s' % instance.config['HOSTNAME']]) + instance.runcmd(['mkdir', '-p', '/src/ws', '/src/cache', + '/src/tmp']) + def morph_cmd(*args, **kwargs): + # TODO: decide whether to use cached artifacts or not by + # adding --artifact-cache-server= --cache-server= + argv = ['morph', '--log=/src/morph.log', '--cachedir=/src/cache', + '--tempdir=/src/tmp', '--log-max=100M', + '--trove-host', trove_host, '--trove-id', trove_id, + '--build-ref-prefix', build_ref_prefix] + argv.extend(args) + instance.runcmd(argv, **kwargs) + + repo = morph_frobber.sb.root_repository_url + ref = morph_frobber.defs_repo.HEAD + sha1 = morph_frobber.defs_repo.resolve_ref_to_commit(ref) + morph_cmd('init', '/src/ws') + chdir = '/src/ws' + + morph_cmd('checkout', repo, ref, chdir=chdir) + # TODO: Add a morph subcommand that gives the path to the root repository. + repo_path = os.path.relpath( + morph_frobber.sb.get_git_directory_name(repo), + morph_frobber.sb.root_directory) + chdir = os.path.join(chdir, ref, repo_path) + + instance.runcmd(['git', 'reset', '--hard', sha1], chdir=chdir) + print 'Building test systems for {sys}'.format(sys=system_path) + for to_build_path, to_build_morph in systems.iteritems(): + if to_build_morph['arch'] == system_morph['arch']: + print 'Test building {path}'.format(path=to_build_path) + morph_cmd('build', to_build_path, chdir=chdir, + stdin=None, stdout=None, stderr=None) + print 'Finished Building test systems' + + def python_smoke_test(instance): + instance.runcmd(['python', '-c', 'print "Hello World"']) + + # TODO: Come up with a better way of determining which tests to run + if 'devel' in system_path: + tests.append(baserock_build_test) + else: + tests.append(python_smoke_test) + + for test in tests: + test(instance) + + def deploy_and_test_systems(self, cluster_path, + deployment_hosts, build_test_config): + """Run the deployments and tests""" + + image_path = '/opt/ci/' + version = 'release-test' + + morph_frobber = MorphologyFrobber() + cluster_morph = morph_frobber.load_morphology(cluster_path) + systems = dict(morph_frobber.load_cluster_systems(cluster_morph)) + + for system_path, deployment_name, deployment_config in \ + morph_frobber.iterate_cluster_deployments(cluster_morph): + + system_morph = systems[system_path] + # We can only test systems in KVM that have a BSP + if not any('bsp' in si['morph'] for si in system_morph['strata']): + continue + + # We can only test systems in KVM that we have a host for + if system_morph['arch'] not in deployment_hosts: + continue + host_machine = deployment_hosts[system_morph['arch']] + deployment = Deployment(cluster_path, deployment_name, + deployment_config, host_machine) + + instance = deployment.deploy() + try: + self._run_tests(instance, system_path, system_morph, + build_test_config, morph_frobber, systems) + finally: + instance.delete() + + def process_args(self, args): + """Process the command line args and kick off the builds/tests""" + if self.settings['build-ref-prefix'] is None: + self.settings['build-ref-prefix'] = ( + os.path.join(self.settings['trove-id'], 'builds')) + for setting in ('deployment-host', 'trove-host', + 'trove-id', 'build-ref-prefix'): + self.settings.require(setting) + + deployment_hosts = {} + for host_config in self.settings['deployment-host']: + arch, address = host_config.split(':', 1) + user, address = address.split('@', 1) + address, disk_path = address.split(':', 1) + if user == '': + user = 'root' + # TODO: Don't assume root is the user with deploy access + deployment_hosts[arch] = VMHost(user, address, disk_path) + + build_test_config = (self.settings['trove-host'], + self.settings['trove-id'], + self.settings['build-ref-prefix']) + + if len(args) != 1: + raise cliapp.AppException('Usage: release-test CLUSTER') + cluster_path = morphlib.util.sanitise_morphology_path(args[0]) + self.deploy_and_test_systems(cluster_path, deployment_hosts, + build_test_config) + + +if __name__ == '__main__': + ReleaseApp().run() diff --git a/trove.configure b/trove.configure index c7a4f3af..4cc9720a 100755 --- a/trove.configure +++ b/trove.configure @@ -25,6 +25,7 @@ # * TROVE_COMPANY # * LORRY_SSH_KEY # * UPSTREAM_TROVE +# * UPSTREAM_TROVE_PROTOCOL # * TROVE_ADMIN_USER # * TROVE_ADMIN_EMAIL # * TROVE_ADMIN_NAME @@ -136,7 +137,8 @@ trove_configuration={ optional_keys = ('MASON_ID', 'HOSTNAME', 'TROVE_HOSTNAME', - 'LORRY_CONTROLLER_MINIONS', 'TROVE_BACKUP_KEYS') + 'LORRY_CONTROLLER_MINIONS', 'TROVE_BACKUP_KEYS', + 'UPSTREAM_TROVE_PROTOCOL') for key in optional_keys: if key in os.environ: |