summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorRichard Maw <richard.maw@codethink.co.uk>2014-08-28 10:02:57 (GMT)
committerRichard Maw <richard.maw@codethink.co.uk>2014-08-28 10:02:57 (GMT)
commit529cf76e1a276e6324ffa38eef71dcf7d700ad12 (patch)
tree466be2587f655aea12ae426fa2345f928e401464
parent864cb7fe28e99f9bfec955e929e29dbd4d1045dd (diff)
parente3097bdcc35588f4a7dcc99e399b0b492d9e12f4 (diff)
downloaddefinitions-529cf76e1a276e6324ffa38eef71dcf7d700ad12.tar.gz
Merge branch 'baserock/richardmaw/mason-loop-time'
Reviewed-by: Sam Thursfield Reviewed-by: Michael Drake
-rw-r--r--clusters/mason.morph13
-rw-r--r--mason.configure10
-rwxr-xr-xmason/mason-generator.sh90
-rwxr-xr-xmason/mason.sh15
-rwxr-xr-xscripts/release-build11
5 files changed, 74 insertions, 65 deletions
diff --git a/clusters/mason.morph b/clusters/mason.morph
index 1aa2dad..8579f7e 100644
--- a/clusters/mason.morph
+++ b/clusters/mason.morph
@@ -14,7 +14,7 @@ systems:
type: kvm
location: kvm+ssh://vm-user@vm-host/red-box-v1-trove/vm-path/red-box-v1-trove.img
AUTOSTART: true
- DISK_SIZE: 100G
+ DISK_SIZE: 20G
HOSTNAME: red-box-v1-trove
LORRY_SSH_KEY: ssh_keys/lorry.key
MASON_SSH_PUBKEY: ssh_keys/mason.key.pub
@@ -27,19 +27,19 @@ systems:
TROVE_HOST: red-box-v1
TROVE_ID: red-box-v1-trove
UPSTREAM_TROVE: upstream-trove
- UPSTREAM_TROVE_PROTOCOL: http
VCPUS: 2
VERSION_LABEL: 45
WORKER_SSH_PUBKEY: ssh_keys/worker.key.pub
- morph: systems/distbuild-system-x86_64.morph
deploy-defaults:
- CONTROLLERHOST: red-box-v1-controller
+ ARTIFACT_CACHE_SERVER: red-box-v1-trove.example.com
+ CONTROLLERHOST: red-box-v1-controller.example.com
DISTBUILD_CONTROLLER: false
DISTBUILD_WORKER: true
INSTALL_FILES: distbuild/manifest
RAM_SIZE: 8G
- TROVE_HOST: red-box-v1-trove.example.com
- TROVE_ID: red-box-v1-trove
+ TROVE_HOST: upstream-trove
+ TROVE_ID: upstream-trove
VCPUS: 2
WORKER_SSH_KEY: ssh_keys/worker.key
deploy:
@@ -49,9 +49,8 @@ systems:
DISK_SIZE: 60G
DISTBUILD_CONTROLLER: true
HOSTNAME: red-box-v1-controller
- MASON_CLUSTER_MORPHOLOGY: ci.morph
+ MASON_CLUSTER_MORPHOLOGY: clusters/ci.morph
MASON_DEFINITIONS_REF: master
MASON_DISTBUILD_ARCH: x86_64
MASON_TEST_HOST: vm-user@vm-host:/vm-path/
- MASON_UPSTREAM_TROVE_ADDRESS: upstream-trove
WORKERS: red-box-v1-controller
diff --git a/mason.configure b/mason.configure
index fb73b01..90d4970 100644
--- a/mason.configure
+++ b/mason.configure
@@ -19,8 +19,8 @@
# a Mason instance at deployment time. It uses the following variables
# from the environment:
#
+# * ARTIFACT_CACHE_SERVER
# * MASON_CLUSTER_MORPHOLOGY
-# * MASON_UPSTREAM_TROVE_ADDRESS
# * MASON_DEFINITIONS_REF
# * MASON_DISTBUILD_ARCH
# * MASON_TEST_HOST
@@ -32,8 +32,8 @@ set -e
ROOT="$1"
-if [ "$MASON_CLUSTER_MORPHOLOGY" = "" \
- -o "$MASON_UPSTREAM_TROVE_ADDRESS" = "" \
+if [ "$ARTIFACT_CACHE_SERVER" = "" \
+ -o "$MASON_CLUSTER_MORPHOLOGY" = "" \
-o "$MASON_DEFINITIONS_REF" = "" \
-o "$MASON_DISTBUILD_ARCH" = "" \
-o "$MASON_TEST_HOST" = "" ]; then
@@ -61,11 +61,11 @@ echo "Creating $MASON_CONFIG"
cat >>"$MASON_CONFIG" <<EOF
#################### START OF DEPLOY TIME CONFIGURATION #######################
-UPSTREAM_TROVE_ADDRESS=$(shellescape "$MASON_UPSTREAM_TROVE_ADDRESS")
+ARTIFACT_CACHE_SERVER=$(shellescape "$ARTIFACT_CACHE_SERVER")
+UPSTREAM_TROVE_ADDRESS=$(shellescape "$TROVE_HOST")
DEFINITIONS_REF=$(shellescape "$MASON_DEFINITIONS_REF")
DISTBUILD_ARCH=$(shellescape "$MASON_DISTBUILD_ARCH")
DISTBUILD_CONTROLLER_ADDRESS=$(shellescape "$CONTROLLERHOST")
-DISTBUILD_TROVE_ADDRESS=$(shellescape "$TROVE_HOST")
TROVE_ID=$(shellescape "$TROVE_ID")
BUILD_CLUSTER_MORPHOLOGY=$(shellescape "$MASON_CLUSTER_MORPHOLOGY")
TEST_VM_HOST_SSH_URL=$(shellescape "$MASON_TEST_HOST")
diff --git a/mason/mason-generator.sh b/mason/mason-generator.sh
index c1ffeb2..1a2ecca 100755
--- a/mason/mason-generator.sh
+++ b/mason/mason-generator.sh
@@ -2,46 +2,48 @@
set -e
-if [ "$1" == "-h" -o "$1" == "--help" ]; then
- echo "Usage:"
- echo " `basename $0` HOST_PREFIX UPSTREAM_TROVE_HOSTNAME VM_USER VM_HOST VM_PATH [HOST_POSTFIX]"
- echo ""
- echo "Where:"
- echo " HOST_PREFIX -- Name of your Mason instance"
- echo " e.g. \"my-mason\" to produce hostnames:"
- echo " my-mason-trove and my-mason-controller"
- echo " UPSTREAM_TROVE_HOSTNAME -- Upstream trove's hostname"
- echo " VM_USER -- User on VM host for VM deployment"
- echo " VM_HOST -- VM host for VM deployment"
- echo " VM_PATH -- Path to store VM images in on VM host"
- echo " HOST_POSTFIX -- e.g. \".example.com\" to get"
- echo " my-mason-trove.example.com"
- echo ""
- echo "This script makes deploying a Mason system simpler by automating"
- echo "the generation of keys for the systems to use, building of the"
- echo "systems, filling out the mason deployment cluster morphology"
- echo "template with useful values, and finally deploying the systems."
- echo ""
- echo "To ensure that the deployed system can deploy test systems, you"
- echo "must supply an ssh key to the VM host. Do so with the following"
- echo "command:"
- echo " ssh-copy-id -i ssh_keys-HOST_PREFIX/worker.key.pub VM_USER@VM_HOST"
- echo ""
- echo "To ensure that the mason can upload artifacts to the upstream trove,"
- echo "you must supply an ssh key to the upstream trove. Do so with the"
- echo "following command:"
- echo " ssh-copy-id -i ssh_keys-HOST_PREFIX/id_rsa.key.pub root@UPSTREAM_TROVE_HOSTNAME"
- echo ""
+if [ "$#" -lt 5 -o "$#" -gt 6 -o "$1" == "-h" -o "$1" == "--help" ]; then
+ cat <<EOF
+Usage:
+ `basename $0` HOST_PREFIX UPSTREAM_TROVE_HOSTNAME VM_USER VM_HOST VM_PATH [HOST_POSTFIX]
+
+Where:
+ HOST_PREFIX -- Name of your Mason instance
+ e.g. "my-mason" to produce hostnames:
+ my-mason-trove and my-mason-controller
+ UPSTREAM_TROVE_HOSTNAME -- Upstream trove's hostname
+ VM_USER -- User on VM host for VM deployment
+ VM_HOST -- VM host for VM deployment
+ VM_PATH -- Path to store VM images in on VM host
+ HOST_POSTFIX -- e.g. ".example.com" to get
+ my-mason-trove.example.com
+
+This script makes deploying a Mason system simpler by automating
+the generation of keys for the systems to use, building of the
+systems, filling out the mason deployment cluster morphology
+template with useful values, and finally deploying the systems.
+
+To ensure that the deployed system can deploy test systems, you
+must supply an ssh key to the VM host. Do so with the following
+command:
+ ssh-copy-id -i ssh_keys-HOST_PREFIX/worker.key.pub VM_USER@VM_HOST
+
+To ensure that the mason can upload artifacts to the upstream trove,
+you must supply an ssh key to the upstream trove. Do so with the
+following command:
+ ssh-copy-id -i ssh_keys-HOST_PREFIX/id_rsa.key.pub root@UPSTREAM_TROVE_HOSTNAME
+
+EOF
exit 0
fi
-HOST_PREFIX=$1
-UPSTREAM_TROVE=$2
-VM_USER=$3
-VM_HOST=$4
-VM_PATH=$5
-HOST_POSTFIX=$6
+HOST_PREFIX="$1"
+UPSTREAM_TROVE="$2"
+VM_USER="$3"
+VM_HOST="$4"
+VM_PATH="$5"
+HOST_POSTFIX="$6"
sedescape() {
# Escape all non-alphanumeric characters
@@ -53,12 +55,12 @@ sedescape() {
# Key generation
##############################################################################
-mkdir "ssh_keys-${HOST_PREFIX}"
+mkdir -p "ssh_keys-${HOST_PREFIX}"
cd "ssh_keys-${HOST_PREFIX}"
-ssh-keygen -t rsa -b 2048 -f mason.key -C mason@TROVE_HOST -N ''
-ssh-keygen -t rsa -b 2048 -f lorry.key -C lorry@TROVE_HOST -N ''
-ssh-keygen -t rsa -b 2048 -f worker.key -C worker@TROVE_HOST -N ''
-ssh-keygen -t rsa -b 2048 -f id_rsa -C trove-admin@TROVE_HOST -N ''
+test -e mason.key || ssh-keygen -t rsa -b 2048 -f mason.key -C mason@TROVE_HOST -N ''
+test -e lorry.key || ssh-keygen -t rsa -b 2048 -f lorry.key -C lorry@TROVE_HOST -N ''
+test -e worker.key || ssh-keygen -t rsa -b 2048 -f worker.key -C worker@TROVE_HOST -N ''
+test -e id_rsa || ssh-keygen -t rsa -b 2048 -f id_rsa -C trove-admin@TROVE_HOST -N ''
cd ../
@@ -66,7 +68,7 @@ cd ../
# Mason setup
##############################################################################
-cp mason.morph mason-${HOST_PREFIX}.morph
+cp clusters/mason.morph mason-${HOST_PREFIX}.morph
sed -i "s/red-box-v1/$(sedescape "$HOST_PREFIX")/g" "mason-$HOST_PREFIX.morph"
sed -i "s/ssh_keys/ssh_keys-$(sedescape "$HOST_PREFIX")/g" "mason-$HOST_PREFIX.morph"
@@ -81,8 +83,8 @@ sed -i "s/\.example\.com/$(sedescape "$HOST_POSTFIX")/g" "mason-$HOST_PREFIX.mor
# System building
##############################################################################
-morph build trove-system-x86_64
-morph build distbuild-system-x86_64
+morph build systems/trove-system-x86_64.morph
+morph build systems/distbuild-system-x86_64.morph
##############################################################################
diff --git a/mason/mason.sh b/mason/mason.sh
index 13b97b7..13ea21d 100755
--- a/mason/mason.sh
+++ b/mason/mason.sh
@@ -11,9 +11,9 @@ if [ ! -e ws ]; then
fi
cd ws
-definitions_repo="$DEFINITIONS_REF"/"$DISTBUILD_TROVE_ADDRESS"/baserock/baserock/definitions
+definitions_repo="$DEFINITIONS_REF"/"$UPSTREAM_TROVE_ADDRESS"/baserock/baserock/definitions
if [ ! -e "$definitions_repo" ]; then
- morph checkout git://"$DISTBUILD_TROVE_ADDRESS"/baserock/baserock/definitions "$DEFINITIONS_REF"
+ morph checkout git://"$UPSTREAM_TROVE_ADDRESS"/baserock/baserock/definitions "$DEFINITIONS_REF"
cd "$definitions_repo"
git config user.name "$TROVE_ID"-mason
git config user.email "$TROVE_ID"-mason@$(hostname)
@@ -41,9 +41,10 @@ rm -f "$HOME/success"
echo INFO: Mason building: $DEFINITIONS_REF at $SHA1
if ! "scripts/release-build" --no-default-configs \
- --trove-host "$DISTBUILD_TROVE_ADDRESS" \
- --controllers "$DISTBUILD_ARCH:$DISTBUILD_CONTROLLER_ADDRESS" \
- "$BUILD_CLUSTER_MORPHOLOGY"; then
+ --trove-host "$UPSTREAM_TROVE_ADDRESS" \
+ --artifact-cache-server "http://$ARTIFACT_CACHE_SERVER:8080/" \
+ --controllers "$DISTBUILD_ARCH:$DISTBUILD_CONTROLLER_ADDRESS" \
+ "$BUILD_CLUSTER_MORPHOLOGY"; then
echo ERROR: Failed to build release images
echo Build logs for chunks:
find builds -type f -exec echo {} \; -exec cat {} \;
@@ -60,11 +61,11 @@ fi
"scripts/release-test" \
--deployment-host "$DISTBUILD_ARCH":"$TEST_VM_HOST_SSH_URL" \
- --trove-host "$DISTBUILD_TROVE_ADDRESS" \
+ --trove-host "$UPSTREAM_TROVE_ADDRESS" \
--trove-id "$TROVE_ID" \
"$BUILD_CLUSTER_MORPHOLOGY"
-"scripts/release-upload" --build-trove-host "$DISTBUILD_TROVE_ADDRESS" \
+"scripts/release-upload" --build-trove-host "$ARTIFACT_CACHE_SERVER" \
--arch "$DISTBUILD_ARCH" \
--log-level=debug --log="$HOME"/release-upload.log \
--public-trove-host "$UPSTREAM_TROVE_ADDRESS" \
diff --git a/scripts/release-build b/scripts/release-build
index a3e25eb..5525e9e 100755
--- a/scripts/release-build
+++ b/scripts/release-build
@@ -53,6 +53,9 @@ class ReleaseApp(cliapp.Application):
self.settings.string(['trove-host'],
'hostname of Trove instance')
+ self.settings.string(['artifact-cache-server'],
+ 'server to fetch artifacts from', default=None)
+
self.settings.string(['release-number'],
'Baserock version of the systems being built',
default='yy.ww')
@@ -157,10 +160,14 @@ class ReleaseApp(cliapp.Application):
def deploy_single_image(self, cluster_path, name, location, version_label):
deploy_command = [
'morph', 'deploy', cluster_path, name,
- '--trove-host=%s' % self.settings['trove-host'],
+ '--trove-host=%s' % self.settings['trove-host']]
+ artifact_server = self.settings['artifact-cache-server']
+ if artifact_server is not None:
+ deploy_command.append('--artifact-cache-server=%s' % artifact_server)
+ deploy_command.extend((
'%s.location=%s' % (name, location),
'%s.VERSION_LABEL=%s' % (name, version_label)
- ]
+ ))
cliapp.runcmd(deploy_command, stdout=sys.stdout)