diff options
author | Richard Maw <richard.maw@codethink.co.uk> | 2014-08-28 10:02:57 +0000 |
---|---|---|
committer | Richard Maw <richard.maw@codethink.co.uk> | 2014-08-28 10:02:57 +0000 |
commit | 529cf76e1a276e6324ffa38eef71dcf7d700ad12 (patch) | |
tree | 466be2587f655aea12ae426fa2345f928e401464 | |
parent | 864cb7fe28e99f9bfec955e929e29dbd4d1045dd (diff) | |
parent | e3097bdcc35588f4a7dcc99e399b0b492d9e12f4 (diff) | |
download | definitions-529cf76e1a276e6324ffa38eef71dcf7d700ad12.tar.gz |
Merge branch 'baserock/richardmaw/mason-loop-time'
Reviewed-by: Sam Thursfield
Reviewed-by: Michael Drake
-rw-r--r-- | clusters/mason.morph | 13 | ||||
-rw-r--r-- | mason.configure | 10 | ||||
-rwxr-xr-x | mason/mason-generator.sh | 90 | ||||
-rwxr-xr-x | mason/mason.sh | 15 | ||||
-rwxr-xr-x | scripts/release-build | 11 |
5 files changed, 74 insertions, 65 deletions
diff --git a/clusters/mason.morph b/clusters/mason.morph index 1aa2dadb..8579f7e4 100644 --- a/clusters/mason.morph +++ b/clusters/mason.morph @@ -14,7 +14,7 @@ systems: type: kvm location: kvm+ssh://vm-user@vm-host/red-box-v1-trove/vm-path/red-box-v1-trove.img AUTOSTART: true - DISK_SIZE: 100G + DISK_SIZE: 20G HOSTNAME: red-box-v1-trove LORRY_SSH_KEY: ssh_keys/lorry.key MASON_SSH_PUBKEY: ssh_keys/mason.key.pub @@ -27,19 +27,19 @@ systems: TROVE_HOST: red-box-v1 TROVE_ID: red-box-v1-trove UPSTREAM_TROVE: upstream-trove - UPSTREAM_TROVE_PROTOCOL: http VCPUS: 2 VERSION_LABEL: 45 WORKER_SSH_PUBKEY: ssh_keys/worker.key.pub - morph: systems/distbuild-system-x86_64.morph deploy-defaults: - CONTROLLERHOST: red-box-v1-controller + ARTIFACT_CACHE_SERVER: red-box-v1-trove.example.com + CONTROLLERHOST: red-box-v1-controller.example.com DISTBUILD_CONTROLLER: false DISTBUILD_WORKER: true INSTALL_FILES: distbuild/manifest RAM_SIZE: 8G - TROVE_HOST: red-box-v1-trove.example.com - TROVE_ID: red-box-v1-trove + TROVE_HOST: upstream-trove + TROVE_ID: upstream-trove VCPUS: 2 WORKER_SSH_KEY: ssh_keys/worker.key deploy: @@ -49,9 +49,8 @@ systems: DISK_SIZE: 60G DISTBUILD_CONTROLLER: true HOSTNAME: red-box-v1-controller - MASON_CLUSTER_MORPHOLOGY: ci.morph + MASON_CLUSTER_MORPHOLOGY: clusters/ci.morph MASON_DEFINITIONS_REF: master MASON_DISTBUILD_ARCH: x86_64 MASON_TEST_HOST: vm-user@vm-host:/vm-path/ - MASON_UPSTREAM_TROVE_ADDRESS: upstream-trove WORKERS: red-box-v1-controller diff --git a/mason.configure b/mason.configure index fb73b01c..90d49709 100644 --- a/mason.configure +++ b/mason.configure @@ -19,8 +19,8 @@ # a Mason instance at deployment time. It uses the following variables # from the environment: # +# * ARTIFACT_CACHE_SERVER # * MASON_CLUSTER_MORPHOLOGY -# * MASON_UPSTREAM_TROVE_ADDRESS # * MASON_DEFINITIONS_REF # * MASON_DISTBUILD_ARCH # * MASON_TEST_HOST @@ -32,8 +32,8 @@ set -e ROOT="$1" -if [ "$MASON_CLUSTER_MORPHOLOGY" = "" \ - -o "$MASON_UPSTREAM_TROVE_ADDRESS" = "" \ +if [ "$ARTIFACT_CACHE_SERVER" = "" \ + -o "$MASON_CLUSTER_MORPHOLOGY" = "" \ -o "$MASON_DEFINITIONS_REF" = "" \ -o "$MASON_DISTBUILD_ARCH" = "" \ -o "$MASON_TEST_HOST" = "" ]; then @@ -61,11 +61,11 @@ echo "Creating $MASON_CONFIG" cat >>"$MASON_CONFIG" <<EOF #################### START OF DEPLOY TIME CONFIGURATION ####################### -UPSTREAM_TROVE_ADDRESS=$(shellescape "$MASON_UPSTREAM_TROVE_ADDRESS") +ARTIFACT_CACHE_SERVER=$(shellescape "$ARTIFACT_CACHE_SERVER") +UPSTREAM_TROVE_ADDRESS=$(shellescape "$TROVE_HOST") DEFINITIONS_REF=$(shellescape "$MASON_DEFINITIONS_REF") DISTBUILD_ARCH=$(shellescape "$MASON_DISTBUILD_ARCH") DISTBUILD_CONTROLLER_ADDRESS=$(shellescape "$CONTROLLERHOST") -DISTBUILD_TROVE_ADDRESS=$(shellescape "$TROVE_HOST") TROVE_ID=$(shellescape "$TROVE_ID") BUILD_CLUSTER_MORPHOLOGY=$(shellescape "$MASON_CLUSTER_MORPHOLOGY") TEST_VM_HOST_SSH_URL=$(shellescape "$MASON_TEST_HOST") diff --git a/mason/mason-generator.sh b/mason/mason-generator.sh index c1ffeb24..1a2eccac 100755 --- a/mason/mason-generator.sh +++ b/mason/mason-generator.sh @@ -2,46 +2,48 @@ set -e -if [ "$1" == "-h" -o "$1" == "--help" ]; then - echo "Usage:" - echo " `basename $0` HOST_PREFIX UPSTREAM_TROVE_HOSTNAME VM_USER VM_HOST VM_PATH [HOST_POSTFIX]" - echo "" - echo "Where:" - echo " HOST_PREFIX -- Name of your Mason instance" - echo " e.g. \"my-mason\" to produce hostnames:" - echo " my-mason-trove and my-mason-controller" - echo " UPSTREAM_TROVE_HOSTNAME -- Upstream trove's hostname" - echo " VM_USER -- User on VM host for VM deployment" - echo " VM_HOST -- VM host for VM deployment" - echo " VM_PATH -- Path to store VM images in on VM host" - echo " HOST_POSTFIX -- e.g. \".example.com\" to get" - echo " my-mason-trove.example.com" - echo "" - echo "This script makes deploying a Mason system simpler by automating" - echo "the generation of keys for the systems to use, building of the" - echo "systems, filling out the mason deployment cluster morphology" - echo "template with useful values, and finally deploying the systems." - echo "" - echo "To ensure that the deployed system can deploy test systems, you" - echo "must supply an ssh key to the VM host. Do so with the following" - echo "command:" - echo " ssh-copy-id -i ssh_keys-HOST_PREFIX/worker.key.pub VM_USER@VM_HOST" - echo "" - echo "To ensure that the mason can upload artifacts to the upstream trove," - echo "you must supply an ssh key to the upstream trove. Do so with the" - echo "following command:" - echo " ssh-copy-id -i ssh_keys-HOST_PREFIX/id_rsa.key.pub root@UPSTREAM_TROVE_HOSTNAME" - echo "" +if [ "$#" -lt 5 -o "$#" -gt 6 -o "$1" == "-h" -o "$1" == "--help" ]; then + cat <<EOF +Usage: + `basename $0` HOST_PREFIX UPSTREAM_TROVE_HOSTNAME VM_USER VM_HOST VM_PATH [HOST_POSTFIX] + +Where: + HOST_PREFIX -- Name of your Mason instance + e.g. "my-mason" to produce hostnames: + my-mason-trove and my-mason-controller + UPSTREAM_TROVE_HOSTNAME -- Upstream trove's hostname + VM_USER -- User on VM host for VM deployment + VM_HOST -- VM host for VM deployment + VM_PATH -- Path to store VM images in on VM host + HOST_POSTFIX -- e.g. ".example.com" to get + my-mason-trove.example.com + +This script makes deploying a Mason system simpler by automating +the generation of keys for the systems to use, building of the +systems, filling out the mason deployment cluster morphology +template with useful values, and finally deploying the systems. + +To ensure that the deployed system can deploy test systems, you +must supply an ssh key to the VM host. Do so with the following +command: + ssh-copy-id -i ssh_keys-HOST_PREFIX/worker.key.pub VM_USER@VM_HOST + +To ensure that the mason can upload artifacts to the upstream trove, +you must supply an ssh key to the upstream trove. Do so with the +following command: + ssh-copy-id -i ssh_keys-HOST_PREFIX/id_rsa.key.pub root@UPSTREAM_TROVE_HOSTNAME + +EOF exit 0 fi -HOST_PREFIX=$1 -UPSTREAM_TROVE=$2 -VM_USER=$3 -VM_HOST=$4 -VM_PATH=$5 -HOST_POSTFIX=$6 +HOST_PREFIX="$1" +UPSTREAM_TROVE="$2" +VM_USER="$3" +VM_HOST="$4" +VM_PATH="$5" +HOST_POSTFIX="$6" sedescape() { # Escape all non-alphanumeric characters @@ -53,12 +55,12 @@ sedescape() { # Key generation ############################################################################## -mkdir "ssh_keys-${HOST_PREFIX}" +mkdir -p "ssh_keys-${HOST_PREFIX}" cd "ssh_keys-${HOST_PREFIX}" -ssh-keygen -t rsa -b 2048 -f mason.key -C mason@TROVE_HOST -N '' -ssh-keygen -t rsa -b 2048 -f lorry.key -C lorry@TROVE_HOST -N '' -ssh-keygen -t rsa -b 2048 -f worker.key -C worker@TROVE_HOST -N '' -ssh-keygen -t rsa -b 2048 -f id_rsa -C trove-admin@TROVE_HOST -N '' +test -e mason.key || ssh-keygen -t rsa -b 2048 -f mason.key -C mason@TROVE_HOST -N '' +test -e lorry.key || ssh-keygen -t rsa -b 2048 -f lorry.key -C lorry@TROVE_HOST -N '' +test -e worker.key || ssh-keygen -t rsa -b 2048 -f worker.key -C worker@TROVE_HOST -N '' +test -e id_rsa || ssh-keygen -t rsa -b 2048 -f id_rsa -C trove-admin@TROVE_HOST -N '' cd ../ @@ -66,7 +68,7 @@ cd ../ # Mason setup ############################################################################## -cp mason.morph mason-${HOST_PREFIX}.morph +cp clusters/mason.morph mason-${HOST_PREFIX}.morph sed -i "s/red-box-v1/$(sedescape "$HOST_PREFIX")/g" "mason-$HOST_PREFIX.morph" sed -i "s/ssh_keys/ssh_keys-$(sedescape "$HOST_PREFIX")/g" "mason-$HOST_PREFIX.morph" @@ -81,8 +83,8 @@ sed -i "s/\.example\.com/$(sedescape "$HOST_POSTFIX")/g" "mason-$HOST_PREFIX.mor # System building ############################################################################## -morph build trove-system-x86_64 -morph build distbuild-system-x86_64 +morph build systems/trove-system-x86_64.morph +morph build systems/distbuild-system-x86_64.morph ############################################################################## diff --git a/mason/mason.sh b/mason/mason.sh index 13b97b7e..13ea21db 100755 --- a/mason/mason.sh +++ b/mason/mason.sh @@ -11,9 +11,9 @@ if [ ! -e ws ]; then fi cd ws -definitions_repo="$DEFINITIONS_REF"/"$DISTBUILD_TROVE_ADDRESS"/baserock/baserock/definitions +definitions_repo="$DEFINITIONS_REF"/"$UPSTREAM_TROVE_ADDRESS"/baserock/baserock/definitions if [ ! -e "$definitions_repo" ]; then - morph checkout git://"$DISTBUILD_TROVE_ADDRESS"/baserock/baserock/definitions "$DEFINITIONS_REF" + morph checkout git://"$UPSTREAM_TROVE_ADDRESS"/baserock/baserock/definitions "$DEFINITIONS_REF" cd "$definitions_repo" git config user.name "$TROVE_ID"-mason git config user.email "$TROVE_ID"-mason@$(hostname) @@ -41,9 +41,10 @@ rm -f "$HOME/success" echo INFO: Mason building: $DEFINITIONS_REF at $SHA1 if ! "scripts/release-build" --no-default-configs \ - --trove-host "$DISTBUILD_TROVE_ADDRESS" \ - --controllers "$DISTBUILD_ARCH:$DISTBUILD_CONTROLLER_ADDRESS" \ - "$BUILD_CLUSTER_MORPHOLOGY"; then + --trove-host "$UPSTREAM_TROVE_ADDRESS" \ + --artifact-cache-server "http://$ARTIFACT_CACHE_SERVER:8080/" \ + --controllers "$DISTBUILD_ARCH:$DISTBUILD_CONTROLLER_ADDRESS" \ + "$BUILD_CLUSTER_MORPHOLOGY"; then echo ERROR: Failed to build release images echo Build logs for chunks: find builds -type f -exec echo {} \; -exec cat {} \; @@ -60,11 +61,11 @@ fi "scripts/release-test" \ --deployment-host "$DISTBUILD_ARCH":"$TEST_VM_HOST_SSH_URL" \ - --trove-host "$DISTBUILD_TROVE_ADDRESS" \ + --trove-host "$UPSTREAM_TROVE_ADDRESS" \ --trove-id "$TROVE_ID" \ "$BUILD_CLUSTER_MORPHOLOGY" -"scripts/release-upload" --build-trove-host "$DISTBUILD_TROVE_ADDRESS" \ +"scripts/release-upload" --build-trove-host "$ARTIFACT_CACHE_SERVER" \ --arch "$DISTBUILD_ARCH" \ --log-level=debug --log="$HOME"/release-upload.log \ --public-trove-host "$UPSTREAM_TROVE_ADDRESS" \ diff --git a/scripts/release-build b/scripts/release-build index a3e25ebb..5525e9e9 100755 --- a/scripts/release-build +++ b/scripts/release-build @@ -53,6 +53,9 @@ class ReleaseApp(cliapp.Application): self.settings.string(['trove-host'], 'hostname of Trove instance') + self.settings.string(['artifact-cache-server'], + 'server to fetch artifacts from', default=None) + self.settings.string(['release-number'], 'Baserock version of the systems being built', default='yy.ww') @@ -157,10 +160,14 @@ class ReleaseApp(cliapp.Application): def deploy_single_image(self, cluster_path, name, location, version_label): deploy_command = [ 'morph', 'deploy', cluster_path, name, - '--trove-host=%s' % self.settings['trove-host'], + '--trove-host=%s' % self.settings['trove-host']] + artifact_server = self.settings['artifact-cache-server'] + if artifact_server is not None: + deploy_command.append('--artifact-cache-server=%s' % artifact_server) + deploy_command.extend(( '%s.location=%s' % (name, location), '%s.VERSION_LABEL=%s' % (name, version_label) - ] + )) cliapp.runcmd(deploy_command, stdout=sys.stdout) |