summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorRichard Maw <richard.maw@codethink.co.uk>2014-08-22 13:01:39 (GMT)
committerRichard Maw <richard.maw@codethink.co.uk>2014-08-28 10:01:44 (GMT)
commitf45695be540c58314ab30847318ab3cb9b861524 (patch)
tree874146075ff99ff968ab7e8c5e6da1ba9a1bf902
parent69af63a4824726cceaf176e6878bc87ca67f76e5 (diff)
downloaddefinitions-f45695be540c58314ab30847318ab3cb9b861524.tar.gz
Deploy Masons to use source on upstream troves
The per-mason trove only needs to worry about being an artifact cache, so we can prevent it populating itself from the upstream trove by making it use the SSH protocol for fetching sources, and not registering its ssh key with the upstream trove. The MASON_UPSTREAM_TROVE_ADDRESS option has been removed, as this is now the TROVE_HOST. The distbuild network is now configured to use the upstream trove for sources, and the local trove for artifacts, with the ARTIFACT_CACHE_SERVER option. mason.configure now uses ARTIFACT_CACHE_SERVER to tell deploy commands which server to fetch artifacts from.
-rw-r--r--clusters/mason.morph9
-rw-r--r--mason.configure10
-rwxr-xr-xmason/mason-generator.sh72
-rwxr-xr-xmason/mason.sh15
-rwxr-xr-xscripts/release-build11
5 files changed, 63 insertions, 54 deletions
diff --git a/clusters/mason.morph b/clusters/mason.morph
index 9bac0bb..fd8c25d 100644
--- a/clusters/mason.morph
+++ b/clusters/mason.morph
@@ -27,19 +27,19 @@ systems:
TROVE_HOST: red-box-v1
TROVE_ID: red-box-v1-trove
UPSTREAM_TROVE: upstream-trove
- UPSTREAM_TROVE_PROTOCOL: http
VCPUS: 2
VERSION_LABEL: 45
WORKER_SSH_PUBKEY: ssh_keys/worker.key.pub
- morph: systems/distbuild-system-x86_64.morph
deploy-defaults:
- CONTROLLERHOST: red-box-v1-controller
+ ARTIFACT_CACHE_SERVER: red-box-v1-trove.example.com
+ CONTROLLERHOST: red-box-v1-controller.example.com
DISTBUILD_CONTROLLER: false
DISTBUILD_WORKER: true
INSTALL_FILES: distbuild/manifest
RAM_SIZE: 8G
- TROVE_HOST: red-box-v1-trove.example.com
- TROVE_ID: red-box-v1-trove
+ TROVE_HOST: upstream-trove
+ TROVE_ID: upstream-trove
VCPUS: 2
WORKER_SSH_KEY: ssh_keys/worker.key
deploy:
@@ -53,5 +53,4 @@ systems:
MASON_DEFINITIONS_REF: master
MASON_DISTBUILD_ARCH: x86_64
MASON_TEST_HOST: vm-user@vm-host:/vm-path/
- MASON_UPSTREAM_TROVE_ADDRESS: upstream-trove
WORKERS: red-box-v1-controller
diff --git a/mason.configure b/mason.configure
index fb73b01..90d4970 100644
--- a/mason.configure
+++ b/mason.configure
@@ -19,8 +19,8 @@
# a Mason instance at deployment time. It uses the following variables
# from the environment:
#
+# * ARTIFACT_CACHE_SERVER
# * MASON_CLUSTER_MORPHOLOGY
-# * MASON_UPSTREAM_TROVE_ADDRESS
# * MASON_DEFINITIONS_REF
# * MASON_DISTBUILD_ARCH
# * MASON_TEST_HOST
@@ -32,8 +32,8 @@ set -e
ROOT="$1"
-if [ "$MASON_CLUSTER_MORPHOLOGY" = "" \
- -o "$MASON_UPSTREAM_TROVE_ADDRESS" = "" \
+if [ "$ARTIFACT_CACHE_SERVER" = "" \
+ -o "$MASON_CLUSTER_MORPHOLOGY" = "" \
-o "$MASON_DEFINITIONS_REF" = "" \
-o "$MASON_DISTBUILD_ARCH" = "" \
-o "$MASON_TEST_HOST" = "" ]; then
@@ -61,11 +61,11 @@ echo "Creating $MASON_CONFIG"
cat >>"$MASON_CONFIG" <<EOF
#################### START OF DEPLOY TIME CONFIGURATION #######################
-UPSTREAM_TROVE_ADDRESS=$(shellescape "$MASON_UPSTREAM_TROVE_ADDRESS")
+ARTIFACT_CACHE_SERVER=$(shellescape "$ARTIFACT_CACHE_SERVER")
+UPSTREAM_TROVE_ADDRESS=$(shellescape "$TROVE_HOST")
DEFINITIONS_REF=$(shellescape "$MASON_DEFINITIONS_REF")
DISTBUILD_ARCH=$(shellescape "$MASON_DISTBUILD_ARCH")
DISTBUILD_CONTROLLER_ADDRESS=$(shellescape "$CONTROLLERHOST")
-DISTBUILD_TROVE_ADDRESS=$(shellescape "$TROVE_HOST")
TROVE_ID=$(shellescape "$TROVE_ID")
BUILD_CLUSTER_MORPHOLOGY=$(shellescape "$MASON_CLUSTER_MORPHOLOGY")
TEST_VM_HOST_SSH_URL=$(shellescape "$MASON_TEST_HOST")
diff --git a/mason/mason-generator.sh b/mason/mason-generator.sh
index 652929f..911d879 100755
--- a/mason/mason-generator.sh
+++ b/mason/mason-generator.sh
@@ -3,45 +3,47 @@
set -e
if [ "$1" == "-h" -o "$1" == "--help" ]; then
- echo "Usage:"
- echo " `basename $0` HOST_PREFIX UPSTREAM_TROVE_HOSTNAME VM_USER VM_HOST VM_PATH [HOST_POSTFIX]"
- echo ""
- echo "Where:"
- echo " HOST_PREFIX -- Name of your Mason instance"
- echo " e.g. \"my-mason\" to produce hostnames:"
- echo " my-mason-trove and my-mason-controller"
- echo " UPSTREAM_TROVE_HOSTNAME -- Upstream trove's hostname"
- echo " VM_USER -- User on VM host for VM deployment"
- echo " VM_HOST -- VM host for VM deployment"
- echo " VM_PATH -- Path to store VM images in on VM host"
- echo " HOST_POSTFIX -- e.g. \".example.com\" to get"
- echo " my-mason-trove.example.com"
- echo ""
- echo "This script makes deploying a Mason system simpler by automating"
- echo "the generation of keys for the systems to use, building of the"
- echo "systems, filling out the mason deployment cluster morphology"
- echo "template with useful values, and finally deploying the systems."
- echo ""
- echo "To ensure that the deployed system can deploy test systems, you"
- echo "must supply an ssh key to the VM host. Do so with the following"
- echo "command:"
- echo " ssh-copy-id -i ssh_keys-HOST_PREFIX/worker.key.pub VM_USER@VM_HOST"
- echo ""
- echo "To ensure that the mason can upload artifacts to the upstream trove,"
- echo "you must supply an ssh key to the upstream trove. Do so with the"
- echo "following command:"
- echo " ssh-copy-id -i ssh_keys-HOST_PREFIX/id_rsa.key.pub root@UPSTREAM_TROVE_HOSTNAME"
- echo ""
+ cat <<EOF
+Usage:
+ `basename $0` HOST_PREFIX UPSTREAM_TROVE_HOSTNAME VM_USER VM_HOST VM_PATH [HOST_POSTFIX]
+
+Where:
+ HOST_PREFIX -- Name of your Mason instance
+ e.g. "my-mason" to produce hostnames:
+ my-mason-trove and my-mason-controller
+ UPSTREAM_TROVE_HOSTNAME -- Upstream trove's hostname
+ VM_USER -- User on VM host for VM deployment
+ VM_HOST -- VM host for VM deployment
+ VM_PATH -- Path to store VM images in on VM host
+ HOST_POSTFIX -- e.g. ".example.com" to get
+ my-mason-trove.example.com
+
+This script makes deploying a Mason system simpler by automating
+the generation of keys for the systems to use, building of the
+systems, filling out the mason deployment cluster morphology
+template with useful values, and finally deploying the systems.
+
+To ensure that the deployed system can deploy test systems, you
+must supply an ssh key to the VM host. Do so with the following
+command:
+ ssh-copy-id -i ssh_keys-HOST_PREFIX/worker.key.pub VM_USER@VM_HOST
+
+To ensure that the mason can upload artifacts to the upstream trove,
+you must supply an ssh key to the upstream trove. Do so with the
+following command:
+ ssh-copy-id -i ssh_keys-HOST_PREFIX/id_rsa.key.pub root@UPSTREAM_TROVE_HOSTNAME
+
+EOF
exit 0
fi
-HOST_PREFIX=$1
-UPSTREAM_TROVE=$2
-VM_USER=$3
-VM_HOST=$4
-VM_PATH=$5
-HOST_POSTFIX=$6
+HOST_PREFIX="$1"
+UPSTREAM_TROVE="$2"
+VM_USER="$3"
+VM_HOST="$4"
+VM_PATH="$5"
+HOST_POSTFIX="$6"
sedescape() {
# Escape all non-alphanumeric characters
diff --git a/mason/mason.sh b/mason/mason.sh
index 13b97b7..13ea21d 100755
--- a/mason/mason.sh
+++ b/mason/mason.sh
@@ -11,9 +11,9 @@ if [ ! -e ws ]; then
fi
cd ws
-definitions_repo="$DEFINITIONS_REF"/"$DISTBUILD_TROVE_ADDRESS"/baserock/baserock/definitions
+definitions_repo="$DEFINITIONS_REF"/"$UPSTREAM_TROVE_ADDRESS"/baserock/baserock/definitions
if [ ! -e "$definitions_repo" ]; then
- morph checkout git://"$DISTBUILD_TROVE_ADDRESS"/baserock/baserock/definitions "$DEFINITIONS_REF"
+ morph checkout git://"$UPSTREAM_TROVE_ADDRESS"/baserock/baserock/definitions "$DEFINITIONS_REF"
cd "$definitions_repo"
git config user.name "$TROVE_ID"-mason
git config user.email "$TROVE_ID"-mason@$(hostname)
@@ -41,9 +41,10 @@ rm -f "$HOME/success"
echo INFO: Mason building: $DEFINITIONS_REF at $SHA1
if ! "scripts/release-build" --no-default-configs \
- --trove-host "$DISTBUILD_TROVE_ADDRESS" \
- --controllers "$DISTBUILD_ARCH:$DISTBUILD_CONTROLLER_ADDRESS" \
- "$BUILD_CLUSTER_MORPHOLOGY"; then
+ --trove-host "$UPSTREAM_TROVE_ADDRESS" \
+ --artifact-cache-server "http://$ARTIFACT_CACHE_SERVER:8080/" \
+ --controllers "$DISTBUILD_ARCH:$DISTBUILD_CONTROLLER_ADDRESS" \
+ "$BUILD_CLUSTER_MORPHOLOGY"; then
echo ERROR: Failed to build release images
echo Build logs for chunks:
find builds -type f -exec echo {} \; -exec cat {} \;
@@ -60,11 +61,11 @@ fi
"scripts/release-test" \
--deployment-host "$DISTBUILD_ARCH":"$TEST_VM_HOST_SSH_URL" \
- --trove-host "$DISTBUILD_TROVE_ADDRESS" \
+ --trove-host "$UPSTREAM_TROVE_ADDRESS" \
--trove-id "$TROVE_ID" \
"$BUILD_CLUSTER_MORPHOLOGY"
-"scripts/release-upload" --build-trove-host "$DISTBUILD_TROVE_ADDRESS" \
+"scripts/release-upload" --build-trove-host "$ARTIFACT_CACHE_SERVER" \
--arch "$DISTBUILD_ARCH" \
--log-level=debug --log="$HOME"/release-upload.log \
--public-trove-host "$UPSTREAM_TROVE_ADDRESS" \
diff --git a/scripts/release-build b/scripts/release-build
index a3e25eb..5525e9e 100755
--- a/scripts/release-build
+++ b/scripts/release-build
@@ -53,6 +53,9 @@ class ReleaseApp(cliapp.Application):
self.settings.string(['trove-host'],
'hostname of Trove instance')
+ self.settings.string(['artifact-cache-server'],
+ 'server to fetch artifacts from', default=None)
+
self.settings.string(['release-number'],
'Baserock version of the systems being built',
default='yy.ww')
@@ -157,10 +160,14 @@ class ReleaseApp(cliapp.Application):
def deploy_single_image(self, cluster_path, name, location, version_label):
deploy_command = [
'morph', 'deploy', cluster_path, name,
- '--trove-host=%s' % self.settings['trove-host'],
+ '--trove-host=%s' % self.settings['trove-host']]
+ artifact_server = self.settings['artifact-cache-server']
+ if artifact_server is not None:
+ deploy_command.append('--artifact-cache-server=%s' % artifact_server)
+ deploy_command.extend((
'%s.location=%s' % (name, location),
'%s.VERSION_LABEL=%s' % (name, version_label)
- ]
+ ))
cliapp.runcmd(deploy_command, stdout=sys.stdout)