From 69af63a4824726cceaf176e6878bc87ca67f76e5 Mon Sep 17 00:00:00 2001
From: Daniel Silverstone <daniel.silverstone@codethink.co.uk>
Date: Thu, 21 Aug 2014 15:27:31 +0000
Subject: Tidy mason deploy

---
 clusters/mason.morph     |  2 +-
 mason/mason-generator.sh | 16 ++++++++--------
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/clusters/mason.morph b/clusters/mason.morph
index 1aa2dadb..9bac0bbc 100644
--- a/clusters/mason.morph
+++ b/clusters/mason.morph
@@ -49,7 +49,7 @@ systems:
       DISK_SIZE: 60G
       DISTBUILD_CONTROLLER: true
       HOSTNAME: red-box-v1-controller
-      MASON_CLUSTER_MORPHOLOGY: ci.morph
+      MASON_CLUSTER_MORPHOLOGY: clusters/ci.morph
       MASON_DEFINITIONS_REF: master
       MASON_DISTBUILD_ARCH: x86_64
       MASON_TEST_HOST: vm-user@vm-host:/vm-path/
diff --git a/mason/mason-generator.sh b/mason/mason-generator.sh
index c1ffeb24..652929f2 100755
--- a/mason/mason-generator.sh
+++ b/mason/mason-generator.sh
@@ -53,12 +53,12 @@ sedescape() {
 # Key generation
 ##############################################################################
 
-mkdir "ssh_keys-${HOST_PREFIX}"
+mkdir -p "ssh_keys-${HOST_PREFIX}"
 cd "ssh_keys-${HOST_PREFIX}"
-ssh-keygen -t rsa -b 2048 -f mason.key -C mason@TROVE_HOST -N ''
-ssh-keygen -t rsa -b 2048 -f lorry.key -C lorry@TROVE_HOST -N ''
-ssh-keygen -t rsa -b 2048 -f worker.key -C worker@TROVE_HOST -N ''
-ssh-keygen -t rsa -b 2048 -f id_rsa -C trove-admin@TROVE_HOST -N ''
+test -e mason.key || ssh-keygen -t rsa -b 2048 -f mason.key -C mason@TROVE_HOST -N ''
+test -e lorry.key || ssh-keygen -t rsa -b 2048 -f lorry.key -C lorry@TROVE_HOST -N ''
+test -e worker.key || ssh-keygen -t rsa -b 2048 -f worker.key -C worker@TROVE_HOST -N ''
+test -e id_rsa || ssh-keygen -t rsa -b 2048 -f id_rsa -C trove-admin@TROVE_HOST -N ''
 cd ../
 
 
@@ -66,7 +66,7 @@ cd ../
 # Mason setup
 ##############################################################################
 
-cp mason.morph mason-${HOST_PREFIX}.morph
+cp clusters/mason.morph mason-${HOST_PREFIX}.morph
 
 sed -i "s/red-box-v1/$(sedescape "$HOST_PREFIX")/g" "mason-$HOST_PREFIX.morph"
 sed -i "s/ssh_keys/ssh_keys-$(sedescape "$HOST_PREFIX")/g" "mason-$HOST_PREFIX.morph"
@@ -81,8 +81,8 @@ sed -i "s/\.example\.com/$(sedescape "$HOST_POSTFIX")/g" "mason-$HOST_PREFIX.mor
 # System building
 ##############################################################################
 
-morph build trove-system-x86_64
-morph build distbuild-system-x86_64
+morph build systems/trove-system-x86_64.morph
+morph build systems/distbuild-system-x86_64.morph
 
 
 ##############################################################################
-- 
cgit v1.2.1


From f45695be540c58314ab30847318ab3cb9b861524 Mon Sep 17 00:00:00 2001
From: Richard Maw <richard.maw@codethink.co.uk>
Date: Fri, 22 Aug 2014 13:01:39 +0000
Subject: Deploy Masons to use source on upstream troves

The per-mason trove only needs to worry about being an artifact cache,
so we can prevent it populating itself from the upstream trove by making
it use the SSH protocol for fetching sources, and not registering its
ssh key with the upstream trove.

The MASON_UPSTREAM_TROVE_ADDRESS option has been removed, as this is now
the TROVE_HOST.

The distbuild network is now configured to use the upstream trove for
sources, and the local trove for artifacts, with the
ARTIFACT_CACHE_SERVER option.

mason.configure now uses ARTIFACT_CACHE_SERVER to tell deploy commands
which server to fetch artifacts from.
---
 clusters/mason.morph     |  9 +++---
 mason.configure          | 10 +++----
 mason/mason-generator.sh | 72 +++++++++++++++++++++++++-----------------------
 mason/mason.sh           | 15 +++++-----
 scripts/release-build    | 11 ++++++--
 5 files changed, 63 insertions(+), 54 deletions(-)

diff --git a/clusters/mason.morph b/clusters/mason.morph
index 9bac0bbc..fd8c25d7 100644
--- a/clusters/mason.morph
+++ b/clusters/mason.morph
@@ -27,19 +27,19 @@ systems:
       TROVE_HOST: red-box-v1
       TROVE_ID: red-box-v1-trove
       UPSTREAM_TROVE: upstream-trove
-      UPSTREAM_TROVE_PROTOCOL: http
       VCPUS: 2
       VERSION_LABEL: 45
       WORKER_SSH_PUBKEY: ssh_keys/worker.key.pub
 - morph: systems/distbuild-system-x86_64.morph
   deploy-defaults:
-    CONTROLLERHOST: red-box-v1-controller
+    ARTIFACT_CACHE_SERVER: red-box-v1-trove.example.com
+    CONTROLLERHOST: red-box-v1-controller.example.com
     DISTBUILD_CONTROLLER: false
     DISTBUILD_WORKER: true
     INSTALL_FILES: distbuild/manifest
     RAM_SIZE: 8G
-    TROVE_HOST: red-box-v1-trove.example.com
-    TROVE_ID: red-box-v1-trove
+    TROVE_HOST: upstream-trove
+    TROVE_ID: upstream-trove
     VCPUS: 2
     WORKER_SSH_KEY: ssh_keys/worker.key
   deploy:
@@ -53,5 +53,4 @@ systems:
       MASON_DEFINITIONS_REF: master
       MASON_DISTBUILD_ARCH: x86_64
       MASON_TEST_HOST: vm-user@vm-host:/vm-path/
-      MASON_UPSTREAM_TROVE_ADDRESS: upstream-trove
       WORKERS: red-box-v1-controller
diff --git a/mason.configure b/mason.configure
index fb73b01c..90d49709 100644
--- a/mason.configure
+++ b/mason.configure
@@ -19,8 +19,8 @@
 # a Mason instance at deployment time. It uses the following variables
 # from the environment:
 #
+#   * ARTIFACT_CACHE_SERVER
 #   * MASON_CLUSTER_MORPHOLOGY
-#   * MASON_UPSTREAM_TROVE_ADDRESS
 #   * MASON_DEFINITIONS_REF
 #   * MASON_DISTBUILD_ARCH
 #   * MASON_TEST_HOST
@@ -32,8 +32,8 @@ set -e
 
 ROOT="$1"
 
-if [ "$MASON_CLUSTER_MORPHOLOGY" = "" \
-     -o "$MASON_UPSTREAM_TROVE_ADDRESS" = "" \
+if [ "$ARTIFACT_CACHE_SERVER" = "" \
+     -o "$MASON_CLUSTER_MORPHOLOGY" = "" \
      -o "$MASON_DEFINITIONS_REF" = "" \
      -o "$MASON_DISTBUILD_ARCH" = "" \
      -o "$MASON_TEST_HOST" = "" ]; then
@@ -61,11 +61,11 @@ echo "Creating $MASON_CONFIG"
 cat >>"$MASON_CONFIG" <<EOF
 #################### START OF DEPLOY TIME CONFIGURATION #######################
 
-UPSTREAM_TROVE_ADDRESS=$(shellescape "$MASON_UPSTREAM_TROVE_ADDRESS")
+ARTIFACT_CACHE_SERVER=$(shellescape "$ARTIFACT_CACHE_SERVER")
+UPSTREAM_TROVE_ADDRESS=$(shellescape "$TROVE_HOST")
 DEFINITIONS_REF=$(shellescape "$MASON_DEFINITIONS_REF")
 DISTBUILD_ARCH=$(shellescape "$MASON_DISTBUILD_ARCH")
 DISTBUILD_CONTROLLER_ADDRESS=$(shellescape "$CONTROLLERHOST")
-DISTBUILD_TROVE_ADDRESS=$(shellescape "$TROVE_HOST")
 TROVE_ID=$(shellescape "$TROVE_ID")
 BUILD_CLUSTER_MORPHOLOGY=$(shellescape "$MASON_CLUSTER_MORPHOLOGY")
 TEST_VM_HOST_SSH_URL=$(shellescape "$MASON_TEST_HOST")
diff --git a/mason/mason-generator.sh b/mason/mason-generator.sh
index 652929f2..911d879f 100755
--- a/mason/mason-generator.sh
+++ b/mason/mason-generator.sh
@@ -3,45 +3,47 @@
 set -e
 
 if [ "$1" == "-h" -o "$1" == "--help" ]; then
-  echo "Usage:"
-  echo "  `basename $0` HOST_PREFIX UPSTREAM_TROVE_HOSTNAME VM_USER VM_HOST VM_PATH [HOST_POSTFIX]"
-  echo ""
-  echo "Where:"
-  echo "  HOST_PREFIX              -- Name of your Mason instance"
-  echo "                              e.g. \"my-mason\" to produce hostnames:"
-  echo "                              my-mason-trove and my-mason-controller"
-  echo "  UPSTREAM_TROVE_HOSTNAME  -- Upstream trove's hostname"
-  echo "  VM_USER                  -- User on VM host for VM deployment"
-  echo "  VM_HOST                  -- VM host for VM deployment"
-  echo "  VM_PATH                  -- Path to store VM images in on VM host"
-  echo "  HOST_POSTFIX             -- e.g. \".example.com\" to get"
-  echo "                              my-mason-trove.example.com"
-  echo ""
-  echo "This script makes deploying a Mason system simpler by automating"
-  echo "the generation of keys for the systems to use, building of the"
-  echo "systems, filling out the mason deployment cluster morphology"
-  echo "template with useful values, and finally deploying the systems."
-  echo ""
-  echo "To ensure that the deployed system can deploy test systems, you"
-  echo "must supply an ssh key to the VM host.  Do so with the following"
-  echo "command:"
-  echo "  ssh-copy-id -i ssh_keys-HOST_PREFIX/worker.key.pub VM_USER@VM_HOST"
-  echo ""
-  echo "To ensure that the mason can upload artifacts to the upstream trove,"
-  echo "you must supply an ssh key to the upstream trove.  Do so with the"
-  echo "following command:"
-  echo "  ssh-copy-id -i ssh_keys-HOST_PREFIX/id_rsa.key.pub root@UPSTREAM_TROVE_HOSTNAME"
-  echo ""
+  cat <<EOF
+Usage:
+    `basename $0` HOST_PREFIX UPSTREAM_TROVE_HOSTNAME VM_USER VM_HOST VM_PATH [HOST_POSTFIX]
+
+Where:
+  HOST_PREFIX              -- Name of your Mason instance
+                              e.g. "my-mason" to produce hostnames:
+                              my-mason-trove and my-mason-controller
+  UPSTREAM_TROVE_HOSTNAME  -- Upstream trove's hostname
+  VM_USER                  -- User on VM host for VM deployment
+  VM_HOST                  -- VM host for VM deployment
+  VM_PATH                  -- Path to store VM images in on VM host
+  HOST_POSTFIX             -- e.g. ".example.com" to get
+                              my-mason-trove.example.com
+
+This script makes deploying a Mason system simpler by automating
+the generation of keys for the systems to use, building of the
+systems, filling out the mason deployment cluster morphology
+template with useful values, and finally deploying the systems.
+
+To ensure that the deployed system can deploy test systems, you
+must supply an ssh key to the VM host.  Do so with the following
+command:
+  ssh-copy-id -i ssh_keys-HOST_PREFIX/worker.key.pub VM_USER@VM_HOST
+
+To ensure that the mason can upload artifacts to the upstream trove,
+you must supply an ssh key to the upstream trove.  Do so with the
+following command:
+  ssh-copy-id -i ssh_keys-HOST_PREFIX/id_rsa.key.pub root@UPSTREAM_TROVE_HOSTNAME
+
+EOF
   exit 0
 fi
 
 
-HOST_PREFIX=$1
-UPSTREAM_TROVE=$2
-VM_USER=$3
-VM_HOST=$4
-VM_PATH=$5
-HOST_POSTFIX=$6
+HOST_PREFIX="$1"
+UPSTREAM_TROVE="$2"
+VM_USER="$3"
+VM_HOST="$4"
+VM_PATH="$5"
+HOST_POSTFIX="$6"
 
 sedescape() {
     # Escape all non-alphanumeric characters
diff --git a/mason/mason.sh b/mason/mason.sh
index 13b97b7e..13ea21db 100755
--- a/mason/mason.sh
+++ b/mason/mason.sh
@@ -11,9 +11,9 @@ if [ ! -e ws ]; then
 fi
 cd ws
 
-definitions_repo="$DEFINITIONS_REF"/"$DISTBUILD_TROVE_ADDRESS"/baserock/baserock/definitions
+definitions_repo="$DEFINITIONS_REF"/"$UPSTREAM_TROVE_ADDRESS"/baserock/baserock/definitions
 if [ ! -e "$definitions_repo" ]; then
-    morph checkout git://"$DISTBUILD_TROVE_ADDRESS"/baserock/baserock/definitions "$DEFINITIONS_REF"
+    morph checkout git://"$UPSTREAM_TROVE_ADDRESS"/baserock/baserock/definitions "$DEFINITIONS_REF"
     cd "$definitions_repo"
     git config user.name "$TROVE_ID"-mason
     git config user.email "$TROVE_ID"-mason@$(hostname)
@@ -41,9 +41,10 @@ rm -f "$HOME/success"
 echo INFO: Mason building: $DEFINITIONS_REF at $SHA1
 
 if ! "scripts/release-build" --no-default-configs \
-        --trove-host "$DISTBUILD_TROVE_ADDRESS" \
-	--controllers "$DISTBUILD_ARCH:$DISTBUILD_CONTROLLER_ADDRESS" \
-	"$BUILD_CLUSTER_MORPHOLOGY"; then
+        --trove-host "$UPSTREAM_TROVE_ADDRESS" \
+        --artifact-cache-server "http://$ARTIFACT_CACHE_SERVER:8080/" \
+        --controllers "$DISTBUILD_ARCH:$DISTBUILD_CONTROLLER_ADDRESS" \
+        "$BUILD_CLUSTER_MORPHOLOGY"; then
     echo ERROR: Failed to build release images
     echo Build logs for chunks:
     find builds -type f -exec echo {} \; -exec cat {} \;
@@ -60,11 +61,11 @@ fi
 
 "scripts/release-test" \
 	--deployment-host "$DISTBUILD_ARCH":"$TEST_VM_HOST_SSH_URL" \
-	--trove-host "$DISTBUILD_TROVE_ADDRESS" \
+	--trove-host "$UPSTREAM_TROVE_ADDRESS" \
 	--trove-id "$TROVE_ID" \
 	"$BUILD_CLUSTER_MORPHOLOGY"
 
-"scripts/release-upload" --build-trove-host "$DISTBUILD_TROVE_ADDRESS" \
+"scripts/release-upload" --build-trove-host "$ARTIFACT_CACHE_SERVER" \
 	--arch "$DISTBUILD_ARCH" \
 	--log-level=debug --log="$HOME"/release-upload.log \
 	--public-trove-host "$UPSTREAM_TROVE_ADDRESS" \
diff --git a/scripts/release-build b/scripts/release-build
index a3e25ebb..5525e9e9 100755
--- a/scripts/release-build
+++ b/scripts/release-build
@@ -53,6 +53,9 @@ class ReleaseApp(cliapp.Application):
         self.settings.string(['trove-host'],
                              'hostname of Trove instance')
 
+        self.settings.string(['artifact-cache-server'],
+                             'server to fetch artifacts from', default=None)
+
         self.settings.string(['release-number'],
                              'Baserock version of the systems being built',
                              default='yy.ww')
@@ -157,10 +160,14 @@ class ReleaseApp(cliapp.Application):
     def deploy_single_image(self, cluster_path, name, location, version_label):
         deploy_command = [
             'morph', 'deploy', cluster_path, name,
-            '--trove-host=%s' % self.settings['trove-host'],
+            '--trove-host=%s' % self.settings['trove-host']]
+        artifact_server = self.settings['artifact-cache-server']
+        if artifact_server is not None:
+            deploy_command.append('--artifact-cache-server=%s' % artifact_server)
+        deploy_command.extend((
             '%s.location=%s' % (name, location),
             '%s.VERSION_LABEL=%s' % (name, version_label)
-        ]
+        ))
 
         cliapp.runcmd(deploy_command, stdout=sys.stdout)
 
-- 
cgit v1.2.1


From 6a5643eef693ee06f51304918647e1a131f6685a Mon Sep 17 00:00:00 2001
From: Richard Maw <richard.maw@codethink.co.uk>
Date: Fri, 22 Aug 2014 13:15:53 +0000
Subject: Reduce the disk size allocated to Mason Troves

We don't need to store gits on there, so we don't need it to be quite so
large.
---
 clusters/mason.morph | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clusters/mason.morph b/clusters/mason.morph
index fd8c25d7..8579f7e4 100644
--- a/clusters/mason.morph
+++ b/clusters/mason.morph
@@ -14,7 +14,7 @@ systems:
       type: kvm
       location: kvm+ssh://vm-user@vm-host/red-box-v1-trove/vm-path/red-box-v1-trove.img
       AUTOSTART: true
-      DISK_SIZE: 100G
+      DISK_SIZE: 20G
       HOSTNAME: red-box-v1-trove
       LORRY_SSH_KEY: ssh_keys/lorry.key
       MASON_SSH_PUBKEY: ssh_keys/mason.key.pub
-- 
cgit v1.2.1


From e3097bdcc35588f4a7dcc99e399b0b492d9e12f4 Mon Sep 17 00:00:00 2001
From: Richard Maw <richard.maw@codethink.co.uk>
Date: Fri, 22 Aug 2014 13:16:27 +0000
Subject: mason-generator: Display help for wrong number of args

---
 mason/mason-generator.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mason/mason-generator.sh b/mason/mason-generator.sh
index 911d879f..1a2eccac 100755
--- a/mason/mason-generator.sh
+++ b/mason/mason-generator.sh
@@ -2,7 +2,7 @@
 
 set -e
 
-if [ "$1" == "-h" -o "$1" == "--help" ]; then
+if [ "$#" -lt 5 -o "$#" -gt 6 -o "$1" == "-h" -o "$1" == "--help" ]; then
   cat <<EOF
 Usage:
     `basename $0` HOST_PREFIX UPSTREAM_TROVE_HOSTNAME VM_USER VM_HOST VM_PATH [HOST_POSTFIX]
-- 
cgit v1.2.1