From e3248542d2045cd3b268bf7a83700dfee4774ba0 Mon Sep 17 00:00:00 2001
From: Richard Maw <richard.maw@codethink.co.uk>
Date: Fri, 22 Aug 2014 13:01:39 +0000
Subject: Deploy Masons to use source on upstream troves

The per-mason trove only needs to worry about being an artifact cache,
so its lorry protocol can be changed back to SSH.

The MASON_UPSTREAM_TROVE_ADDRESS option has been removed, as this is now
the TROVE_HOST.

The distbuild network is now configured to use the upstream trove for
sources, and the local trove for artifacts, with the
ARTIFACT_CACHE_SERVER option.

mason.configure now uses ARTIFACT_CACHE_SERVER to tell deploy commands
which server to fetch artifacts from.
---
 clusters/mason.morph     |  9 ++++---
 mason.configure          |  6 ++---
 mason/mason-generator.sh | 62 +++++++++++++++++++++++++-----------------------
 mason/mason.sh           | 15 ++++++------
 scripts/release-build    | 11 +++++++--
 5 files changed, 55 insertions(+), 48 deletions(-)

diff --git a/clusters/mason.morph b/clusters/mason.morph
index 9bac0bbc..fd8c25d7 100644
--- a/clusters/mason.morph
+++ b/clusters/mason.morph
@@ -27,19 +27,19 @@ systems:
       TROVE_HOST: red-box-v1
       TROVE_ID: red-box-v1-trove
       UPSTREAM_TROVE: upstream-trove
-      UPSTREAM_TROVE_PROTOCOL: http
       VCPUS: 2
       VERSION_LABEL: 45
       WORKER_SSH_PUBKEY: ssh_keys/worker.key.pub
 - morph: systems/distbuild-system-x86_64.morph
   deploy-defaults:
-    CONTROLLERHOST: red-box-v1-controller
+    ARTIFACT_CACHE_SERVER: red-box-v1-trove.example.com
+    CONTROLLERHOST: red-box-v1-controller.example.com
     DISTBUILD_CONTROLLER: false
     DISTBUILD_WORKER: true
     INSTALL_FILES: distbuild/manifest
     RAM_SIZE: 8G
-    TROVE_HOST: red-box-v1-trove.example.com
-    TROVE_ID: red-box-v1-trove
+    TROVE_HOST: upstream-trove
+    TROVE_ID: upstream-trove
     VCPUS: 2
     WORKER_SSH_KEY: ssh_keys/worker.key
   deploy:
@@ -53,5 +53,4 @@ systems:
       MASON_DEFINITIONS_REF: master
       MASON_DISTBUILD_ARCH: x86_64
       MASON_TEST_HOST: vm-user@vm-host:/vm-path/
-      MASON_UPSTREAM_TROVE_ADDRESS: upstream-trove
       WORKERS: red-box-v1-controller
diff --git a/mason.configure b/mason.configure
index fb73b01c..5f35566d 100644
--- a/mason.configure
+++ b/mason.configure
@@ -20,7 +20,6 @@
 # from the environment:
 #
 #   * MASON_CLUSTER_MORPHOLOGY
-#   * MASON_UPSTREAM_TROVE_ADDRESS
 #   * MASON_DEFINITIONS_REF
 #   * MASON_DISTBUILD_ARCH
 #   * MASON_TEST_HOST
@@ -33,7 +32,6 @@ set -e
 ROOT="$1"
 
 if [ "$MASON_CLUSTER_MORPHOLOGY" = "" \
-     -o "$MASON_UPSTREAM_TROVE_ADDRESS" = "" \
      -o "$MASON_DEFINITIONS_REF" = "" \
      -o "$MASON_DISTBUILD_ARCH" = "" \
      -o "$MASON_TEST_HOST" = "" ]; then
@@ -61,11 +59,11 @@ echo "Creating $MASON_CONFIG"
 cat >>"$MASON_CONFIG" <<EOF
 #################### START OF DEPLOY TIME CONFIGURATION #######################
 
-UPSTREAM_TROVE_ADDRESS=$(shellescape "$MASON_UPSTREAM_TROVE_ADDRESS")
+ARTIFACT_CACHE_SERVER=$(shellescape "$ARTIFACT_CACHE_SERVER")
+UPSTREAM_TROVE_ADDRESS=$(shellescape "$TROVE_HOST")
 DEFINITIONS_REF=$(shellescape "$MASON_DEFINITIONS_REF")
 DISTBUILD_ARCH=$(shellescape "$MASON_DISTBUILD_ARCH")
 DISTBUILD_CONTROLLER_ADDRESS=$(shellescape "$CONTROLLERHOST")
-DISTBUILD_TROVE_ADDRESS=$(shellescape "$TROVE_HOST")
 TROVE_ID=$(shellescape "$TROVE_ID")
 BUILD_CLUSTER_MORPHOLOGY=$(shellescape "$MASON_CLUSTER_MORPHOLOGY")
 TEST_VM_HOST_SSH_URL=$(shellescape "$MASON_TEST_HOST")
diff --git a/mason/mason-generator.sh b/mason/mason-generator.sh
index d93d89f7..501e5956 100755
--- a/mason/mason-generator.sh
+++ b/mason/mason-generator.sh
@@ -3,40 +3,42 @@
 set -e
 
 if [ "$1" == "-h" -o "$1" == "--help" ]; then
-  echo "Usage:"
-  echo "  `basename $0` HOST_PREFIX UPSTREAM_TROVE_HOSTNAME VM_USER VM_HOST VM_PATH [HOST_POSTFIX]"
-  echo ""
-  echo "Where:"
-  echo "  HOST_PREFIX              -- Name of your Mason instance"
-  echo "                              e.g. \"my-mason\" to produce hostnames:"
-  echo "                              my-mason-trove and my-mason-controller"
-  echo "  UPSTREAM_TROVE_HOSTNAME  -- Upstream trove's hostname"
-  echo "  VM_USER                  -- User on VM host for VM deployment"
-  echo "  VM_HOST                  -- VM host for VM deployment"
-  echo "  VM_PATH                  -- Path to store VM images in on VM host"
-  echo "  HOST_POSTFIX             -- e.g. \".example.com\" to get"
-  echo "                              my-mason-trove.example.com"
-  echo ""
-  echo "This script makes deploying a Mason system simpler by automating"
-  echo "the generation of keys for the systems to use, building of the"
-  echo "systems, filling out the mason deployment cluster morphology"
-  echo "template with useful values, and finally deploying the systems."
-  echo ""
-  echo "To ensure that the deployed system can deploy test systems, you"
-  echo "must supply an ssh key to the VM host.  Do so with the following"
-  echo "command:"
-  echo "  ssh-copy-id -i ssh_keys-HOST_PREFIX/worker.key.pub VM_USER@VM_HOST"
-  echo ""
+  cat <<EOF
+Usage:
+    `basename $0` HOST_PREFIX UPSTREAM_TROVE_HOSTNAME VM_USER VM_HOST VM_PATH [HOST_POSTFIX]
+
+Where:
+  HOST_PREFIX              -- Name of your Mason instance
+                              e.g. "my-mason" to produce hostnames:
+                              my-mason-trove and my-mason-controller
+  UPSTREAM_TROVE_HOSTNAME  -- Upstream trove's hostname
+  VM_USER                  -- User on VM host for VM deployment
+  VM_HOST                  -- VM host for VM deployment
+  VM_PATH                  -- Path to store VM images in on VM host
+  HOST_POSTFIX             -- e.g. ".example.com" to get
+                              my-mason-trove.example.com
+
+This script makes deploying a Mason system simpler by automating
+the generation of keys for the systems to use, building of the
+systems, filling out the mason deployment cluster morphology
+template with useful values, and finally deploying the systems.
+
+To ensure that the deployed system can deploy test systems, you
+must supply an ssh key to the VM host.  Do so with the following
+command:
+  ssh-copy-id -i ssh_keys-HOST_PREFIX/worker.key.pub VM_USER@VM_HOST
+
+EOF
   exit 0
 fi
 
 
-HOST_PREFIX=$1
-UPSTREAM_TROVE=$2
-VM_USER=$3
-VM_HOST=$4
-VM_PATH=$5
-HOST_POSTFIX=$6
+HOST_PREFIX="$1"
+UPSTREAM_TROVE="$2"
+VM_USER="$3"
+VM_HOST="$4"
+VM_PATH="$5"
+HOST_POSTFIX="$6"
 
 sedescape() {
     # Escape all non-alphanumeric characters
diff --git a/mason/mason.sh b/mason/mason.sh
index 13b97b7e..13ea21db 100755
--- a/mason/mason.sh
+++ b/mason/mason.sh
@@ -11,9 +11,9 @@ if [ ! -e ws ]; then
 fi
 cd ws
 
-definitions_repo="$DEFINITIONS_REF"/"$DISTBUILD_TROVE_ADDRESS"/baserock/baserock/definitions
+definitions_repo="$DEFINITIONS_REF"/"$UPSTREAM_TROVE_ADDRESS"/baserock/baserock/definitions
 if [ ! -e "$definitions_repo" ]; then
-    morph checkout git://"$DISTBUILD_TROVE_ADDRESS"/baserock/baserock/definitions "$DEFINITIONS_REF"
+    morph checkout git://"$UPSTREAM_TROVE_ADDRESS"/baserock/baserock/definitions "$DEFINITIONS_REF"
     cd "$definitions_repo"
     git config user.name "$TROVE_ID"-mason
     git config user.email "$TROVE_ID"-mason@$(hostname)
@@ -41,9 +41,10 @@ rm -f "$HOME/success"
 echo INFO: Mason building: $DEFINITIONS_REF at $SHA1
 
 if ! "scripts/release-build" --no-default-configs \
-        --trove-host "$DISTBUILD_TROVE_ADDRESS" \
-	--controllers "$DISTBUILD_ARCH:$DISTBUILD_CONTROLLER_ADDRESS" \
-	"$BUILD_CLUSTER_MORPHOLOGY"; then
+        --trove-host "$UPSTREAM_TROVE_ADDRESS" \
+        --artifact-cache-server "http://$ARTIFACT_CACHE_SERVER:8080/" \
+        --controllers "$DISTBUILD_ARCH:$DISTBUILD_CONTROLLER_ADDRESS" \
+        "$BUILD_CLUSTER_MORPHOLOGY"; then
     echo ERROR: Failed to build release images
     echo Build logs for chunks:
     find builds -type f -exec echo {} \; -exec cat {} \;
@@ -60,11 +61,11 @@ fi
 
 "scripts/release-test" \
 	--deployment-host "$DISTBUILD_ARCH":"$TEST_VM_HOST_SSH_URL" \
-	--trove-host "$DISTBUILD_TROVE_ADDRESS" \
+	--trove-host "$UPSTREAM_TROVE_ADDRESS" \
 	--trove-id "$TROVE_ID" \
 	"$BUILD_CLUSTER_MORPHOLOGY"
 
-"scripts/release-upload" --build-trove-host "$DISTBUILD_TROVE_ADDRESS" \
+"scripts/release-upload" --build-trove-host "$ARTIFACT_CACHE_SERVER" \
 	--arch "$DISTBUILD_ARCH" \
 	--log-level=debug --log="$HOME"/release-upload.log \
 	--public-trove-host "$UPSTREAM_TROVE_ADDRESS" \
diff --git a/scripts/release-build b/scripts/release-build
index a3e25ebb..5525e9e9 100755
--- a/scripts/release-build
+++ b/scripts/release-build
@@ -53,6 +53,9 @@ class ReleaseApp(cliapp.Application):
         self.settings.string(['trove-host'],
                              'hostname of Trove instance')
 
+        self.settings.string(['artifact-cache-server'],
+                             'server to fetch artifacts from', default=None)
+
         self.settings.string(['release-number'],
                              'Baserock version of the systems being built',
                              default='yy.ww')
@@ -157,10 +160,14 @@ class ReleaseApp(cliapp.Application):
     def deploy_single_image(self, cluster_path, name, location, version_label):
         deploy_command = [
             'morph', 'deploy', cluster_path, name,
-            '--trove-host=%s' % self.settings['trove-host'],
+            '--trove-host=%s' % self.settings['trove-host']]
+        artifact_server = self.settings['artifact-cache-server']
+        if artifact_server is not None:
+            deploy_command.append('--artifact-cache-server=%s' % artifact_server)
+        deploy_command.extend((
             '%s.location=%s' % (name, location),
             '%s.VERSION_LABEL=%s' % (name, version_label)
-        ]
+        ))
 
         cliapp.runcmd(deploy_command, stdout=sys.stdout)
 
-- 
cgit v1.2.1