Rework `morph deploy` to work with cluster morphologies.

From now on, `morph deploy` will work to only accept a cluster
morphology as argument. A cluster morphology defines a list of
systems to built, and for each system a list of ways to deploy
them. We will not support the old syntax anymore.

- Update `morph deploy` docstring with the new syntax, including
  an explanation of cluster morphologies (also document `tar`
  deployments).
- Fix tests that used the old syntax.
- Add tests for cluster deployments.

5 files changed, 261 insertions, 81 deletions

diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py
index 8530cb57..dc9f5158 100644
--- a/morphlib/plugins/deploy_plugin.py
+++ b/morphlib/plugins/deploy_plugin.py
@@ -39,7 +39,7 @@ class DeployPlugin(cliapp.Plugin):
     def enable(self):
         self.app.add_subcommand(
             'deploy', self.deploy,
-            arg_synopsis='TYPE SYSTEM LOCATION [KEY=VALUE]')
+            arg_synopsis='CLUSTER [SYSTEM.KEY=VALUE]')
         self.other = \
             morphlib.plugins.branch_and_merge_plugin.BranchAndMergePlugin()
         self.other.app = self.app
@@ -52,30 +52,82 @@ class DeployPlugin(cliapp.Plugin):
 
         Command line arguments:
 
-        * `TYPE` is the type of deployment: to a raw disk image,
-          VirtualBox, or something else. See below.
-
-        * `SYSTEM` is the name of the system morphology to deploy.
-
-        * `LOCATION` is where the deployed system should end up at. The
-          syntax depends on the deployment type. See below.
-
-        * `KEY=VALUE` is a configuration parameter to pass onto the
-          configuration extension. See below.
-
-        Morph can deploy a system image. The deployment mechanism is
-        quite flexible, and can be extended by the user. "Deployment"
-        here is quite a general concept: it covers anything where a system
-        image is taken, configured, and then put somewhere where it can
-        be run.
-
-        `TYPE` specifies the exact way in which the deployment happens.
-        Morph provides four deployment types:
-
-        * `rawdisk` where Morph builds a raw disk image at `LOCATION`,
-          and sets up the image with a bootloader and configuration
-          so that it can be booted. Disk size is set with `DISK_SIZE`
-          (see below).
+        * `CLUSTER` is the name of the cluster to deploy.
+
+        * `SYSTEM.KEY=VALUE` can be used to assign `VALUE` to a parameter
+          named `KEY` for the system identified by `SYSTEM` in the cluster
+          morphology (see below). This will override parameters defined
+          in the morphology.
+
+        Morph deploys a set of systems listed in a cluster morphology.
+        "Deployment" here is quite a general concept: it covers anything
+        where a system image is taken, configured, and then put somewhere
+        where it can be run. The deployment mechanism is quite flexible,
+        and can be extended by the user.
+
+        A cluster morphology defines a list of systems to deploy, and
+        for each system a list of ways to deploy them. It contains the
+        following fields:
+
+        * **name**: MUST be the same as the basename of the morphology
+         filename, sans .morph suffix.
+
+        * **kind**: MUST be `cluster`.
+
+        * **systems**: a list of systems to deploy;
+         the value is a list of mappings, where each mapping has the
+         following keys:
+
+           * **morph**: the system morphology to use in the specified
+             commit.
+
+           * **deploy**: a mapping where each key identifies a
+             system and each system has at least the following keys:
+
+               * **type**: identifies the type of development e.g. (kvm,
+                 nfsboot) (see below).
+               * **location**: where the deployed system should end up
+                 at. The syntax depends on the deployment type (see below).
+                 Any additional item on the dictionary will be added to the
+                 environment as `KEY=VALUE`.
+
+            * **deploy-defaults**: allows multiple deployments of the same
+             system to share some settings, when they can. Default settings
+             will be overridden by those defined inside the deploy mapping.
+
+         # Example
+
+            name: cluster-foo
+            kind: cluster
+            systems:
+                - morph: devel-system-x86_64-generic
+                  deploy:
+                      cluster-foo-x86_64-1:
+                          type: kvm
+                          location: kvm+ssh://user@host/x86_64-1/x86_64-1.img
+                          HOSTNAME: cluster-foo-x86_64-1
+                          DISK_SIZE: 4G
+                          RAM_SIZE: 4G
+                          VCPUS: 2
+                - morph: devel-system-armv7-highbank
+                  deploy-defaults:
+                      type: nfsboot
+                      location: cluster-foo-nfsboot-server
+                  deploy:
+                      cluster-foo-armv7-1:
+                          HOSTNAME: cluster-foo-armv7-1
+                      cluster-foo-armv7-2:
+                          HOSTNAME: cluster-foo-armv7-2
+
+        Each system defined in a cluster morphology can be deployed in
+        multiple ways (`type` in a cluster morphology). Morph provides
+        five types of deployment:
+
+        * `tar` where Morph builds a tar archive of the root file system.
+
+        * `rawdisk` where Morph builds a raw disk image and sets up the
+          image with a bootloader and configuration so that it can be
+          booted. Disk size is set with `DISK_SIZE` (see below).
 
         * `virtualbox-ssh` where Morph creates a VirtualBox disk image,
           and creates a new virtual machine on a remote host, accessed
@@ -87,29 +139,15 @@ class DeployPlugin(cliapp.Plugin):
           `DISK_SIZE` and `RAM_SIZE` (see below).
 
         * `nfsboot` where Morph creates a system to be booted over
-          a network
-
-        The following `KEY=VALUE` parameters are supported for all
-        deployment types:
-
-        * `DISK_SIZE=X` to set the size of the disk image. `X`
-          should use a suffix of `K`, `M`, or `G` (in upper or lower
-          case) to indicate kilo-, mega-, or gigabytes. For example,
-          `DISK_SIZE=100G` would create a 100 gigabyte disk image. **This
-          parameter is mandatory**.
-
-        * `RAM_SIZE=X` to set the size of virtual RAM for the virtual
-          machine.  `X` is interpreted in the same was as `DISK_SIZE`,
-          and defaults to `1G`.
-
-        The `kvm` and `virtualbox-ssh` deployment types support an
-        additional parameter:
+          a network.
 
-        * `AUTOSTART=<VALUE>` - allowed values are `yes` and `no`
-          (default)
+        In addition to the deployment type, the user must also give
+        a value for `location`. Its syntax depends on the deployment
+        types. The deployment types provided by Morph use the
+        following syntaxes:
 
-        The syntax of the `LOCATION` depends on the deployment types. The
-        deployment types provided by Morph use the following syntaxes:
+        * `tar`: pathname to the tar archive to be created; for
+          example, `/home/alice/testsystem.tar`
 
         * `rawdisk`: pathname to the disk image to be created; for
           example, `/home/alice/testsystem.img`
@@ -132,39 +170,48 @@ class DeployPlugin(cliapp.Plugin):
               * `/home/alice/testsys.vdi` and `/home/alice/testys.img` are
                 the pathnames of the disk image files on the target host.
 
-        For the `nfsboot` write extension,
+        * `nfsboot`: the address of the nfsboot server. (Note this is just
+          the _address_ of the trove, _not_ `user@...`, since `root@` will
+          automatically be prepended to the server address.)
+
+        The following `KEY=VALUE` parameters are supported for `rawdisk`,
+        `virtualbox-ssh` and `kvm` and deployment types:
+
+        * `DISK_SIZE=X` to set the size of the disk image. `X` should use a
+          suffix of `K`, `M`, or `G` (in upper or lower case) to indicate
+          kilo-, mega-, or gigabytes. For example, `DISK_SIZE=100G` would
+          create a 100 gigabyte disk image. **This parameter is mandatory**.
+
+        The `kvm` and `virtualbox-ssh` deployment types support an additional
+        parameter:
 
-        * LOCATION is the address of the nfsboot server. (Note this
-          is just the _address_ of the trove, _not_ `user@...`, since
-          `root@` will automatically be prepended to the server address.)
+        * `RAM_SIZE=X` to set the size of virtual RAM for the virtual
+          machine. `X` is interpreted in the same was as `DISK_SIZE`,
+          and defaults to `1G`.
 
-        * the following KEY=VALUE PAIRS are mandatory
+        * `AUTOSTART=<VALUE>` - allowed values are `yes` and `no`
+          (default)
 
-              * NFSBOOT_CONFIGURE=yes (or any non-empty value). This
+        For the `nfsboot` write extension,
+
+        * the following `KEY=VALUE` pairs are mandatory
+
+              * `NFSBOOT_CONFIGURE=yes` (or any non-empty value). This
                 enables the `nfsboot` configuration extension (see
                 below) which MUST be used when using the `nfsboot`
                 write extension.
 
-              * HOSTNAME=<STRING> a unique identifier for that system's
+              * `HOSTNAME=<STRING>` a unique identifier for that system's
                 `nfs` root when it's deployed on the nfsboot server - the
                 extension creates a directory with that name for the `nfs`
                 root, and stores kernels by that name for the tftp server.
 
-        * the following KEY=VALUE PAIRS are optional
+        * the following `KEY=VALUE` pairs are optional
 
-              * VERSION_LABEL=<STRING> - set the name of the system
+              * `VERSION_LABEL=<STRING>` - set the name of the system
                 version being deployed, when upgrading. Defaults to
                 "factory".
 
-        An example command line using `morph deploy with the nfsboot`
-        type is
-
-            morph deploy nfsboot devel-system-x86_64-generic \
-            customer-trove \
-            NFSBOOT_CONFIGURE=yes \
-            HOSTNAME=test-deployment-1 \
-            VERSION_LABEL=inital-test
-
         Each deployment type is implemented by a **write extension**. The
         ones provided by Morph are listed above, but users may also
         create their own by adding them in the same git repository
@@ -172,9 +219,7 @@ class DeployPlugin(cliapp.Plugin):
         script that does whatever is needed for the deployment. A write
         extension is passed two command line parameters: the name of an
         unpacked directory tree that contains the system files (after
-        configuration, see below), and the `LOCATION` argument. Any
-        additional `KEY=VALUE` arguments given to `morph deploy` are
-        set as environment variables when the write extension runs.
+        configuration, see below), and the `location` parameter.
 
         Regardless of the type of deployment, the image may be
         configured for a specific deployment by using **configuration
@@ -201,13 +246,14 @@ class DeployPlugin(cliapp.Plugin):
         * `nfsboot` configures the system for nfsbooting. This MUST
           be used when deploying with the `nfsboot` write extension.
 
-        Any `KEY=VALUE` parameters given to `morph deploy` are set as
-        environment variables when either the configuration or the write
-        extension runs.
+        Any `KEY=VALUE` parameters given in `deploy` or `deploy-defaults`
+        sections of the cluster morphology, or given through the command line
+        are set as environment variables when either the configuration or the
+        write extension runs (except `type` and `location`).
 
         '''
 
-        if len(args) < 3:
+        if not args:
             raise cliapp.AppException(
                 'Too few arguments to deploy command (see help)')
 
@@ -220,14 +266,47 @@ class DeployPlugin(cliapp.Plugin):
             self.app.settings['tempdir-min-space'],
             '/', 0)
 
-        deployment_type = args[0]
-        system_name = args[1]
-        location = args[2]
-        env_vars = args[3:]
+        cluster = args[0]
+        env_vars = args[1:]
+
+        branch_dir = self.other.deduce_system_branch()[1]
+        root_repo = self.other.get_branch_config(branch_dir, 'branch.root')
+        root_repo_dir = self.other.find_repository(branch_dir, root_repo)
+        data = self.other.load_morphology(root_repo_dir, cluster)
+
+        for system in data['systems']:
+            self.deploy_system(system, env_vars)
+
+    def deploy_system(self, system, env_vars):
+        morph = system['morph']
+        deploy_defaults = system['deploy-defaults']
+        deployments = system['deploy']
+
+        for system_id, deploy_params in deployments.iteritems():
+            user_env = morphlib.util.parse_environment_pairs(
+                    os.environ,
+                    [pair[len(system_id)+1:]
+                    for pair in env_vars
+                    if pair.startswith(system_id)])
+
+            final_env = dict(deploy_defaults.items() +
+                             deploy_params.items() +
+                             user_env.items())
+
+            deployment_type = final_env.pop('type', None)
+            if not deployment_type:
+                raise morphlib.Error('"type" is undefined '
+                                     'for system "%s"' % system_id)
+
+            location = final_env.pop('location', None)
+            if not location:
+                raise morphlib.Error('"location" is undefined '
+                                     'for system "%s"' % system_id)
 
-        # Set up environment for running extensions.
-        env = morphlib.util.parse_environment_pairs(os.environ, env_vars)
+            morphlib.util.sanitize_environment(final_env)
+            self.do_deploy(morph, deployment_type, location, final_env)
 
+    def do_deploy(self, system_name, deployment_type, location, env):
         # Deduce workspace and system branch and branch root repository.
         workspace = self.other.deduce_workspace()
         branch, branch_dir = self.other.deduce_system_branch()
diff --git a/tests.deploy/deploy-cluster.script b/tests.deploy/deploy-cluster.script
new file mode 100755
index 00000000..917ac717
--- /dev/null
+++ b/tests.deploy/deploy-cluster.script
@@ -0,0 +1,46 @@
+#!/bin/bash
+#
+# Copyright (C) 2013  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+# Test "morph deploy" by deploying the systems in cluster morphology.
+
+
+set -eu
+
+
+. "$SRCDIR/tests.deploy/setup-build"
+
+cd "$DATADIR/workspace/branch1"
+
+"$SRCDIR/scripts/test-morph" build hello-system
+
+"$SRCDIR/scripts/test-morph" build linux-system
+
+"$SRCDIR/scripts/test-morph" --log "$DATADIR/deploy.log" \
+    deploy test_cluster \
+    linux-system-2.HOSTNAME="baserock-rocks-even-more" \
+    > /dev/null
+
+test -e hello-system.img
+test -e linux-system-1.tar
+test -e linux-system-2.tar
+
+hostname1=$(tar -xf linux-system-1.tar ./etc/hostname -O)
+hostname2=$(tar -xf linux-system-2.tar ./etc/hostname -O)
+
+[ "$hostname1" = baserock-rocks ]
+[ "$hostname2" = baserock-rocks-even-more ]
diff --git a/tests.deploy/deploy-rawdisk-without-disk-size-fails.script b/tests.deploy/deploy-rawdisk-without-disk-size-fails.script
index 94084a5c..035557dd 100755
--- a/tests.deploy/deploy-rawdisk-without-disk-size-fails.script
+++ b/tests.deploy/deploy-rawdisk-without-disk-size-fails.script
@@ -25,6 +25,6 @@ set -eu
 cd "$DATADIR/workspace/branch1"
 "$SRCDIR/scripts/test-morph" build linux-system
 ! "$SRCDIR/scripts/test-morph" --log "$DATADIR/deploy.log" \
-    deploy rawdisk linux-system "$DATADIR/disk.img" > /dev/null 2>&1
+    deploy rawdisk_test_cluster_without_disk_size > /dev/null 2>&1
 test ! -e "$DATADIR/disk.img"
 
diff --git a/tests.deploy/deploy-rawdisk.script b/tests.deploy/deploy-rawdisk.script
index bead945b..ebda50c7 100755
--- a/tests.deploy/deploy-rawdisk.script
+++ b/tests.deploy/deploy-rawdisk.script
@@ -26,6 +26,6 @@ set -eu
 cd "$DATADIR/workspace/branch1"
 "$SRCDIR/scripts/test-morph" build linux-system
 "$SRCDIR/scripts/test-morph" --log "$DATADIR/deploy.log" \
-    deploy rawdisk linux-system "$DATADIR/disk.img" DISK_SIZE=1G > /dev/null
-test -e "$DATADIR/disk.img"
+    deploy rawdisk_test_cluster > /dev/null
+test -e disk.img
 
diff --git a/tests.deploy/setup b/tests.deploy/setup
index 584ce039..88488a91 100755
--- a/tests.deploy/setup
+++ b/tests.deploy/setup
@@ -166,11 +166,67 @@ cat <<EOF > linux-system.morph
             "repo": "test:morphs",
             "ref": "master"
         }
+    ],
+    "configuration-extensions": [
+        set-hostname
     ]
+
 }
 EOF
 git add linux-system.morph
 
+
+cat <<EOF > rawdisk_test_cluster.morph
+name: rawdisk_test_cluster
+kind: cluster
+systems:
+    - morph: linux-system
+      deploy:
+          linux-system-1:
+              type: rawdisk
+              location: disk.img
+              DISK_SIZE: 1G
+EOF
+git add rawdisk_test_cluster.morph
+
+cat <<EOF > rawdisk_test_cluster_without_disk_size.morph
+name: rawdisk_test_cluster_without_disk_size
+kind: cluster
+systems:
+    - morph: linux-system
+      deploy:
+          linux-system-1:
+              type: rawdisk
+              location: disk.img
+EOF
+git add rawdisk_test_cluster_without_disk_size.morph
+
+cat <<EOF > test_cluster.morph
+name: test_cluster
+kind: cluster
+systems:
+    - morph: hello-system
+      deploy-defaults:
+          type: tar
+      deploy:
+          hello-system:
+              type: rawdisk
+              location: hello-system.img
+              DISK_SIZE: 1G
+    - morph: linux-system
+      deploy-defaults:
+          HOSTNAME: "baserock-rocks"
+      deploy:
+          linux-system-1:
+              type: tar
+              location: linux-system-1.tar
+          linux-system-2:
+              type: tar
+              location: linux-system-2.tar
+EOF
+git add test_cluster.morph
+
+
 git commit --quiet -m "add morphs"
 
 # Make a dummy kernel chunk.
@@ -202,4 +258,3 @@ no-distcc = true
 quiet = true
 log = /tmp/morph.log
 EOF
-