Merge branch 'baserock/adamcoldrick/deploy-specific-systems-v5'

Reviewed by: Richard Maw <richard.maw@codethink.co.uk>
             Lars Wirzenius <lars.wirzenius@codethink.co.uk>

6 files changed, 203 insertions, 11 deletions

diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py
index ca5902a6..b5c8168d 100644
--- a/morphlib/morphloader.py
+++ b/morphlib/morphloader.py
@@ -205,6 +205,15 @@ class MultipleValidationErrors(morphlib.Error):
             self.msg += ('\t' + str(error))
 
 
+class DuplicateDeploymentNameError(morphlib.Error):
+
+    def __init__(self, duplicates):
+        self.duplicates = duplicates
+        morphlib.Error.__init__(
+            self, 'Cluster morphology contains the following non-unique '
+            'deployment names:\n%s' % '\n    '.join(duplicates))
+
+
 class OrderedDumper(yaml.SafeDumper):
     keyorder = (
         'name',
@@ -412,7 +421,23 @@ class MorphologyLoader(object):
         getattr(self, '_validate_%s' % kind)(morph)
 
     def _validate_cluster(self, morph):
-        pass
+        # Deployment names must be unique within a cluster
+        deployments = collections.Counter()
+        for system in morph['systems']:
+            deployments.update(system['deploy'].iterkeys())
+            if 'subsystems' in system:
+                deployments.update(self._get_subsystem_names(system))
+        duplicates = set(deployment for deployment, count
+                         in deployments.iteritems() if count > 1)
+        if duplicates:
+            raise DuplicateDeploymentNameError(duplicates)
+
+    def _get_subsystem_names(self, system): # pragma: no cover
+        for subsystem in system.get('subsystems', []):
+            for name in subsystem['deploy'].iterkeys():
+                yield name
+            for name in self._get_subsystem_names(subsystem):
+                yield name
 
     def _validate_system(self, morph):
         # A system must contain at least one stratum
diff --git a/morphlib/morphloader_tests.py b/morphlib/morphloader_tests.py
index a050e10b..82663298 100644
--- a/morphlib/morphloader_tests.py
+++ b/morphlib/morphloader_tests.py
@@ -458,6 +458,24 @@ build-system: dummy
             self.loader.validate(m)
         self.assertEqual(cm.exception.strata, ["foo"])
 
+    def test_validate_requires_unique_deployment_names_in_cluster(self):
+        subsystem = [{'morph': 'baz', 'deploy': {'foobar': None}}]
+        m = morphlib.morph3.Morphology(
+            name='cluster',
+            kind='cluster',
+            systems=[{'morph': 'foo',
+                      'deploy': {'deployment': {}},
+                      'subsystems': subsystem},
+                     {'morph': 'bar',
+                      'deploy': {'deployment': {}},
+                      'subsystems': subsystem}])
+        with self.assertRaises(
+                morphlib.morphloader.DuplicateDeploymentNameError) as cm:
+            self.loader.validate(m)
+        ex = cm.exception
+        self.assertIn('foobar', ex.duplicates)
+        self.assertIn('deployment', ex.duplicates)
+
     def test_loads_yaml_from_string(self):
         string = '''\
 name: foo
diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py
index 3afb7b17..6fc0998c 100644
--- a/morphlib/plugins/deploy_plugin.py
+++ b/morphlib/plugins/deploy_plugin.py
@@ -35,10 +35,9 @@ class DeployPlugin(cliapp.Plugin):
                                   'existing cluster of systems rather than do '
                                   'an initial deployment',
                                   group=group_deploy)
-
         self.app.add_subcommand(
             'deploy', self.deploy,
-            arg_synopsis='CLUSTER [SYSTEM.KEY=VALUE]')
+            arg_synopsis='CLUSTER [DEPLOYMENT...] [SYSTEM.KEY=VALUE]')
 
     def disable(self):
         pass
@@ -50,6 +49,10 @@ class DeployPlugin(cliapp.Plugin):
 
         * `CLUSTER` is the name of the cluster to deploy.
 
+        * `DEPLOYMENT...` is the name of zero or more deployments in the
+          morphology to deploy. If none are specified then all deployments
+          in the morphology are deployed.
+
         * `SYSTEM.KEY=VALUE` can be used to assign `VALUE` to a parameter
           named `KEY` for the system identified by `SYSTEM` in the cluster
           morphology (see below). This will override parameters defined
@@ -249,7 +252,7 @@ class DeployPlugin(cliapp.Plugin):
 
         Deployment configuration is stored in the deployed system as
         /baserock/deployment.meta. THIS CONTAINS ALL ENVIRONMENT VARIABLES SET
-        DURINGR DEPLOYMENT, so make sure you have no sensitive information in
+        DURING DEPLOYMENT, so make sure you have no sensitive information in
         your environment that is being leaked. As a special case, any
         environment/deployment variable that contains 'PASSWORD' in its name is
         stripped out and not stored in the final system.
@@ -276,7 +279,6 @@ class DeployPlugin(cliapp.Plugin):
 
         self.app.settings['no-git-update'] = True
         cluster_name = morphlib.util.strip_morph_extension(args[0])
-        env_vars = args[1:]
 
         ws = morphlib.workspace.open('.')
         sb = morphlib.sysbranchdir.open_from_within('.')
@@ -303,6 +305,22 @@ class DeployPlugin(cliapp.Plugin):
                 "Error: morph deploy is only supported for cluster"
                 " morphologies.")
 
+        # parse the rest of the args
+        all_subsystems = set()
+        all_deployments = set()
+        deployments = set()
+        for system in cluster_morphology['systems']:
+            all_deployments.update([sys_id for sys_id in system['deploy']])
+            if 'subsystems' in system:
+                all_subsystems.update(loader._get_subsystem_names(system))
+        for item in args[1:]:
+            if not item in all_deployments:
+                break
+            deployments.add(item)
+        env_vars = args[len(deployments) + 1:]
+        self.validate_deployment_options(
+            env_vars, all_deployments, all_subsystems)
+
         bb = morphlib.buildbranch.BuildBranch(sb, build_ref_prefix,
                                               push_temporary=False)
         with contextlib.closing(bb) as bb:
@@ -336,15 +354,35 @@ class DeployPlugin(cliapp.Plugin):
                     self.deploy_system(build_command, deploy_tempdir,
                                        root_repo_dir, bb.root_repo_url,
                                        bb.root_ref, system, env_vars,
-                                       parent_location='')
+                                       deployments, parent_location='')
             finally:
                 shutil.rmtree(deploy_tempdir)
 
         self.app.status(msg='Finished deployment')
 
+    def validate_deployment_options(
+            self, env_vars, all_deployments, all_subsystems):
+        for var in env_vars:
+            for subsystem in all_subsystems:
+                if subsystem == var:
+                    raise cliapp.AppException(
+                        'Cannot directly deploy subsystems. Create a top '
+                        'level deployment for the subsystem %s instead.' %
+                        subsystem)
+                if not any(deployment in var
+                        for deployment in all_deployments) \
+                        and not subsystem in var:
+                    raise cliapp.AppException(
+                        'Variable referenced a non-existent deployment '
+                        'name: %s' % var)
+
     def deploy_system(self, build_command, deploy_tempdir,
                       root_repo_dir, build_repo, ref, system, env_vars,
-                      parent_location):
+                      deployment_filter, parent_location):
+        sys_ids = set(sys_id for sys_id, _ in system['deploy'].iteritems())
+        if deployment_filter and not \
+                any(sys_id in deployment_filter for sys_id in sys_ids):
+            return
         old_status_prefix = self.app.status_prefix
         system_status_prefix = '%s[%s]' % (old_status_prefix, system['morph'])
         self.app.status_prefix = system_status_prefix
@@ -357,8 +395,9 @@ class DeployPlugin(cliapp.Plugin):
             artifact = build_command.resolve_artifacts(srcpool)
 
             deploy_defaults = system.get('deploy-defaults', {})
-            deployments = system['deploy']
-            for system_id, deploy_params in deployments.iteritems():
+            for system_id, deploy_params in system['deploy'].iteritems():
+                if not system_id in deployment_filter and deployment_filter:
+                    continue
                 deployment_status_prefix = '%s[%s]' % (
                     system_status_prefix, system_id)
                 self.app.status_prefix = deployment_status_prefix
@@ -399,7 +438,7 @@ class DeployPlugin(cliapp.Plugin):
                     for subsystem in system.get('subsystems', []):
                         self.deploy_system(build_command, deploy_tempdir,
                                            root_repo_dir, build_repo,
-                                           ref, subsystem, env_vars,
+                                           ref, subsystem, env_vars, [],
                                            parent_location=system_tree)
                     if parent_location:
                         deploy_location = os.path.join(parent_location,
diff --git a/scripts/edit-morph b/scripts/edit-morph
index 465a3ea3..2b81747c 100755
--- a/scripts/edit-morph
+++ b/scripts/edit-morph
@@ -209,7 +209,7 @@ class EditMorph(cliapp.Application):
             d = yaml.load(f)
         yield d
         with open(path, 'w') as f:
-            yaml.dump(d, f)
+            yaml.dump(d, f, default_flow_style=False)
 
     def cmd_set_system_artifact_depends(self, args):
         '''Change the artifacts used by a System.
diff --git a/yarns/deployment.yarn b/yarns/deployment.yarn
index 67aecce2..b30aa4cf 100644
--- a/yarns/deployment.yarn
+++ b/yarns/deployment.yarn
@@ -185,3 +185,94 @@ will mention the initramfs, and the UUID of the disk.
     AND file mnt/extlinux.conf matches initramfs
     AND file mnt/extlinux.conf matches root=UUID=
     FINALLY mnt is unmounted
+
+Partial deployments
+===================
+
+Deploy part of a cluster
+------------------------
+
+Starting from the well-defined position of having a cluster morphology
+with only one definition.
+
+    SCENARIO partially deploying a cluster morphology
+    GIVEN a workspace
+    AND a git server
+    WHEN the user checks out the system branch called master
+    AND the user builds the system test-system in branch master
+    GIVEN a cluster called test-cluster in system branch master
+    AND a system in cluster test-cluster in branch master called test-system
+    AND system test-system in cluster test-cluster in branch master builds test-system
+    AND system test-system in cluster test-cluster in branch master has deployment type: tar
+    AND system test-system in cluster test-cluster in branch master has deployment location: test-system.tar
+
+It is useful to group related deployments together, so we support adding
+another deployment to the same cluster morphology.
+
+    GIVEN a system in cluster test-cluster in branch master called second-system
+    AND system second-system in cluster test-cluster in branch master builds test-system
+    AND system second-system in cluster test-cluster in branch master has deployment type: tar
+    AND system second-system in cluster test-cluster in branch master has deployment location: second-system.tar
+
+When we don't tell `morph deploy` which system we want to deploy, all
+of the systems in the cluster are deployed. Here a successful deployment
+will have morph exit sucessfully and in the case of tarball deployments,
+the tarballs for both the systems will be created.
+
+    WHEN the user attempts to deploy the cluster test-cluster in branch master
+    THEN morph succeeded
+    AND file workspace/master/test/morphs/test-system.tar exists
+    AND file workspace/master/test/morphs/second-system.tar exists
+
+However, we don't need to deploy every system defined in a cluster at
+once. This is useful for cases such as having a cluster morphology for
+deploying a whole distbuild network, and re-deploying only nodes that
+have failed.
+
+    GIVEN the files workspace/master/test/morphs/test-system.tar and workspace/master/test/morphs/second-system.tar are removed
+    WHEN the user attempts to deploy test-system from cluster test-cluster in branch master
+
+A successful deployment will have morph exit successfully, and in the
+case of tarball deployments, only the tarball for the system we asked
+for will be created.
+
+    THEN morph succeeded
+    AND file workspace/master/test/morphs/test-system.tar exists
+    AND file workspace/master/test/morphs/second-system.tar does not exist
+
+Cluster morphs can contain "nested systems", i.e. systems which have
+subsystems to deploy as part of them.
+
+We need to add a subsystem to the cluster to test this.
+
+    GIVEN a subsystem in cluster test-cluster in branch master called test-system.sysroot
+    AND subsystem test-system.sysroot in cluster test-cluster in branch master builds test-system
+    AND subsystem test-system.sysroot in cluster test-cluster in branch master has deployment type: sysroot
+
+We specify the location as a file path, this is relative to the parent
+system's extracted rootfs, before it is configured.
+
+    AND subsystem test-system.sysroot in cluster test-cluster in branch master has deployment location: var/lib/sysroots/test-system
+
+The system which contains a nested system is deployed the same as
+before, we don't need to mention the nested deployment.
+
+    AND the file workspace/master/test/morphs/test-system.tar is removed
+    WHEN the user attempts to deploy test-system from cluster test-cluster in branch master
+    THEN morph succeeded
+    AND file workspace/master/test/morphs/test-system.tar exists
+    AND tarball workspace/master/test/morphs/test-system.tar contains var/lib/sysroots/test-system/baserock
+
+Morph will abort deployment if the system to deploy that is specified
+on the command line is not defined in the morphology.
+
+    WHEN the user attempts to deploy not-a-system from cluster test-cluster in branch master
+    THEN morph failed
+
+It is not valid to deploy a nested system on its own. If it becomes
+desirable to deploy a system that is identical to a system that already
+exists but is nested in another, it should be redefined as a top-level
+deployment.
+
+    WHEN the user attempts to deploy test-system.sysroot from cluster test-cluster in branch master
+    THEN morph failed
diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn
index 66d47bfd..b5b6a253 100644
--- a/yarns/implementations.yarn
+++ b/yarns/implementations.yarn
@@ -657,6 +657,14 @@ them, so they can be added to the end of the implements section.
     if [ $MATCH_1 == "deploys" ]; then run_morph "$@"
     else attempt_morph "$@"; fi
 
+    IMPLEMENTS WHEN the user (attempts to deploy|deploys) (.*) from cluster (\S+) in branch (\S+)
+    cd "$DATADIR/workspace/$MATCH_4"
+    set -- deploy "$MATCH_3"
+    systems=$(echo "$MATCH_2" | sed -e 's/, /\n/g' -e 's/ and /\n/g')
+    set -- "$@" $systems
+    if [ $MATCH_1 == "deploys" ]; then run_morph "$@"
+    else attempt_morph "$@"; fi
+
     IMPLEMENTS WHEN the user (attempts to upgrade|upgrades) the (system|cluster) (\S+) in branch (\S+)( with options (.*))?
     cd "$DATADIR/workspace/$MATCH_4"
     set -- deploy --upgrade "$MATCH_3"
@@ -694,6 +702,14 @@ The file contents is used as a `printf`(1) format string.
     IMPLEMENTS GIVEN a file called (\S+) containing "(.*)"
     printf "$MATCH_2" > "$DATADIR/$MATCH_1"
 
+Remove a file
+-------------
+
+    IMPLEMENTS GIVEN the file(s)? (.*) (is|are) removed
+    cd "$DATADIR"
+    files=$(echo "$MATCH_2" | sed -e 's/, /\n/g' -e 's/ and /\n/g')
+    rm $files
+
 Set attributes on a file or directory
 -------------------------------------
 
@@ -712,6 +728,9 @@ Check attributes of a file on the filesystem
     IMPLEMENTS THEN file (\S+) exists
     test -e "$DATADIR/$MATCH_1"
 
+    IMPLEMENTS THEN file (\S+) does not exist
+    test ! -e "$DATADIR/$MATCH_1"
+
     IMPLEMENTS THEN file (\S+) has permissions (\S+)
     stat -c %A "$DATADIR/$MATCH_1" | grep -Fx -e "$MATCH_2"