From 6b1bf6c586ed2304c39f6d8572f08f62c1743368 Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Thu, 22 May 2014 15:28:06 +0000 Subject: Validate cluster morphologies The names of deployments in cluster morphologies will need to be unique in order for the deployment of selected systems to work. --- morphlib/morphloader.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) (limited to 'morphlib') diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py index ca5902a6..b5c8168d 100644 --- a/morphlib/morphloader.py +++ b/morphlib/morphloader.py @@ -205,6 +205,15 @@ class MultipleValidationErrors(morphlib.Error): self.msg += ('\t' + str(error)) +class DuplicateDeploymentNameError(morphlib.Error): + + def __init__(self, duplicates): + self.duplicates = duplicates + morphlib.Error.__init__( + self, 'Cluster morphology contains the following non-unique ' + 'deployment names:\n%s' % '\n '.join(duplicates)) + + class OrderedDumper(yaml.SafeDumper): keyorder = ( 'name', @@ -412,7 +421,23 @@ class MorphologyLoader(object): getattr(self, '_validate_%s' % kind)(morph) def _validate_cluster(self, morph): - pass + # Deployment names must be unique within a cluster + deployments = collections.Counter() + for system in morph['systems']: + deployments.update(system['deploy'].iterkeys()) + if 'subsystems' in system: + deployments.update(self._get_subsystem_names(system)) + duplicates = set(deployment for deployment, count + in deployments.iteritems() if count > 1) + if duplicates: + raise DuplicateDeploymentNameError(duplicates) + + def _get_subsystem_names(self, system): # pragma: no cover + for subsystem in system.get('subsystems', []): + for name in subsystem['deploy'].iterkeys(): + yield name + for name in self._get_subsystem_names(subsystem): + yield name def _validate_system(self, morph): # A system must contain at least one stratum -- cgit v1.2.1 From 2bd9ed61ebeb621b7f47c2e05a370c6d330fc27d Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Fri, 23 May 2014 16:26:51 +0000 Subject: Add unit test to test the validation of cluster morphs --- morphlib/morphloader_tests.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'morphlib') diff --git a/morphlib/morphloader_tests.py b/morphlib/morphloader_tests.py index a050e10b..82663298 100644 --- a/morphlib/morphloader_tests.py +++ b/morphlib/morphloader_tests.py @@ -458,6 +458,24 @@ build-system: dummy self.loader.validate(m) self.assertEqual(cm.exception.strata, ["foo"]) + def test_validate_requires_unique_deployment_names_in_cluster(self): + subsystem = [{'morph': 'baz', 'deploy': {'foobar': None}}] + m = morphlib.morph3.Morphology( + name='cluster', + kind='cluster', + systems=[{'morph': 'foo', + 'deploy': {'deployment': {}}, + 'subsystems': subsystem}, + {'morph': 'bar', + 'deploy': {'deployment': {}}, + 'subsystems': subsystem}]) + with self.assertRaises( + morphlib.morphloader.DuplicateDeploymentNameError) as cm: + self.loader.validate(m) + ex = cm.exception + self.assertIn('foobar', ex.duplicates) + self.assertIn('deployment', ex.duplicates) + def test_loads_yaml_from_string(self): string = '''\ name: foo -- cgit v1.2.1 From d6e2ce66621891a3758da7f49411a41ebc001f57 Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Thu, 22 May 2014 15:30:19 +0000 Subject: Allow the user to specify deployments in a cluster Instead of taking the name of a cluster morphology and zero or more parameters for overriding the cluster morphology, morph deploy should take the name of a cluster morphology and the names of zero or more system deployments that are defined in the cluster morphology. If no deployment names are given then all deployments are deployed. --- morphlib/plugins/deploy_plugin.py | 51 +++++++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 8 deletions(-) (limited to 'morphlib') diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index 3afb7b17..eb3a1988 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -35,10 +35,9 @@ class DeployPlugin(cliapp.Plugin): 'existing cluster of systems rather than do ' 'an initial deployment', group=group_deploy) - self.app.add_subcommand( 'deploy', self.deploy, - arg_synopsis='CLUSTER [SYSTEM.KEY=VALUE]') + arg_synopsis='CLUSTER [DEPLOYMENT...] [SYSTEM.KEY=VALUE]') def disable(self): pass @@ -276,7 +275,6 @@ class DeployPlugin(cliapp.Plugin): self.app.settings['no-git-update'] = True cluster_name = morphlib.util.strip_morph_extension(args[0]) - env_vars = args[1:] ws = morphlib.workspace.open('.') sb = morphlib.sysbranchdir.open_from_within('.') @@ -303,6 +301,22 @@ class DeployPlugin(cliapp.Plugin): "Error: morph deploy is only supported for cluster" " morphologies.") + # parse the rest of the args + all_subsystems = set() + all_deployments = set() + deployments = set() + for system in cluster_morphology['systems']: + all_deployments.update([sys_id for sys_id in system['deploy']]) + if 'subsystems' in system: + all_subsystems.update(loader._get_subsystem_names(system)) + for item in args[1:]: + if not item in all_deployments: + break + deployments.add(item) + env_vars = args[len(deployments) + 1:] + self.validate_deployment_options( + env_vars, all_deployments, all_subsystems) + bb = morphlib.buildbranch.BuildBranch(sb, build_ref_prefix, push_temporary=False) with contextlib.closing(bb) as bb: @@ -336,15 +350,35 @@ class DeployPlugin(cliapp.Plugin): self.deploy_system(build_command, deploy_tempdir, root_repo_dir, bb.root_repo_url, bb.root_ref, system, env_vars, - parent_location='') + deployments, parent_location='') finally: shutil.rmtree(deploy_tempdir) self.app.status(msg='Finished deployment') + def validate_deployment_options( + self, env_vars, all_deployments, all_subsystems): + for var in env_vars: + for subsystem in all_subsystems: + if subsystem == var: + raise cliapp.AppException( + 'Cannot directly deploy subsystems. Create a top ' + 'level deployment for the subsystem %s instead.' % + subsystem) + if not any(deployment in var + for deployment in all_deployments) \ + and not subsystem in var: + raise cliapp.AppException( + 'Variable referenced a non-existent deployment ' + 'name: %s' % var) + def deploy_system(self, build_command, deploy_tempdir, root_repo_dir, build_repo, ref, system, env_vars, - parent_location): + deployment_filter, parent_location): + sys_ids = set(sys_id for sys_id, _ in system['deploy'].iteritems()) + if deployment_filter and not \ + any(sys_id in deployment_filter for sys_id in sys_ids): + return old_status_prefix = self.app.status_prefix system_status_prefix = '%s[%s]' % (old_status_prefix, system['morph']) self.app.status_prefix = system_status_prefix @@ -357,8 +391,9 @@ class DeployPlugin(cliapp.Plugin): artifact = build_command.resolve_artifacts(srcpool) deploy_defaults = system.get('deploy-defaults', {}) - deployments = system['deploy'] - for system_id, deploy_params in deployments.iteritems(): + for system_id, deploy_params in system['deploy'].iteritems(): + if not system_id in deployment_filter and deployment_filter: + continue deployment_status_prefix = '%s[%s]' % ( system_status_prefix, system_id) self.app.status_prefix = deployment_status_prefix @@ -399,7 +434,7 @@ class DeployPlugin(cliapp.Plugin): for subsystem in system.get('subsystems', []): self.deploy_system(build_command, deploy_tempdir, root_repo_dir, build_repo, - ref, subsystem, env_vars, + ref, subsystem, env_vars, [], parent_location=system_tree) if parent_location: deploy_location = os.path.join(parent_location, -- cgit v1.2.1 From f435d78d61491e2acd3107a78eba289977938b38 Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Thu, 22 May 2014 15:34:19 +0000 Subject: Update the help text for morph deploy --- morphlib/plugins/deploy_plugin.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'morphlib') diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index eb3a1988..6fc0998c 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -49,6 +49,10 @@ class DeployPlugin(cliapp.Plugin): * `CLUSTER` is the name of the cluster to deploy. + * `DEPLOYMENT...` is the name of zero or more deployments in the + morphology to deploy. If none are specified then all deployments + in the morphology are deployed. + * `SYSTEM.KEY=VALUE` can be used to assign `VALUE` to a parameter named `KEY` for the system identified by `SYSTEM` in the cluster morphology (see below). This will override parameters defined @@ -248,7 +252,7 @@ class DeployPlugin(cliapp.Plugin): Deployment configuration is stored in the deployed system as /baserock/deployment.meta. THIS CONTAINS ALL ENVIRONMENT VARIABLES SET - DURINGR DEPLOYMENT, so make sure you have no sensitive information in + DURING DEPLOYMENT, so make sure you have no sensitive information in your environment that is being leaked. As a special case, any environment/deployment variable that contains 'PASSWORD' in its name is stripped out and not stored in the final system. -- cgit v1.2.1