diff options
author | Adam Coldrick <adam.coldrick@codethink.co.uk> | 2014-06-18 14:57:48 +0000 |
---|---|---|
committer | Adam Coldrick <adam.coldrick@codethink.co.uk> | 2014-06-18 14:57:48 +0000 |
commit | d5ee8bdc636f5830f897b1846522b64bd5f06ebf (patch) | |
tree | 297139ef3710ee4ebfec7b83ae5d31113fc984e2 | |
parent | 5bf3a96bd125548f058907001904b552f952e279 (diff) | |
parent | 64c96a31d0d7c59d37703edaa08e85b452eb7f22 (diff) | |
download | morph-d5ee8bdc636f5830f897b1846522b64bd5f06ebf.tar.gz |
Merge branch 'baserock/adamcoldrick/deploy-specific-systems-v5'
Reviewed by: Richard Maw <richard.maw@codethink.co.uk>
Lars Wirzenius <lars.wirzenius@codethink.co.uk>
-rw-r--r-- | morphlib/morphloader.py | 27 | ||||
-rw-r--r-- | morphlib/morphloader_tests.py | 18 | ||||
-rw-r--r-- | morphlib/plugins/deploy_plugin.py | 57 | ||||
-rwxr-xr-x | scripts/edit-morph | 2 | ||||
-rw-r--r-- | yarns/deployment.yarn | 91 | ||||
-rw-r--r-- | yarns/implementations.yarn | 19 |
6 files changed, 203 insertions, 11 deletions
diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py index ca5902a6..b5c8168d 100644 --- a/morphlib/morphloader.py +++ b/morphlib/morphloader.py @@ -205,6 +205,15 @@ class MultipleValidationErrors(morphlib.Error): self.msg += ('\t' + str(error)) +class DuplicateDeploymentNameError(morphlib.Error): + + def __init__(self, duplicates): + self.duplicates = duplicates + morphlib.Error.__init__( + self, 'Cluster morphology contains the following non-unique ' + 'deployment names:\n%s' % '\n '.join(duplicates)) + + class OrderedDumper(yaml.SafeDumper): keyorder = ( 'name', @@ -412,7 +421,23 @@ class MorphologyLoader(object): getattr(self, '_validate_%s' % kind)(morph) def _validate_cluster(self, morph): - pass + # Deployment names must be unique within a cluster + deployments = collections.Counter() + for system in morph['systems']: + deployments.update(system['deploy'].iterkeys()) + if 'subsystems' in system: + deployments.update(self._get_subsystem_names(system)) + duplicates = set(deployment for deployment, count + in deployments.iteritems() if count > 1) + if duplicates: + raise DuplicateDeploymentNameError(duplicates) + + def _get_subsystem_names(self, system): # pragma: no cover + for subsystem in system.get('subsystems', []): + for name in subsystem['deploy'].iterkeys(): + yield name + for name in self._get_subsystem_names(subsystem): + yield name def _validate_system(self, morph): # A system must contain at least one stratum diff --git a/morphlib/morphloader_tests.py b/morphlib/morphloader_tests.py index a050e10b..82663298 100644 --- a/morphlib/morphloader_tests.py +++ b/morphlib/morphloader_tests.py @@ -458,6 +458,24 @@ build-system: dummy self.loader.validate(m) self.assertEqual(cm.exception.strata, ["foo"]) + def test_validate_requires_unique_deployment_names_in_cluster(self): + subsystem = [{'morph': 'baz', 'deploy': {'foobar': None}}] + m = morphlib.morph3.Morphology( + name='cluster', + kind='cluster', + systems=[{'morph': 'foo', + 'deploy': {'deployment': {}}, + 'subsystems': subsystem}, + {'morph': 'bar', + 'deploy': {'deployment': {}}, + 'subsystems': subsystem}]) + with self.assertRaises( + morphlib.morphloader.DuplicateDeploymentNameError) as cm: + self.loader.validate(m) + ex = cm.exception + self.assertIn('foobar', ex.duplicates) + self.assertIn('deployment', ex.duplicates) + def test_loads_yaml_from_string(self): string = '''\ name: foo diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index 3afb7b17..6fc0998c 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -35,10 +35,9 @@ class DeployPlugin(cliapp.Plugin): 'existing cluster of systems rather than do ' 'an initial deployment', group=group_deploy) - self.app.add_subcommand( 'deploy', self.deploy, - arg_synopsis='CLUSTER [SYSTEM.KEY=VALUE]') + arg_synopsis='CLUSTER [DEPLOYMENT...] [SYSTEM.KEY=VALUE]') def disable(self): pass @@ -50,6 +49,10 @@ class DeployPlugin(cliapp.Plugin): * `CLUSTER` is the name of the cluster to deploy. + * `DEPLOYMENT...` is the name of zero or more deployments in the + morphology to deploy. If none are specified then all deployments + in the morphology are deployed. + * `SYSTEM.KEY=VALUE` can be used to assign `VALUE` to a parameter named `KEY` for the system identified by `SYSTEM` in the cluster morphology (see below). This will override parameters defined @@ -249,7 +252,7 @@ class DeployPlugin(cliapp.Plugin): Deployment configuration is stored in the deployed system as /baserock/deployment.meta. THIS CONTAINS ALL ENVIRONMENT VARIABLES SET - DURINGR DEPLOYMENT, so make sure you have no sensitive information in + DURING DEPLOYMENT, so make sure you have no sensitive information in your environment that is being leaked. As a special case, any environment/deployment variable that contains 'PASSWORD' in its name is stripped out and not stored in the final system. @@ -276,7 +279,6 @@ class DeployPlugin(cliapp.Plugin): self.app.settings['no-git-update'] = True cluster_name = morphlib.util.strip_morph_extension(args[0]) - env_vars = args[1:] ws = morphlib.workspace.open('.') sb = morphlib.sysbranchdir.open_from_within('.') @@ -303,6 +305,22 @@ class DeployPlugin(cliapp.Plugin): "Error: morph deploy is only supported for cluster" " morphologies.") + # parse the rest of the args + all_subsystems = set() + all_deployments = set() + deployments = set() + for system in cluster_morphology['systems']: + all_deployments.update([sys_id for sys_id in system['deploy']]) + if 'subsystems' in system: + all_subsystems.update(loader._get_subsystem_names(system)) + for item in args[1:]: + if not item in all_deployments: + break + deployments.add(item) + env_vars = args[len(deployments) + 1:] + self.validate_deployment_options( + env_vars, all_deployments, all_subsystems) + bb = morphlib.buildbranch.BuildBranch(sb, build_ref_prefix, push_temporary=False) with contextlib.closing(bb) as bb: @@ -336,15 +354,35 @@ class DeployPlugin(cliapp.Plugin): self.deploy_system(build_command, deploy_tempdir, root_repo_dir, bb.root_repo_url, bb.root_ref, system, env_vars, - parent_location='') + deployments, parent_location='') finally: shutil.rmtree(deploy_tempdir) self.app.status(msg='Finished deployment') + def validate_deployment_options( + self, env_vars, all_deployments, all_subsystems): + for var in env_vars: + for subsystem in all_subsystems: + if subsystem == var: + raise cliapp.AppException( + 'Cannot directly deploy subsystems. Create a top ' + 'level deployment for the subsystem %s instead.' % + subsystem) + if not any(deployment in var + for deployment in all_deployments) \ + and not subsystem in var: + raise cliapp.AppException( + 'Variable referenced a non-existent deployment ' + 'name: %s' % var) + def deploy_system(self, build_command, deploy_tempdir, root_repo_dir, build_repo, ref, system, env_vars, - parent_location): + deployment_filter, parent_location): + sys_ids = set(sys_id for sys_id, _ in system['deploy'].iteritems()) + if deployment_filter and not \ + any(sys_id in deployment_filter for sys_id in sys_ids): + return old_status_prefix = self.app.status_prefix system_status_prefix = '%s[%s]' % (old_status_prefix, system['morph']) self.app.status_prefix = system_status_prefix @@ -357,8 +395,9 @@ class DeployPlugin(cliapp.Plugin): artifact = build_command.resolve_artifacts(srcpool) deploy_defaults = system.get('deploy-defaults', {}) - deployments = system['deploy'] - for system_id, deploy_params in deployments.iteritems(): + for system_id, deploy_params in system['deploy'].iteritems(): + if not system_id in deployment_filter and deployment_filter: + continue deployment_status_prefix = '%s[%s]' % ( system_status_prefix, system_id) self.app.status_prefix = deployment_status_prefix @@ -399,7 +438,7 @@ class DeployPlugin(cliapp.Plugin): for subsystem in system.get('subsystems', []): self.deploy_system(build_command, deploy_tempdir, root_repo_dir, build_repo, - ref, subsystem, env_vars, + ref, subsystem, env_vars, [], parent_location=system_tree) if parent_location: deploy_location = os.path.join(parent_location, diff --git a/scripts/edit-morph b/scripts/edit-morph index 465a3ea3..2b81747c 100755 --- a/scripts/edit-morph +++ b/scripts/edit-morph @@ -209,7 +209,7 @@ class EditMorph(cliapp.Application): d = yaml.load(f) yield d with open(path, 'w') as f: - yaml.dump(d, f) + yaml.dump(d, f, default_flow_style=False) def cmd_set_system_artifact_depends(self, args): '''Change the artifacts used by a System. diff --git a/yarns/deployment.yarn b/yarns/deployment.yarn index 67aecce2..b30aa4cf 100644 --- a/yarns/deployment.yarn +++ b/yarns/deployment.yarn @@ -185,3 +185,94 @@ will mention the initramfs, and the UUID of the disk. AND file mnt/extlinux.conf matches initramfs AND file mnt/extlinux.conf matches root=UUID= FINALLY mnt is unmounted + +Partial deployments +=================== + +Deploy part of a cluster +------------------------ + +Starting from the well-defined position of having a cluster morphology +with only one definition. + + SCENARIO partially deploying a cluster morphology + GIVEN a workspace + AND a git server + WHEN the user checks out the system branch called master + AND the user builds the system test-system in branch master + GIVEN a cluster called test-cluster in system branch master + AND a system in cluster test-cluster in branch master called test-system + AND system test-system in cluster test-cluster in branch master builds test-system + AND system test-system in cluster test-cluster in branch master has deployment type: tar + AND system test-system in cluster test-cluster in branch master has deployment location: test-system.tar + +It is useful to group related deployments together, so we support adding +another deployment to the same cluster morphology. + + GIVEN a system in cluster test-cluster in branch master called second-system + AND system second-system in cluster test-cluster in branch master builds test-system + AND system second-system in cluster test-cluster in branch master has deployment type: tar + AND system second-system in cluster test-cluster in branch master has deployment location: second-system.tar + +When we don't tell `morph deploy` which system we want to deploy, all +of the systems in the cluster are deployed. Here a successful deployment +will have morph exit sucessfully and in the case of tarball deployments, +the tarballs for both the systems will be created. + + WHEN the user attempts to deploy the cluster test-cluster in branch master + THEN morph succeeded + AND file workspace/master/test/morphs/test-system.tar exists + AND file workspace/master/test/morphs/second-system.tar exists + +However, we don't need to deploy every system defined in a cluster at +once. This is useful for cases such as having a cluster morphology for +deploying a whole distbuild network, and re-deploying only nodes that +have failed. + + GIVEN the files workspace/master/test/morphs/test-system.tar and workspace/master/test/morphs/second-system.tar are removed + WHEN the user attempts to deploy test-system from cluster test-cluster in branch master + +A successful deployment will have morph exit successfully, and in the +case of tarball deployments, only the tarball for the system we asked +for will be created. + + THEN morph succeeded + AND file workspace/master/test/morphs/test-system.tar exists + AND file workspace/master/test/morphs/second-system.tar does not exist + +Cluster morphs can contain "nested systems", i.e. systems which have +subsystems to deploy as part of them. + +We need to add a subsystem to the cluster to test this. + + GIVEN a subsystem in cluster test-cluster in branch master called test-system.sysroot + AND subsystem test-system.sysroot in cluster test-cluster in branch master builds test-system + AND subsystem test-system.sysroot in cluster test-cluster in branch master has deployment type: sysroot + +We specify the location as a file path, this is relative to the parent +system's extracted rootfs, before it is configured. + + AND subsystem test-system.sysroot in cluster test-cluster in branch master has deployment location: var/lib/sysroots/test-system + +The system which contains a nested system is deployed the same as +before, we don't need to mention the nested deployment. + + AND the file workspace/master/test/morphs/test-system.tar is removed + WHEN the user attempts to deploy test-system from cluster test-cluster in branch master + THEN morph succeeded + AND file workspace/master/test/morphs/test-system.tar exists + AND tarball workspace/master/test/morphs/test-system.tar contains var/lib/sysroots/test-system/baserock + +Morph will abort deployment if the system to deploy that is specified +on the command line is not defined in the morphology. + + WHEN the user attempts to deploy not-a-system from cluster test-cluster in branch master + THEN morph failed + +It is not valid to deploy a nested system on its own. If it becomes +desirable to deploy a system that is identical to a system that already +exists but is nested in another, it should be redefined as a top-level +deployment. + + WHEN the user attempts to deploy test-system.sysroot from cluster test-cluster in branch master + THEN morph failed diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn index 66d47bfd..b5b6a253 100644 --- a/yarns/implementations.yarn +++ b/yarns/implementations.yarn @@ -657,6 +657,14 @@ them, so they can be added to the end of the implements section. if [ $MATCH_1 == "deploys" ]; then run_morph "$@" else attempt_morph "$@"; fi + IMPLEMENTS WHEN the user (attempts to deploy|deploys) (.*) from cluster (\S+) in branch (\S+) + cd "$DATADIR/workspace/$MATCH_4" + set -- deploy "$MATCH_3" + systems=$(echo "$MATCH_2" | sed -e 's/, /\n/g' -e 's/ and /\n/g') + set -- "$@" $systems + if [ $MATCH_1 == "deploys" ]; then run_morph "$@" + else attempt_morph "$@"; fi + IMPLEMENTS WHEN the user (attempts to upgrade|upgrades) the (system|cluster) (\S+) in branch (\S+)( with options (.*))? cd "$DATADIR/workspace/$MATCH_4" set -- deploy --upgrade "$MATCH_3" @@ -694,6 +702,14 @@ The file contents is used as a `printf`(1) format string. IMPLEMENTS GIVEN a file called (\S+) containing "(.*)" printf "$MATCH_2" > "$DATADIR/$MATCH_1" +Remove a file +------------- + + IMPLEMENTS GIVEN the file(s)? (.*) (is|are) removed + cd "$DATADIR" + files=$(echo "$MATCH_2" | sed -e 's/, /\n/g' -e 's/ and /\n/g') + rm $files + Set attributes on a file or directory ------------------------------------- @@ -712,6 +728,9 @@ Check attributes of a file on the filesystem IMPLEMENTS THEN file (\S+) exists test -e "$DATADIR/$MATCH_1" + IMPLEMENTS THEN file (\S+) does not exist + test ! -e "$DATADIR/$MATCH_1" + IMPLEMENTS THEN file (\S+) has permissions (\S+) stat -c %A "$DATADIR/$MATCH_1" | grep -Fx -e "$MATCH_2" |