diff options
author | Lars Wirzenius <lars.wirzenius@codethink.co.uk> | 2013-08-16 14:26:08 +0000 |
---|---|---|
committer | Lars Wirzenius <lars.wirzenius@codethink.co.uk> | 2013-08-16 14:26:08 +0000 |
commit | 76f2294a749b186f729f9d3634a43ea9268437b0 (patch) | |
tree | 7ac42828eac543e27591e814064e08d9a590b7f5 /morphlib | |
parent | 0a5ef89e107c22a5d1931838526fe3b22af42b95 (diff) | |
parent | 9c281e003d3f6f048f065879f5504803f1ec85b3 (diff) | |
download | morph-76f2294a749b186f729f9d3634a43ea9268437b0.tar.gz |
Merge remote-tracking branch 'origin/baserock/tiagogomes/new_deployment_v4'
A tab fixed
Diffstat (limited to 'morphlib')
-rw-r--r-- | morphlib/morph2.py | 38 | ||||
-rw-r--r-- | morphlib/morph2_tests.py | 58 | ||||
-rw-r--r-- | morphlib/morphloader.py | 19 | ||||
-rw-r--r-- | morphlib/morphloader_tests.py | 14 | ||||
-rw-r--r-- | morphlib/plugins/branch_and_merge_plugin.py | 9 | ||||
-rw-r--r-- | morphlib/plugins/deploy_plugin.py | 233 | ||||
-rw-r--r-- | morphlib/util.py | 6 | ||||
-rw-r--r-- | morphlib/util_tests.py | 5 |
8 files changed, 296 insertions, 86 deletions
diff --git a/morphlib/morph2.py b/morphlib/morph2.py index d949c696..a733ce77 100644 --- a/morphlib/morph2.py +++ b/morphlib/morph2.py @@ -60,7 +60,8 @@ class Morphology(object): ('arch', None), ('system-kind', None), ('configuration-extensions', []), - ] + ], + 'cluster': [] } @staticmethod @@ -129,6 +130,27 @@ class Morphology(object): if name in names: raise ValueError('Duplicate chunk "%s"' % name) names.add(name) + elif self['kind'] == 'cluster': + if not 'systems' in self: + raise KeyError('"systems" not found') + if not self['systems']: + raise ValueError('"systems" is empty') + for system in self['systems']: + if 'morph' not in system: + raise KeyError('"morph" not found') + if 'deploy-defaults' in system: + if not isinstance(system['deploy-defaults'], dict): + raise ValueError('deploy defaults for morph "%s" ' + 'are not a mapping: %r' + % (system['morph'], + system['deploy-defaults'])) + if 'deploy' in system: + for system_id, deploy_params in system['deploy'].items(): + if not isinstance(deploy_params, dict): + raise ValueError('deployment parameters for ' + 'system "%s" are not a mapping:' + ' %r' + % (system_id, deploy_params)) def _set_default_value(self, target_dict, key, value): '''Change a value in the in-memory representation of the morphology @@ -157,6 +179,8 @@ class Morphology(object): if self['kind'] == 'stratum': self._set_stratum_defaults() + elif self['kind'] == 'cluster': + self._set_cluster_defaults() def _set_stratum_defaults(self): for source in self['chunks']: @@ -171,6 +195,18 @@ class Morphology(object): if 'prefix' not in source: self._set_default_value(source, 'prefix', '/usr') + def _set_cluster_defaults(self): + if 'systems' in self and self['systems']: + for system in self['systems']: + if 'deploy-defaults' not in system: + self._set_default_value(system, + 'deploy-defaults', + dict()) + if 'deploy' not in system: + self._set_default_value(system, + 'deploy', + dict()) + def _parse_size(self, size): if isinstance(size, basestring): size = size.lower() diff --git a/morphlib/morph2_tests.py b/morphlib/morph2_tests.py index 9de23e56..bf32d3c2 100644 --- a/morphlib/morph2_tests.py +++ b/morphlib/morph2_tests.py @@ -379,3 +379,61 @@ class MorphologyTests(unittest.TestCase): ''') cmds = m.get_commands('build-commands') self.assertEqual(cmds, []) + + ## Cluster morphologies tests + + def test_parses_simple_cluster_morph(self): + m = Morphology(''' + name: foo + kind: cluster + systems: + - morph: bar + ''') + self.assertEqual(m['name'], 'foo') + self.assertEqual(m['kind'], 'cluster') + self.assertEqual(m['systems'][0]['morph'], 'bar') + + def test_fails_without_systems(self): + text = ''' + name: foo + kind: cluster + ''' + self.assertRaises(KeyError, Morphology, text) + + def test_fails_with_empty_systems(self): + text = ''' + name: foo + kind: cluster + systems: + ''' + self.assertRaises(ValueError, Morphology, text) + + def test_fails_without_morph(self): + text = ''' + name: foo + kind: cluster + systems: + - deploy: + ''' + self.assertRaises(KeyError, Morphology, text) + + def test_fails_with_invalid_deploy_defaults(self): + text = ''' + name: foo + kind: cluster + systems: + - morph: bar + deploy-defaults: ooops_i_am_not_a_mapping + ''' + self.assertRaises(ValueError, Morphology, text) + + def test_fails_with_invalid_deployment_params(self): + text = ''' + name: foo + kind: cluster + systems: + - morph: bar + deploy: + qux: ooops_i_am_not_a_mapping + ''' + self.assertRaises(ValueError, Morphology, text) diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py index 9b00ded5..70f46064 100644 --- a/morphlib/morphloader.py +++ b/morphlib/morphloader.py @@ -109,6 +109,10 @@ class MorphologyLoader(object): 'name', 'arch', ], + 'cluster': [ + 'name', + 'systems', + ], } _static_defaults = { @@ -144,6 +148,7 @@ class MorphologyLoader(object): 'configuration-extensions': [], 'disk-size': '1G', }, + 'cluster': {}, } def parse_morphology_text(self, text, whence): @@ -217,8 +222,10 @@ class MorphologyLoader(object): # The rest of the validation is dependent on the kind. + # FIXME: move validation of clusters from morph2 to + # here, and use morphload to load the morphology kind = morph['kind'] - if kind not in ('system', 'stratum', 'chunk'): + if kind not in ('system', 'stratum', 'chunk', 'cluster'): raise UnknownKindError(morph['kind'], morph.filename) required = ['kind'] + self._required_fields[kind] @@ -230,9 +237,10 @@ class MorphologyLoader(object): self._validate_system(morph) elif kind == 'stratum': self._validate_stratum(morph) - else: - assert kind == 'chunk' + elif kind == 'chunk': self._validate_chunk(morph) + else: + assert kind == 'cluster' def _validate_system(self, morph): # All stratum names should be unique within a system. @@ -322,9 +330,10 @@ class MorphologyLoader(object): self._set_system_defaults(morphology) elif kind == 'stratum': self._set_stratum_defaults(morphology) - else: - assert kind == 'chunk' + elif kind == 'chunk': self._set_chunk_defaults(morphology) + else: + assert kind == 'cluster' def _set_system_defaults(self, morph): pass diff --git a/morphlib/morphloader_tests.py b/morphlib/morphloader_tests.py index 0f115eb1..6e957510 100644 --- a/morphlib/morphloader_tests.py +++ b/morphlib/morphloader_tests.py @@ -471,4 +471,16 @@ name: foo self.loader.set_defaults(m) self.assertEqual(m['max-jobs'], 42) - + def test_parses_simple_cluster_morph(self): + string = ''' + name: foo + kind: cluster + systems: + - morph: bar + ''' + m = self.loader.parse_morphology_text(string, 'test') + self.loader.set_defaults(m) + self.loader.validate(m) + self.assertEqual(m['name'], 'foo') + self.assertEqual(m['kind'], 'cluster') + self.assertEqual(m['systems'][0]['morph'], 'bar') diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py index 38b882a0..85e74501 100644 --- a/morphlib/plugins/branch_and_merge_plugin.py +++ b/morphlib/plugins/branch_and_merge_plugin.py @@ -349,7 +349,11 @@ class BranchAndMergePlugin(cliapp.Plugin): ], 'chunk': [ 'name', - ] + ], + 'cluster': [ + 'name', + 'systems', + ], } also_known = { @@ -376,6 +380,9 @@ class BranchAndMergePlugin(cliapp.Plugin): 'max-jobs', 'chunks', 'devices', + ], + 'cluster': [ + 'kind' ] } diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index 8530cb57..dc9f5158 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -39,7 +39,7 @@ class DeployPlugin(cliapp.Plugin): def enable(self): self.app.add_subcommand( 'deploy', self.deploy, - arg_synopsis='TYPE SYSTEM LOCATION [KEY=VALUE]') + arg_synopsis='CLUSTER [SYSTEM.KEY=VALUE]') self.other = \ morphlib.plugins.branch_and_merge_plugin.BranchAndMergePlugin() self.other.app = self.app @@ -52,30 +52,82 @@ class DeployPlugin(cliapp.Plugin): Command line arguments: - * `TYPE` is the type of deployment: to a raw disk image, - VirtualBox, or something else. See below. - - * `SYSTEM` is the name of the system morphology to deploy. - - * `LOCATION` is where the deployed system should end up at. The - syntax depends on the deployment type. See below. - - * `KEY=VALUE` is a configuration parameter to pass onto the - configuration extension. See below. - - Morph can deploy a system image. The deployment mechanism is - quite flexible, and can be extended by the user. "Deployment" - here is quite a general concept: it covers anything where a system - image is taken, configured, and then put somewhere where it can - be run. - - `TYPE` specifies the exact way in which the deployment happens. - Morph provides four deployment types: - - * `rawdisk` where Morph builds a raw disk image at `LOCATION`, - and sets up the image with a bootloader and configuration - so that it can be booted. Disk size is set with `DISK_SIZE` - (see below). + * `CLUSTER` is the name of the cluster to deploy. + + * `SYSTEM.KEY=VALUE` can be used to assign `VALUE` to a parameter + named `KEY` for the system identified by `SYSTEM` in the cluster + morphology (see below). This will override parameters defined + in the morphology. + + Morph deploys a set of systems listed in a cluster morphology. + "Deployment" here is quite a general concept: it covers anything + where a system image is taken, configured, and then put somewhere + where it can be run. The deployment mechanism is quite flexible, + and can be extended by the user. + + A cluster morphology defines a list of systems to deploy, and + for each system a list of ways to deploy them. It contains the + following fields: + + * **name**: MUST be the same as the basename of the morphology + filename, sans .morph suffix. + + * **kind**: MUST be `cluster`. + + * **systems**: a list of systems to deploy; + the value is a list of mappings, where each mapping has the + following keys: + + * **morph**: the system morphology to use in the specified + commit. + + * **deploy**: a mapping where each key identifies a + system and each system has at least the following keys: + + * **type**: identifies the type of development e.g. (kvm, + nfsboot) (see below). + * **location**: where the deployed system should end up + at. The syntax depends on the deployment type (see below). + Any additional item on the dictionary will be added to the + environment as `KEY=VALUE`. + + * **deploy-defaults**: allows multiple deployments of the same + system to share some settings, when they can. Default settings + will be overridden by those defined inside the deploy mapping. + + # Example + + name: cluster-foo + kind: cluster + systems: + - morph: devel-system-x86_64-generic + deploy: + cluster-foo-x86_64-1: + type: kvm + location: kvm+ssh://user@host/x86_64-1/x86_64-1.img + HOSTNAME: cluster-foo-x86_64-1 + DISK_SIZE: 4G + RAM_SIZE: 4G + VCPUS: 2 + - morph: devel-system-armv7-highbank + deploy-defaults: + type: nfsboot + location: cluster-foo-nfsboot-server + deploy: + cluster-foo-armv7-1: + HOSTNAME: cluster-foo-armv7-1 + cluster-foo-armv7-2: + HOSTNAME: cluster-foo-armv7-2 + + Each system defined in a cluster morphology can be deployed in + multiple ways (`type` in a cluster morphology). Morph provides + five types of deployment: + + * `tar` where Morph builds a tar archive of the root file system. + + * `rawdisk` where Morph builds a raw disk image and sets up the + image with a bootloader and configuration so that it can be + booted. Disk size is set with `DISK_SIZE` (see below). * `virtualbox-ssh` where Morph creates a VirtualBox disk image, and creates a new virtual machine on a remote host, accessed @@ -87,29 +139,15 @@ class DeployPlugin(cliapp.Plugin): `DISK_SIZE` and `RAM_SIZE` (see below). * `nfsboot` where Morph creates a system to be booted over - a network - - The following `KEY=VALUE` parameters are supported for all - deployment types: - - * `DISK_SIZE=X` to set the size of the disk image. `X` - should use a suffix of `K`, `M`, or `G` (in upper or lower - case) to indicate kilo-, mega-, or gigabytes. For example, - `DISK_SIZE=100G` would create a 100 gigabyte disk image. **This - parameter is mandatory**. - - * `RAM_SIZE=X` to set the size of virtual RAM for the virtual - machine. `X` is interpreted in the same was as `DISK_SIZE`, - and defaults to `1G`. - - The `kvm` and `virtualbox-ssh` deployment types support an - additional parameter: + a network. - * `AUTOSTART=<VALUE>` - allowed values are `yes` and `no` - (default) + In addition to the deployment type, the user must also give + a value for `location`. Its syntax depends on the deployment + types. The deployment types provided by Morph use the + following syntaxes: - The syntax of the `LOCATION` depends on the deployment types. The - deployment types provided by Morph use the following syntaxes: + * `tar`: pathname to the tar archive to be created; for + example, `/home/alice/testsystem.tar` * `rawdisk`: pathname to the disk image to be created; for example, `/home/alice/testsystem.img` @@ -132,39 +170,48 @@ class DeployPlugin(cliapp.Plugin): * `/home/alice/testsys.vdi` and `/home/alice/testys.img` are the pathnames of the disk image files on the target host. - For the `nfsboot` write extension, + * `nfsboot`: the address of the nfsboot server. (Note this is just + the _address_ of the trove, _not_ `user@...`, since `root@` will + automatically be prepended to the server address.) + + The following `KEY=VALUE` parameters are supported for `rawdisk`, + `virtualbox-ssh` and `kvm` and deployment types: + + * `DISK_SIZE=X` to set the size of the disk image. `X` should use a + suffix of `K`, `M`, or `G` (in upper or lower case) to indicate + kilo-, mega-, or gigabytes. For example, `DISK_SIZE=100G` would + create a 100 gigabyte disk image. **This parameter is mandatory**. + + The `kvm` and `virtualbox-ssh` deployment types support an additional + parameter: - * LOCATION is the address of the nfsboot server. (Note this - is just the _address_ of the trove, _not_ `user@...`, since - `root@` will automatically be prepended to the server address.) + * `RAM_SIZE=X` to set the size of virtual RAM for the virtual + machine. `X` is interpreted in the same was as `DISK_SIZE`, + and defaults to `1G`. - * the following KEY=VALUE PAIRS are mandatory + * `AUTOSTART=<VALUE>` - allowed values are `yes` and `no` + (default) - * NFSBOOT_CONFIGURE=yes (or any non-empty value). This + For the `nfsboot` write extension, + + * the following `KEY=VALUE` pairs are mandatory + + * `NFSBOOT_CONFIGURE=yes` (or any non-empty value). This enables the `nfsboot` configuration extension (see below) which MUST be used when using the `nfsboot` write extension. - * HOSTNAME=<STRING> a unique identifier for that system's + * `HOSTNAME=<STRING>` a unique identifier for that system's `nfs` root when it's deployed on the nfsboot server - the extension creates a directory with that name for the `nfs` root, and stores kernels by that name for the tftp server. - * the following KEY=VALUE PAIRS are optional + * the following `KEY=VALUE` pairs are optional - * VERSION_LABEL=<STRING> - set the name of the system + * `VERSION_LABEL=<STRING>` - set the name of the system version being deployed, when upgrading. Defaults to "factory". - An example command line using `morph deploy with the nfsboot` - type is - - morph deploy nfsboot devel-system-x86_64-generic \ - customer-trove \ - NFSBOOT_CONFIGURE=yes \ - HOSTNAME=test-deployment-1 \ - VERSION_LABEL=inital-test - Each deployment type is implemented by a **write extension**. The ones provided by Morph are listed above, but users may also create their own by adding them in the same git repository @@ -172,9 +219,7 @@ class DeployPlugin(cliapp.Plugin): script that does whatever is needed for the deployment. A write extension is passed two command line parameters: the name of an unpacked directory tree that contains the system files (after - configuration, see below), and the `LOCATION` argument. Any - additional `KEY=VALUE` arguments given to `morph deploy` are - set as environment variables when the write extension runs. + configuration, see below), and the `location` parameter. Regardless of the type of deployment, the image may be configured for a specific deployment by using **configuration @@ -201,13 +246,14 @@ class DeployPlugin(cliapp.Plugin): * `nfsboot` configures the system for nfsbooting. This MUST be used when deploying with the `nfsboot` write extension. - Any `KEY=VALUE` parameters given to `morph deploy` are set as - environment variables when either the configuration or the write - extension runs. + Any `KEY=VALUE` parameters given in `deploy` or `deploy-defaults` + sections of the cluster morphology, or given through the command line + are set as environment variables when either the configuration or the + write extension runs (except `type` and `location`). ''' - if len(args) < 3: + if not args: raise cliapp.AppException( 'Too few arguments to deploy command (see help)') @@ -220,14 +266,47 @@ class DeployPlugin(cliapp.Plugin): self.app.settings['tempdir-min-space'], '/', 0) - deployment_type = args[0] - system_name = args[1] - location = args[2] - env_vars = args[3:] + cluster = args[0] + env_vars = args[1:] + + branch_dir = self.other.deduce_system_branch()[1] + root_repo = self.other.get_branch_config(branch_dir, 'branch.root') + root_repo_dir = self.other.find_repository(branch_dir, root_repo) + data = self.other.load_morphology(root_repo_dir, cluster) + + for system in data['systems']: + self.deploy_system(system, env_vars) + + def deploy_system(self, system, env_vars): + morph = system['morph'] + deploy_defaults = system['deploy-defaults'] + deployments = system['deploy'] + + for system_id, deploy_params in deployments.iteritems(): + user_env = morphlib.util.parse_environment_pairs( + os.environ, + [pair[len(system_id)+1:] + for pair in env_vars + if pair.startswith(system_id)]) + + final_env = dict(deploy_defaults.items() + + deploy_params.items() + + user_env.items()) + + deployment_type = final_env.pop('type', None) + if not deployment_type: + raise morphlib.Error('"type" is undefined ' + 'for system "%s"' % system_id) + + location = final_env.pop('location', None) + if not location: + raise morphlib.Error('"location" is undefined ' + 'for system "%s"' % system_id) - # Set up environment for running extensions. - env = morphlib.util.parse_environment_pairs(os.environ, env_vars) + morphlib.util.sanitize_environment(final_env) + self.do_deploy(morph, deployment_type, location, final_env) + def do_deploy(self, system_name, deployment_type, location, env): # Deduce workspace and system branch and branch root repository. workspace = self.other.deduce_workspace() branch, branch_dir = self.other.deduce_system_branch() diff --git a/morphlib/util.py b/morphlib/util.py index 7526c93c..22288cac 100644 --- a/morphlib/util.py +++ b/morphlib/util.py @@ -361,7 +361,6 @@ def parse_environment_pairs(env, pairs): return dict(env.items() + extra_env.items()) - def get_host_architecture(): # pragma: no cover '''Get the canonical Morph name for the host's architecture.''' @@ -381,3 +380,8 @@ def get_host_architecture(): # pragma: no cover raise morphlib.Error('Unknown host architecture %s' % machine) return table[machine] + + +def sanitize_environment(env): + for k in env: + env[k] = str(env[k]) diff --git a/morphlib/util_tests.py b/morphlib/util_tests.py index ca9fe5ae..2ad9e8aa 100644 --- a/morphlib/util_tests.py +++ b/morphlib/util_tests.py @@ -105,3 +105,8 @@ class ParseEnvironmentPairsTests(unittest.TestCase): morphlib.util.parse_environment_pairs, {"foo": "bar"}, ["foo=bar"]) + + def test_sanitize_environment(self): + d = { 'a': 1 } + morphlib.util.sanitize_environment(d) + self.assertTrue(isinstance(d['a'], str)) |