diff options
Diffstat (limited to 'morphlib/plugins/deploy_plugin.py')
| -rw-r--r-- | morphlib/plugins/deploy_plugin.py | 256 |
1 files changed, 187 insertions, 69 deletions
diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index 90c658a0..ae62b75d 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -16,6 +16,7 @@ import cliapp import contextlib +import json import os import shutil import stat @@ -25,23 +26,24 @@ import uuid import morphlib -# UGLY HACK: We need to re-use some code from the branch and merge -# plugin, so we import and instantiate that plugin. This needs to -# be fixed by refactoring the codebase so the shared code is in -# morphlib, not in a plugin. However, this hack lets us re-use -# code without copying it. -import morphlib.plugins.branch_and_merge_plugin + +class ExtensionNotFoundError(morphlib.Error): + pass class DeployPlugin(cliapp.Plugin): def enable(self): + group_deploy = 'Deploy Options' + self.app.settings.boolean(['upgrade'], + 'specify that you want to upgrade an ' + 'existing cluster of systems rather than do ' + 'an initial deployment', + group=group_deploy) + self.app.add_subcommand( 'deploy', self.deploy, arg_synopsis='CLUSTER [SYSTEM.KEY=VALUE]') - self.other = \ - morphlib.plugins.branch_and_merge_plugin.BranchAndMergePlugin() - self.other.app = self.app def disable(self): pass @@ -250,8 +252,20 @@ class DeployPlugin(cliapp.Plugin): are set as environment variables when either the configuration or the write extension runs (except `type` and `location`). + Deployment configuration is stored in the deployed system as + /baserock/deployment.meta. THIS CONTAINS ALL ENVIRONMENT VARIABLES SET + DURINGR DEPLOYMENT, so make sure you have no sensitive information in + your environment that is being leaked. As a special case, any + environment/deployment variable that contains 'PASSWORD' in its name is + stripped out and not stored in the final system. + ''' + # Nasty hack to allow deploying things of a different architecture + def validate(self, root_artifact): + pass + morphlib.buildcommand.BuildCommand._validate_architecture = validate + if not args: raise cliapp.AppException( 'Too few arguments to deploy command (see help)') @@ -318,65 +332,112 @@ class DeployPlugin(cliapp.Plugin): ref=build_ref, dirname=gd.dirname, remote=remote.get_push_url(), chatty=True) - for system in cluster_morphology['systems']: - self.deploy_system(build_command, root_repo_dir, - bb.root_repo_url, bb.root_ref, - system, env_vars) - - def deploy_system(self, build_command, root_repo_dir, build_repo, ref, - system, env_vars): - # Find the artifact to build - morph = system['morph'] - srcpool = build_command.create_source_pool(build_repo, ref, - morph + '.morph') - def validate(self, root_artifact): - pass - morphlib.buildcommand.BuildCommand._validate_architecture = validate + # Create a tempdir for this deployment to work in + deploy_tempdir = tempfile.mkdtemp( + dir=os.path.join(self.app.settings['tempdir'], 'deployments')) + try: + for system in cluster_morphology['systems']: + self.deploy_system(build_command, deploy_tempdir, + root_repo_dir, bb.root_repo_url, + bb.root_ref, system, env_vars, + parent_location='') + finally: + shutil.rmtree(deploy_tempdir) + + self.app.status(msg='Finished deployment') - artifact = build_command.resolve_artifacts(srcpool) - - deploy_defaults = system['deploy-defaults'] - deployments = system['deploy'] - for system_id, deploy_params in deployments.iteritems(): - user_env = morphlib.util.parse_environment_pairs( - os.environ, - [pair[len(system_id)+1:] - for pair in env_vars - if pair.startswith(system_id)]) - - final_env = dict(deploy_defaults.items() + - deploy_params.items() + - user_env.items()) - - deployment_type = final_env.pop('type', None) - if not deployment_type: - raise morphlib.Error('"type" is undefined ' - 'for system "%s"' % system_id) - - location = final_env.pop('location', None) - if not location: - raise morphlib.Error('"location" is undefined ' - 'for system "%s"' % system_id) - - morphlib.util.sanitize_environment(final_env) - self.do_deploy(build_command, root_repo_dir, ref, artifact, - deployment_type, location, final_env) - - def do_deploy(self, build_command, root_repo_dir, ref, artifact, - deployment_type, location, env): - - # Create a tempdir for this deployment to work in - deploy_tempdir = tempfile.mkdtemp( - dir=os.path.join(self.app.settings['tempdir'], 'deployments')) + def deploy_system(self, build_command, deploy_tempdir, + root_repo_dir, build_repo, ref, system, env_vars, + parent_location): + old_status_prefix = self.app.status_prefix + system_status_prefix = '%s[%s]' % (old_status_prefix, system['morph']) + self.app.status_prefix = system_status_prefix try: - # Create a tempdir to extract the rootfs in - system_tree = tempfile.mkdtemp(dir=deploy_tempdir) + # Find the artifact to build + morph = system['morph'] + srcpool = build_command.create_source_pool(build_repo, ref, + morph + '.morph') + + artifact = build_command.resolve_artifacts(srcpool) + + deploy_defaults = system.get('deploy-defaults', {}) + deployments = system['deploy'] + for system_id, deploy_params in deployments.iteritems(): + deployment_status_prefix = '%s[%s]' % ( + system_status_prefix, system_id) + self.app.status_prefix = deployment_status_prefix + try: + user_env = morphlib.util.parse_environment_pairs( + os.environ, + [pair[len(system_id)+1:] + for pair in env_vars + if pair.startswith(system_id)]) + + final_env = dict(deploy_defaults.items() + + deploy_params.items() + + user_env.items()) + + is_upgrade = ('yes' if self.app.settings['upgrade'] + else 'no') + final_env['UPGRADE'] = is_upgrade + + deployment_type = final_env.pop('type', None) + if not deployment_type: + raise morphlib.Error('"type" is undefined ' + 'for system "%s"' % system_id) + + location = final_env.pop('location', None) + if not location: + raise morphlib.Error('"location" is undefined ' + 'for system "%s"' % system_id) + + morphlib.util.sanitize_environment(final_env) + self.check_deploy(root_repo_dir, ref, deployment_type, + location, final_env) + system_tree = self.setup_deploy(build_command, + deploy_tempdir, + root_repo_dir, + ref, artifact, + deployment_type, + location, final_env) + for subsystem in system.get('subsystems', []): + self.deploy_system(build_command, deploy_tempdir, + root_repo_dir, build_repo, + ref, subsystem, env_vars, + parent_location=system_tree) + if parent_location: + deploy_location = os.path.join(parent_location, + location.lstrip('/')) + else: + deploy_location = location + self.run_deploy_commands(deploy_tempdir, final_env, + artifact, root_repo_dir, + ref, deployment_type, + system_tree, deploy_location) + finally: + self.app.status_prefix = system_status_prefix + finally: + self.app.status_prefix = old_status_prefix - # Extensions get a private tempdir so we can more easily clean - # up any files an extension left behind - deploy_private_tempdir = tempfile.mkdtemp(dir=deploy_tempdir) - env['TMPDIR'] = deploy_private_tempdir + def check_deploy(self, root_repo_dir, ref, deployment_type, location, env): + # Run optional write check extension. These are separate from the write + # extension because it may be several minutes before the write + # extension itself has the chance to raise an error. + try: + self._run_extension( + root_repo_dir, ref, deployment_type, '.check', + [location], env) + except ExtensionNotFoundError: + pass + + def setup_deploy(self, build_command, deploy_tempdir, root_repo_dir, ref, + artifact, deployment_type, location, env): + # deployment_type, location and env are only used for saving metadata + # Create a tempdir to extract the rootfs in + system_tree = tempfile.mkdtemp(dir=deploy_tempdir) + + try: # Unpack the artifact (tarball) to a temporary directory. self.app.status(msg='Unpacking system for configuration') @@ -397,7 +458,27 @@ class DeployPlugin(cliapp.Plugin): msg='System unpacked at %(system_tree)s', system_tree=system_tree) + self.app.status( + msg='Writing deployment metadata file') + metadata = self.create_metadata( + artifact, root_repo_dir, deployment_type, location, env) + metadata_path = os.path.join( + system_tree, 'baserock', 'deployment.meta') + with morphlib.savefile.SaveFile(metadata_path, 'w') as f: + f.write(json.dumps(metadata, indent=4, sort_keys=True)) + return system_tree + except Exception: + shutil.rmtree(system_tree) + raise + + def run_deploy_commands(self, deploy_tempdir, env, artifact, root_repo_dir, + ref, deployment_type, system_tree, location): + # Extensions get a private tempdir so we can more easily clean + # up any files an extension left behind + deploy_private_tempdir = tempfile.mkdtemp(dir=deploy_tempdir) + env['TMPDIR'] = deploy_private_tempdir + try: # Run configuration extensions. self.app.status(msg='Configure system') names = artifact.source.morphology['configuration-extensions'] @@ -423,9 +504,7 @@ class DeployPlugin(cliapp.Plugin): finally: # Cleanup. self.app.status(msg='Cleaning up') - shutil.rmtree(deploy_tempdir) - - self.app.status(msg='Finished deployment') + shutil.rmtree(deploy_private_tempdir) def _run_extension(self, gd, ref, name, kind, args, env): '''Run an extension. @@ -446,7 +525,7 @@ class DeployPlugin(cliapp.Plugin): code_dir = os.path.dirname(morphlib.__file__) ext_filename = os.path.join(code_dir, 'exts', name + kind) if not os.path.exists(ext_filename): - raise morphlib.Error( + raise ExtensionNotFoundError( 'Could not find extension %s%s' % (name, kind)) if not self._is_executable(ext_filename): raise morphlib.Error( @@ -464,7 +543,8 @@ class DeployPlugin(cliapp.Plugin): name=name, kind=kind) self.app.runcmd( [ext_filename] + args, - ['sh', '-c', 'while read l; do echo `date "+%F %T"` $l; done'], + ['sh', '-c', 'while read l; do echo `date "+%F %T"` "$1$l"; done', + '-', '%s[%s]' % (self.app.status_prefix, name + kind)], cwd=gd.dirname, env=env, stdout=None, stderr=None) if delete_ext: @@ -474,3 +554,41 @@ class DeployPlugin(cliapp.Plugin): st = os.stat(filename) mask = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH return (stat.S_IMODE(st.st_mode) & mask) != 0 + + def create_metadata(self, system_artifact, root_repo_dir, deployment_type, + location, env): + '''Deployment-specific metadata. + + The `build` and `deploy` operations must be from the same ref, so full + info on the root repo that the system came from is in + /baserock/${system_artifact}.meta and is not duplicated here. We do + store a `git describe` of the definitions.git repo as a convenience for + post-upgrade hooks that we may need to implement at a future date: + the `git describe` output lists the last tag, which will hopefully help + us to identify which release of a system was deployed without having to + keep a list of SHA1s somewhere or query a Trove. + + ''' + + def remove_passwords(env): + def is_password(key): + return 'PASSWORD' in key + return { k:v for k, v in env.iteritems() if not is_password(k) } + + meta = { + 'system-artifact-name': system_artifact.name, + 'configuration': remove_passwords(env), + 'deployment-type': deployment_type, + 'location': location, + 'definitions-version': { + 'describe': root_repo_dir.describe(), + }, + 'morph-version': { + 'ref': morphlib.gitversion.ref, + 'tree': morphlib.gitversion.tree, + 'commit': morphlib.gitversion.commit, + 'version': morphlib.gitversion.version, + }, + } + + return meta |