diff options
Diffstat (limited to 'morphlib/plugins/deploy_plugin.py')
| -rw-r--r-- | morphlib/plugins/deploy_plugin.py | 223 |
1 files changed, 180 insertions, 43 deletions
diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index efe6735b..3c19553e 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -13,6 +13,7 @@ # with this program. If not, see <http://www.gnu.org/licenses/>. +import collections import json import logging import os @@ -27,6 +28,14 @@ import morphlib from morphlib.artifactcachereference import ArtifactCacheReference +class NotYetBuiltError(morphlib.Error): + + def __init__(self, name): + self.msg = ('Deployment failed as %s is not yet built.\n' + 'Please ensure the system is built before deployment.' + % name) + + class DeployPlugin(cliapp.Plugin): def enable(self): @@ -385,7 +394,7 @@ class DeployPlugin(cliapp.Plugin): name=name, email=email, build_uuid=build_uuid, status=self.app.status) with pbb as (repo, commit, original_ref): - self.deploy_cluster(build_command, cluster_morphology, + self.deploy_cluster(sb, build_command, cluster_morphology, root_repo_dir, repo, commit, env_vars, deployments) else: @@ -398,6 +407,11 @@ class DeployPlugin(cliapp.Plugin): deployments) self.app.status(msg='Finished deployment') + if self.app.settings['partial']: + self.app.status(msg='WARNING: This was a partial deployment. ' + 'Configuration extensions have not been ' + 'run. Applying the result to an existing ' + 'system may not have reproducible results.') def validate_deployment_options( self, env_vars, all_deployments, all_subsystems): @@ -415,21 +429,54 @@ class DeployPlugin(cliapp.Plugin): 'Variable referenced a non-existent deployment ' 'name: %s' % var) - def deploy_cluster(self, build_command, cluster_morphology, root_repo_dir, - repo, commit, env_vars, deployments): + def deploy_cluster(self, sb, build_command, cluster_morphology, + root_repo_dir, repo, commit, env_vars, deployments): # Create a tempdir for this deployment to work in deploy_tempdir = tempfile.mkdtemp( dir=os.path.join(self.app.settings['tempdir'], 'deployments')) try: for system in cluster_morphology['systems']: - self.deploy_system(build_command, deploy_tempdir, + self.deploy_system(sb, build_command, deploy_tempdir, root_repo_dir, repo, commit, system, env_vars, deployments, parent_location='') finally: shutil.rmtree(deploy_tempdir) - def deploy_system(self, build_command, deploy_tempdir, + def _sanitise_morphology_paths(self, paths, sb): + sanitised_paths = [] + for path in paths: + path = morphlib.util.sanitise_morphology_path(path) + sanitised_paths.append(sb.relative_to_root_repo(path)) + return sanitised_paths + + def _find_artifacts(self, filenames, root_artifact): + found = collections.OrderedDict() + not_found = filenames + for a in root_artifact.walk(): + if a.source.filename in filenames and a.source.name not in found: + found[a.source.name] = a + not_found.remove(a.source.filename) + return found, not_found + + def _validate_partial_deployment(self, deployment_type, + artifact, component_names): + supported_types = ('tar', 'sysroot') + if deployment_type not in supported_types: + raise cliapp.AppException('Not deploying %s, --partial was ' + 'set and partial deployment only ' + 'supports %s deployments.' % + (artifact.source.name, + ', '.join(supported_types))) + components, not_found = self._find_artifacts(component_names, + artifact) + if not_found: + raise cliapp.AppException('Components %s not found in system %s.' % + (', '.join(not_found), + artifact.source.name)) + return components + + def deploy_system(self, sb, build_command, deploy_tempdir, root_repo_dir, build_repo, ref, system, env_vars, deployment_filter, parent_location): sys_ids = set(system['deploy'].iterkeys()) @@ -475,6 +522,12 @@ class DeployPlugin(cliapp.Plugin): raise morphlib.Error('"type" is undefined ' 'for system "%s"' % system_id) + components = self._sanitise_morphology_paths( + deploy_params.get('partial-deploy-components', []), sb) + if self.app.settings['partial']: + components = self._validate_partial_deployment( + deployment_type, artifact, components) + location = final_env.pop('location', None) if not location: raise morphlib.Error('"location" is undefined ' @@ -488,9 +541,10 @@ class DeployPlugin(cliapp.Plugin): root_repo_dir, ref, artifact, deployment_type, - location, final_env) + location, final_env, + components=components) for subsystem in system.get('subsystems', []): - self.deploy_system(build_command, deploy_tempdir, + self.deploy_system(sb, build_command, deploy_tempdir, root_repo_dir, build_repo, ref, subsystem, env_vars, [], parent_location=system_tree) @@ -542,13 +596,27 @@ class DeployPlugin(cliapp.Plugin): pass def checkout_stratum(self, path, artifact, lac, rac): + """Pull the chunks in a stratum, and checkout them into `path`. + + This reads a stratum artifact and pulls the chunks it contains from + the remote into the local artifact cache if they are not already + cached locally. Each of these chunks is then checked out into `path`. + + Also download the stratum metadata into the local cache, then place + it in the /baserock directory of the system checkout indicated by + `path`. + + If any of the chunks have not been cached either locally or remotely, + a morphlib.remoteartifactcache.GetError is raised. + + """ with open(lac.get(artifact), 'r') as stratum: chunks = [ArtifactCacheReference(c) for c in json.load(stratum)] morphlib.builder.download_depends(chunks, lac, rac) for chunk in chunks: self.app.status(msg='Checkout chunk %(name)s.', name=chunk.basename(), chatty=True) - lac.get(chunk, path, self.app.status) + lac.get(chunk, path) metadata = os.path.join(path, 'baserock', '%s.meta' % artifact.name) with lac.get_artifact_metadata(artifact, 'meta') as meta_src: @@ -556,14 +624,94 @@ class DeployPlugin(cliapp.Plugin): shutil.copyfileobj(meta_src, meta_dst) def checkout_strata(self, path, artifact, lac, rac): + """Pull the dependencies of `artifact` and checkout them into `path`. + + This assumes that `artifact` is a system artifact. If any of the + dependencies aren't cached remotely or locally, this raises a + morphlib.remoteartifactcache.GetError. + + """ deps = artifact.source.dependencies morphlib.builder.download_depends(deps, lac, rac) for stratum in deps: self.checkout_stratum(path, stratum, lac, rac) morphlib.builder.ldconfig(self.app.runcmd, path) + def checkout_system(self, build_command, artifact, path): + """Checkout a system into `path`. + + This checks out each of the strata into the directory given by `path`, + then checks out the system artifact into the same directory. This uses + OSTree's `union` checkout mode to overwrite duplicate files but not + need an empty directory. Artifacts which aren't cached locally are + fetched from the remote cache. + + Raises a NotYetBuiltError if either the system artifact or any of the + chunk artifacts in the strata which make up the system aren't cached + either locally or remotely. + + """ + # Check if the system artifact is in the local or remote cache. + # If it isn't, we don't need to bother checking out strata before + # we fail. + if not (build_command.lac.has(artifact) + or build_command.rac.has(artifact)): + raise NotYetBuiltError(artifact.name) + + # Checkout the strata involved in the artifact into a tempdir + self.app.status(msg='Checking out strata in system') + try: + self.checkout_strata(path, artifact, + build_command.lac, build_command.rac) + + self.app.status(msg='Checking out system for configuration') + build_command.cache_artifacts_locally([artifact]) + build_command.lac.get(artifact, path) + except (morphlib.ostreeartifactcache.NotCachedError, + morphlib.remoteartifactcache.GetError): + raise NotYetBuiltError(artifact.name) + + self.app.status( + msg='System checked out at %(system_tree)s', + system_tree=path) + + def checkout_components(self, bc, components, path): + if not components: + raise cliapp.AppException('Deployment failed as no components ' + 'were specified for deployment and ' + '--partial was set.') + for name, artifact in components.iteritems(): + deps = artifact.source.dependencies + morphlib.builder.download_depends(deps, bc.lac, bc.rac) + for dep in deps: + if dep.source.morphology['kind'] == 'stratum': + self.checkout_stratum(path, dep, bc.lac, bc.rac) + elif dep.source.morphology['kind'] == 'chunk': + self.app.status(msg='Checkout chunk %(name)s.', + name=dep.basename(), chatty=True) + bc.lac.get(dep, path) + if artifact.source.morphology['kind'] == 'stratum': + self.checkout_stratum(path, artifact, bc.lac, bc.rac) + elif artifact.source.morphology['kind'] == 'chunk': + self.app.status(msg='Checkout chunk %(name)s.', + name=name, chatty=True) + bc.lac.get(artifact, path) + self.app.status( + msg='Components %(components)s checkout out at %(path)s', + components=', '.join(components), path=path) + def setup_deploy(self, build_command, deploy_tempdir, root_repo_dir, ref, - artifact, deployment_type, location, env): + artifact, deployment_type, location, env, components=[]): + """Checkout the artifact, create metadata and return the location. + + This checks out the system into a temporary directory, and then mounts + this temporary directory alongside a different temporary directory + using a union filesystem. This allows changes to be made without + touching the checked out artifacts. The deployment metadata file is + created and then the directory at which the two temporary directories + are mounted is returned. + + """ # deployment_type, location and env are only used for saving metadata deployment_dir = tempfile.mkdtemp(dir=deploy_tempdir) @@ -583,26 +731,11 @@ class DeployPlugin(cliapp.Plugin): deploy_tree = os.path.join(deployment_dir, 'overlay-deploy-%s' % artifact.name) try: - # Checkout the strata involved in the artifact into a tempdir - self.app.status(msg='Checking out strata in system') - self.checkout_strata(system_tree, artifact, - build_command.lac, build_command.rac) - - self.app.status(msg='Checking out system for configuration') - if build_command.lac.has(artifact): - build_command.lac.get(artifact, system_tree) - elif build_command.rac.has(artifact): - build_command.cache_artifacts_locally([artifact]) - build_command.lac.get(artifact, system_tree) + if self.app.settings['partial']: + self.checkout_components(build_command, components, + system_tree) else: - raise cliapp.AppException('Deployment failed as system is' - ' not yet built.\nPlease ensure' - ' the system is built before' - ' deployment.') - - self.app.status( - msg='System checked out at %(system_tree)s', - system_tree=system_tree) + self.checkout_system(build_command, artifact, system_tree) union_filesystem = self.app.settings['union-filesystem'] morphlib.fsutils.overlay_mount(self.app.runcmd, @@ -625,10 +758,7 @@ class DeployPlugin(cliapp.Plugin): except Exception: if deploy_tree and os.path.exists(deploy_tree): morphlib.fsutils.unmount(self.app.runcmd, deploy_tree) - shutil.rmtree(deploy_tree) - shutil.rmtree(system_tree) - shutil.rmtree(overlay_dir) - shutil.rmtree(work_dir) + shutil.rmtree(deployment_dir) raise def run_deploy_commands(self, deploy_tempdir, env, artifact, root_repo_dir, @@ -640,15 +770,19 @@ class DeployPlugin(cliapp.Plugin): try: # Run configuration extensions. - self.app.status(msg='Configure system') - names = artifact.source.morphology['configuration-extensions'] - for name in names: - self._run_extension( - root_repo_dir, - name, - '.configure', - [system_tree], - env) + if not self.app.settings['partial']: + self.app.status(msg='Configure system') + names = artifact.source.morphology['configuration-extensions'] + for name in names: + self._run_extension( + root_repo_dir, + name, + '.configure', + [system_tree], + env) + else: + self.app.status(msg='WARNING: Not running configuration ' + 'extensions as --partial is set!') # Run write extension. self.app.status(msg='Writing to device') @@ -665,7 +799,7 @@ class DeployPlugin(cliapp.Plugin): shutil.rmtree(deploy_private_tempdir) def _report_extension_stdout(self, line): - self.app.status(msg=line.replace('%s', '%%')) + self.app.status(msg=line.replace('%', '%%')) def _report_extension_stderr(self, error_list): def cb(line): error_list.append(line) @@ -699,7 +833,7 @@ class DeployPlugin(cliapp.Plugin): raise cliapp.AppException(message) def create_metadata(self, system_artifact, root_repo_dir, deployment_type, - location, env): + location, env, components=[]): '''Deployment-specific metadata. The `build` and `deploy` operations must be from the same ref, so full @@ -731,6 +865,9 @@ class DeployPlugin(cliapp.Plugin): 'commit': morphlib.gitversion.commit, 'version': morphlib.gitversion.version, }, + 'partial': self.app.settings['partial'], } + if self.app.settings['partial']: + meta['partial-components'] = components return meta |