From e343b6a3725a5a6790eda19e52a55fcb8ed162c4 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 25 Jan 2013 14:13:14 +0000 Subject: Add deployment plugin This adds a new optional field to system morphologies: "configuration-extensions". The deployment plugin relies heavily on code from the branch and merge plugin. This needs to be eventually fixed by refactoring the codebase so that the shared code is in morphlib and not in plugins. However, doing that is beyond the scope of adding a deployment plugin. --- morphlib/plugins/__init__.py | 0 morphlib/plugins/branch_and_merge_plugin.py | 1 + morphlib/plugins/deploy_plugin.py | 200 ++++++++++++++++++++++++++++ 3 files changed, 201 insertions(+) create mode 100644 morphlib/plugins/__init__.py create mode 100644 morphlib/plugins/deploy_plugin.py (limited to 'morphlib/plugins') diff --git a/morphlib/plugins/__init__.py b/morphlib/plugins/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py index 14dc9782..1179d1e2 100644 --- a/morphlib/plugins/branch_and_merge_plugin.py +++ b/morphlib/plugins/branch_and_merge_plugin.py @@ -329,6 +329,7 @@ class BranchAndMergePlugin(cliapp.Plugin): 'description', 'disk-size', '_disk-size', + 'configuration-extensions', ], 'stratum': [ 'kind', diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py new file mode 100644 index 00000000..89f7ed07 --- /dev/null +++ b/morphlib/plugins/deploy_plugin.py @@ -0,0 +1,200 @@ +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import cliapp +import gzip +import os +import shutil +import tarfile +import tempfile +import uuid + +import morphlib +import morphlib.plugins.branch_and_merge_plugin + + +class DeployPlugin(cliapp.Plugin): + + def enable(self): + self.app.add_subcommand( + 'deploy', self.deploy, + arg_synopsis='TYPE SYSTEM LOCATION [KEY=VALUE]') + self.other = \ + morphlib.plugins.branch_and_merge_plugin.BranchAndMergePlugin() + + def disable(self): + pass + + def deploy(self, args): + '''Build a system from the current system branch''' + + if len(args) < 3: + raise cliapp.AppException('morph build expects exactly one ' + 'parameter: the system to build') + + deployment_type = args[0] + system_name = args[1] + location = args[2] + env_vars = args[3:] + + # Deduce workspace and system branch and branch root repository. + workspace = self.other.deduce_workspace() + branch, branch_dir = self.other.deduce_system_branch() + branch_root = self.other.get_branch_config(branch_dir, 'branch.root') + branch_uuid = self.other.get_branch_config(branch_dir, 'branch.uuid') + + # Generate a UUID for the build. + build_uuid = uuid.uuid4().hex + + build_command = morphlib.buildcommand.BuildCommand(self.app) + build_command = self.app.hookmgr.call('new-build-command', + build_command) + push = self.app.settings['push-build-branches'] + + self.app.status(msg='Starting build %(uuid)s', uuid=build_uuid) + + self.app.status(msg='Collecting morphologies involved in ' + 'building %(system)s from %(branch)s', + system=system_name, branch=branch) + + # Get repositories of morphologies involved in building this system + # from the current system branch. + build_repos = self.other.get_system_build_repos( + branch, branch_dir, branch_root, system_name) + + # Generate temporary build ref names for all these repositories. + self.other.generate_build_ref_names(build_repos, branch_uuid) + + # Create the build refs for all these repositories and commit + # all uncommitted changes to them, updating all references + # to system branch refs to point to the build refs instead. + self.other.update_build_refs(build_repos, branch, build_uuid, push) + + if push: + self.other.push_build_refs(build_repos) + build_branch_root = branch_root + else: + dirname = build_repos[branch_root]['dirname'] + build_branch_root = urlparse.urljoin('file://', dirname) + + # Run the build. + build_ref = build_repos[branch_root]['build-ref'] + order = build_command.compute_build_order( + build_branch_root, + build_ref, + system_name + '.morph') + artifact = order.groups[-1][-1] + + if push: + self.other.delete_remote_build_refs(build_repos) + + # Unpack the artifact (tarball) to a temporary directory. + self.app.status(msg='Unpacking system for configuration') + + system_tree = tempfile.mkdtemp() + + if build_command.lac.has(artifact): + f = build_command.lac.get(artifact) + else: + f = build_command.rac.get(artifact) + ff = gzip.GzipFile(fileobj=f) + tf = tarfile.TarFile(fileobj=ff) + tf.extractall(path=system_tree) + + self.app.status( + msg='System unpacked at %(system_tree)s', + system_tree=system_tree) + + # Set up environment for running extensions. + env = dict(os.environ) + for spec in env_vars: + name, value = spec.split('=', 1) + if name in env: + raise morphlib.Error( + '%s is already set in the enviroment' % name) + env[name] = value + + # Run configuration extensions. + self.app.status(msg='Configure system') + m = artifact.source.morphology + if 'configuration-extensions' in m: + names = m['configuration-extensions'] + for name in names: + self._run_extension( + build_branch_root, + build_ref, + name, + '.configure', + [system_tree], + env) + + # Run write extension. + self.app.status(msg='Writing to device') + self._run_extension( + build_branch_root, + build_ref, + name, + '.write', + [system_tree, location], + env) + + # Cleanup. + self.app.status(msg='Cleaning up') + shutil.rmtree(system_tree) + + self.app.status(msg='Finished deployment') + + def _run_extension(self, repo_dir, ref, name, kind, args, env): + '''Run an extension. + + The ``kind`` should be either ``.configure`` or ``.write``, + depending on the kind of extension that is sought. + + The extension is found either in the git repository of the + system morphology (repo, ref), or with the Morph code. + + ''' + + ext = self._cat_file(repo_dir, ref, name + kind) + if ext is None: + code_dir = os.path.dirname(morphlib.__file__) + ext_filename = os.path.join(code_dir, 'exts', name + kind) + if not os.path.exists(ext_filename): + raise morphlib.Error( + 'Could not find extenstion %s%s' % (name, kind)) + delete_ext = False + else: + fd, ext_filename = tempfile.mkstemp() + os.write(fd, ext) + os.close(fd) + os.chmod(ext_filename, 0700) + delete_ext = True + + self.app.runcmd( + [ext_filename] + args, env=env, stdout=None, stderr=None) + + if delete_ext: + os.remove(ext_filename) + + def _cat_file(self, repo_dir, ref, pathname): + '''Retrieve contents of a file from a git repository.''' + + argv = ['git', 'cat-file', 'blob', '%s:%s' % (ref, filename)] + try: + return self.app.runcmd(argv, cwd=repo_dir) + except cliapp.AppException: + return None + -- cgit v1.2.1 From 90d52f4205be5ce1eab04f97b0e4a71973258ddd Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Mon, 28 Jan 2013 18:12:52 +0000 Subject: Add cmdtest for "morph deploy" and rawdisk.write --- morphlib/plugins/deploy_plugin.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'morphlib/plugins') diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index 89f7ed07..eff1377c 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -20,6 +20,7 @@ import os import shutil import tarfile import tempfile +import urlparse import uuid import morphlib @@ -34,6 +35,7 @@ class DeployPlugin(cliapp.Plugin): arg_synopsis='TYPE SYSTEM LOCATION [KEY=VALUE]') self.other = \ morphlib.plugins.branch_and_merge_plugin.BranchAndMergePlugin() + self.other.app = self.app def disable(self): pass @@ -134,7 +136,7 @@ class DeployPlugin(cliapp.Plugin): names = m['configuration-extensions'] for name in names: self._run_extension( - build_branch_root, + branch_dir, build_ref, name, '.configure', @@ -144,9 +146,9 @@ class DeployPlugin(cliapp.Plugin): # Run write extension. self.app.status(msg='Writing to device') self._run_extension( - build_branch_root, + branch_dir, build_ref, - name, + deployment_type, '.write', [system_tree, location], env) @@ -174,7 +176,7 @@ class DeployPlugin(cliapp.Plugin): ext_filename = os.path.join(code_dir, 'exts', name + kind) if not os.path.exists(ext_filename): raise morphlib.Error( - 'Could not find extenstion %s%s' % (name, kind)) + 'Could not find extension %s%s' % (name, kind)) delete_ext = False else: fd, ext_filename = tempfile.mkstemp() @@ -192,7 +194,7 @@ class DeployPlugin(cliapp.Plugin): def _cat_file(self, repo_dir, ref, pathname): '''Retrieve contents of a file from a git repository.''' - argv = ['git', 'cat-file', 'blob', '%s:%s' % (ref, filename)] + argv = ['git', 'cat-file', 'blob', '%s:%s' % (ref, pathname)] try: return self.app.runcmd(argv, cwd=repo_dir) except cliapp.AppException: -- cgit v1.2.1 From 2df1538cc3fdbfbd9a0a5a9d6d3640856ea6730c Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 7 Feb 2013 11:00:13 +0000 Subject: Describe how we re-use code from branch+merge plugin Suggested-By: Sam Thursfield --- morphlib/plugins/deploy_plugin.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'morphlib/plugins') diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index eff1377c..63fe647b 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -24,6 +24,12 @@ import urlparse import uuid import morphlib + +# UGLY HACK: We need to re-use some code from the branch and merge +# plugin, so we import and instantiate that plugin. This needs to +# be fixed by refactoring the codebase so the shared code is in +# morphlib, not in a plugin. However, this hack lets us re-use +# code without copying it. import morphlib.plugins.branch_and_merge_plugin -- cgit v1.2.1 From e50d00fe20126c05c96d77bfadcf99ef3f58acf5 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 7 Feb 2013 11:02:18 +0000 Subject: Fix docstring and error message Suggested-By: Sam Thursfield --- morphlib/plugins/deploy_plugin.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'morphlib/plugins') diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index 63fe647b..e921b0af 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -47,11 +47,11 @@ class DeployPlugin(cliapp.Plugin): pass def deploy(self, args): - '''Build a system from the current system branch''' + '''Deploy a built system image.''' if len(args) < 3: - raise cliapp.AppException('morph build expects exactly one ' - 'parameter: the system to build') + raise cliapp.AppException( + 'Too few arguments to deploy command (see help)') deployment_type = args[0] system_name = args[1] -- cgit v1.2.1 From cc7ac2b0820222053e15c953e8e02c7bbff1ad4d Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 7 Feb 2013 11:09:00 +0000 Subject: Make configuration-extensions have a default value This saves a check (and an indentation) in the deployment plugin, making the code a tiny bit simpler. Suggested-By: Sam Thursfield --- morphlib/plugins/deploy_plugin.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) (limited to 'morphlib/plugins') diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index e921b0af..fbc8f145 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -137,17 +137,15 @@ class DeployPlugin(cliapp.Plugin): # Run configuration extensions. self.app.status(msg='Configure system') - m = artifact.source.morphology - if 'configuration-extensions' in m: - names = m['configuration-extensions'] - for name in names: - self._run_extension( - branch_dir, - build_ref, - name, - '.configure', - [system_tree], - env) + names = artifact.source.morphology['configuration-extensions'] + for name in names: + self._run_extension( + branch_dir, + build_ref, + name, + '.configure', + [system_tree], + env) # Run write extension. self.app.status(msg='Writing to device') -- cgit v1.2.1 From 55c1922f52cff88ec1c08f350cab88e8a2957322 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 7 Feb 2013 11:12:25 +0000 Subject: Comment logic in _run_extension Suggested-By: Sam Thursfield --- morphlib/plugins/deploy_plugin.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'morphlib/plugins') diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index fbc8f145..79715e13 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -174,8 +174,10 @@ class DeployPlugin(cliapp.Plugin): ''' + # Look for extension in the system morphology's repository. ext = self._cat_file(repo_dir, ref, name + kind) if ext is None: + # Not found: look for it in the Morph code. code_dir = os.path.dirname(morphlib.__file__) ext_filename = os.path.join(code_dir, 'exts', name + kind) if not os.path.exists(ext_filename): @@ -183,6 +185,7 @@ class DeployPlugin(cliapp.Plugin): 'Could not find extension %s%s' % (name, kind)) delete_ext = False else: + # Found it in the system morphology's repository. fd, ext_filename = tempfile.mkstemp() os.write(fd, ext) os.close(fd) -- cgit v1.2.1