diff options
Diffstat (limited to 'morphlib/plugins')
| -rw-r--r-- | morphlib/plugins/__init__.py | 0 | ||||
| -rw-r--r-- | morphlib/plugins/branch_and_merge_plugin.py | 240 | ||||
| -rw-r--r-- | morphlib/plugins/deploy_plugin.py | 209 |
3 files changed, 393 insertions, 56 deletions
diff --git a/morphlib/plugins/__init__.py b/morphlib/plugins/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/morphlib/plugins/__init__.py diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py index 71082445..e17ef740 100644 --- a/morphlib/plugins/branch_and_merge_plugin.py +++ b/morphlib/plugins/branch_and_merge_plugin.py @@ -51,6 +51,8 @@ class BranchAndMergePlugin(cliapp.Plugin): self.app.add_subcommand('build', self.build, arg_synopsis='SYSTEM') self.app.add_subcommand('status', self.status) + self.app.add_subcommand('branch-from-image', self.branch_from_image, + arg_synopsis='REPO BRANCH [METADATADIR]') # Advanced commands self.app.add_subcommand('foreach', self.foreach, @@ -329,6 +331,7 @@ class BranchAndMergePlugin(cliapp.Plugin): 'description', 'disk-size', '_disk-size', + 'configuration-extensions', ], 'stratum': [ 'kind', @@ -478,6 +481,36 @@ class BranchAndMergePlugin(cliapp.Plugin): if max_subdirs > 0 and len(subdirs) > max_subdirs: break + def read_metadata(self, metadata_path): + '''Load every metadata file in `metadata_path`. + + Given a directory containing metadata, load them into memory + and retain the id of the system. + + Returns the cache_key of the system and a mapping of cache_key + to metadata. + ''' + self.app.status(msg='Reading metadata', chatty=True) + metadata_cache_id_lookup = {} + system_key = None + for path in sorted(glob.iglob(os.path.join(metadata_path, '*.meta'))): + with open(path) as f: + metadata = morphlib.util.json.load(f) + cache_key = metadata['cache-key'] + metadata_cache_id_lookup[cache_key] = metadata + + if metadata['kind'] == 'system': + if system_key is not None: + raise morphlib.Error( + "Metadata directory contains multiple systems.") + system_key = cache_key + + if system_key is None: + raise morphlib.Error( + "Metadata directory does not contain any systems.") + + return system_key, metadata_cache_id_lookup + def init(self, args): '''Initialize a workspace directory.''' @@ -505,27 +538,15 @@ class BranchAndMergePlugin(cliapp.Plugin): os.mkdir(os.path.join(dirname, '.morph')) self.app.status(msg='Initialized morph workspace', chatty=True) - def branch(self, args): - '''Create a new system branch.''' - - if len(args) not in [2, 3]: - raise cliapp.AppException('morph branch needs name of branch ' - 'as parameter') - - repo = args[0] - new_branch = args[1] - commit = 'master' if len(args) == 2 else args[2] - - self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app) - if self.lrc.get_repo(repo).ref_exists(new_branch): - raise cliapp.AppException('branch %s already exists in ' - 'repository %s' % (new_branch, repo)) - - # Create the system branch directory. - workspace = self.deduce_workspace() - branch_dir = os.path.join(workspace, new_branch) + def _create_branch(self, workspace, branch_name, repo, original_ref): + '''Create a branch called branch_name based off original_ref. + + NOTE: self.lrc and self.rrc need to be initialized before + calling since clone_to_directory uses them indirectly via + get_cached_repo + ''' + branch_dir = os.path.join(workspace, branch_name) os.makedirs(branch_dir) - try: # Create a .morph-system-branch directory to clearly identify # this directory as a morph system branch. @@ -533,7 +554,7 @@ class BranchAndMergePlugin(cliapp.Plugin): # Remember the system branch name and the repository we branched # off from initially. - self.set_branch_config(branch_dir, 'branch.name', new_branch) + self.set_branch_config(branch_dir, 'branch.name', branch_name) self.set_branch_config(branch_dir, 'branch.root', repo) # Generate a UUID for the branch. We will use this for naming @@ -542,15 +563,38 @@ class BranchAndMergePlugin(cliapp.Plugin): # Clone into system branch directory. repo_dir = os.path.join(branch_dir, self.convert_uri_to_path(repo)) - self.clone_to_directory(repo_dir, repo, commit) + self.clone_to_directory(repo_dir, repo, original_ref) # Create a new branch in the local morphs repository. - self.app.runcmd(['git', 'checkout', '-b', new_branch, commit], - cwd=repo_dir) + if original_ref != branch_name: + self.app.runcmd(['git', 'checkout', '-b', branch_name, + original_ref], cwd=repo_dir) + + return branch_dir except: - self.remove_branch_dir_safe(workspace, new_branch) + self.remove_branch_dir_safe(workspace, branch_name) raise + def branch(self, args): + '''Create a new system branch.''' + + if len(args) not in [2, 3]: + raise cliapp.AppException('morph branch needs name of branch ' + 'as parameter') + + repo = args[0] + new_branch = args[1] + commit = 'master' if len(args) == 2 else args[2] + + self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app) + if self.lrc.get_repo(repo).ref_exists(new_branch): + raise cliapp.AppException('branch %s already exists in ' + 'repository %s' % (new_branch, repo)) + + # Create the system branch directory. + workspace = self.deduce_workspace() + self._create_branch(workspace, new_branch, repo, commit) + def checkout(self, args): '''Check out an existing system branch.''' @@ -561,31 +605,11 @@ class BranchAndMergePlugin(cliapp.Plugin): repo = args[0] system_branch = args[1] - # Create the system branch directory. - workspace = self.deduce_workspace() - branch_dir = os.path.join(workspace, system_branch) - os.makedirs(branch_dir) self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app) - try: - # Create a .morph-system-branch directory to clearly identify - # this directory as a morph system branch. - os.mkdir(os.path.join(branch_dir, '.morph-system-branch')) - - # Remember the system branch name and the repository we - # branched off from. - self.set_branch_config(branch_dir, 'branch.name', system_branch) - self.set_branch_config(branch_dir, 'branch.root', repo) - - # Generate a UUID for the branch. - self.set_branch_config(branch_dir, 'branch.uuid', uuid.uuid4().hex) - - # Clone into system branch directory. - repo_dir = os.path.join(branch_dir, self.convert_uri_to_path(repo)) - self.clone_to_directory(repo_dir, repo, system_branch) - except: - self.remove_branch_dir_safe(workspace, system_branch) - raise + # Create the system branch directory. + workspace = self.deduce_workspace() + self._create_branch(workspace, system_branch, repo, system_branch) def checkout_repository(self, branch_dir, repo, ref, parent_ref=None): '''Make a chunk or stratum repository available for a system branch @@ -755,6 +779,100 @@ class BranchAndMergePlugin(cliapp.Plugin): self.print_changelog('The following changes were made but have not ' 'been committed') + def _get_repo_name(self, alias_resolver, metadata): + '''Attempt to find the best name for the repository. + + A defined repo-alias is preferred, but older builds may + not have it. + + A guessed repo-alias is the next best thing, but there may + be none or more alilases that would resolve to that URL, so + if there are any, use the shortest as it is likely to be + the most specific. + + If all else fails just use the URL. + ''' + if 'repo-alias' in metadata: + return metadata['repo-alias'] + + repo_url = metadata['repo'] + aliases = alias_resolver.aliases_from_url(repo_url) + + if len(aliases) >= 1: + # If there are multiple valid aliases, use the shortest + return min(aliases, key=len) + + # If there are no aliases, just return the url + return repo_url + + def _resolve_refs_from_metadata(self, alias_resolver, + metadata_cache_id_lookup): + '''Pre-resolve a set of refs from metadata. + + Resolved refs are a dict as {(repo, ref): sha1}. + + If the metadata contains the repo-alias then every + metadata item adds the mapping of its repo-alias and ref + to the commit it was built with. + + If the repo-alias does not exist, such as if the image was + built before that field was added, then mappings of every + possible repo url are added. + ''' + resolved_refs = {} + for md in metadata_cache_id_lookup.itervalues(): + if 'repo-alias' in md: + repourls = [md['repo-alias']] + else: + repourls = [md['repo']] + repourls.extend(alias_resolver.aliases_from_url(md['repo'])) + for repourl in repourls: + resolved_refs[repourl, md['original_ref']] = md['sha1'] + return resolved_refs + + def branch_from_image(self, args): + '''Given the contents of a /baserock directory, produce a branch + of the System, petrified to when the System was made. + ''' + if len(args) not in (2, 3): + raise cliapp.AppException( + 'branch-from-image needs repository, ref and path to metadata') + root_repo = args[0] + branch = args[1] + metadata_path = '/baserock' if len(args) == 2 else args[2] + workspace = self.deduce_workspace() + self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app) + + alias_resolver = morphlib.repoaliasresolver.RepoAliasResolver( + self.app.settings['repo-alias']) + + system_key, metadata_cache_id_lookup = self.read_metadata( + metadata_path) + + system_metadata = metadata_cache_id_lookup[system_key] + repo = self._get_repo_name(alias_resolver, system_metadata) + + # Which repo to use? Specified or deduced? + branch_dir = self._create_branch(workspace, branch, repo, + system_metadata['sha1']) + + # Resolve refs from metadata so petrify substitutes these refs + # into morphologies instead of the current state of the branches + resolved_refs = self._resolve_refs_from_metadata( + alias_resolver, + metadata_cache_id_lookup) + + branch_root_dir = self.find_repository(branch_dir, repo) + name = system_metadata['morphology'][:-len('.morph')] + morphology = self.load_morphology(branch_root_dir, name) + self.petrify_morphology(branch, branch_dir, + repo, branch_root_dir, + repo, branch_root_dir, # not a typo + branch, name, morphology, + petrified_morphologies=set(), + resolved_refs=resolved_refs, + update_working_tree=True) + def petrify(self, args): '''Convert all chunk refs in a system branch to be fixed SHA1s @@ -957,9 +1075,10 @@ class BranchAndMergePlugin(cliapp.Plugin): return False def petrify_everything(self, branch, branch_dir, - branch_root, branch_root_dir, tagref, env): + branch_root, branch_root_dir, tagref, env=os.environ, + resolved_refs=None, update_working_tree=False): petrified_morphologies = set() - resolved_refs = {} + resolved_refs = resolved_refs or {} for f in sorted(glob.iglob(os.path.join(branch_root_dir, '*.morph'))): name = os.path.basename(f)[:-len('.morph')] morphology = self.load_morphology(branch_root_dir, name) @@ -967,12 +1086,14 @@ class BranchAndMergePlugin(cliapp.Plugin): branch_root, branch_root_dir, branch_root, branch_root_dir, tagref, name, morphology, - petrified_morphologies, resolved_refs, env) + petrified_morphologies, resolved_refs, + env, update_working_tree) def petrify_morphology(self, branch, branch_dir, branch_root, branch_root_dir, repo, repo_dir, tagref, name, morphology, - petrified_morphologies, resolved_refs, env): + petrified_morphologies, resolved_refs, + env=os.environ, update_working_tree=False): self.app.status(msg='%(repo)s: Petrifying morphology \"%(morph)s\"', repo=repo, morph=name) @@ -988,7 +1109,7 @@ class BranchAndMergePlugin(cliapp.Plugin): strata += morphology['strata'] for info in strata: # Obtain the commit SHA1 this stratum would be built from. - commit, tree = self.resolve_info(info, resolved_refs) + commit = self.resolve_info(info, resolved_refs) stratum_repo_dir = self.make_available( info, branch, branch_dir, repo, repo_dir) info['ref'] = branch @@ -1002,7 +1123,8 @@ class BranchAndMergePlugin(cliapp.Plugin): info['repo'], stratum_repo_dir, tagref, info['morph'], stratum, petrified_morphologies, - resolved_refs, env) + resolved_refs, env, + update_working_tree) # Change the ref for this morphology to the tag we're creating. if info['ref'] != tagref: @@ -1020,7 +1142,7 @@ class BranchAndMergePlugin(cliapp.Plugin): # chunks into SHA1s. if morphology['kind'] == 'stratum': for info in morphology['chunks']: - commit, tree = self.resolve_info(info, resolved_refs) + commit = self.resolve_info(info, resolved_refs) if info['ref'] != commit: info['unpetrify-ref'] = info['ref'] info['ref'] = commit @@ -1040,6 +1162,12 @@ class BranchAndMergePlugin(cliapp.Plugin): '100644', sha1, '%s.morph' % name], cwd=branch_root_dir, env=env) + # Update the working tree if requested. This can be done with + # git-checkout-index, but we still have the file, so use that + if update_working_tree: + shutil.copy(tmpfile, + os.path.join(branch_root_dir, '%s.morph' % name)) + # Delete the temporary file again. os.remove(tmpfile) @@ -1051,7 +1179,7 @@ class BranchAndMergePlugin(cliapp.Plugin): commit_sha1, tree_sha1 = self.app.resolve_ref( self.lrc, self.rrc, info['repo'], info['ref'], update=not self.app.settings['no-git-update']) - resolved_refs[key] = (commit_sha1, tree_sha1) + resolved_refs[key] = commit_sha1 return resolved_refs[key] def create_tag_commit(self, repo_dir, tagname, msg, env): diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py new file mode 100644 index 00000000..79715e13 --- /dev/null +++ b/morphlib/plugins/deploy_plugin.py @@ -0,0 +1,209 @@ +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import cliapp +import gzip +import os +import shutil +import tarfile +import tempfile +import urlparse +import uuid + +import morphlib + +# UGLY HACK: We need to re-use some code from the branch and merge +# plugin, so we import and instantiate that plugin. This needs to +# be fixed by refactoring the codebase so the shared code is in +# morphlib, not in a plugin. However, this hack lets us re-use +# code without copying it. +import morphlib.plugins.branch_and_merge_plugin + + +class DeployPlugin(cliapp.Plugin): + + def enable(self): + self.app.add_subcommand( + 'deploy', self.deploy, + arg_synopsis='TYPE SYSTEM LOCATION [KEY=VALUE]') + self.other = \ + morphlib.plugins.branch_and_merge_plugin.BranchAndMergePlugin() + self.other.app = self.app + + def disable(self): + pass + + def deploy(self, args): + '''Deploy a built system image.''' + + if len(args) < 3: + raise cliapp.AppException( + 'Too few arguments to deploy command (see help)') + + deployment_type = args[0] + system_name = args[1] + location = args[2] + env_vars = args[3:] + + # Deduce workspace and system branch and branch root repository. + workspace = self.other.deduce_workspace() + branch, branch_dir = self.other.deduce_system_branch() + branch_root = self.other.get_branch_config(branch_dir, 'branch.root') + branch_uuid = self.other.get_branch_config(branch_dir, 'branch.uuid') + + # Generate a UUID for the build. + build_uuid = uuid.uuid4().hex + + build_command = morphlib.buildcommand.BuildCommand(self.app) + build_command = self.app.hookmgr.call('new-build-command', + build_command) + push = self.app.settings['push-build-branches'] + + self.app.status(msg='Starting build %(uuid)s', uuid=build_uuid) + + self.app.status(msg='Collecting morphologies involved in ' + 'building %(system)s from %(branch)s', + system=system_name, branch=branch) + + # Get repositories of morphologies involved in building this system + # from the current system branch. + build_repos = self.other.get_system_build_repos( + branch, branch_dir, branch_root, system_name) + + # Generate temporary build ref names for all these repositories. + self.other.generate_build_ref_names(build_repos, branch_uuid) + + # Create the build refs for all these repositories and commit + # all uncommitted changes to them, updating all references + # to system branch refs to point to the build refs instead. + self.other.update_build_refs(build_repos, branch, build_uuid, push) + + if push: + self.other.push_build_refs(build_repos) + build_branch_root = branch_root + else: + dirname = build_repos[branch_root]['dirname'] + build_branch_root = urlparse.urljoin('file://', dirname) + + # Run the build. + build_ref = build_repos[branch_root]['build-ref'] + order = build_command.compute_build_order( + build_branch_root, + build_ref, + system_name + '.morph') + artifact = order.groups[-1][-1] + + if push: + self.other.delete_remote_build_refs(build_repos) + + # Unpack the artifact (tarball) to a temporary directory. + self.app.status(msg='Unpacking system for configuration') + + system_tree = tempfile.mkdtemp() + + if build_command.lac.has(artifact): + f = build_command.lac.get(artifact) + else: + f = build_command.rac.get(artifact) + ff = gzip.GzipFile(fileobj=f) + tf = tarfile.TarFile(fileobj=ff) + tf.extractall(path=system_tree) + + self.app.status( + msg='System unpacked at %(system_tree)s', + system_tree=system_tree) + + # Set up environment for running extensions. + env = dict(os.environ) + for spec in env_vars: + name, value = spec.split('=', 1) + if name in env: + raise morphlib.Error( + '%s is already set in the enviroment' % name) + env[name] = value + + # Run configuration extensions. + self.app.status(msg='Configure system') + names = artifact.source.morphology['configuration-extensions'] + for name in names: + self._run_extension( + branch_dir, + build_ref, + name, + '.configure', + [system_tree], + env) + + # Run write extension. + self.app.status(msg='Writing to device') + self._run_extension( + branch_dir, + build_ref, + deployment_type, + '.write', + [system_tree, location], + env) + + # Cleanup. + self.app.status(msg='Cleaning up') + shutil.rmtree(system_tree) + + self.app.status(msg='Finished deployment') + + def _run_extension(self, repo_dir, ref, name, kind, args, env): + '''Run an extension. + + The ``kind`` should be either ``.configure`` or ``.write``, + depending on the kind of extension that is sought. + + The extension is found either in the git repository of the + system morphology (repo, ref), or with the Morph code. + + ''' + + # Look for extension in the system morphology's repository. + ext = self._cat_file(repo_dir, ref, name + kind) + if ext is None: + # Not found: look for it in the Morph code. + code_dir = os.path.dirname(morphlib.__file__) + ext_filename = os.path.join(code_dir, 'exts', name + kind) + if not os.path.exists(ext_filename): + raise morphlib.Error( + 'Could not find extension %s%s' % (name, kind)) + delete_ext = False + else: + # Found it in the system morphology's repository. + fd, ext_filename = tempfile.mkstemp() + os.write(fd, ext) + os.close(fd) + os.chmod(ext_filename, 0700) + delete_ext = True + + self.app.runcmd( + [ext_filename] + args, env=env, stdout=None, stderr=None) + + if delete_ext: + os.remove(ext_filename) + + def _cat_file(self, repo_dir, ref, pathname): + '''Retrieve contents of a file from a git repository.''' + + argv = ['git', 'cat-file', 'blob', '%s:%s' % (ref, pathname)] + try: + return self.app.runcmd(argv, cwd=repo_dir) + except cliapp.AppException: + return None + |