diff options
Diffstat (limited to 'morphlib/plugins')
| -rw-r--r-- | morphlib/plugins/branch_and_merge_new_plugin.py | 156 | ||||
| -rw-r--r-- | morphlib/plugins/copy-artifacts_plugin.py | 142 | ||||
| -rw-r--r-- | morphlib/plugins/deploy_plugin.py | 1 | ||||
| -rw-r--r-- | morphlib/plugins/gc_plugin.py | 7 | ||||
| -rw-r--r-- | morphlib/plugins/list_artifacts_plugin.py | 124 |
5 files changed, 179 insertions, 251 deletions
diff --git a/morphlib/plugins/branch_and_merge_new_plugin.py b/morphlib/plugins/branch_and_merge_new_plugin.py index 51cba401..5ac8353a 100644 --- a/morphlib/plugins/branch_and_merge_new_plugin.py +++ b/morphlib/plugins/branch_and_merge_new_plugin.py @@ -363,124 +363,32 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): Command line arguments: - * `SYSTEM` is the name of a system morphology in the root repository - of the current system branch. - * `STRATUM` is the name of a stratum inside the system. - * `CHUNK` is the name of a chunk inside the stratum. + * `CHUNK` is the name of a chunk - This marks the specified stratum or chunk (if given) as being - changed within the system branch, by creating the git branches in - the affected repositories, and changing the relevant morphologies - to point at those branches. It also creates a local clone of - the git repository of the stratum or chunk. - - For example: - - morph edit devel-system-x86-64-generic devel - - The above command will mark the `devel` stratum as being - modified in the current system branch. In this case, the stratum's - morphology is in the same git repository as the system morphology, - so there is no need to create a new git branch. However, the - system morphology is modified to point at the stratum morphology - in the same git branch, rather than the original branch. - - In other words, where the system morphology used to say this: - - morph: devel - repo: baserock:baserock/morphs - ref: master - - The updated system morphology will now say this instead: - - morph: devel - repo: baserock:baserock/morphs - ref: jrandom/new-feature - - (Assuming the system branch is called `jrandom/new-feature`.) - - Another example: - - morph edit devel-system-x86_64-generic devel gcc - - The above command will mark the `gcc` chunk as being edited in - the current system branch. Morph will clone the `gcc` repository - locally, into the current workspace, and create a new (local) - branch named after the system branch. It will also change the - stratum morphology to refer to the new git branch, instead of - whatever branch it was referring to originally. - - If the `gcc` repository already had a git branch named after - the system branch, that is reused. Similarly, if the stratum - morphology was already pointing that that branch, it doesn't - need to be changed again. In that case, the only action Morph - does is to clone the chunk repository locally, and if that was - also done already, Morph does nothing. + This makes a local checkout of CHUNK in the current system branch + and edits any stratum morphology file(s) containing the chunk ''' - if len(args) not in (2, 3): - raise cliapp.AppException('morph edit needs the names of a system,' - ' a stratum and optionally a chunk' - ' as parameters') - - system_name = morphlib.util.strip_morph_extension(args[0]) - stratum_name = morphlib.util.strip_morph_extension(args[1]) - chunk_name = None - if len(args) == 3: - chunk_name = morphlib.util.strip_morph_extension(args[2]) + if len(args) != 1: + raise cliapp.AppException('morph edit needs a chunk ' + 'as parameter') ws = morphlib.workspace.open('.') sb = morphlib.sysbranchdir.open_from_within('.') loader = morphlib.morphloader.MorphologyLoader() + morphs = self._load_all_sysbranch_morphologies(sb, loader) - # Load the system morphology, and all stratum morphologies, including - # all the strata that are being build-depended on. - - logging.debug('Loading system morphology') - system_morph = loader.load_from_file( - sb.get_filename(sb.root_repository_url, system_name + '.morph')) - if system_morph['kind'] != 'system': - raise cliapp.AppException("%s is not a system" % system_name) - system_morph.repo_url = sb.root_repository_url - system_morph.ref = sb.system_branch_name - system_morph.filename = system_name + '.morph' - - logging.debug('Loading stratum morphologies') - morphs = self._load_stratum_morphologies(loader, sb, system_morph) - morphs.add_morphology(system_morph) - logging.debug('morphs: %s' % repr(morphs.morphologies)) - - # Change refs to the stratum to be to the system branch. - # Note: this currently only supports strata in root repository. - - logging.debug('Changing refs to stratum %s' % stratum_name) - stratum_morph = morphs.get_stratum_in_system( - system_morph, stratum_name) - morphs.change_ref( - stratum_morph.repo_url, stratum_morph.ref, stratum_morph.filename, - sb.system_branch_name) - logging.debug('morphs: %s' % repr(morphs.morphologies)) - - # If we're editing a chunk, make it available locally, with the - # relevant git branch checked out. This also invents the new branch - # name. - - if chunk_name: - logging.debug('Editing chunk %s' % chunk_name) - - chunk_url, chunk_ref, chunk_morph = morphs.get_chunk_triplet( - stratum_morph, chunk_name) + def edit_chunk(morph, chunk_name): + chunk_url, chunk_ref, chunk_morph = ( + morphs.get_chunk_triplet(morph, chunk_name)) chunk_dirname = sb.get_git_directory_name(chunk_url) + if not os.path.exists(chunk_dirname): lrc, rrc = morphlib.util.new_repo_caches(self.app) cached_repo = lrc.get_updated_repo(chunk_url) - # FIXME: This makes the simplifying assumption that - # a chunk branch must have the same name as the system - # branch. - gd = sb.clone_cached_repo(cached_repo, chunk_ref) if chunk_ref != sb.system_branch_name: gd.branch(sb.system_branch_name, chunk_ref) @@ -491,16 +399,48 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): gd.fat_init() gd.fat_pull() - # Change the refs to the chunk. - if chunk_ref != sb.system_branch_name: - morphs.change_ref( - chunk_url, chunk_ref, chunk_morph + '.morph', - sb.system_branch_name) + # Change the refs to the chunk. + if chunk_ref != sb.system_branch_name: + morphs.change_ref( + chunk_url, chunk_ref, chunk_morph + '.morph', + sb.system_branch_name) + + return chunk_dirname + + chunk_name = morphlib.util.strip_morph_extension(args[0]) + dirs = set() + found = 0 + + for morph in morphs.morphologies: + if morph['kind'] == 'stratum': + for chunk in morph['chunks']: + if chunk['name'] == chunk_name: + self.app.status( + msg='Editing %(chunk)s in %(stratum)s stratum', + chunk=chunk_name, stratum=morph['name']) + chunk_dirname = edit_chunk(morph, chunk_name) + dirs.add(chunk_dirname) + found = found + 1 # Save any modified strata. self._save_dirty_morphologies(loader, sb, morphs.morphologies) + if found == 0: + self.app.status( + msg="No chunk %(chunk)s found. If you want to create one, add " + "an entry to a stratum morph file.", chunk=chunk_name) + + if found >= 1: + dirs_list = ', '.join(sorted(dirs)) + self.app.status( + msg="Chunk %(chunk)s source is available at %(dirs)s", + chunk=chunk_name, dirs=dirs_list) + + if found > 1: + self.app.status( + msg="Notice that this chunk appears in more than one stratum") + def show_system_branch(self, args): '''Show the name of the current system branch.''' diff --git a/morphlib/plugins/copy-artifacts_plugin.py b/morphlib/plugins/copy-artifacts_plugin.py deleted file mode 100644 index 577d0ef2..00000000 --- a/morphlib/plugins/copy-artifacts_plugin.py +++ /dev/null @@ -1,142 +0,0 @@ -# Copyright (C) 2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - - -# FIXME: no tests - - - -import os -import glob -import json - - -import cliapp - - - -import morphlib - - - -class CopyArtifactsPlugin(cliapp.Plugin): - - def enable(self): - self.app.add_subcommand( - 'list-artifacts', self.list_artifacts, - arg_synopsis='SYSTEM-ARTIFACT') - self.app.add_subcommand( - 'copy-artifacts', self.copy_artifacts, - arg_synopsis='SYSTEM-ARTIFACT DESTINATION') - - def disable(self): - pass - - def list_artifacts(self, args): - '''List every artifact that makes up a system artifact. - - Command line arguments: - - * `SYSTEM-ARTIFACT` is the filename for a build artifact for - a system (ending in `-rootfs`). - - list-artifacts reads the system artifact and writes out a list - of the filenames, in the local artifact cache, of all the - component artifacts included in the system. It does not include - build-dependencies of the system, unless they're included in - the system. - - ''' - - if len(args) != 1: - raise cliapp.AppException( - 'Wrong number of arguments to list-artifacts command' - '(see help)') - - system = args[0] - artifacts = self.find_artifacts(system) - for artifact in artifacts: - self.app.output.write(artifact + "\n") - - def copy_artifacts(self, args): - '''Copy every artifact that makes up a system to an rsync path. - - Command line arguments: - - * `SYSTEM-ARTIFACT` is the filename for a build artifact for - a system (ending in `-rootfs`). - * `DESTINATION` is a URL for where the artifacts are to be - copied to, in a form suitable for the rsync program. - - This command is useful for copying artifacts from a local - system to a Morph artifact cache server, so they can be - shared by other people. It can also be used to archive - artifacts used for a release. Note, however, that it does - not include artifacts that are build-dependencies of the - strata included in the system, but not actually in cluded - in the system. - - Example: - - morph copy-artifacts /src/cache/artifacts/*.system.* \ - cache.example.com:/srv/cache/ - - ''' - - if len(args) != 2: - raise cliapp.AppException( - 'Wrong number of arguments to copy-artifacts command' - '(see help)') - - system = args[0] - rsync_dest = args[1] - artifacts = self.find_artifacts(system) - cmdline = ['rsync'] - cmdline.extend(artifacts) - cmdline.append(rsync_dest) - cliapp.runcmd(cmdline) - - def find_artifacts(self, system): - artifacts_dir = os.path.join( - self.app.settings['cachedir'], 'artifacts') - artifacts = [] - - def find_in_system(dirname): - metadirs = [ - os.path.join(dirname, 'factory', 'baserock'), - os.path.join(dirname, 'baserock') - ] - existing_metadirs = [x for x in metadirs if os.path.isdir(x)] - if not existing_metadirs: - raise NotASystemArtifactError(system) - metadir = existing_metadirs[0] - for basename in glob.glob(os.path.join(metadir, '*.meta')): - metafile = os.path.join(metadir, basename) - metadata = json.load(open(metafile)) - cache_key = metadata['cache-key'] - artifact_glob = os.path.join(artifacts_dir, cache_key) + '*' - found_artifacts = glob.glob(artifact_glob) - if not found_artifacts: - raise cliapp.AppException('Could not find cache-key ' - + cache_key + 'for artifact ' - + metajson['artifact-name']) - artifacts.extend(found_artifacts) - - morphlib.bins.call_in_artifact_directory( - self.app, system, find_in_system) - - return artifacts - diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index 1d582949..3afb7b17 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -274,6 +274,7 @@ class DeployPlugin(cliapp.Plugin): self.app.settings['tempdir-min-space'], '/', 0) + self.app.settings['no-git-update'] = True cluster_name = morphlib.util.strip_morph_extension(args[0]) env_vars = args[1:] diff --git a/morphlib/plugins/gc_plugin.py b/morphlib/plugins/gc_plugin.py index df736afe..43de173e 100644 --- a/morphlib/plugins/gc_plugin.py +++ b/morphlib/plugins/gc_plugin.py @@ -61,6 +61,11 @@ class GCPlugin(cliapp.Plugin): It also removes any left over temporary chunks and staging areas from failed builds. + In addition we remove failed deployments, generally these are + cleared up by morph during deployment but in some cases they + won't be e.g. if morph gets a SIGKILL or the machine running + morph loses power. + ''' tempdir = self.app.settings['tempdir'] @@ -76,7 +81,7 @@ class GCPlugin(cliapp.Plugin): def cleanup_tempdir(self, temp_path, min_space): self.app.status(msg='Cleaning up temp dir %(temp_path)s', temp_path=temp_path, chatty=True) - for subdir in ('failed', 'chunks'): + for subdir in ('deployments', 'failed', 'chunks'): if morphlib.util.get_bytes_free_in_path(temp_path) >= min_space: self.app.status(msg='Not Removing subdirectory ' '%(subdir)s, enough space already cleared', diff --git a/morphlib/plugins/list_artifacts_plugin.py b/morphlib/plugins/list_artifacts_plugin.py new file mode 100644 index 00000000..5e64f708 --- /dev/null +++ b/morphlib/plugins/list_artifacts_plugin.py @@ -0,0 +1,124 @@ +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# This plugin is used as part of the Baserock automated release process. +# +# See: <http://wiki.baserock.org/guides/release-process> for more information. + + +import cliapp +import morphlib + + +class ListArtifactsPlugin(cliapp.Plugin): + + def enable(self): + self.app.add_subcommand( + 'list-artifacts', self.list_artifacts, + arg_synopsis='REPO REF MORPH [MORPH]...') + + def disable(self): + pass + + def list_artifacts(self, args): + '''List every artifact in the build graph of a system. + + Command line arguments: + + * `REPO` is a git repository URL. + * `REF` is a branch or other commit reference in that repository. + * `MORPH` is a system morphology name at that ref. + + You can pass multiple values for `MORPH`, in which case the command + outputs the union of the build graphs of all the systems passed in. + + The output includes any meta-artifacts such as .meta and .build-log + files. + + ''' + + if len(args) < 3: + raise cliapp.AppException( + 'Wrong number of arguments to list-artifacts command ' + '(see help)') + + repo, ref = args[0], args[1] + system_names = map(morphlib.util.strip_morph_extension, args[2:]) + + self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app) + self.resolver = morphlib.artifactresolver.ArtifactResolver() + + artifact_files = set() + for system_name in system_names: + system_artifact_files = self.list_artifacts_for_system( + repo, ref, system_name) + artifact_files.update(system_artifact_files) + + for artifact_file in sorted(artifact_files): + print artifact_file + + def list_artifacts_for_system(self, repo, ref, system_name): + '''List all artifact files in the build graph of a single system.''' + + # Sadly, we must use a fresh source pool and a fresh list of artifacts + # for each system. Creating a source pool is slow (queries every Git + # repo involved in the build) and resolving artifacts isn't so quick + # either. Unfortunately, each Source object can only have one set of + # Artifact objects associated, which means the source pool cannot mix + # sources that are being built for multiple architectures: the build + # graph representation does not distinguish chunks or strata of + # different architectures right now. + + self.app.status( + msg='Creating source pool for %s' % system_name, chatty=True) + source_pool = self.app.create_source_pool( + self.lrc, self.rrc, (repo, ref, system_name + '.morph')) + + self.app.status( + msg='Resolving artifacts for %s' % system_name, chatty=True) + artifacts = self.resolver.resolve_artifacts(source_pool) + + def find_artifact_by_name(artifacts_list, name): + for a in artifacts_list: + if a.source.filename == name + '.morph': + return a + raise ValueError + + system_artifact = find_artifact_by_name(artifacts, system_name) + + self.app.status( + msg='Computing cache keys for %s' % system_name, chatty=True) + build_env = morphlib.buildenvironment.BuildEnvironment( + self.app.settings, system_artifact.source.morphology['arch']) + ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env) + + artifact_files = set() + for artifact in system_artifact.walk(): + artifact.cache_key = ckc.compute_key(artifact) + artifact.cache_id = ckc.get_cache_id(artifact) + + artifact_files.add(artifact.basename()) + + if artifact.source.morphology.needs_artifact_metadata_cached: + artifact_files.add('%s.meta' % artifact.basename()) + + # This is unfortunate hardwiring of behaviour; in future we + # should list all artifacts in the meta-artifact file, so we + # don't have to guess what files there will be. + artifact_files.add('%s.meta' % artifact.cache_key) + if artifact.source.morphology['kind'] == 'chunk': + artifact_files.add('%s.build-log' % artifact.cache_key) + + return artifact_files |