From 0c33da01dcd98c56a14c7d8ae90b7d5a5fee99dd Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Date: Thu, 4 Aug 2016 16:45:59 +0000 Subject: begining Change-Id: I0453930690b04eeba808b6bf46375c2a06df227e --- morphlib/plugins/fix_ref.py | 127 -------------------- morphlib/plugins/fix_ref_plugin.py | 234 +++++++++++++++++++++++++++++++++++++ 2 files changed, 234 insertions(+), 127 deletions(-) delete mode 100644 morphlib/plugins/fix_ref.py create mode 100644 morphlib/plugins/fix_ref_plugin.py diff --git a/morphlib/plugins/fix_ref.py b/morphlib/plugins/fix_ref.py deleted file mode 100644 index 7e65647d..00000000 --- a/morphlib/plugins/fix_ref.py +++ /dev/null @@ -1,127 +0,0 @@ -# Copyright (C) 2016 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see . - - -import cliapp -import contextlib -import glob -import logging -import os -import shutil - -import morphlib - - -class FixRefPlugin(cliapp.Plugin): - '''Add subcommands for handling workspaces and system branches.''' - - def enable(self): - self.app.add_subcommand( - 'fix', self.petrify, arg_synopsis='') - self.app.add_subcommand( - 'unfix', self.unpetrify, arg_synopsis='') - - def petrify(self, args): - '''Convert all chunk refs in a system branch to be fixed SHA1s. - - This modifies all git commit references in system and stratum - morphologies, in the current system branch, to be fixed SHA - commit identifiers, rather than symbolic branch or tag names. - This is useful for making sure none of the components in a system - branch change accidentally. - - Consider the following scenario: - - * The `master` system branch refers to `gcc` using the - `baserock/morph` ref. This is appropriate, since the main line - of development should use the latest curated code. - - * You create a system branch to prepare for a release, called - `TROVE_ID/release/2.0`. The reference to `gcc` is still - `baserock/morph`. - - * You test everything, and make a release. You deploy the release - images onto devices, which get shipped to your customers. - - * A new version GCC is committed to the `baserock/morph` branch. - - * Your release branch suddenly uses a new compiler, which may - or may not work for your particular system at that release. - - To avoid this, you need to _petrify_ all git references - so that they do not change accidentally. If you've tested - your release with the GCC release that is stored in commit - `94c50665324a7aeb32f3096393ec54b2e63bfb28`, then you should - continue to use that version of GCC, regardless of what might - happen in the master system branch. If, and only if, you decide - that a new compiler would be good for your release should you - include it in your release branch. This way, only the things - that you change intentionally change in your release branch. - - ''' - - if args: - raise cliapp.AppException('morph petrify takes no arguments') - - ws = morphlib.workspace.open('.') - sb = morphlib.sysbranchdir.open_from_within('.') - loader = morphlib.morphloader.MorphologyLoader() - lrc, rrc = morphlib.util.new_repo_caches(self.app) - update_repos = not self.app.settings['no-git-update'] - - morphs = self._load_all_sysbranch_morphologies(sb, loader) - - #TODO: Stop using app.resolve_ref - def resolve_refs(morphs): - for repo, ref in morphs.list_refs(): - # You can't resolve null refs, so don't attempt to. - if repo is None or ref is None: - continue - # TODO: Handle refs that are only in workspace in general - if (repo == sb.root_repository_url - and ref == sb.system_branch_name): - continue - commit_sha1, tree_sha1 = self.app.resolve_ref( - lrc, rrc, repo, ref, update=update_repos) - yield ((repo, ref), commit_sha1) - - morphs.repoint_refs(sb.root_repository_url, - sb.system_branch_name) - - morphs.petrify_chunks(dict(resolve_refs(morphs))) - - # Write morphologies back out again. - self._save_dirty_morphologies(loader, sb, morphs.morphologies) - - def unpetrify(self, args): - '''Reverse the process of petrification. - - This undoes the changes `morph petrify` did. - - ''' - - if args: - raise cliapp.AppException('morph petrify takes no arguments') - - ws = morphlib.workspace.open('.') - sb = morphlib.sysbranchdir.open_from_within('.') - loader = morphlib.morphloader.MorphologyLoader() - - morphs = self._load_all_sysbranch_morphologies(sb, loader) - - # Restore the ref for each stratum and chunk - morphs.unpetrify_all() - - # Write morphologies back out again. - self._save_dirty_morphologies(loader, sb, morphs.morphologies) diff --git a/morphlib/plugins/fix_ref_plugin.py b/morphlib/plugins/fix_ref_plugin.py new file mode 100644 index 00000000..40130ad0 --- /dev/null +++ b/morphlib/plugins/fix_ref_plugin.py @@ -0,0 +1,234 @@ +# Copyright (C) 2016 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + + +import cliapp +import contextlib +import glob +import logging +import os +import shutil + +import morphlib + + +class FixRefPlugin(cliapp.Plugin): + + def enable(self): + self.app.add_subcommand( + 'fix', self.fix, arg_synopsis='FIX YOUR REFS') + self.app.add_subcommand( + 'unfix', self.unfix, arg_synopsis='UNFIX YOUR REFS') + + def disable(self): + pass + + def fix(self, args): + '''Convert all chunk refs in a system branch to be fixed SHA1s. + + This modifies all git commit references in system and stratum + morphologies, in the current system branch, to be fixed SHA + commit identifiers, rather than symbolic branch or tag names. + This is useful for making sure none of the components in a system + branch change accidentally. + + Consider the following scenario: + + * The `master` system branch refers to `gcc` using the + `baserock/morph` ref. This is appropriate, since the main line + of development should use the latest curated code. + + * You create a system branch to prepare for a release, called + `TROVE_ID/release/2.0`. The reference to `gcc` is still + `baserock/morph`. + + * You test everything, and make a release. You deploy the release + images onto devices, which get shipped to your customers. + + * A new version GCC is committed to the `baserock/morph` branch. + + * Your release branch suddenly uses a new compiler, which may + or may not work for your particular system at that release. + + To avoid this, you need to _petrify_ all git references + so that they do not change accidentally. If you've tested + your release with the GCC release that is stored in commit + `94c50665324a7aeb32f3096393ec54b2e63bfb28`, then you should + continue to use that version of GCC, regardless of what might + happen in the master system branch. If, and only if, you decide + that a new compiler would be good for your release should you + include it in your release branch. This way, only the things + that you change intentionally change in your release branch. + + ''' + +# if args: +# raise cliapp.AppException('morph petrify takes no arguments') + + + filename = morphlib.util.sanitise_morphology_path(args[0]) + + definitions_repo = morphlib.definitions_repo.open( + '.', search_for_root=True, app=self.app) + loader = definitions_repo.get_morphology_loader() + + filename = definitions_repo.relative_path(filename, cwd='.') + + +# source_pool_context = definitions_repo.source_pool( +# definitions_repo.HEAD, filename) +# with source_pool_context as source_pool: +# bc = morphlib.buildcommand.BuildCommand(self.app) +# root = bc.resolve_artifacts(source_pool) +# print root + def unset_defaults(self, morphology): + '''If a field is equal to its default, delete it. + + The morphology is assumed to be valid. + + ''' + + kind = morphology['kind'] + defaults = morphlib.morphloader._static_defaults[kind] + for key in defaults: + if key in morphology and morphology[key] == defaults[key]: + del morphology[key] + getattr(self, '_unset_%s_defaults' % kind)(morphology) + + + for morph in definitions_repo.load_all_morphologies(): + print morph + print morph.filename + + if morph['kind'] != 'stratum': + continue + for chunk in morph['chunks']: + repo = chunk.get('repo') + ref = chunk.get('ref') + if repo is None or ref is None: + print "FAIL" + continue + chunk['ref'] = 'xxx' + chunk['unpetrify-ref'] = ref + unset_defaults(morph) + loader.save_to_file(morph.filename, morph) + + + continue + # TODO: Handle refs that are only in workspace in general + if (repo == sb.root_repository_url + and ref == sb.system_branch_name): + continue + commit_sha1, tree_sha1 = self.app.resolve_ref( + lrc, rrc, repo, ref, update=update_repos) + + def _load_all_sysbranch_morphologies(root): + '''Read in all the morphologies in the root repository.''' + self.app.status(msg='Loading in all morphologies') + morphs = morphlib.morphset.MorphologySet() + for source in set(a.source for a in root.walk()): + morphs.add_morphology(source) + return morphs + + morphs = _load_all_sysbranch_morphologies(root) + for repo, ref in morphs.list_refs(): + print repo + #print morphs + + + for source in set(a.source for a in root.walk()): + if source.morphology['kind'] != 'stratum': + continue + + name = source.morphology['name'] + print name + ref = source.original_ref + print ref + print source.morphology + unpetrify = source.morphology['unpetrify-ref'] + print unpetrify + + + # Test that chunk has a sha1 ref + # TODO: Could allow either sha1 or existent tag. + if not morphlib.git.is_valid_sha1(ref): + warnings.warn('Chunk "{}" has non-sha1 ref: "{}"\n' + .format(name, ref)) + certified = False + + cached = self.repo_cache.get_updated_repo(source.repo_name, ref) + + # Test that sha1 ref is anchored in a tag or branch, + # and thus not a candidate for removal on `git gc`. + if (morphlib.git.is_valid_sha1(ref) and + not len(cached.tags_containing_sha1(ref)) and + not len(cached.branches_containing_sha1(ref))): + warnings.warn('Chunk "{}" has unanchored ref: "{}"\n' + .format(name, ref)) + certified = False + + +########### + + ws = morphlib.workspace.open('.') + sb = morphlib.sysbranchdir.open_from_within('.') + loader = morphlib.morphloader.MorphologyLoader() + lrc, rrc = morphlib.util.new_repo_caches(self.app) + update_repos = not self.app.settings['no-git-update'] + + morphs = self._load_all_sysbranch_morphologies(sb, loader) + + #TODO: Stop using app.resolve_ref + def resolve_refs(morphs): + for repo, ref in morphs.list_refs(): + # You can't resolve null refs, so don't attempt to. + if repo is None or ref is None: + continue + # TODO: Handle refs that are only in workspace in general + if (repo == sb.root_repository_url + and ref == sb.system_branch_name): + continue + commit_sha1, tree_sha1 = self.app.resolve_ref( + lrc, rrc, repo, ref, update=update_repos) + yield ((repo, ref), commit_sha1) + + morphs.repoint_refs(sb.root_repository_url, + sb.system_branch_name) + + morphs.petrify_chunks(dict(resolve_refs(morphs))) + + # Write morphologies back out again. + self._save_dirty_morphologies(loader, sb, morphs.morphologies) + + def unfix(self, args): + '''Reverse the process of petrification. + + This undoes the changes `morph petrify` did. + + ''' + + if args: + raise cliapp.AppException('morph petrify takes no arguments') + + ws = morphlib.workspace.open('.') + sb = morphlib.sysbranchdir.open_from_within('.') + loader = morphlib.morphloader.MorphologyLoader() + + morphs = self._load_all_sysbranch_morphologies(sb, loader) + + # Restore the ref for each stratum and chunk + morphs.unpetrify_all() + + # Write morphologies back out again. + self._save_dirty_morphologies(loader, sb, morphs.morphologies) -- cgit v1.2.1