diff options
Diffstat (limited to 'morphlib')
-rw-r--r-- | morphlib/__init__.py | 1 | ||||
-rw-r--r-- | morphlib/git.py | 4 | ||||
-rw-r--r-- | morphlib/gitdir.py | 69 | ||||
-rw-r--r-- | morphlib/gitdir_tests.py | 73 | ||||
-rw-r--r-- | morphlib/morphologyfinder.py | 71 | ||||
-rw-r--r-- | morphlib/morphologyfinder_tests.py | 111 | ||||
-rw-r--r-- | morphlib/morphset.py | 9 | ||||
-rw-r--r-- | morphlib/morphset_tests.py | 8 | ||||
-rw-r--r-- | morphlib/plugins/branch_and_merge_new_plugin.py | 137 | ||||
-rw-r--r-- | morphlib/plugins/branch_and_merge_plugin.py | 4 | ||||
-rw-r--r-- | morphlib/sysbranchdir.py | 3 |
11 files changed, 478 insertions, 12 deletions
diff --git a/morphlib/__init__.py b/morphlib/__init__.py index bcdd733b..b1e3c7c3 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -65,6 +65,7 @@ import localrepocache import mountableimage import morph2 import morphologyfactory +import morphologyfinder import morph3 import morphloader import morphset diff --git a/morphlib/git.py b/morphlib/git.py index 4ff08a72..27146206 100644 --- a/morphlib/git.py +++ b/morphlib/git.py @@ -20,6 +20,7 @@ import ConfigParser import logging import os import re +import string import StringIO import time @@ -311,8 +312,7 @@ def clone_into(runcmd, srcpath, targetpath, ref=None): def is_valid_sha1(ref): '''Checks whether a string is a valid SHA1.''' - valid_chars = 'abcdefABCDEF0123456789' - return len(ref) == 40 and all([x in valid_chars for x in ref]) + return len(ref) == 40 and all(x in string.hexdigits for x in ref) def rev_parse(runcmd, gitdir, ref): '''Find the sha1 for the given ref''' diff --git a/morphlib/gitdir.py b/morphlib/gitdir.py index f40190ff..cb247303 100644 --- a/morphlib/gitdir.py +++ b/morphlib/gitdir.py @@ -17,10 +17,27 @@ import cliapp +import glob +import os import morphlib +class NoWorkingTreeError(cliapp.AppException): + + def __init__(self, repo): + cliapp.AppException.__init__( + self, 'Git directory %s has no working tree ' + '(is bare).' % repo.dirname) + + +class InvalidRefError(cliapp.AppException): + def __init__(self, repo, ref): + cliapp.AppException.__init__( + self, 'Git directory %s has no commit ' + 'at ref %s.' %(repo.dirname, ref)) + + class GitDirectory(object): '''Represent a git working tree + .git directory. @@ -126,6 +143,58 @@ class GitDirectory(object): '''Run "git remote update --prune".''' self._runcmd(['git', 'remote', 'update', '--prune']) + def is_bare(self): + '''Determine whether the repository has no work tree (is bare)''' + return self.get_config('core.bare') == 'true' + + def list_files(self, ref=None): + '''Return an iterable of the files in the repository. + + If `ref` is specified, list files at that ref, otherwise + use the working tree. + + If this is a bare repository and no ref is specified, raises + an exception. + + ''' + if ref is None and self.is_bare(): + raise NoWorkingTreeError(self) + if ref is None: + return self._list_files_in_work_tree() + else: + return self._list_files_in_ref(ref) + + def _rev_parse_tree(self, ref): + try: + return self._runcmd(['git', 'rev-parse', '--verify', + '%s^{tree}' % ref]).strip() + except cliapp.AppException as e: + raise InvalidRefError(self, ref) + + def _list_files_in_work_tree(self): + for dirpath, subdirs, filenames in os.walk(self.dirname): + if dirpath == self.dirname and '.git' in subdirs: + subdirs.remove('.git') + for filename in filenames: + yield os.path.join(dirpath, filename)[len(self.dirname)+1:] + + def _list_files_in_ref(self, ref): + tree = self._rev_parse_tree(ref) + output = self._runcmd(['git', 'ls-tree', '--name-only', '-rz', tree]) + # ls-tree appends \0 instead of interspersing, so we need to + # strip the trailing \0 before splitting + paths = output.strip('\0').split('\0') + return paths + + def read_file(self, filename, ref=None): + if ref is None and self.is_bare(): + raise NoWorkingTreeError(self) + if ref is None: + with open(os.path.join(self.dirname, filename)) as f: + return f.read() + tree = self._rev_parse_tree(ref) + return self.cat_file('blob', tree, filename) + def init(dirname): '''Initialise a new git repository.''' diff --git a/morphlib/gitdir_tests.py b/morphlib/gitdir_tests.py index 2494981a..175b8ee7 100644 --- a/morphlib/gitdir_tests.py +++ b/morphlib/gitdir_tests.py @@ -64,3 +64,76 @@ class GitDirectoryTests(unittest.TestCase): gitdir.set_remote_fetch_url('origin', url) self.assertEqual(gitdir.get_remote_fetch_url('origin'), url) +class GitDirectoryContentsTests(unittest.TestCase): + + def setUp(self): + self.tempdir = tempfile.mkdtemp() + self.dirname = os.path.join(self.tempdir, 'foo') + os.mkdir(self.dirname) + gd = morphlib.gitdir.init(self.dirname) + for fn in ('foo', 'bar.morph', 'baz.morph', 'quux'): + with open(os.path.join(self.dirname, fn), "w") as f: + f.write('dummy morphology text') + gd._runcmd(['git', 'add', '.']) + gd._runcmd(['git', 'commit', '-m', 'Initial commit']) + os.rename(os.path.join(self.dirname, 'foo'), + os.path.join(self.dirname, 'foo.morph')) + self.mirror = os.path.join(self.tempdir, 'mirror') + gd._runcmd(['git', 'clone', '--mirror', self.dirname, self.mirror]) + + def tearDown(self): + shutil.rmtree(self.tempdir) + + def test_lists_files_in_work_tree(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + self.assertEqual(sorted(gd.list_files()), + ['bar.morph', 'baz.morph', 'foo.morph', 'quux']) + + def test_read_file_in_work_tree(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + self.assertEqual(gd.read_file('bar.morph'), + 'dummy morphology text') + + def test_list_raises_no_ref_no_work_tree(self): + gd = morphlib.gitdir.GitDirectory(self.mirror) + self.assertRaises(morphlib.gitdir.NoWorkingTreeError, + gd.list_files) + + def test_read_raises_no_ref_no_work_tree(self): + gd = morphlib.gitdir.GitDirectory(self.mirror) + self.assertRaises(morphlib.gitdir.NoWorkingTreeError, + gd.read_file, 'bar.morph') + + def test_lists_files_in_HEAD(self): + for gitdir in (self.dirname, self.mirror): + gd = morphlib.gitdir.GitDirectory(gitdir) + self.assertEqual(sorted(gd.list_files('HEAD')), + ['bar.morph', 'baz.morph', 'foo', 'quux']) + + def test_read_files_in_HEAD(self): + for gitdir in (self.dirname, self.mirror): + gd = morphlib.gitdir.GitDirectory(gitdir) + self.assertEqual(gd.read_file('bar.morph', 'HEAD'), + 'dummy morphology text') + + def test_lists_files_in_named_ref(self): + for gitdir in (self.dirname, self.mirror): + gd = morphlib.gitdir.GitDirectory(gitdir) + self.assertEqual(sorted(gd.list_files('master')), + ['bar.morph', 'baz.morph', 'foo', 'quux']) + + def test_read_file_in_named_ref(self): + for gitdir in (self.dirname, self.mirror): + gd = morphlib.gitdir.GitDirectory(gitdir) + self.assertEqual(gd.read_file('bar.morph', 'master'), + 'dummy morphology text') + + def test_list_raises_invalid_ref(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + self.assertRaises(morphlib.gitdir.InvalidRefError, + gd.list_files, 'no-such-ref') + + def test_read_raises_invalid_ref(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + self.assertRaises(morphlib.gitdir.InvalidRefError, + gd.read_file, 'bar', 'no-such-ref') diff --git a/morphlib/morphologyfinder.py b/morphlib/morphologyfinder.py new file mode 100644 index 00000000..affa0e97 --- /dev/null +++ b/morphlib/morphologyfinder.py @@ -0,0 +1,71 @@ +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# =*= License: GPL-2 =*= + + +import cliapp + +import morphlib + + +class MorphologyFinder(object): + + '''Abstract away finding morphologies in a git repository. + + This class provides an abstraction layer between a git repository + and the morphologies contained in it. + + ''' + + def __init__(self, gitdir, ref=None): + self.gitdir = gitdir + self.ref = ref + + def read_morphology(self, name): + '''Return the un-parsed text of a morphology. + + For the given morphology name, locate and return the contents + of the morphology as a string. + + Also returns a string describing where in the repository the + morphology is located. + + Parsing of this morphology into a form useful for manipulating + is handled by the MorphologyLoader class. + + ''' + filename = '%s.morph' % name + return self.gitdir.read_file(filename, self.ref), filename + + def list_morphologies(self): + '''Return the names of all morphologies in the (repo, ref). + + Finds all morphologies in the git directory at the specified + ref. Morphology names are returned instead of filenames, + so the implementation may change how morphologies are stored + in git repositories. + + ''' + + def is_morphology_path(path): + return path.endswith('.morph') + + def transform_path_to_name(path): + return path[:-len('.morph')] + + return (transform_path_to_name(path) + for path in self.gitdir.list_files(self.ref) + if is_morphology_path(path)) diff --git a/morphlib/morphologyfinder_tests.py b/morphlib/morphologyfinder_tests.py new file mode 100644 index 00000000..ff83ccff --- /dev/null +++ b/morphlib/morphologyfinder_tests.py @@ -0,0 +1,111 @@ +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# =*= License: GPL-2 =*= + + +import os +import shutil +import tempfile +import unittest + +import morphlib + + +class MorphologyFinderTests(unittest.TestCase): + + def setUp(self): + self.tempdir = tempfile.mkdtemp() + self.dirname = os.path.join(self.tempdir, 'repo') + os.mkdir(self.dirname) + gd = morphlib.gitdir.init(self.dirname) + for fn in ('foo', 'bar.morph', 'baz.morph', 'quux'): + with open(os.path.join(self.dirname, fn), "w") as f: + f.write('dummy morphology text') + gd._runcmd(['git', 'add', '.']) + gd._runcmd(['git', 'commit', '-m', 'Initial commit']) + + # Changes for difference between commited and work tree + newmorphpath = os.path.join(self.dirname, 'foo.morph') + os.unlink(os.path.join(self.dirname, 'foo')) + with open(newmorphpath, 'w') as f: + f.write("altered morphology text") + + # Changes for bare repository + self.mirror = os.path.join(self.tempdir, 'mirror') + gd._runcmd(['git', 'clone', '--mirror', self.dirname, self.mirror]) + + def tearDown(self): + shutil.rmtree(self.tempdir) + + def test_list_morphs_in_HEAD(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + mf = morphlib.morphologyfinder.MorphologyFinder(gd, 'HEAD') + self.assertEqual(sorted(mf.list_morphologies()), + ['bar', 'baz']) + + def test_list_morphs_in_master(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + mf = morphlib.morphologyfinder.MorphologyFinder(gd, 'master') + self.assertEqual(sorted(mf.list_morphologies()), + ['bar', 'baz']) + + def test_list_morphs_raises_with_invalid_ref(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + mf = morphlib.morphologyfinder.MorphologyFinder(gd, 'invalid_ref') + self.assertRaises(morphlib.gitdir.InvalidRefError, + mf.list_morphologies) + + def test_list_morphs_in_work_tree(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + mf = morphlib.morphologyfinder.MorphologyFinder(gd) + self.assertEqual(sorted(mf.list_morphologies()), + ['bar', 'baz', 'foo']) + + def test_list_morphs_raises_no_worktree_no_ref(self): + gd = morphlib.gitdir.GitDirectory(self.mirror) + mf = morphlib.morphologyfinder.MorphologyFinder(gd) + self.assertRaises(morphlib.gitdir.NoWorkingTreeError, + mf.list_morphologies) + + def test_read_morph_in_HEAD(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + mf = morphlib.morphologyfinder.MorphologyFinder(gd, 'HEAD') + self.assertEqual(mf.read_morphology('bar')[0], + "dummy morphology text") + + def test_read_morph_in_master(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + mf = morphlib.morphologyfinder.MorphologyFinder(gd, 'master') + self.assertEqual(mf.read_morphology('bar')[0], + "dummy morphology text") + + def test_read_morph_raises_with_invalid_ref(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + mf = morphlib.morphologyfinder.MorphologyFinder(gd, 'invalid_ref') + self.assertRaises(morphlib.gitdir.InvalidRefError, + mf.read_morphology, 'bar') + + def test_read_morph_in_work_tree(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + mf = morphlib.morphologyfinder.MorphologyFinder(gd) + self.assertEqual(mf.read_morphology('foo')[0], + "altered morphology text") + + def test_read_morph_raises_no_worktree_no_ref(self): + gd = morphlib.gitdir.GitDirectory(self.mirror) + mf = morphlib.morphologyfinder.MorphologyFinder(gd) + self.assertRaises(morphlib.gitdir.NoWorkingTreeError, + mf.read_morphology, 'bar') diff --git a/morphlib/morphset.py b/morphlib/morphset.py index 98a4b8f9..c256760e 100644 --- a/morphlib/morphset.py +++ b/morphlib/morphset.py @@ -135,18 +135,19 @@ class MorphologySet(object): spec['ref'] == orig_ref and spec['morph'] + '.morph' == morph_filename) - def change_specs(specs): + def change_specs(specs, m): for spec in specs: if wanted_spec(spec): + spec['unpetrify-ref'] = spec['ref'] spec['ref'] = new_ref m.dirty = True def change(m): if m['kind'] == 'system': - change_specs(m['strata']) + change_specs(m['strata'], m) elif m['kind'] == 'stratum': - change_specs(m['chunks']) - change_specs(m['build-depends']) + change_specs(m['chunks'], m) + change_specs(m['build-depends'], m) for m in self.morphologies: change(m) diff --git a/morphlib/morphset_tests.py b/morphlib/morphset_tests.py index 7dbc861a..65fe2058 100644 --- a/morphlib/morphset_tests.py +++ b/morphlib/morphset_tests.py @@ -127,7 +127,8 @@ class MorphologySetTests(unittest.TestCase): { 'repo': 'test:morphs', 'ref': 'new-ref', - 'morph': 'foo-stratum' + 'morph': 'foo-stratum', + 'unpetrify-ref': 'master', }) def test_changes_stratum_ref_in_build_depends(self): @@ -140,6 +141,7 @@ class MorphologySetTests(unittest.TestCase): 'repo': self.stratum.repo_url, 'ref': self.stratum.ref, 'morph': self.stratum['name'], + 'unpetrify-ref': 'master', }, ] }) @@ -157,7 +159,8 @@ class MorphologySetTests(unittest.TestCase): { 'repo': 'test:morphs', 'ref': 'new-ref', - 'morph': 'foo-stratum' + 'morph': 'foo-stratum', + 'unpetrify-ref': 'master', }) def test_changes_chunk_ref(self): @@ -175,6 +178,7 @@ class MorphologySetTests(unittest.TestCase): 'repo': 'test:foo-chunk', 'ref': 'new-ref', 'morph': 'foo-chunk', + 'unpetrify-ref': 'master', } ]) diff --git a/morphlib/plugins/branch_and_merge_new_plugin.py b/morphlib/plugins/branch_and_merge_new_plugin.py index 61cd40c0..66231de9 100644 --- a/morphlib/plugins/branch_and_merge_new_plugin.py +++ b/morphlib/plugins/branch_and_merge_new_plugin.py @@ -44,6 +44,10 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): self.app.add_subcommand( 'edit', self.edit, arg_synopsis='SYSTEM STRATUM [CHUNK]') self.app.add_subcommand( + 'petrify', self.petrify, arg_synopsis='') + self.app.add_subcommand( + 'unpetrify', self.unpetrify, arg_synopsis='') + self.app.add_subcommand( 'show-system-branch', self.show_system_branch, arg_synopsis='') self.app.add_subcommand( 'show-branch-root', self.show_branch_root, arg_synopsis='') @@ -604,3 +608,136 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): % (repo, pretty_command)) self.app.output.write('\n') self.app.output.flush() + + def _load_all_sysbranch_morphologies(self, sb, loader): + '''Read in all the morphologies in the root repository.''' + self.app.status(msg='Loading in all morphologies') + morphs = morphlib.morphset.MorphologySet() + mf = morphlib.morphologyfinder.MorphologyFinder( + morphlib.gitdir.GitDirectory( + sb.get_git_directory_name(sb.root_repository_url))) + for morph in mf.list_morphologies(): + text, filename = mf.read_morphology(morph) + m = loader.load_from_string(text, filename=filename) + m.repo_url = sb.root_repository_url + m.ref = sb.system_branch_name + morphs.add_morphology(m) + return morphs + + def petrify(self, args): + '''Convert all chunk refs in a system branch to be fixed SHA1s. + + This modifies all git commit references in system and stratum + morphologies, in the current system branch, to be fixed SHA + commit identifiers, rather than symbolic branch or tag names. + This is useful for making sure none of the components in a system + branch change accidentally. + + Consider the following scenario: + + * The `master` system branch refers to `gcc` using the + `baserock/morph` ref. This is appropriate, since the main line + of development should use the latest curated code. + + * You create a system branch to prepare for a release, called + `TROVE_ID/release/2.0`. The reference to `gcc` is still + `baserock/morph`. + + * You test everything, and make a release. You deploy the release + images onto devices, which get shipped to your customers. + + * A new version GCC is committed to the `baserock/morph` branch. + + * Your release branch suddenly uses a new compiler, which may + or may not work for your particular system at that release. + + To avoid this, you need to _petrify_ all git references + so that they do not change accidentally. If you've tested + your release with the GCC release that is stored in commit + `94c50665324a7aeb32f3096393ec54b2e63bfb28`, then you should + continue to use that version of GCC, regardless of what might + happen in the master system branch. If, and only if, you decide + that a new compiler would be good for your release should you + include it in your release branch. This way, only the things + that you change intentionally change in your release branch. + + ''' + + if args: + raise cliapp.AppException('morph petrify takes no arguments') + + ws = morphlib.workspace.open('.') + sb = morphlib.sysbranchdir.open_from_within('.') + loader = morphlib.morphloader.MorphologyLoader() + lrc, rrc = morphlib.util.new_repo_caches(self.app) + update_repos = not self.app.settings['no-git-update'] + done = set() + + morphs = self._load_all_sysbranch_morphologies(sb, loader) + + # Petrify the ref to each stratum and chunk. + def petrify_specs(specs): + for spec in specs: + ref = spec['ref'] + # Do not double petrify refs + if morphlib.git.is_valid_sha1(ref): + continue + commit_sha1, tree_sha1 = self.app.resolve_ref( + lrc, rrc, spec['repo'], ref, update=update_repos) + assert 'name' in spec or 'morph' in spec + filename = '%s.morph' % spec.get('morph', spec.get('name')) + morphs.change_ref(spec['repo'], ref, filename, commit_sha1) + + for m in morphs.morphologies: + if m['kind'] == 'system': + petrify_specs(m['strata']) + elif m['kind'] == 'stratum': + petrify_specs(m['build-depends']) + petrify_specs(m['chunks']) + + # Write morphologies back out again. + self._save_dirty_morphologies(loader, sb, morphs.morphologies) + + def unpetrify(self, args): + '''Reverse the process of petrification. + + This undoes the changes `morph petrify` did. + + ''' + + if args: + raise cliapp.AppException('morph petrify takes no arguments') + + ws = morphlib.workspace.open('.') + sb = morphlib.sysbranchdir.open_from_within('.') + loader = morphlib.morphloader.MorphologyLoader() + lrc, rrc = morphlib.util.new_repo_caches(self.app) + update_repos = not self.app.settings['no-git-update'] + done = set() + + morphs = self._load_all_sysbranch_morphologies(sb, loader) + + # Restore the ref for each stratum and chunk + def unpetrify_specs(specs): + dirty = False + for spec in specs: + ref = spec['ref'] + # Don't attempt to unpetrify refs which aren't petrified + if not ('unpetrify-ref' in spec + and morphlib.git.is_valid_sha1(ref)): + continue + spec['ref'] = spec.pop('unpetrify-ref') + dirty = True + return dirty + + for m in morphs.morphologies: + dirty = False + if m['kind'] == 'system': + dirty = dirty or unpetrify_specs(m['strata']) + elif m['kind'] == 'stratum': + dirty = dirty or unpetrify_specs(m['build-depends']) + dirty = dirty or unpetrify_specs(m['chunks']) + m.dirty = True + + # Write morphologies back out again. + self._save_dirty_morphologies(loader, sb, morphs.morphologies) diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py index c1549a5d..671071b6 100644 --- a/morphlib/plugins/branch_and_merge_plugin.py +++ b/morphlib/plugins/branch_and_merge_plugin.py @@ -61,8 +61,8 @@ class BranchAndMergePlugin(cliapp.Plugin): arg_synopsis='BRANCH') # self.app.add_subcommand('edit', self.edit, # arg_synopsis='SYSTEM STRATUM [CHUNK]') - self.app.add_subcommand('petrify', self.petrify) - self.app.add_subcommand('unpetrify', self.unpetrify) + self.app.add_subcommand('old-petrify', self.petrify) + self.app.add_subcommand('old-unpetrify', self.unpetrify) self.app.add_subcommand( 'tag', self.tag, arg_synopsis='TAG-NAME -- [GIT-COMMIT-ARG...]') self.app.add_subcommand('build', self.build, diff --git a/morphlib/sysbranchdir.py b/morphlib/sysbranchdir.py index 9ad1e2fd..0b3c859a 100644 --- a/morphlib/sysbranchdir.py +++ b/morphlib/sysbranchdir.py @@ -95,8 +95,7 @@ class SystemBranchDirectory(object): # Remove anyleading slashes, or os.path.join below will only # use the relative part (since it's absolute, not relative). - while relative.startswith('/'): - relative = relative[1:] + relative = relative.lstrip('/') return os.path.join(self.root_directory, relative) |