diff options
author | Richard Maw <richard.maw@codethink.co.uk> | 2014-07-11 14:25:40 +0100 |
---|---|---|
committer | Richard Maw <richard.maw@codethink.co.uk> | 2014-07-11 14:25:40 +0100 |
commit | ce1fedb4e5ab82105853c4f3a8e05fb83f62c18e (patch) | |
tree | 6a738f64b517992eed10c3d5891bbb27840202e4 /morphlib | |
parent | fe4486c1ec69c520a2c0aee7b52c35b57808120e (diff) | |
parent | 597f2a19363b7e91b928d814627c9236779998fb (diff) | |
download | morph-ce1fedb4e5ab82105853c4f3a8e05fb83f62c18e.tar.gz |
Merge remote-tracking branch 'origin/baserock/richardmaw/S11284/morphologies-by-path-v4'
Reviewed-by: Sam Thursfield
Reviewed-by: Lars Wirzenius
Diffstat (limited to 'morphlib')
-rw-r--r-- | morphlib/app.py | 28 | ||||
-rw-r--r-- | morphlib/artifactresolver.py | 6 | ||||
-rw-r--r-- | morphlib/buildcommand.py | 2 | ||||
-rw-r--r-- | morphlib/gitdir.py | 10 | ||||
-rw-r--r-- | morphlib/gitdir_tests.py | 37 | ||||
-rw-r--r-- | morphlib/morphloader.py | 64 | ||||
-rw-r--r-- | morphlib/morphologyfactory.py | 56 | ||||
-rw-r--r-- | morphlib/morphologyfinder.py | 21 | ||||
-rw-r--r-- | morphlib/morphologyfinder_tests.py | 16 | ||||
-rw-r--r-- | morphlib/morphset.py | 49 | ||||
-rw-r--r-- | morphlib/morphset_tests.py | 29 | ||||
-rw-r--r-- | morphlib/plugins/branch_and_merge_new_plugin.py | 829 | ||||
-rw-r--r-- | morphlib/plugins/branch_and_merge_plugin.py | 2242 | ||||
-rw-r--r-- | morphlib/plugins/build_plugin.py | 15 | ||||
-rw-r--r-- | morphlib/plugins/cross-bootstrap_plugin.py | 2 | ||||
-rw-r--r-- | morphlib/plugins/deploy_plugin.py | 13 | ||||
-rw-r--r-- | morphlib/plugins/list_artifacts_plugin.py | 23 | ||||
-rw-r--r-- | morphlib/source.py | 2 | ||||
-rw-r--r-- | morphlib/sysbranchdir.py | 5 | ||||
-rw-r--r-- | morphlib/util.py | 23 | ||||
-rw-r--r-- | morphlib/util_tests.py | 29 |
21 files changed, 756 insertions, 2745 deletions
diff --git a/morphlib/app.py b/morphlib/app.py index df8c360a..e0874317 100644 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -282,7 +282,8 @@ class Morph(cliapp.Application): while args: assert len(args) >= 2, args - yield args[0], args[1], args[2] + ".morph" + yield (args[0], args[1], + morphlib.util.sanitise_morphology_path(args[2])) args = args[3:] def create_source_pool(self, lrc, rrc, triplet): @@ -367,18 +368,23 @@ class Morph(cliapp.Application): raise cliapp.AppException( "Cannot build a morphology of type 'cluster'.") elif morphology['kind'] == 'system': - queue.extend((s.get('repo') or reponame, - s.get('ref') or ref, - '%s.morph' % s['morph']) - for s in morphology['strata']) + queue.extend( + (s.get('repo') or reponame, + s.get('ref') or ref, + morphlib.util.sanitise_morphology_path(s['morph'])) + for s in morphology['strata']) elif morphology['kind'] == 'stratum': if morphology['build-depends']: - queue.extend((s.get('repo') or reponame, - s.get('ref') or ref, - '%s.morph' % s['morph']) - for s in morphology['build-depends']) - queue.extend((c['repo'], c['ref'], '%s.morph' % c['morph']) - for c in morphology['chunks']) + queue.extend( + (s.get('repo') or reponame, + s.get('ref') or ref, + morphlib.util.sanitise_morphology_path(s['morph'])) + for s in morphology['build-depends']) + queue.extend( + (c['repo'], + c['ref'], + morphlib.util.sanitise_morphology_path(c['morph'])) + for c in morphology['chunks']) def cache_repo_and_submodules(self, cache, url, ref, done): subs_to_process = set() diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py index 00976eb7..c18042a3 100644 --- a/morphlib/artifactresolver.py +++ b/morphlib/artifactresolver.py @@ -142,7 +142,7 @@ class ArtifactResolver(object): stratum_source = self._source_pool.lookup( info.get('repo') or source.repo_name, info.get('ref') or source.original_ref, - '%s.morph' % info['morph']) + morphlib.util.sanitise_morphology_path(info['morph'])) stratum_name = stratum_source.morphology['name'] matches, overlaps, unmatched = source.split_rules.partition( @@ -167,7 +167,7 @@ class ArtifactResolver(object): other_source = self._source_pool.lookup( stratum_info.get('repo') or source.repo_name, stratum_info.get('ref') or source.original_ref, - '%s.morph' % stratum_info['morph']) + morphlib.util.sanitise_morphology_path(stratum_info['morph'])) # Make every stratum artifact this stratum source produces # depend on every stratum artifact the other stratum source @@ -194,7 +194,7 @@ class ArtifactResolver(object): chunk_source = self._source_pool.lookup( info['repo'], info['ref'], - '%s.morph' % info['morph']) + morphlib.util.sanitise_morphology_path(info['morph'])) chunk_name = chunk_source.morphology['name'] diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index f68046e3..45c6ef00 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -231,7 +231,7 @@ class BuildCommand(object): for spec in specs: repo_name = spec.get('repo') or src.repo_name ref = spec.get('ref') or src.original_ref - filename = '%s.morph' % spec['morph'] + filename = morphlib.util.sanitise_morphology_path(spec['morph']) logging.debug( 'Validating cross ref to %s:%s:%s' % (repo_name, ref, filename)) diff --git a/morphlib/gitdir.py b/morphlib/gitdir.py index 8f6d69d7..5b0693cb 100644 --- a/morphlib/gitdir.py +++ b/morphlib/gitdir.py @@ -515,6 +515,16 @@ class GitDirectory(object): tree = self.resolve_ref_to_tree(ref) return self.get_file_from_ref(tree, filename) + def is_symlink(self, filename, ref=None): + if ref is None and self.is_bare(): + raise NoWorkingTreeError(self) + if ref is None: + filepath = os.path.join(self.dirname, filename.lstrip('/')) + return os.path.islink(filepath) + tree_entry = self._runcmd(['git', 'ls-tree', ref, filename]) + file_mode = tree_entry.split(' ', 1)[0] + return file_mode == '120000' + @property def HEAD(self): output = self._runcmd(['git', 'rev-parse', '--abbrev-ref', 'HEAD']) diff --git a/morphlib/gitdir_tests.py b/morphlib/gitdir_tests.py index 14b2a57a..b3b4a8ab 100644 --- a/morphlib/gitdir_tests.py +++ b/morphlib/gitdir_tests.py @@ -216,6 +216,43 @@ class GitDirectoryContentsTests(unittest.TestCase): self.assertEqual(gd.describe(), 'example') +class GitDirectoryFileTypeTests(unittest.TestCase): + + def setUp(self): + self.tempdir = tempfile.mkdtemp() + self.dirname = os.path.join(self.tempdir, 'foo') + os.mkdir(self.dirname) + gd = morphlib.gitdir.init(self.dirname) + with open(os.path.join(self.dirname, 'file'), "w") as f: + f.write('dummy morphology text') + os.symlink('file', os.path.join(self.dirname, 'link')) + os.symlink('no file', os.path.join(self.dirname, 'broken')) + gd._runcmd(['git', 'add', '.']) + gd._runcmd(['git', 'commit', '-m', 'Initial commit']) + self.mirror = os.path.join(self.tempdir, 'mirror') + gd._runcmd(['git', 'clone', '--mirror', self.dirname, self.mirror]) + + def tearDown(self): + shutil.rmtree(self.tempdir) + + def test_working_tree_symlinks(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + self.assertTrue(gd.is_symlink('link')) + self.assertTrue(gd.is_symlink('broken')) + self.assertFalse(gd.is_symlink('file')) + + def test_bare_symlinks(self): + gd = morphlib.gitdir.GitDirectory(self.mirror) + self.assertTrue(gd.is_symlink('link', 'HEAD')) + self.assertTrue(gd.is_symlink('broken', 'HEAD')) + self.assertFalse(gd.is_symlink('file', 'HEAD')) + + def test_is_symlink_raises_no_ref_no_work_tree(self): + gd = morphlib.gitdir.GitDirectory(self.mirror) + self.assertRaises(morphlib.gitdir.NoWorkingTreeError, + gd.is_symlink, 'file') + + class GitDirectoryRefTwiddlingTests(unittest.TestCase): def setUp(self): diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py index 368e5477..45416a19 100644 --- a/morphlib/morphloader.py +++ b/morphlib/morphloader.py @@ -40,25 +40,33 @@ class MorphologyObsoleteFieldWarning(UserWarning): class MorphologySyntaxError(morphlib.Error): + pass + + +class MorphologyNotYamlError(MorphologySyntaxError): def __init__(self, morphology, errmsg): self.msg = 'Syntax error in morphology %s:\n%s' % (morphology, errmsg) -class NotADictionaryError(morphlib.Error): +class NotADictionaryError(MorphologySyntaxError): def __init__(self, morph_filename): self.msg = 'Not a dictionary: morphology %s' % morph_filename -class UnknownKindError(morphlib.Error): +class MorphologyValidationError(morphlib.Error): + pass + + +class UnknownKindError(MorphologyValidationError): def __init__(self, kind, morph_filename): self.msg = ( 'Unknown kind %s in morphology %s' % (kind, morph_filename)) -class MissingFieldError(morphlib.Error): +class MissingFieldError(MorphologyValidationError): def __init__(self, field, morphology_name): self.field = field @@ -67,7 +75,7 @@ class MissingFieldError(morphlib.Error): 'Missing field %s from morphology %s' % (field, morphology_name)) -class InvalidFieldError(morphlib.Error): +class InvalidFieldError(MorphologyValidationError): def __init__(self, field, morphology_name): self.field = field @@ -76,7 +84,7 @@ class InvalidFieldError(morphlib.Error): 'Field %s not allowed in morphology %s' % (field, morphology_name)) -class InvalidTypeError(morphlib.Error): +class InvalidTypeError(MorphologyValidationError): def __init__(self, field, expected, actual, morphology_name): self.field = field @@ -88,7 +96,7 @@ class InvalidTypeError(morphlib.Error): (field, expected, actual, morphology_name)) -class ObsoleteFieldsError(morphlib.Error): +class ObsoleteFieldsError(MorphologyValidationError): def __init__(self, fields, morph_filename): self.msg = ( @@ -96,14 +104,14 @@ class ObsoleteFieldsError(morphlib.Error): (morph_filename, ' '.join(fields))) -class UnknownArchitectureError(morphlib.Error): +class UnknownArchitectureError(MorphologyValidationError): def __init__(self, arch, morph_filename): self.msg = ('Unknown architecture %s in morphology %s' % (arch, morph_filename)) -class NoBuildDependenciesError(morphlib.Error): +class NoBuildDependenciesError(MorphologyValidationError): def __init__(self, stratum_name, chunk_name, morph_filename): self.msg = ( @@ -111,7 +119,7 @@ class NoBuildDependenciesError(morphlib.Error): (stratum_name, chunk_name, morph_filename)) -class NoStratumBuildDependenciesError(morphlib.Error): +class NoStratumBuildDependenciesError(MorphologyValidationError): def __init__(self, stratum_name, morph_filename): self.msg = ( @@ -119,7 +127,7 @@ class NoStratumBuildDependenciesError(morphlib.Error): (stratum_name, morph_filename)) -class EmptyStratumError(morphlib.Error): +class EmptyStratumError(MorphologyValidationError): def __init__(self, stratum_name, morph_filename): self.msg = ( @@ -127,86 +135,86 @@ class EmptyStratumError(morphlib.Error): (stratum_name, morph_filename)) -class DuplicateChunkError(morphlib.Error): +class DuplicateChunkError(MorphologyValidationError): def __init__(self, stratum_name, chunk_name): self.stratum_name = stratum_name self.chunk_name = chunk_name - morphlib.Error.__init__( + MorphologyValidationError.__init__( self, 'Duplicate chunk %(chunk_name)s '\ 'in stratum %(stratum_name)s' % locals()) -class EmptyRefError(morphlib.Error): +class EmptyRefError(MorphologyValidationError): def __init__(self, ref_location, morph_filename): self.ref_location = ref_location self.morph_filename = morph_filename - morphlib.Error.__init__( + MorphologyValidationError.__init__( self, 'Empty ref found for %(ref_location)s '\ 'in %(morph_filename)s' % locals()) -class ChunkSpecRefNotStringError(morphlib.Error): +class ChunkSpecRefNotStringError(MorphologyValidationError): def __init__(self, ref_value, chunk_name, stratum_name): self.ref_value = ref_value self.chunk_name = chunk_name self.stratum_name = stratum_name - morphlib.Error.__init__( + MorphologyValidationError.__init__( self, 'Ref %(ref_value)s for %(chunk_name)s '\ 'in stratum %(stratum_name)s is not a string' % locals()) -class SystemStrataNotListError(morphlib.Error): +class SystemStrataNotListError(MorphologyValidationError): def __init__(self, system_name, strata_type): self.system_name = system_name self.strata_type = strata_type typename = strata_type.__name__ - morphlib.Error.__init__( + MorphologyValidationError.__init__( self, 'System %(system_name)s has the wrong type for its strata: '\ '%(typename)s, expected list' % locals()) -class DuplicateStratumError(morphlib.Error): +class DuplicateStratumError(MorphologyValidationError): def __init__(self, system_name, stratum_name): self.system_name = system_name self.stratum_name = stratum_name - morphlib.Error.__init__( + MorphologyValidationError.__init__( self, 'Duplicate stratum %(stratum_name)s '\ 'in system %(system_name)s' % locals()) -class SystemStratumSpecsNotMappingError(morphlib.Error): +class SystemStratumSpecsNotMappingError(MorphologyValidationError): def __init__(self, system_name, strata): self.system_name = system_name self.strata = strata - morphlib.Error.__init__( + MorphologyValidationError.__init__( self, 'System %(system_name)s has stratum specs '\ 'that are not mappings.' % locals()) -class EmptySystemError(morphlib.Error): +class EmptySystemError(MorphologyValidationError): def __init__(self, system_name): - morphlib.Error.__init__( + MorphologyValidationError.__init__( self, 'System %(system_name)s has no strata.' % locals()) -class MultipleValidationErrors(morphlib.Error): +class MultipleValidationErrors(MorphologyValidationError): def __init__(self, name, errors): self.name = name self.errors = errors self.msg = 'Multiple errors when validating %(name)s:' for error in errors: - self.msg += ('\t' + str(error)) + self.msg += ('\n' + str(error)) -class DuplicateDeploymentNameError(morphlib.Error): +class DuplicateDeploymentNameError(MorphologyValidationError): def __init__(self, cluster_filename, duplicates): self.duplicates = duplicates @@ -352,7 +360,7 @@ class MorphologyLoader(object): try: obj = yaml.safe_load(text) except yaml.error.YAMLError as e: - raise MorphologySyntaxError(morph_filename, e) + raise MorphologyNotYamlError(morph_filename, e) if not isinstance(obj, dict): raise NotADictionaryError(morph_filename) diff --git a/morphlib/morphologyfactory.py b/morphlib/morphologyfactory.py index cd59d4ec..1cde2c77 100644 --- a/morphlib/morphologyfactory.py +++ b/morphlib/morphologyfactory.py @@ -14,6 +14,8 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import os + import morphlib import cliapp @@ -74,43 +76,41 @@ class MorphologyFactory(object): self._app.status(*args, **kwargs) def _get_morphology_text(self, reponame, sha1, filename): + morph_name = os.path.splitext(os.path.basename(filename))[0] if self._lrc.has_repo(reponame): self.status(msg="Looking for %s in local repo cache" % filename, chatty=True) - repo = self._lrc.get_repo(reponame) - file_list = repo.ls_tree(sha1) - - if filename in file_list: - return repo.cat(sha1, filename) + try: + repo = self._lrc.get_repo(reponame) + text = repo.cat(sha1, filename) + except IOError: + text = None + file_list = repo.ls_tree(sha1) elif self._rrc is not None: - self.status(msg="Looking for %s in remote repo cache" % filename, + self.status(msg="Retrieving %(reponame)s %(sha1)s %(filename)s" + " from the remote artifact cache.", + reponame=reponame, sha1=sha1, filename=filename, chatty=True) - file_list = self._rrc.ls_tree(reponame, sha1) - - if filename in file_list: - self.status(msg='Retrieving %s %s %s' - 'from the remote artifact cache.' - % (reponame, sha1, filename), chatty=True) - return self._rrc.cat_file(reponame, sha1, filename) + try: + text = self._rrc.cat_file(reponame, sha1, filename) + except morphlib.remoterepocache.CatFileError: + text = None + file_list = self._rrc.ls_tree(reponame, sha1) else: raise NotcachedError(reponame) - self.status(msg="File %s doesn't exist: " - "attempting to infer chunk morph from repo's build system" - % filename, chatty=True) - bs = morphlib.buildsystem.detect_build_system(file_list) - if bs is None: - raise MorphologyNotFoundError(filename) - # TODO consider changing how morphs are located to be by morph - # name rather than filename, it would save creating a - # filename only to strip it back to its morph name again - # and would allow future changes like morphologies being - # stored as git metadata instead of as a file in the repo - morph_name = filename[:-len('.morph')] - return bs.get_morphology_text(morph_name) + if text is None: + self.status(msg="File %s doesn't exist: attempting to infer " + "chunk morph from repo's build system" + % filename, chatty=True) + bs = morphlib.buildsystem.detect_build_system(file_list) + if bs is None: + raise MorphologyNotFoundError(filename) + text = bs.get_morphology_text(morph_name) + return morph_name, text def get_morphology(self, reponame, sha1, filename): - text = self._get_morphology_text(reponame, sha1, filename) + morph_name, text = self._get_morphology_text(reponame, sha1, filename) try: morphology = morphlib.morph2.Morphology(text) @@ -118,7 +118,7 @@ class MorphologyFactory(object): raise morphlib.Error("Error parsing %s: %s" % (filename, str(e))) - if filename != morphology['name'] + '.morph': + if morph_name != morphology['name']: raise morphlib.Error( "Name %s does not match basename of morphology file %s" % (morphology['name'], filename)) diff --git a/morphlib/morphologyfinder.py b/morphlib/morphologyfinder.py index affa0e97..87c0de1a 100644 --- a/morphlib/morphologyfinder.py +++ b/morphlib/morphologyfinder.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -34,38 +34,29 @@ class MorphologyFinder(object): self.gitdir = gitdir self.ref = ref - def read_morphology(self, name): + def read_morphology(self, filename): '''Return the un-parsed text of a morphology. For the given morphology name, locate and return the contents of the morphology as a string. - Also returns a string describing where in the repository the - morphology is located. - Parsing of this morphology into a form useful for manipulating is handled by the MorphologyLoader class. ''' - filename = '%s.morph' % name - return self.gitdir.read_file(filename, self.ref), filename + return self.gitdir.read_file(filename, self.ref) def list_morphologies(self): - '''Return the names of all morphologies in the (repo, ref). + '''Return the filenames of all morphologies in the (repo, ref). Finds all morphologies in the git directory at the specified - ref. Morphology names are returned instead of filenames, - so the implementation may change how morphologies are stored - in git repositories. + ref. ''' def is_morphology_path(path): return path.endswith('.morph') - def transform_path_to_name(path): - return path[:-len('.morph')] - - return (transform_path_to_name(path) + return (path for path in self.gitdir.list_files(self.ref) if is_morphology_path(path)) diff --git a/morphlib/morphologyfinder_tests.py b/morphlib/morphologyfinder_tests.py index ff83ccff..b07b2613 100644 --- a/morphlib/morphologyfinder_tests.py +++ b/morphlib/morphologyfinder_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -54,13 +54,13 @@ class MorphologyFinderTests(unittest.TestCase): gd = morphlib.gitdir.GitDirectory(self.dirname) mf = morphlib.morphologyfinder.MorphologyFinder(gd, 'HEAD') self.assertEqual(sorted(mf.list_morphologies()), - ['bar', 'baz']) + ['bar.morph', 'baz.morph']) def test_list_morphs_in_master(self): gd = morphlib.gitdir.GitDirectory(self.dirname) mf = morphlib.morphologyfinder.MorphologyFinder(gd, 'master') self.assertEqual(sorted(mf.list_morphologies()), - ['bar', 'baz']) + ['bar.morph', 'baz.morph']) def test_list_morphs_raises_with_invalid_ref(self): gd = morphlib.gitdir.GitDirectory(self.dirname) @@ -72,7 +72,7 @@ class MorphologyFinderTests(unittest.TestCase): gd = morphlib.gitdir.GitDirectory(self.dirname) mf = morphlib.morphologyfinder.MorphologyFinder(gd) self.assertEqual(sorted(mf.list_morphologies()), - ['bar', 'baz', 'foo']) + ['bar.morph', 'baz.morph', 'foo.morph']) def test_list_morphs_raises_no_worktree_no_ref(self): gd = morphlib.gitdir.GitDirectory(self.mirror) @@ -83,13 +83,13 @@ class MorphologyFinderTests(unittest.TestCase): def test_read_morph_in_HEAD(self): gd = morphlib.gitdir.GitDirectory(self.dirname) mf = morphlib.morphologyfinder.MorphologyFinder(gd, 'HEAD') - self.assertEqual(mf.read_morphology('bar')[0], + self.assertEqual(mf.read_morphology('bar.morph'), "dummy morphology text") def test_read_morph_in_master(self): gd = morphlib.gitdir.GitDirectory(self.dirname) mf = morphlib.morphologyfinder.MorphologyFinder(gd, 'master') - self.assertEqual(mf.read_morphology('bar')[0], + self.assertEqual(mf.read_morphology('bar.morph'), "dummy morphology text") def test_read_morph_raises_with_invalid_ref(self): @@ -101,11 +101,11 @@ class MorphologyFinderTests(unittest.TestCase): def test_read_morph_in_work_tree(self): gd = morphlib.gitdir.GitDirectory(self.dirname) mf = morphlib.morphologyfinder.MorphologyFinder(gd) - self.assertEqual(mf.read_morphology('foo')[0], + self.assertEqual(mf.read_morphology('foo.morph'), "altered morphology text") def test_read_morph_raises_no_worktree_no_ref(self): gd = morphlib.gitdir.GitDirectory(self.mirror) mf = morphlib.morphologyfinder.MorphologyFinder(gd) self.assertRaises(morphlib.gitdir.NoWorkingTreeError, - mf.read_morphology, 'bar') + mf.read_morphology, 'bar.morph') diff --git a/morphlib/morphset.py b/morphlib/morphset.py index dedbabd5..590ac51e 100644 --- a/morphlib/morphset.py +++ b/morphlib/morphset.py @@ -19,19 +19,6 @@ import morphlib -class StratumNotInSystemError(morphlib.Error): - - def __init__(self, system_name, stratum_name): - self.msg = ( - 'System %s does not contain %s' % (system_name, stratum_name)) - - -class StratumNotInSetError(morphlib.Error): - - def __init__(self, stratum_name): - self.msg = 'Stratum %s is not in MorphologySet' % stratum_name - - class ChunkNotInStratumError(morphlib.Error): def __init__(self, stratum_name, chunk_name): @@ -79,31 +66,11 @@ class MorphologySet(object): def _find_spec(self, specs, wanted_name): for spec in specs: - name = spec.get('morph', spec.get('name')) + name = spec.get('name', spec.get('morph')) if name == wanted_name: return spec.get('repo'), spec.get('ref'), name return None, None, None - def get_stratum_in_system(self, system_morph, stratum_name): - '''Return morphology for a stratum that is in a system. - - If the stratum is not in the system, raise StratumNotInSystemError. - If the stratum morphology has not been added to the set, - raise StratumNotInSetError. - - ''' - - repo_url, ref, morph = self._find_spec( - system_morph['strata'], stratum_name) - if (repo_url, ref, morph) == (None, None, None): - raise StratumNotInSystemError(system_morph['name'], stratum_name) - m = self._get_morphology(repo_url or system_morph.repo_url, - ref or system_morph.ref, - '%s.morph' % morph) - if m is None: - raise StratumNotInSetError(stratum_name) - return m - def get_chunk_triplet(self, stratum_morph, chunk_name): '''Return the repo url, ref, morph name triplet for a chunk. @@ -160,8 +127,8 @@ class MorphologySet(object): specs = m[kind] for spec in specs: if cb_filter(m, kind, spec): - orig_spec = (spec.get('repo'), spec.get('ref'), - spec['morph']) + fn = morphlib.util.sanitise_morphology_path(spec['morph']) + orig_spec = (spec.get('repo'), spec.get('ref'), fn) dirtied = cb_process(m, kind, spec) if dirtied: m.dirty = True @@ -175,17 +142,18 @@ class MorphologySet(object): process_spec_list(m, 'chunks') for m in self.morphologies: - tup = (m.repo_url, m.ref, m.filename[:-len('.morph')]) + tup = (m.repo_url, m.ref, m.filename) if tup in altered_references: spec = altered_references[tup] if m.ref != spec.get('ref'): m.ref = spec.get('ref') m.dirty = True - assert (m.filename == spec['morph'] + '.morph' + file = morphlib.util.sanitise_morphology_path(spec['morph']) + assert (m.filename == file or m.repo_url == spec.get('repo')), \ 'Moving morphologies is not supported.' - def change_ref(self, repo_url, orig_ref, morph_filename, new_ref): + def change_ref(self, repo_url, orig_ref, morph_name, new_ref): '''Change a triplet's ref to a new one in all morphologies in a ref. Change orig_ref to new_ref in any morphology that references the @@ -194,9 +162,10 @@ class MorphologySet(object): ''' def wanted_spec(m, kind, spec): + spec_name = spec.get('name', spec['morph']) return (spec.get('repo') == repo_url and spec.get('ref') == orig_ref and - spec['morph'] + '.morph' == morph_filename) + spec_name == morph_name) def process_spec(m, kind, spec): spec['unpetrify-ref'] = spec.get('ref') diff --git a/morphlib/morphset_tests.py b/morphlib/morphset_tests.py index d6908844..8679c64a 100644 --- a/morphlib/morphset_tests.py +++ b/morphlib/morphset_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013, 2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -80,27 +80,6 @@ class MorphologySetTests(unittest.TestCase): self.morphs.add_morphology(self.system) self.assertEqual(self.morphs.morphologies, [self.system]) - def test_get_stratum_in_system(self): - self.morphs.add_morphology(self.system) - self.morphs.add_morphology(self.stratum) - self.assertEqual( - self.morphs.get_stratum_in_system( - self.system, self.stratum['name']), - self.stratum) - - def test_raises_stratum_not_in_system_error(self): - self.morphs.add_morphology(self.system) - self.morphs.add_morphology(self.stratum) - self.assertRaises( - morphlib.morphset.StratumNotInSystemError, - self.morphs.get_stratum_in_system, self.system, 'unknown-stratum') - - def test_raises_stratum_not_in_set_error(self): - self.morphs.add_morphology(self.system) - self.assertRaises( - morphlib.morphset.StratumNotInSetError, - self.morphs.get_stratum_in_system, self.system, 'foo-stratum') - def test_get_chunk_triplet(self): self.morphs.add_morphology(self.system) self.morphs.add_morphology(self.stratum) @@ -119,7 +98,7 @@ class MorphologySetTests(unittest.TestCase): self.morphs.change_ref( self.stratum.repo_url, self.stratum.ref, - self.stratum.filename, + self.stratum['name'], 'new-ref') self.assertEqual(self.stratum.ref, 'new-ref') self.assertEqual( @@ -155,7 +134,7 @@ class MorphologySetTests(unittest.TestCase): self.morphs.change_ref( self.stratum.repo_url, self.stratum.ref, - self.stratum.filename, + self.stratum['name'], 'new-ref') self.assertEqual( other_stratum['build-depends'][0], @@ -172,7 +151,7 @@ class MorphologySetTests(unittest.TestCase): self.morphs.change_ref( 'test:foo-chunk', 'master', - 'foo-chunk.morph', + 'foo-chunk', 'new-ref') self.assertEqual( self.stratum['chunks'], diff --git a/morphlib/plugins/branch_and_merge_new_plugin.py b/morphlib/plugins/branch_and_merge_new_plugin.py deleted file mode 100644 index 5ac8353a..00000000 --- a/morphlib/plugins/branch_and_merge_new_plugin.py +++ /dev/null @@ -1,829 +0,0 @@ -# Copyright (C) 2012,2013,2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import cliapp -import contextlib -import glob -import logging -import os -import shutil - -import morphlib - - -class BranchRootHasNoSystemsError(cliapp.AppException): - def __init__(self, repo, ref): - cliapp.AppException.__init__( - self, 'System branch root repository %s ' - 'has no system morphologies at ref %s' % (repo, ref)) - - -class SimpleBranchAndMergePlugin(cliapp.Plugin): - - '''Add subcommands for handling workspaces and system branches.''' - - def enable(self): - self.app.add_subcommand('init', self.init, arg_synopsis='[DIR]') - self.app.add_subcommand('workspace', self.workspace, arg_synopsis='') - self.app.add_subcommand( - 'checkout', self.checkout, arg_synopsis='REPO BRANCH') - self.app.add_subcommand( - 'branch', self.branch, arg_synopsis='REPO NEW [OLD]') - self.app.add_subcommand( - 'edit', self.edit, arg_synopsis='SYSTEM STRATUM [CHUNK]') - self.app.add_subcommand( - 'petrify', self.petrify, arg_synopsis='') - self.app.add_subcommand( - 'unpetrify', self.unpetrify, arg_synopsis='') - self.app.add_subcommand( - 'show-system-branch', self.show_system_branch, arg_synopsis='') - self.app.add_subcommand( - 'show-branch-root', self.show_branch_root, arg_synopsis='') - self.app.add_subcommand('foreach', self.foreach, - arg_synopsis='-- COMMAND [ARGS...]') - self.app.add_subcommand('status', self.status, - arg_synopsis='') - self.app.add_subcommand('branch-from-image', self.branch_from_image, - arg_synopsis='BRANCH') - group_branch = 'Branching Options' - self.app.settings.string(['metadata-dir'], - 'Set metadata location for branch-from-image' - ' (default: /baserock)', - metavar='DIR', - default='/baserock', - group=group_branch) - - def disable(self): - pass - - def init(self, args): - '''Initialize a workspace directory. - - Command line argument: - - * `DIR` is the directory to use as a workspace, and defaults to - the current directory. - - This creates a workspace, either in the current working directory, - or if `DIR` is given, in that directory. If the directory doesn't - exist, it is created. If it does exist, it must be empty. - - You need to run `morph init` to initialise a workspace, or none - of the other system branching tools will work: they all assume - an existing workspace. Note that a workspace only exists on your - machine, not on the git server. - - Example: - - morph init /src/workspace - cd /src/workspace - - ''' - - if not args: - args = ['.'] - elif len(args) > 1: - raise morphlib.Error('init must get at most one argument') - - ws = morphlib.workspace.create(args[0]) - self.app.status(msg='Initialized morph workspace', chatty=True) - - def workspace(self, args): - '''Show the toplevel directory of the current workspace.''' - - ws = morphlib.workspace.open('.') - self.app.output.write('%s\n' % ws.root) - - # TODO: Move this somewhere nicer - @contextlib.contextmanager - def _initializing_system_branch(self, ws, root_url, system_branch, - cached_repo, base_ref): - '''A context manager for system branches under construction. - - The purpose of this context manager is to factor out the branch - cleanup code for if an exception occurs while a branch is being - constructed. - - This could be handled by a higher order function which takes - a function to initialize the branch as a parameter, but with - statements look nicer and are more obviously about resource - cleanup. - - ''' - root_dir = ws.get_default_system_branch_directory_name(system_branch) - try: - sb = morphlib.sysbranchdir.create( - root_dir, root_url, system_branch) - gd = sb.clone_cached_repo(cached_repo, base_ref) - - yield (sb, gd) - - gd.update_submodules(self.app) - gd.update_remotes() - - except morphlib.sysbranchdir.SystemBranchDirectoryAlreadyExists as e: - logging.error('Caught exception: %s' % str(e)) - raise - except BaseException as e: - # Oops. Clean up. - logging.error('Caught exception: %s' % str(e)) - logging.info('Removing half-finished branch %s' % system_branch) - self._remove_branch_dir_safe(ws.root, root_dir) - raise - - def checkout(self, args): - '''Check out an existing system branch. - - Command line arguments: - - * `REPO` is the URL to the repository to the root repository of - a system branch. - * `BRANCH` is the name of the system branch. - - This will check out an existing system branch to an existing - workspace. You must create the workspace first. This only checks - out the root repository, not the repositories for individual - components. You need to use `morph edit` to check out those. - - Example: - - cd /src/workspace - morph checkout baserock:baserock/morphs master - - ''' - - if len(args) != 2: - raise cliapp.AppException('morph checkout needs a repo and the ' - 'name of a branch as parameters') - - root_url = args[0] - system_branch = args[1] - base_ref = system_branch - - self._require_git_user_config() - - # Open the workspace first thing, so user gets a quick error if - # we're not inside a workspace. - ws = morphlib.workspace.open('.') - - # Make sure the root repository is in the local git repository - # cache, and is up to date. - lrc, rrc = morphlib.util.new_repo_caches(self.app) - cached_repo = lrc.get_updated_repo(root_url) - - # Check the git branch exists. - cached_repo.resolve_ref(system_branch) - - with self._initializing_system_branch( - ws, root_url, system_branch, cached_repo, base_ref) as (sb, gd): - - if gd.has_fat(): - gd.fat_init() - gd.fat_pull() - - if not self._checkout_has_systems(gd): - raise BranchRootHasNoSystemsError(root_url, base_ref) - - - def branch(self, args): - '''Create a new system branch. - - Command line arguments: - - * `REPO` is a repository URL. - * `NEW` is the name of the new system branch. - * `OLD` is the point from which to branch, and defaults to `master`. - - This creates a new system branch. It needs to be run in an - existing workspace (see `morph workspace`). It creates a new - git branch in the clone of the repository in the workspace. The - system branch will not be visible on the git server until you - push your changes to the repository. - - Example: - - cd /src/workspace - morph branch baserock:baserock/morphs jrandom/new-feature - - ''' - - if len(args) not in [2, 3]: - raise cliapp.AppException( - 'morph branch needs name of branch as parameter') - - root_url = args[0] - system_branch = args[1] - base_ref = 'master' if len(args) == 2 else args[2] - origin_base_ref = 'origin/%s' % base_ref - - self._require_git_user_config() - - # Open the workspace first thing, so user gets a quick error if - # we're not inside a workspace. - ws = morphlib.workspace.open('.') - - # Make sure the root repository is in the local git repository - # cache, and is up to date. - lrc, rrc = morphlib.util.new_repo_caches(self.app) - cached_repo = lrc.get_updated_repo(root_url) - - # Make sure the system branch doesn't exist yet. - if cached_repo.ref_exists(system_branch): - raise cliapp.AppException( - 'branch %s already exists in repository %s' % - (system_branch, root_url)) - - # Make sure the base_ref exists. - cached_repo.resolve_ref(base_ref) - - with self._initializing_system_branch( - ws, root_url, system_branch, cached_repo, base_ref) as (sb, gd): - - gd.branch(system_branch, base_ref) - gd.checkout(system_branch) - if gd.has_fat(): - gd.fat_init() - gd.fat_pull() - - if not self._checkout_has_systems(gd): - raise BranchRootHasNoSystemsError(root_url, base_ref) - - def _save_dirty_morphologies(self, loader, sb, morphs): - logging.debug('Saving dirty morphologies: start') - for morph in morphs: - if morph.dirty: - logging.debug( - 'Saving morphology: %s %s %s' % - (morph.repo_url, morph.ref, morph.filename)) - loader.unset_defaults(morph) - loader.save_to_file( - sb.get_filename(morph.repo_url, morph.filename), morph) - morph.dirty = False - logging.debug('Saving dirty morphologies: done') - - def _get_stratum_triplets(self, morph): - # Gather all references to other strata from a morphology. The - # morphology must be either a system or a stratum one. In a - # stratum one, the refs are all for build dependencies of the - # stratum. In a system one, they're the list of strata in the - # system. - - assert morph['kind'] in ('system', 'stratum') - if morph['kind'] == 'system': - specs = morph.get('strata', []) - elif morph['kind'] == 'stratum': - specs = morph.get('build-depends', []) - - # Given a list of dicts that reference strata, return a list - # of triplets (repo url, ref, filename). - - return [ - (spec.get('repo') or morph.repo_url, - spec.get('ref') or morph.ref, - '%s.morph' % spec['morph']) - for spec in specs - ] - - def _checkout(self, lrc, sb, repo_url, ref): - logging.debug( - 'Checking out %s (%s) into %s' % - (repo_url, ref, sb.root_directory)) - cached_repo = lrc.get_updated_repo(repo_url) - gd = sb.clone_cached_repo(cached_repo, ref) - gd.update_submodules(self.app) - gd.update_remotes() - - def _load_morphology_from_file(self, loader, dirname, filename): - full_filename = os.path.join(dirname, filename) - return loader.load_from_file(full_filename) - - def _load_morphology_from_git(self, loader, gd, ref, filename): - try: - text = gd.get_file_from_ref(ref, filename) - except cliapp.AppException: - text = gd.get_file_from_ref('origin/%s' % ref, filename) - return loader.load_from_string(text, filename) - - def _load_stratum_morphologies(self, loader, sb, system_morph): - logging.debug('Starting to load strata for %s' % system_morph.filename) - lrc, rrc = morphlib.util.new_repo_caches(self.app) - morphset = morphlib.morphset.MorphologySet() - queue = self._get_stratum_triplets(system_morph) - while queue: - repo_url, ref, filename = queue.pop() - if not morphset.has(repo_url, ref, filename): - logging.debug('Loading: %s %s %s' % (repo_url, ref, filename)) - dirname = sb.get_git_directory_name(repo_url) - - # Get the right morphology. The right ref might not be - # checked out, in which case we get the file from git. - # However, if it is checked out, we get it from the - # filesystem directly, in case the user has made any - # changes to it. If the entire repo hasn't been checked - # out yet, do that first. - - if not os.path.exists(dirname): - self._checkout(lrc, sb, repo_url, ref) - m = self._load_morphology_from_file( - loader, dirname, filename) - else: - gd = morphlib.gitdir.GitDirectory(dirname) - if gd.is_currently_checked_out(ref): - m = self._load_morphology_from_file( - loader, dirname, filename) - else: - m = self._load_morphology_from_git( - loader, gd, ref, filename) - - m.repo_url = repo_url - m.ref = ref - m.filename = filename - - morphset.add_morphology(m) - queue.extend(self._get_stratum_triplets(m)) - - logging.debug('All strata loaded') - return morphset - - def edit(self, args): - '''Edit or checkout a component in a system branch. - - Command line arguments: - - * `CHUNK` is the name of a chunk - - This makes a local checkout of CHUNK in the current system branch - and edits any stratum morphology file(s) containing the chunk - - ''' - - if len(args) != 1: - raise cliapp.AppException('morph edit needs a chunk ' - 'as parameter') - - ws = morphlib.workspace.open('.') - sb = morphlib.sysbranchdir.open_from_within('.') - loader = morphlib.morphloader.MorphologyLoader() - morphs = self._load_all_sysbranch_morphologies(sb, loader) - - def edit_chunk(morph, chunk_name): - chunk_url, chunk_ref, chunk_morph = ( - morphs.get_chunk_triplet(morph, chunk_name)) - - chunk_dirname = sb.get_git_directory_name(chunk_url) - - if not os.path.exists(chunk_dirname): - lrc, rrc = morphlib.util.new_repo_caches(self.app) - cached_repo = lrc.get_updated_repo(chunk_url) - - gd = sb.clone_cached_repo(cached_repo, chunk_ref) - if chunk_ref != sb.system_branch_name: - gd.branch(sb.system_branch_name, chunk_ref) - gd.checkout(sb.system_branch_name) - gd.update_submodules(self.app) - gd.update_remotes() - if gd.has_fat(): - gd.fat_init() - gd.fat_pull() - - # Change the refs to the chunk. - if chunk_ref != sb.system_branch_name: - morphs.change_ref( - chunk_url, chunk_ref, chunk_morph + '.morph', - sb.system_branch_name) - - return chunk_dirname - - chunk_name = morphlib.util.strip_morph_extension(args[0]) - dirs = set() - found = 0 - - for morph in morphs.morphologies: - if morph['kind'] == 'stratum': - for chunk in morph['chunks']: - if chunk['name'] == chunk_name: - self.app.status( - msg='Editing %(chunk)s in %(stratum)s stratum', - chunk=chunk_name, stratum=morph['name']) - chunk_dirname = edit_chunk(morph, chunk_name) - dirs.add(chunk_dirname) - found = found + 1 - - # Save any modified strata. - - self._save_dirty_morphologies(loader, sb, morphs.morphologies) - - if found == 0: - self.app.status( - msg="No chunk %(chunk)s found. If you want to create one, add " - "an entry to a stratum morph file.", chunk=chunk_name) - - if found >= 1: - dirs_list = ', '.join(sorted(dirs)) - self.app.status( - msg="Chunk %(chunk)s source is available at %(dirs)s", - chunk=chunk_name, dirs=dirs_list) - - if found > 1: - self.app.status( - msg="Notice that this chunk appears in more than one stratum") - - def show_system_branch(self, args): - '''Show the name of the current system branch.''' - - ws = morphlib.workspace.open('.') - sb = morphlib.sysbranchdir.open_from_within('.') - self.app.output.write('%s\n' % sb.system_branch_name) - - def show_branch_root(self, args): - '''Show the name of the repository holding the system morphologies. - - This would, for example, write out something like: - - /src/ws/master/baserock:baserock/morphs - - when the master branch of the `baserock:baserock/morphs` - repository is checked out. - - ''' - - ws = morphlib.workspace.open('.') - sb = morphlib.sysbranchdir.open_from_within('.') - self.app.output.write('%s\n' % sb.get_config('branch.root')) - - def _remove_branch_dir_safe(self, workspace_root, system_branch_root): - # This function avoids throwing any exceptions, so it is safe to call - # inside an 'except' block without altering the backtrace. - - def handle_error(function, path, excinfo): - logging.warning ("Error while trying to clean up %s: %s" % - (path, excinfo)) - - shutil.rmtree(system_branch_root, onerror=handle_error) - - # Remove parent directories that are empty too, avoiding exceptions - parent = os.path.dirname(system_branch_root) - while parent != os.path.abspath(workspace_root): - if len(os.listdir(parent)) > 0 or os.path.islink(parent): - break - os.rmdir(parent) - parent = os.path.dirname(parent) - - def _require_git_user_config(self): - '''Warn if the git user.name and user.email variables are not set.''' - - keys = { - 'user.name': 'My Name', - 'user.email': 'me@example.com', - } - - try: - morphlib.git.check_config_set(self.app.runcmd, keys) - except morphlib.git.ConfigNotSetException as e: - self.app.status( - msg="WARNING: %(message)s", - message=str(e), error=True) - - @staticmethod - def _checkout_has_systems(gd): - loader = morphlib.morphloader.MorphologyLoader() - for filename in glob.iglob(os.path.join(gd.dirname, '*.morph')): - m = loader.load_from_file(filename) - if m['kind'] == 'system': - return True - return False - - def foreach(self, args): - '''Run a command in each repository checked out in a system branch. - - Use -- before specifying the command to separate its arguments from - Morph's own arguments. - - Command line arguments: - - * `--` indicates the end of option processing for Morph. - * `COMMAND` is a command to run. - * `ARGS` is a list of arguments or options to be passed onto - `COMMAND`. - - This runs the given `COMMAND` in each git repository belonging - to the current system branch that exists locally in the current - workspace. This can be a handy way to do the same thing in all - the local git repositories. - - For example: - - morph foreach -- git push - - The above command would push any committed changes in each - repository to the git server. - - ''' - - if not args: - raise cliapp.AppException('morph foreach expects a command to run') - - ws = morphlib.workspace.open('.') - sb = morphlib.sysbranchdir.open_from_within('.') - - for gd in sorted(sb.list_git_directories(), key=lambda gd: gd.dirname): - # Get the repository's original name - # Continue in the case of error, since the previous iteration - # worked in the case of the user cloning a repository in the - # system branch's directory. - try: - repo = gd.get_config('morph.repository') - except cliapp.AppException: - continue - - self.app.output.write('%s\n' % repo) - status, output, error = self.app.runcmd_unchecked( - args, cwd=gd.dirname) - self.app.output.write(output) - if status != 0: - self.app.output.write(error) - pretty_command = ' '.join(cliapp.shell_quote(arg) - for arg in args) - raise cliapp.AppException( - 'Command failed at repo %s: %s' - % (repo, pretty_command)) - self.app.output.write('\n') - self.app.output.flush() - - def _load_all_sysbranch_morphologies(self, sb, loader): - '''Read in all the morphologies in the root repository.''' - self.app.status(msg='Loading in all morphologies') - morphs = morphlib.morphset.MorphologySet() - for morph in sb.load_all_morphologies(loader): - morphs.add_morphology(morph) - return morphs - - def petrify(self, args): - '''Convert all chunk refs in a system branch to be fixed SHA1s. - - This modifies all git commit references in system and stratum - morphologies, in the current system branch, to be fixed SHA - commit identifiers, rather than symbolic branch or tag names. - This is useful for making sure none of the components in a system - branch change accidentally. - - Consider the following scenario: - - * The `master` system branch refers to `gcc` using the - `baserock/morph` ref. This is appropriate, since the main line - of development should use the latest curated code. - - * You create a system branch to prepare for a release, called - `TROVE_ID/release/2.0`. The reference to `gcc` is still - `baserock/morph`. - - * You test everything, and make a release. You deploy the release - images onto devices, which get shipped to your customers. - - * A new version GCC is committed to the `baserock/morph` branch. - - * Your release branch suddenly uses a new compiler, which may - or may not work for your particular system at that release. - - To avoid this, you need to _petrify_ all git references - so that they do not change accidentally. If you've tested - your release with the GCC release that is stored in commit - `94c50665324a7aeb32f3096393ec54b2e63bfb28`, then you should - continue to use that version of GCC, regardless of what might - happen in the master system branch. If, and only if, you decide - that a new compiler would be good for your release should you - include it in your release branch. This way, only the things - that you change intentionally change in your release branch. - - ''' - - if args: - raise cliapp.AppException('morph petrify takes no arguments') - - ws = morphlib.workspace.open('.') - sb = morphlib.sysbranchdir.open_from_within('.') - loader = morphlib.morphloader.MorphologyLoader() - lrc, rrc = morphlib.util.new_repo_caches(self.app) - update_repos = not self.app.settings['no-git-update'] - - morphs = self._load_all_sysbranch_morphologies(sb, loader) - - #TODO: Stop using app.resolve_ref - def resolve_refs(morphs): - for repo, ref in morphs.list_refs(): - # You can't resolve null refs, so don't attempt to. - if repo is None or ref is None: - continue - # TODO: Handle refs that are only in workspace in general - if (repo == sb.root_repository_url - and ref == sb.system_branch_name): - continue - commit_sha1, tree_sha1 = self.app.resolve_ref( - lrc, rrc, repo, ref, update=update_repos) - yield ((repo, ref), commit_sha1) - - morphs.repoint_refs(sb.root_repository_url, - sb.system_branch_name) - - morphs.petrify_chunks(dict(resolve_refs(morphs))) - - # Write morphologies back out again. - self._save_dirty_morphologies(loader, sb, morphs.morphologies) - - def unpetrify(self, args): - '''Reverse the process of petrification. - - This undoes the changes `morph petrify` did. - - ''' - - if args: - raise cliapp.AppException('morph petrify takes no arguments') - - ws = morphlib.workspace.open('.') - sb = morphlib.sysbranchdir.open_from_within('.') - loader = morphlib.morphloader.MorphologyLoader() - - morphs = self._load_all_sysbranch_morphologies(sb, loader) - - # Restore the ref for each stratum and chunk - morphs.unpetrify_all() - - # Write morphologies back out again. - self._save_dirty_morphologies(loader, sb, morphs.morphologies) - - def status(self, args): - '''Show information about the current system branch or workspace - - This shows the status of every local git repository of the - current system branch. This is similar to running `git status` - in each repository separately. - - If run in a Morph workspace, but not in a system branch checkout, - it lists all checked out system branches in the workspace. - - ''' - - if args: - raise cliapp.AppException('morph status takes no arguments') - - ws = morphlib.workspace.open('.') - try: - sb = morphlib.sysbranchdir.open_from_within('.') - except morphlib.sysbranchdir.NotInSystemBranch: - self._workspace_status(ws) - else: - self._branch_status(ws, sb) - - def _workspace_status(self, ws): - '''Show information about the current workspace - - This lists all checked out system branches in the workspace. - - ''' - self.app.output.write("System branches in current workspace:\n") - branches = sorted(ws.list_system_branches(), - key=lambda x: x.root_directory) - for sb in branches: - self.app.output.write(" %s\n" % sb.get_config('branch.name')) - - def _branch_status(self, ws, sb): - '''Show information about the current branch - - This shows the status of every local git repository of the - current system branch. This is similar to running `git status` - in each repository separately. - - ''' - branch = sb.get_config('branch.name') - root = sb.get_config('branch.root') - - self.app.output.write("On branch %s, root %s\n" % (branch, root)) - - has_uncommitted_changes = False - for gd in sorted(sb.list_git_directories(), key=lambda x: x.dirname): - try: - repo = gd.get_config('morph.repository') - except cliapp.AppException: - self.app.output.write( - ' %s: not part of system branch\n' % gd.dirname) - # TODO: make this less vulnerable to a branch using - # refs/heads/foo instead of foo - head = gd.HEAD - if head != branch: - self.app.output.write( - ' %s: unexpected ref checked out %r\n' % (repo, head)) - if any(gd.get_index().get_uncommitted_changes()): - has_uncommitted_changes = True - self.app.output.write(' %s: uncommitted changes\n' % repo) - - if not has_uncommitted_changes: - self.app.output.write("\nNo repos have outstanding changes.\n") - - def branch_from_image(self, args): - '''Produce a branch of an existing system image. - - Given the metadata specified by --metadata-dir, create a new - branch then petrify it to the state of the commits at the time - the system was built. - - If --metadata-dir is not specified, it defaults to your currently - running system. - - ''' - if len(args) != 1: - raise cliapp.AppException( - "branch-from-image needs exactly 1 argument " - "of the new system branch's name") - system_branch = args[0] - metadata_path = self.app.settings['metadata-dir'] - alias_resolver = morphlib.repoaliasresolver.RepoAliasResolver( - self.app.settings['repo-alias']) - - self._require_git_user_config() - - ws = morphlib.workspace.open('.') - - system, metadata = self._load_system_metadata(metadata_path) - resolved_refs = dict(self._resolve_refs_from_metadata(alias_resolver, - metadata)) - self.app.status(msg='Resolved refs: %r' % resolved_refs) - base_ref = system['sha1'] - # The previous version would fall back to deducing this from the repo - # url and the repo alias resolver, but this does not always work, and - # new systems always have repo-alias in the metadata - root_url = system['repo-alias'] - - lrc, rrc = morphlib.util.new_repo_caches(self.app) - cached_repo = lrc.get_updated_repo(root_url) - - - with self._initializing_system_branch( - ws, root_url, system_branch, cached_repo, base_ref) as (sb, gd): - - # TODO: It's nasty to clone to a sha1 then create a branch - # of that sha1 then check it out, a nicer API may be the - # initial clone not checking out a branch at all, then - # the user creates and checks out their own branches - gd.branch(system_branch, base_ref) - gd.checkout(system_branch) - - loader = morphlib.morphloader.MorphologyLoader() - morphs = self._load_all_sysbranch_morphologies(sb, loader) - - morphs.repoint_refs(sb.root_repository_url, - sb.system_branch_name) - - morphs.petrify_chunks(resolved_refs) - - self._save_dirty_morphologies(loader, sb, morphs.morphologies) - - @staticmethod - def _load_system_metadata(path): - '''Load all metadata in `path` corresponding to a single System. - ''' - - smd = morphlib.systemmetadatadir.SystemMetadataDir(path) - metadata = smd.values() - systems = [md for md in metadata - if 'kind' in md and md['kind'] == 'system'] - - if not systems: - raise cliapp.AppException( - 'Metadata directory does not contain any systems.') - if len(systems) > 1: - raise cliapp.AppException( - 'Metadata directory contains multiple systems.') - system_metadatum = systems[0] - - metadata_cache_id_lookup = dict((md['cache-key'], md) - for md in metadata) - - return system_metadatum, metadata_cache_id_lookup - - @staticmethod - def _resolve_refs_from_metadata(alias_resolver, metadata): - '''Pre-resolve a set of refs from existing metadata. - - Given the metadata, generate a mapping of all the (repo, ref) - pairs defined in the metadata and the commit id they resolved to. - - ''' - for md in metadata.itervalues(): - repourls = set((md['repo-alias'], md['repo'])) - repourls.update(alias_resolver.aliases_from_url(md['repo'])) - for repourl in repourls: - yield ((repourl, md['original_ref']), md['sha1']) diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py index d268decf..a66098b8 100644 --- a/morphlib/plugins/branch_and_merge_plugin.py +++ b/morphlib/plugins/branch_and_merge_plugin.py @@ -15,882 +15,476 @@ import cliapp -import copy -import functools +import contextlib import glob import logging import os import shutil -import socket -import tempfile -import time -import urlparse -import uuid import morphlib -def warns_git_config(keys): - def decorator(func): - @functools.wraps(func) - def check_config(self, *args, **kwargs): - try: - morphlib.git.check_config_set(self.app.runcmd, keys) - except cliapp.AppException, e: - self.app.status(msg="WARNING: %(message)s", - message=str(e), error=True) - return func(self, *args, **kwargs) - return check_config - - return decorator - - -warns_git_identity = warns_git_config({'user.name': 'My Name', - 'user.email': 'me@example.com'}) - - class BranchAndMergePlugin(cliapp.Plugin): - def __init__(self): - # Start recording changes. - self.init_changelog() + '''Add subcommands for handling workspaces and system branches.''' def enable(self): - # User-facing commands - self.app.add_subcommand('merge', self.merge, - arg_synopsis='BRANCH') -# self.app.add_subcommand('edit', self.edit, -# arg_synopsis='SYSTEM STRATUM [CHUNK]') - self.app.add_subcommand('old-petrify', self.petrify) - self.app.add_subcommand('old-unpetrify', self.unpetrify) + self.app.add_subcommand('init', self.init, arg_synopsis='[DIR]') + self.app.add_subcommand('workspace', self.workspace, arg_synopsis='') + self.app.add_subcommand( + 'checkout', self.checkout, arg_synopsis='REPO BRANCH') + self.app.add_subcommand( + 'branch', self.branch, arg_synopsis='REPO NEW [OLD]') + self.app.add_subcommand( + 'edit', self.edit, arg_synopsis='SYSTEM STRATUM [CHUNK]') self.app.add_subcommand( - 'tag', self.tag, arg_synopsis='TAG-NAME -- [GIT-COMMIT-ARG...]') - self.app.add_subcommand('old-build', self.build, - arg_synopsis='SYSTEM') - self.app.add_subcommand('old-status', self.status) - self.app.add_subcommand('old-branch-from-image', - self.branch_from_image, - arg_synopsis='REPO BRANCH') - - # Advanced commands - self.app.add_subcommand('old-foreach', self.foreach, + 'petrify', self.petrify, arg_synopsis='') + self.app.add_subcommand( + 'unpetrify', self.unpetrify, arg_synopsis='') + self.app.add_subcommand( + 'show-system-branch', self.show_system_branch, arg_synopsis='') + self.app.add_subcommand( + 'show-branch-root', self.show_branch_root, arg_synopsis='') + self.app.add_subcommand('foreach', self.foreach, arg_synopsis='-- COMMAND [ARGS...]') + self.app.add_subcommand('status', self.status, + arg_synopsis='') + self.app.add_subcommand('branch-from-image', self.branch_from_image, + arg_synopsis='BRANCH') + group_branch = 'Branching Options' + self.app.settings.string(['metadata-dir'], + 'Set metadata location for branch-from-image' + ' (default: /baserock)', + metavar='DIR', + default='/baserock', + group=group_branch) def disable(self): pass - def init_changelog(self): - self.changelog = {} - - def log_change(self, repo, text): - if not repo in self.changelog: - self.changelog[repo] = [] - self.changelog[repo].append(text) - - def print_changelog(self, title, early_keys=[]): - if self.changelog and self.app.settings['verbose']: - msg = '\n%s:\n\n' % title - keys = [x for x in early_keys if x in self.changelog] - keys.extend([x for x in self.changelog if x not in early_keys]) - for key in keys: - messages = self.changelog[key] - msg += ' %s:\n' % key - msg += '\n'.join([' %s' % x for x in messages]) - msg += '\n\n' - self.app.output.write(msg) + def init(self, args): + '''Initialize a workspace directory. - @staticmethod - def deduce_workspace(): - dirname = os.getcwd() - while dirname != '/': - dot_morph = os.path.join(dirname, '.morph') - if os.path.isdir(dot_morph): - return dirname - dirname = os.path.dirname(dirname) - raise cliapp.AppException("Can't find the workspace directory.\n" - "Morph must be built and deployed within " - "the system branch checkout within the " - "workspace directory.") - - def deduce_system_branch(self): - # 1. Deduce the workspace. If this fails, we're not inside a workspace. - workspace = self.deduce_workspace() - - # 2. We're in a workspace. Check if we're inside a system branch. - # If we are, return its name. - dirname = os.getcwd() - while dirname != workspace and dirname != '/': - if os.path.isdir(os.path.join(dirname, '.morph-system-branch')): - branch_name = self.get_branch_config(dirname, 'branch.name') - return branch_name, dirname - dirname = os.path.dirname(dirname) - - # 3. We're in a workspace but not inside a branch. Try to find a - # branch directory in the directories below the current working - # directory. Avoid ambiguity by only recursing deeper if there - # is only one subdirectory. - for dirname in self.walk_special_directories( - os.getcwd(), special_subdir='.morph-system-branch', - max_subdirs=1): - branch_name = self.get_branch_config(dirname, 'branch.name') - return branch_name, dirname - - raise cliapp.AppException("Can't find the system branch directory.\n" - "Morph must be built and deployed within " - "the system branch checkout.") - - def find_repository(self, branch_dir, repo): - for dirname in self.walk_special_directories(branch_dir, - special_subdir='.git'): - try: - original_repo = self.get_repo_config( - dirname, 'morph.repository') - except cliapp.AppException: - # The user may have manually put a git repo in the branch - continue - if repo == original_repo: - return dirname - return None - - def find_system_branch(self, workspace, branch_name): - for dirname in self.walk_special_directories( - workspace, special_subdir='.morph-system-branch'): - branch = self.get_branch_config(dirname, 'branch.name') - if branch_name == branch: - return dirname - return None - - def set_branch_config(self, branch_dir, option, value): - filename = os.path.join(branch_dir, '.morph-system-branch', 'config') - self.app.runcmd(['git', 'config', '-f', filename, option, value], - print_command=False) - - def get_branch_config(self, branch_dir, option): - filename = os.path.join(branch_dir, '.morph-system-branch', 'config') - value = self.app.runcmd(['git', 'config', '-f', filename, option], - print_command=False) - return value.strip() - - def set_repo_config(self, repo_dir, option, value): - self.app.runcmd(['git', 'config', option, value], cwd=repo_dir, - print_command=False) - - def get_repo_config(self, repo_dir, option): - value = self.app.runcmd(['git', 'config', option], cwd=repo_dir, - print_command=False) - return value.strip() - - def get_head(self, repo_path): - '''Return the ref that the working tree is on for a repo''' - - ref = self.app.runcmd(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], - cwd=repo_path).strip() - if ref == 'HEAD': - ref = 'detached HEAD' - return ref - - def get_uncommitted_changes(self, repo_dir, env={}): - status = self.app.runcmd(['git', 'status', '--porcelain'], - cwd=repo_dir, env=env) - changes = [] - for change in status.strip().splitlines(): - xy, paths = change.strip().split(' ', 1) - if xy != '??': - changes.append(paths.split()[0]) - return changes - - def get_unmerged_changes(self, repo_dir, env={}): - '''Identifies files which have unresolved merge conflicts''' - - # The second column of the git command output is set either if the - # file has changes in the working tree or if it has conflicts. - status = self.app.runcmd(['git', 'status', '--porcelain'], - cwd=repo_dir, env=env) - changes = [] - for change in status.strip().splitlines(): - xy, paths = change[0:2], change[2:].strip() - if xy[1] != ' ' and xy != '??': - changes.append(paths.split()[0]) - return changes - - def resolve_ref(self, repodir, ref): - try: - return self.app.runcmd(['git', 'rev-parse', '--verify', ref], - cwd=repodir)[0:40] - except cliapp.AppException, e: - logging.info( - 'Ignoring error executing git rev-parse: %s' % str(e)) - return None + Command line argument: - def resolve_reponame(self, reponame): - '''Return the full pull URL of a reponame.''' + * `DIR` is the directory to use as a workspace, and defaults to + the current directory. - resolver = morphlib.repoaliasresolver.RepoAliasResolver( - self.app.settings['repo-alias']) - return resolver.pull_url(reponame) + This creates a workspace, either in the current working directory, + or if `DIR` is given, in that directory. If the directory doesn't + exist, it is created. If it does exist, it must be empty. + + You need to run `morph init` to initialise a workspace, or none + of the other system branching tools will work: they all assume + an existing workspace. Note that a workspace only exists on your + machine, not on the git server. + + Example: - def get_cached_repo(self, repo_name): - '''Return CachedRepo object from the local repository cache + morph init /src/workspace + cd /src/workspace - Repo is cached and updated if necessary. The cache itself has a - mechanism in place to avoid multiple updates per Morph invocation. ''' - self.app.status(msg='Updating git repository %s in cache' % repo_name) - if not self.app.settings['no-git-update']: - repo = self.lrc.cache_repo(repo_name) - repo.update() - else: - repo = self.lrc.get_repo(repo_name) - return repo + if not args: + args = ['.'] + elif len(args) > 1: + raise morphlib.Error('init must get at most one argument') - def clone_to_directory(self, dirname, reponame, ref): - '''Clone a repository below a directory. + ws = morphlib.workspace.create(args[0]) + self.app.status(msg='Initialized morph workspace', chatty=True) - As a side effect, clone it into the local repo cache. + def workspace(self, args): + '''Show the toplevel directory of the current workspace.''' - ''' + ws = morphlib.workspace.open('.') + self.app.output.write('%s\n' % ws.root) - # Setup. - resolver = morphlib.repoaliasresolver.RepoAliasResolver( - self.app.settings['repo-alias']) - repo = self.get_cached_repo(reponame) - - # Make sure the parent directories needed for the repo dir exist. - parent_dir = os.path.dirname(dirname) - if not os.path.exists(parent_dir): - os.makedirs(parent_dir) - - # Clone it from cache to target directory. - target_path = os.path.abspath(dirname) - repo.clone_checkout(ref, target_path) - - # Remember the repo name we cloned from in order to be able - # to identify the repo again later using the same name, even - # if the user happens to rename the directory. - self.set_repo_config(dirname, 'morph.repository', reponame) - - # Create a UUID for the clone. We will use this for naming - # temporary refs, e.g. for building. - self.set_repo_config(dirname, 'morph.uuid', uuid.uuid4().hex) - - # URL configuration - morphlib.git.set_remote(self.app.runcmd, dirname, 'origin', repo.url) - self.set_repo_config( - dirname, 'url.%s.pushInsteadOf' % resolver.push_url(reponame), - resolver.pull_url(reponame)) - morphlib.git.update_submodules(self.app, target_path) - - self.app.runcmd(['git', 'remote', 'update'], cwd=dirname) - - def load_morphology(self, repo_dir, name, ref=None): - '''Loads a morphology from a repo in a system branch - - If 'ref' is specified, the version is taken from there instead of the - working tree. Note that you shouldn't use this to fetch files on - branches other than the current system branch, because the remote in - the system branch repo may be completely out of date. Use the local - repository cache instead for this. - ''' + # TODO: Move this somewhere nicer + @contextlib.contextmanager + def _initializing_system_branch(self, ws, root_url, system_branch, + cached_repo, base_ref): + '''A context manager for system branches under construction. - if ref is None: - filename = os.path.join(repo_dir, '%s.morph' % name) - with open(filename) as f: - text = f.read() - else: - filename = '%s.morph at ref %s in %s' % (name, ref, repo_dir) - if not morphlib.git.is_valid_sha1(ref): - ref = morphlib.git.rev_parse(self.app.runcmd, repo_dir, ref) - try: - text = self.app.runcmd(['git', 'cat-file', 'blob', - '%s:%s.morph' % (ref, name)], - cwd=repo_dir) - except cliapp.AppException as e: - msg = '%s.morph was not found in %s' % (name, repo_dir) - if ref is not None: - msg += ' at ref %s' % ref - raise cliapp.AppException(msg) + The purpose of this context manager is to factor out the branch + cleanup code for if an exception occurs while a branch is being + constructed. + This could be handled by a higher order function which takes + a function to initialize the branch as a parameter, but with + statements look nicer and are more obviously about resource + cleanup. + + ''' + root_dir = ws.get_default_system_branch_directory_name(system_branch) try: - morphology = morphlib.morph2.Morphology(text) - except ValueError as e: - raise morphlib.Error("Error parsing %s: %s" % - (filename, str(e))) - - self._validate_morphology(morphology, '%s.morph' % name) - - return morphology - - def _validate_morphology(self, morphology, basename): - # FIXME: This really should be in MorphologyFactory. Later. - - def require(field): - if field not in morphology: - raise morphlib.Error( - 'Required field "%s" is missing from morphology %s' % - (field, basename)) - - required = { - 'system': [ - 'name', - 'arch', - 'strata', - ], - 'stratum': [ - 'name', - 'chunks', - ], - 'chunk': [ - 'name', - ], - 'cluster': [ - 'name', - 'systems', - ], - } - - also_known = { - 'system': [ - 'kind', - 'description', - 'configuration-extensions', - ], - 'stratum': [ - 'kind', - 'description', - 'build-depends', - ], - 'chunk': [ - 'kind', - 'description', - 'build-system', - 'configure-commands', - 'build-commands', - 'test-commands', - 'install-commands', - 'max-jobs', - 'chunks', - 'devices', - ], - 'cluster': [ - 'kind' - ] - } - - require('kind') - kind = morphology['kind'] - if kind not in required: - raise morphlib.Error( - 'Unknown morphology kind "%s" in %s' % (kind, basename)) - for field in required[kind]: - require(field) - - known = required[kind] + also_known[kind] - for field in morphology.keys(): - if field not in known and not field.startswith('_orig_'): - msg = 'Unknown field "%s" in %s' % (field, basename) - logging.warning(msg) - self.app.status(msg=msg) - - def reset_work_tree_safe(self, repo_dir): - # This function avoids throwing any exceptions, so it is safe to call - # inside an 'except' block without altering the backtrace. + sb = morphlib.sysbranchdir.create( + root_dir, root_url, system_branch) + gd = sb.clone_cached_repo(cached_repo, base_ref) - command = 'git', 'reset', '--hard' - status, output, error = self.app.runcmd_unchecked(command, - cwd=repo_dir) - if status != 0: - logging.warning ("Warning: error while trying to clean up %s: %s" % - (repo_dir, error)) + yield (sb, gd) - @staticmethod - def update_morphology(repo_dir, name, morphology, output_fd=None): - if not name.endswith('.morph'): - name = '%s.morph' % name - filename = os.path.join(repo_dir, '%s' % name) - morphology.update_file(filename, output_fd=output_fd) + gd.update_submodules(self.app) + gd.update_remotes() - if name != morphology['name'] + '.morph': - logging.warning('%s: morphology "name" should match filename' % - filename) + except morphlib.sysbranchdir.SystemBranchDirectoryAlreadyExists as e: + logging.error('Caught exception: %s' % str(e)) + raise + except BaseException as e: + # Oops. Clean up. + logging.error('Caught exception: %s' % str(e)) + logging.info('Removing half-finished branch %s' % system_branch) + self._remove_branch_dir_safe(ws.root, root_dir) + raise - @staticmethod - def get_edit_info(morphology_name, morphology, name, collection='strata'): - try: - return morphology.lookup_child_by_name(name) - except KeyError: - if collection is 'strata': - raise cliapp.AppException( - 'Stratum "%s" not found in system "%s"' % - (name, morphology_name)) - else: - raise cliapp.AppException( - 'Chunk "%s" not found in stratum "%s"' % - (name, morphology_name)) + def checkout(self, args): + '''Check out an existing system branch. - @staticmethod - def convert_uri_to_path(uri): - parts = urlparse.urlparse(uri) - - # If the URI path is relative, assume it is an aliased repo (e.g. - # baserock:morphs). Otherwise assume it is a full URI where we need - # to strip off the scheme and .git suffix. - if not os.path.isabs(parts.path): - return uri - else: - path = parts.netloc - if parts.path.endswith('.git'): - path = os.path.join(path, parts.path[1:-len('.git')]) - else: - path = os.path.join(path, parts.path[1:]) - return path + Command line arguments: - @staticmethod - def remove_branch_dir_safe(workspace, branch): - # This function avoids throwing any exceptions, so it is safe to call - # inside an 'except' block without altering the backtrace. + * `REPO` is the URL to the repository to the root repository of + a system branch. + * `BRANCH` is the name of the system branch. - def handle_error(function, path, excinfo): - logging.warning ("Warning: error while trying to clean up %s: %s" % - (path, excinfo)) + This will check out an existing system branch to an existing + workspace. You must create the workspace first. This only checks + out the root repository, not the repositories for individual + components. You need to use `morph edit` to check out those. - branch_dir = os.path.join(workspace, branch) - shutil.rmtree(branch_dir, onerror=handle_error) + Example: - # Remove parent directories that are empty too, avoiding exceptions - parent = os.path.dirname(branch_dir) - while parent != os.path.abspath(workspace): - if len(os.listdir(parent)) > 0 or os.path.islink(parent): - break - os.rmdir(parent) - parent = os.path.dirname(parent) + cd /src/workspace + morph checkout baserock:baserock/morphs master - @staticmethod - def iterate_branch_repos(branch_path, root_repo_path): - '''Produces a sorted list of component repos in a branch checkout''' + ''' - dirs = [d for d in BranchAndMergePlugin.walk_special_directories( - branch_path, special_subdir='.git') - if not os.path.samefile(d, root_repo_path)] - dirs.sort() + if len(args) != 2: + raise cliapp.AppException('morph checkout needs a repo and the ' + 'name of a branch as parameters') - for d in [root_repo_path] + dirs: - yield d + root_url = args[0] + system_branch = args[1] + base_ref = system_branch - @staticmethod - def walk_special_directories(root_dir, special_subdir=None, max_subdirs=0): - assert(special_subdir is not None) - assert(max_subdirs >= 0) - - visited = set() - for dirname, subdirs, files in os.walk(root_dir, followlinks=True): - # Avoid infinite recursion due to symlinks. - if dirname in visited: - subdirs[:] = [] - continue - visited.add(dirname) + self._require_git_user_config() - # Check if the current directory has the special subdirectory. - if special_subdir in subdirs: - yield dirname + # Open the workspace first thing, so user gets a quick error if + # we're not inside a workspace. + ws = morphlib.workspace.open('.') - # Do not recurse into hidden directories. - subdirs[:] = [x for x in subdirs if not x.startswith('.')] + # Make sure the root repository is in the local git repository + # cache, and is up to date. + lrc, rrc = morphlib.util.new_repo_caches(self.app) + cached_repo = lrc.get_updated_repo(root_url) - # Do not recurse if there is more than the maximum number of - # subdirectories allowed. - if max_subdirs > 0 and len(subdirs) > max_subdirs: - break + # Check the git branch exists. + cached_repo.resolve_ref(system_branch) - def read_metadata(self, metadata_path): - '''Load every metadata file in `metadata_path`. - - Given a directory containing metadata, load them into memory - and retain the id of the system. + with self._initializing_system_branch( + ws, root_url, system_branch, cached_repo, base_ref) as (sb, gd): - Returns the cache_key of the system and a mapping of cache_key - to metadata. - ''' - self.app.status(msg='Reading metadata', chatty=True) - metadata_cache_id_lookup = {} - system_key = None - for path in sorted(glob.iglob(os.path.join(metadata_path, '*.meta'))): - with open(path) as f: - metadata = morphlib.util.json.load(f) - cache_key = metadata['cache-key'] - metadata_cache_id_lookup[cache_key] = metadata - - if metadata['kind'] == 'system': - if system_key is not None: - raise morphlib.Error( - "Metadata directory contains multiple systems.") - system_key = cache_key - - if system_key is None: - raise morphlib.Error( - "Metadata directory does not contain any systems.") - - return system_key, metadata_cache_id_lookup - - def _create_branch(self, workspace, branch_name, repo, original_ref): - '''Create a branch called branch_name based off original_ref. - - NOTE: self.lrc and self.rrc need to be initialized before - calling since clone_to_directory uses them indirectly via - get_cached_repo - ''' - branch_dir = os.path.join(workspace, branch_name) - os.makedirs(branch_dir) - try: - # Create a .morph-system-branch directory to clearly identify - # this directory as a morph system branch. - os.mkdir(os.path.join(branch_dir, '.morph-system-branch')) - - # Remember the system branch name and the repository we branched - # off from initially. - self.set_branch_config(branch_dir, 'branch.name', branch_name) - self.set_branch_config(branch_dir, 'branch.root', repo) - - # Generate a UUID for the branch. We will use this for naming - # temporary refs, e.g. building. - self.set_branch_config(branch_dir, 'branch.uuid', uuid.uuid4().hex) - - # Clone into system branch directory. - repo_dir = os.path.join(branch_dir, self.convert_uri_to_path(repo)) - self.clone_to_directory(repo_dir, repo, original_ref) - - # Create a new branch in the local morphs repository. - if original_ref != branch_name: - self.app.runcmd(['git', 'checkout', '-b', branch_name, - original_ref], cwd=repo_dir) - - return branch_dir - except BaseException, e: - logging.error('Caught exception: %s' % str(e)) - logging.info('Removing half-finished branch %s' % branch_name) - self.remove_branch_dir_safe(workspace, branch_name) - raise + if gd.has_fat(): + gd.fat_init() + gd.fat_pull() - def checkout_repository(self, branch_dir, repo, ref, parent_ref=None): - '''Make a chunk or stratum repository available for a system branch - We ensure the 'system_branch' ref within 'repo' is checked out, - creating it from 'parent_ref' if required. + def branch(self, args): + '''Create a new system branch. - The function aims for permissiveness, so users can try to fix any - weirdness they have caused in the repos with another call to 'morph - edit'. + Command line arguments: + + * `REPO` is a repository URL. + * `NEW` is the name of the new system branch. + * `OLD` is the point from which to branch, and defaults to `master`. + + This creates a new system branch. It needs to be run in an + existing workspace (see `morph workspace`). It creates a new + git branch in the clone of the repository in the workspace. The + system branch will not be visible on the git server until you + push your changes to the repository. + + Example: + + cd /src/workspace + morph branch baserock:baserock/morphs jrandom/new-feature ''' - parent_ref = parent_ref or ref + if len(args) not in [2, 3]: + raise cliapp.AppException( + 'morph branch needs name of branch as parameter') - repo_dir = self.find_repository(branch_dir, repo) - if repo_dir is None: - repo_url = self.resolve_reponame(repo) - repo_dir = os.path.join(branch_dir, self.convert_uri_to_path(repo)) - self.clone_to_directory(repo_dir, repo, parent_ref) + root_url = args[0] + system_branch = args[1] + base_ref = 'master' if len(args) == 2 else args[2] + origin_base_ref = 'origin/%s' % base_ref - if self.resolve_ref(repo_dir, ref) is None: - self.log_change(repo, 'branch "%s" created from "%s"' % - (ref, parent_ref)) - command = ['git', 'checkout', '-b', ref] - else: - # git copes even if the system_branch ref is already checked out - command = ['git', 'checkout', ref] - - status, output, error = self.app.runcmd_unchecked( - command, cwd=repo_dir) - if status != 0: - raise cliapp.AppException('Command failed: %s in repo %s\n%s' % - (' '.join(command), repo, error)) - return repo_dir - - def make_available(self, spec, branch, branch_dir, root_repo, - root_repo_dir): - '''Check out the morphology that 'spec' refers to, for editing''' - - if spec.get('repo') in (None, root_repo): - # This is only possible for stratum morphologies - repo_dir = root_repo_dir - if spec.get('ref') not in (None, root_repo): - # Bring the morphology forward from its ref to the current HEAD - repo = self.lrc.get_repo(root_repo) - m = repo.load_morphology(spec['ref'], spec['morph']) - self.update_morphology(root_repo_dir, spec['morph'], m) - self.log_change(spec['repo'], - '"%s" copied from "%s" to "%s"' % - (spec['morph'], spec['ref'], branch)) - else: - repo_dir = self.checkout_repository( - branch_dir, spec['repo'], branch, parent_ref=spec['ref']) - return repo_dir + self._require_git_user_config() + + # Open the workspace first thing, so user gets a quick error if + # we're not inside a workspace. + ws = morphlib.workspace.open('.') + + # Make sure the root repository is in the local git repository + # cache, and is up to date. + lrc, rrc = morphlib.util.new_repo_caches(self.app) + cached_repo = lrc.get_updated_repo(root_url) + + # Make sure the system branch doesn't exist yet. + if cached_repo.ref_exists(system_branch): + raise cliapp.AppException( + 'branch %s already exists in repository %s' % + (system_branch, root_url)) + + # Make sure the base_ref exists. + cached_repo.resolve_ref(base_ref) + + with self._initializing_system_branch( + ws, root_url, system_branch, cached_repo, base_ref) as (sb, gd): + + gd.branch(system_branch, base_ref) + gd.checkout(system_branch) + if gd.has_fat(): + gd.fat_init() + gd.fat_pull() + + def _save_dirty_morphologies(self, loader, sb, morphs): + logging.debug('Saving dirty morphologies: start') + for morph in morphs: + if morph.dirty: + logging.debug( + 'Saving morphology: %s %s %s' % + (morph.repo_url, morph.ref, morph.filename)) + loader.unset_defaults(morph) + loader.save_to_file( + sb.get_filename(morph.repo_url, morph.filename), morph) + morph.dirty = False + logging.debug('Saving dirty morphologies: done') + + def _checkout(self, lrc, sb, repo_url, ref): + logging.debug( + 'Checking out %s (%s) into %s' % + (repo_url, ref, sb.root_directory)) + cached_repo = lrc.get_updated_repo(repo_url) + gd = sb.clone_cached_repo(cached_repo, ref) + gd.update_submodules(self.app) + gd.update_remotes() + + def _load_morphology_from_file(self, loader, dirname, filename): + full_filename = os.path.join(dirname, filename) + return loader.load_from_file(full_filename) + + def _load_morphology_from_git(self, loader, gd, ref, filename): + try: + text = gd.get_file_from_ref(ref, filename) + except cliapp.AppException: + text = gd.get_file_from_ref('origin/%s' % ref, filename) + return loader.load_from_string(text, filename) - @warns_git_identity def edit(self, args): '''Edit or checkout a component in a system branch. Command line arguments: - * `SYSTEM` is the name of a system morphology in the root repository - of the current system branch. - * `STRATUM` is the name of a stratum inside the system. - * `CHUNK` is the name of a chunk inside the stratum. + * `CHUNK` is the name of a chunk - This marks the specified stratum or chunk (if given) as being - changed within the system branch, by creating the git branches in - the affected repositories, and changing the relevant morphologies - to point at those branches. It also creates a local clone of - the git repository of the stratum or chunk. + This makes a local checkout of CHUNK in the current system branch + and edits any stratum morphology file(s) containing the chunk - For example: + ''' - morph edit devel-system-x86-64-generic devel + if len(args) != 1: + raise cliapp.AppException('morph edit needs a chunk ' + 'as parameter') + + ws = morphlib.workspace.open('.') + sb = morphlib.sysbranchdir.open_from_within('.') + loader = morphlib.morphloader.MorphologyLoader() + morphs = self._load_all_sysbranch_morphologies(sb, loader) + + def edit_chunk(morph, chunk_name): + chunk_url, chunk_ref, chunk_morph = ( + morphs.get_chunk_triplet(morph, chunk_name)) + + chunk_dirname = sb.get_git_directory_name(chunk_url) + + if not os.path.exists(chunk_dirname): + lrc, rrc = morphlib.util.new_repo_caches(self.app) + cached_repo = lrc.get_updated_repo(chunk_url) + + gd = sb.clone_cached_repo(cached_repo, chunk_ref) + if chunk_ref != sb.system_branch_name: + gd.branch(sb.system_branch_name, chunk_ref) + gd.checkout(sb.system_branch_name) + gd.update_submodules(self.app) + gd.update_remotes() + if gd.has_fat(): + gd.fat_init() + gd.fat_pull() + + # Change the refs to the chunk. + if chunk_ref != sb.system_branch_name: + morphs.change_ref( + chunk_url, chunk_ref, + chunk_morph, + sb.system_branch_name) + + return chunk_dirname + + chunk_name = args[0] + dirs = set() + found = 0 + + for morph in morphs.morphologies: + if morph['kind'] == 'stratum': + for chunk in morph['chunks']: + if chunk['name'] == chunk_name: + self.app.status( + msg='Editing %(chunk)s in %(stratum)s stratum', + chunk=chunk_name, stratum=morph['name']) + chunk_dirname = edit_chunk(morph, chunk_name) + dirs.add(chunk_dirname) + found = found + 1 + + # Save any modified strata. + + self._save_dirty_morphologies(loader, sb, morphs.morphologies) + + if found == 0: + self.app.status( + msg="No chunk %(chunk)s found. If you want to create one, add " + "an entry to a stratum morph file.", chunk=chunk_name) + + if found >= 1: + dirs_list = ', '.join(sorted(dirs)) + self.app.status( + msg="Chunk %(chunk)s source is available at %(dirs)s", + chunk=chunk_name, dirs=dirs_list) + + if found > 1: + self.app.status( + msg="Notice that this chunk appears in more than one stratum") + + def show_system_branch(self, args): + '''Show the name of the current system branch.''' + + ws = morphlib.workspace.open('.') + sb = morphlib.sysbranchdir.open_from_within('.') + self.app.output.write('%s\n' % sb.system_branch_name) + + def show_branch_root(self, args): + '''Show the name of the repository holding the system morphologies. + + This would, for example, write out something like: + + /src/ws/master/baserock:baserock/morphs + + when the master branch of the `baserock:baserock/morphs` + repository is checked out. - The above command will mark the `devel` stratum as being - modified in the current system branch. In this case, the stratum's - morphology is in the same git repository as the system morphology, - so there is no need to create a new git branch. However, the - system morphology is modified to point at the stratum morphology - in the same git branch, rather than the original branch. + ''' - In other words, where the system morphology used to say this: + ws = morphlib.workspace.open('.') + sb = morphlib.sysbranchdir.open_from_within('.') + self.app.output.write('%s\n' % sb.get_config('branch.root')) - morph: devel - repo: baserock:baserock/morphs - ref: master + def _remove_branch_dir_safe(self, workspace_root, system_branch_root): + # This function avoids throwing any exceptions, so it is safe to call + # inside an 'except' block without altering the backtrace. - The updated system morphology will now say this instead: + def handle_error(function, path, excinfo): + logging.warning ("Error while trying to clean up %s: %s" % + (path, excinfo)) - morph: devel - repo: baserock:baserock/morphs - ref: jrandom/new-feature + shutil.rmtree(system_branch_root, onerror=handle_error) - (Assuming the system branch is called `jrandom/new-feature`.) + # Remove parent directories that are empty too, avoiding exceptions + parent = os.path.dirname(system_branch_root) + while parent != os.path.abspath(workspace_root): + if len(os.listdir(parent)) > 0 or os.path.islink(parent): + break + os.rmdir(parent) + parent = os.path.dirname(parent) - Another example: + def _require_git_user_config(self): + '''Warn if the git user.name and user.email variables are not set.''' - morph edit devel-system-x86_64-generic devel gcc + keys = { + 'user.name': 'My Name', + 'user.email': 'me@example.com', + } - The above command will mark the `gcc` chunk as being edited in - the current system branch. Morph will clone the `gcc` repository - locally, into the current workspace, and create a new (local) - branch named after the system branch. It will also change the - stratum morphology to refer to the new git branch, instead of - whatever branch it was referring to originally. + try: + morphlib.git.check_config_set(self.app.runcmd, keys) + except morphlib.git.ConfigNotSetException as e: + self.app.status( + msg="WARNING: %(message)s", + message=str(e), error=True) - If the `gcc` repository already had a git branch named after - the system branch, that is reused. Similarly, if the stratum - morphology was already pointing at that branch, it doesn't - need to be changed again. In that case, the only action Morph - does is to clone the chunk repository locally, and if that was - also done already, Morph does nothing. + def foreach(self, args): + '''Run a command in each repository checked out in a system branch. - ''' + Use -- before specifying the command to separate its arguments from + Morph's own arguments. - if len(args) not in (2, 3): - raise cliapp.AppException( - 'morph edit must either get a system and a stratum ' - 'or a system, a stratum and a chunk as arguments') - - workspace = self.deduce_workspace() - branch, branch_dir = self.deduce_system_branch() - - # Find out which repository we branched off from. - root_repo = self.get_branch_config(branch_dir, 'branch.root') - root_repo_dir = self.find_repository(branch_dir, root_repo) - - system_name = args[0] - stratum_name = args[1] - chunk_name = args[2] if len(args) > 2 else None - - self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app) - - # We need to touch every stratum in the system, not just the target - # the user specified, because others may have build-depends that - # point to the edited stratum. - system_morphology = self.load_morphology(root_repo_dir, system_name) - - # Test that the specified stratum exists in the system - self.get_edit_info(system_name, system_morphology, stratum_name) - - all_strata = system_morphology['strata'] - stratum_refs_to_correct = set() - - # Petrify the chunk - for stratum_spec in (s for s in all_strata - if s['morph'] == stratum_name): - stratum_repo_dir = self.make_available( - stratum_spec, branch, branch_dir, root_repo, root_repo_dir) - stratum_morphology = self.load_morphology( - stratum_repo_dir, stratum_spec['morph']) - - if chunk_name is not None: - # Change the stratum's ref to the chunk - chunk_spec = self.get_edit_info( - stratum_name, stratum_morphology, chunk_name, - collection='chunks') - - if 'unpetrify-ref' in chunk_spec: - chunk_spec['ref'] = chunk_spec['unpetrify-ref'] - del chunk_spec['unpetrify-ref'] - - self.make_available( - chunk_spec, branch, branch_dir, root_repo, - root_repo_dir) - - if chunk_spec['ref'] != branch: - chunk_spec['ref'] = branch - - self.log_change(stratum_spec['repo'], - '"%s" now includes "%s" from "%s"' % - (stratum_name, chunk_name, branch)) - stratum_refs_to_correct.add((stratum_spec['repo'], - stratum_spec['ref'], - stratum_spec['morph'])) - # Correct the System Morphology's reference - stratum_spec['ref'] = branch - self.update_morphology(stratum_repo_dir, stratum_spec['morph'], - stratum_morphology) - self.log_change(root_repo, - '"%s" now includes "%s" from "%s"' % - (system_name, stratum_name, branch)) - - # Correct all references to altered strata - while stratum_refs_to_correct: - repo, ref, morph = stratum_refs_to_correct.pop() - spec = {"repo": repo, "ref": ref, "morph": morph} - for stratum_spec in all_strata: - changed = False - if repo == root_repo: - stratum_repo_dir = root_repo_dir - else: - stratum_repo_dir = self.checkout_repository( - branch_dir, stratum_spec['repo'], - branch, stratum_spec['ref']) - stratum_morphology = self.load_morphology( - stratum_repo_dir, stratum_spec['morph']) - if ('build-depends' in stratum_morphology - and stratum_morphology['build-depends'] is not None): - for bd_spec in stratum_morphology['build-depends']: - bd_triplet = (bd_spec['repo'], - bd_spec['ref'], - bd_spec['morph']) - if (bd_triplet == (repo, ref, morph)): - bd_spec['ref'] = branch - changed = True - if changed: - stratum_refs_to_correct.add((stratum_spec['repo'], - stratum_spec['ref'], - stratum_spec['morph'])) - # Update the System morphology to use - # the modified version of the Stratum - stratum_spec['ref'] = branch - self.update_morphology(stratum_repo_dir, - stratum_spec['morph'], - stratum_morphology) - self.log_change(root_repo, - '"%s" now includes "%s" from "%s"' % - (system_name, stratum_name, branch)) - - self.update_morphology(root_repo_dir, system_name, system_morphology) - - self.print_changelog('The following changes were made but have not ' - 'been committed') - - def _get_repo_name(self, alias_resolver, metadata): - '''Attempt to find the best name for the repository. - - A defined repo-alias is preferred, but older builds may - not have it. - - A guessed repo-alias is the next best thing, but there may - be none or more alilases that would resolve to that URL, so - if there are any, use the shortest as it is likely to be - the most specific. - - If all else fails just use the URL. - ''' - if 'repo-alias' in metadata: - return metadata['repo-alias'] - - repo_url = metadata['repo'] - aliases = alias_resolver.aliases_from_url(repo_url) - - if len(aliases) >= 1: - # If there are multiple valid aliases, use the shortest - return min(aliases, key=len) - - # If there are no aliases, just return the url - return repo_url - - def _resolve_refs_from_metadata(self, alias_resolver, - metadata_cache_id_lookup): - '''Pre-resolve a set of refs from metadata. - - Resolved refs are a dict as {(repo, ref): sha1}. - - If the metadata contains the repo-alias then every - metadata item adds the mapping of its repo-alias and ref - to the commit it was built with. - - If the repo-alias does not exist, such as if the image was - built before that field was added, then mappings of every - possible repo url are added. - ''' - resolved_refs = {} - for md in metadata_cache_id_lookup.itervalues(): - if 'repo-alias' in md: - repourls = [md['repo-alias']] - else: - repourls = [md['repo']] - repourls.extend(alias_resolver.aliases_from_url(md['repo'])) - for repourl in repourls: - resolved_refs[repourl, md['original_ref']] = md['sha1'] - return resolved_refs + Command line arguments: + + * `--` indicates the end of option processing for Morph. + * `COMMAND` is a command to run. + * `ARGS` is a list of arguments or options to be passed onto + `COMMAND`. + + This runs the given `COMMAND` in each git repository belonging + to the current system branch that exists locally in the current + workspace. This can be a handy way to do the same thing in all + the local git repositories. + + For example: + + morph foreach -- git push + + The above command would push any committed changes in each + repository to the git server. - def branch_from_image(self, args): - '''Given the contents of a /baserock directory, produce a branch - of the System, petrified to when the System was made. ''' - if len(args) not in (2, 3): - raise cliapp.AppException( - 'branch-from-image needs repository, ref and path to metadata') - root_repo = args[0] - branch = args[1] - metadata_path = self.app.settings['metadata-dir'] - workspace = self.deduce_workspace() - self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app) - alias_resolver = morphlib.repoaliasresolver.RepoAliasResolver( - self.app.settings['repo-alias']) + if not args: + raise cliapp.AppException('morph foreach expects a command to run') + + ws = morphlib.workspace.open('.') + sb = morphlib.sysbranchdir.open_from_within('.') - system_key, metadata_cache_id_lookup = self.read_metadata( - metadata_path) - - system_metadata = metadata_cache_id_lookup[system_key] - repo = self._get_repo_name(alias_resolver, system_metadata) - - # Which repo to use? Specified or deduced? - branch_dir = self._create_branch(workspace, branch, repo, - system_metadata['sha1']) - - # Resolve refs from metadata so petrify substitutes these refs - # into morphologies instead of the current state of the branches - resolved_refs = self._resolve_refs_from_metadata( - alias_resolver, - metadata_cache_id_lookup) - - branch_root_dir = self.find_repository(branch_dir, repo) - name = system_metadata['morphology'][:-len('.morph')] - morphology = self.load_morphology(branch_root_dir, name) - self.petrify_morphology(branch, branch_dir, - repo, branch_root_dir, - repo, branch_root_dir, # not a typo - branch, name, morphology, - petrified_morphologies=set(), - resolved_refs=resolved_refs, - update_working_tree=True) + for gd in sorted(sb.list_git_directories(), key=lambda gd: gd.dirname): + # Get the repository's original name + # Continue in the case of error, since the previous iteration + # worked in the case of the user cloning a repository in the + # system branch's directory. + try: + repo = gd.get_config('morph.repository') + except cliapp.AppException: + continue + + self.app.output.write('%s\n' % repo) + status, output, error = self.app.runcmd_unchecked( + args, cwd=gd.dirname) + self.app.output.write(output) + if status != 0: + self.app.output.write(error) + pretty_command = ' '.join(cliapp.shell_quote(arg) + for arg in args) + raise cliapp.AppException( + 'Command failed at repo %s: %s' + % (repo, pretty_command)) + self.app.output.write('\n') + self.app.output.flush() + + def _load_all_sysbranch_morphologies(self, sb, loader): + '''Read in all the morphologies in the root repository.''' + self.app.status(msg='Loading in all morphologies') + morphs = morphlib.morphset.MorphologySet() + for morph in sb.load_all_morphologies(loader): + morphs.add_morphology(morph) + return morphs def petrify(self, args): '''Convert all chunk refs in a system branch to be fixed SHA1s. @@ -931,976 +525,220 @@ class BranchAndMergePlugin(cliapp.Plugin): ''' - # Stratum refs are not petrified, because they must all be edited to - # set the new chunk refs, which requires branching them all for the - # current branch - so they will not be updated outside of the user's - # control in any case. Chunks that have already been edited on the - # current branch are also not petrified. - - if len(args) != 0: + if args: raise cliapp.AppException('morph petrify takes no arguments') - branch, branch_path = self.deduce_system_branch() - root_repo = self.get_branch_config(branch_path, 'branch.root') - root_repo_dir = self.find_repository(branch_path, root_repo) - self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app) - - self.petrify_everything(branch, branch_path, root_repo, root_repo_dir, - branch, os.environ, None, True) - - def unpetrify(self, args): - '''Reverse the process of petrification. - - This undoes the changes `morph petrify` did. - - ''' - - # This function makes no attempt to 'unedit' strata that were branched - # solely so they could be petrified. - - if len(args) != 0: - raise cliapp.AppException('morph unpetrify takes no arguments') - - self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app) + ws = morphlib.workspace.open('.') + sb = morphlib.sysbranchdir.open_from_within('.') + loader = morphlib.morphloader.MorphologyLoader() + lrc, rrc = morphlib.util.new_repo_caches(self.app) + update_repos = not self.app.settings['no-git-update'] - workspace = self.deduce_workspace() - branch, branch_path = self.deduce_system_branch() - root_repo = self.get_branch_config(branch_path, 'branch.root') - root_repo_dir = self.find_repository(branch_path, root_repo) + morphs = self._load_all_sysbranch_morphologies(sb, loader) - for f in sorted(glob.iglob(os.path.join(root_repo_dir, '*.morph'))): - name = os.path.basename(f)[:-len('.morph')] - morphology = self.load_morphology(root_repo_dir, name) - if morphology['kind'] != 'system': - continue - - for stratum_info in morphology['strata']: - repo_dir = self.make_available( - stratum_info, branch, branch_path, root_repo, - root_repo_dir) - stratum_info['ref'] = branch - - stratum = self.load_morphology(repo_dir, stratum_info['morph']) + #TODO: Stop using app.resolve_ref + def resolve_refs(morphs): + for repo, ref in morphs.list_refs(): + # You can't resolve null refs, so don't attempt to. + if repo is None or ref is None: + continue + # TODO: Handle refs that are only in workspace in general + if (repo == sb.root_repository_url + and ref == sb.system_branch_name): + continue + commit_sha1, tree_sha1 = self.app.resolve_ref( + lrc, rrc, repo, ref, update=update_repos) + yield ((repo, ref), commit_sha1) - for chunk_info in stratum['chunks']: - if 'unpetrify-ref' in chunk_info: - chunk_info['ref'] = chunk_info['unpetrify-ref'] - del chunk_info['unpetrify-ref'] - self.update_morphology(repo_dir, stratum_info['morph'], - stratum) + morphs.repoint_refs(sb.root_repository_url, + sb.system_branch_name) - self.update_morphology(root_repo_dir, name, morphology) + morphs.petrify_chunks(dict(resolve_refs(morphs))) - self.print_changelog('The following changes were made but have not ' - 'been committed') + # Write morphologies back out again. + self._save_dirty_morphologies(loader, sb, morphs.morphologies) - @warns_git_identity - def tag(self, args): - '''Create an annotated Git tag of a petrified system branch. - - Command line arguments: - - * `TAG-NAME` is the name of the Git tag to be created. - * `--` separates the Git arguments and options from the ones - Morph parses for itself. - * `GIT-COMMIT-ARG` is a `git commit` option or argument, - e.g., '-m' or '-F'. These should provide the commit message. - - This command creates an annotated Git tag that points at a commit - where all system and stratum morphologies have been petrified. - The working tree won't be petrified, only the commit. - - Example: - - morph tag release-12.765 -- -m "Release 12.765" - - ''' + def unpetrify(self, args): + '''Reverse the process of petrification. - if len(args) < 1: - raise cliapp.AppException('morph tag expects a tag name') - - tagname = args[0] - - # Deduce workspace, system branch and branch root repository. - workspace = self.deduce_workspace() - branch, branch_dir = self.deduce_system_branch() - branch_root = self.get_branch_config(branch_dir, 'branch.root') - branch_root_dir = self.find_repository(branch_dir, branch_root) - - # Prepare an environment for our internal index file. - # This index file allows us to commit changes to a tree without - # git noticing any change in the working tree or its own index. - env = dict(os.environ) - env['GIT_INDEX_FILE'] = os.path.join( - branch_root_dir, '.git', 'morph-tag-index') - - # Extract git arguments that deal with the commit message. - # This is so that we can use them for creating the tag commit. - msg = None - msg_args = [] - for i in xrange(0, len(args)): - if args[i] == '-m' or args[i] == '-F': - if i < len(args)-1: - msg_args.append(args[i]) - msg_args.append(args[i+1]) - if args[i] == '-m': - msg = args[i+1] - else: - msg = open(args[i+1]).read() - elif args[i].startswith('--message='): - msg_args.append(args[i]) - msg = args[i][len('--message='):] - - # Fail if no commit message was provided. - if not msg or not msg_args: - raise cliapp.AppException( - 'Commit message expected. Please run one of ' - 'the following commands to provide one:\n' - ' morph tag NAME -- -m "Message"\n' - ' morph tag NAME -- --message="Message"\n' - ' morph tag NAME -- -F <message file>') - - # Abort if the tag already exists. - # FIXME At the moment this only checks the local repo in the - # workspace, not the remote repo cache or the local repo cache. - if self.ref_exists_locally(branch_root_dir, 'refs/tags/%s' % tagname): - raise cliapp.AppException('%s: Tag "%s" already exists' % - (branch_root, tagname)) - - self.app.status(msg='%(repo)s: Preparing tag commit', - repo=branch_root) - - # Read current tree into the internal index. - parent_sha1 = self.resolve_ref(branch_root_dir, branch) - self.app.runcmd(['git', 'read-tree', parent_sha1], - cwd=branch_root_dir, env=env) - - self.app.status(msg='%(repo)s: Petrifying everything', - repo=branch_root) - - # Petrify everything. - self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app) - self.petrify_everything(branch, branch_dir, - branch_root, branch_root_dir, - tagname, env) - - self.app.status(msg='%(repo)s: Creating tag commit', - repo=branch_root) - - # Create a dangling commit. - commit = self.create_tag_commit( - branch_root_dir, tagname, msg, env) - - self.app.status(msg='%(repo)s: Creating annotated tag "%(tag)s"', - repo=branch_root, tag=tagname) - - # Create an annotated tag for this commit. - self.create_annotated_tag(branch_root_dir, commit, env, args) - - def ref_exists_locally(self, repo_dir, ref): - try: - morphlib.git.rev_parse(self.app.runcmd, repo_dir, ref) - return True - except cliapp.AppException: - return False - - def petrify_everything(self, branch, branch_dir, - branch_root, branch_root_dir, tagref, env=os.environ, - resolved_refs=None, update_working_tree=False): - petrified_morphologies = set() - resolved_refs = resolved_refs or {} - for f in sorted(glob.iglob(os.path.join(branch_root_dir, '*.morph'))): - name = os.path.basename(f)[:-len('.morph')] - morphology = self.load_morphology(branch_root_dir, name) - self.petrify_morphology(branch, branch_dir, - branch_root, branch_root_dir, - branch_root, branch_root_dir, - tagref, name, morphology, - petrified_morphologies, resolved_refs, - env, update_working_tree) - - def petrify_morphology(self, branch, branch_dir, - branch_root, branch_root_dir, repo, repo_dir, - tagref, name, morphology, - petrified_morphologies, resolved_refs, - env=os.environ, update_working_tree=False): - self.app.status(msg='%(repo)s: Petrifying morphology \"%(morph)s\"', - repo=repo, morph=name) - - # Mark morphology as petrified so we don't petrify it twice. - petrified_morphologies.add(morphology) - - # Resolve the refs of all build dependencies (strata) and strata - # in the morphology into commit SHA1s. - strata = [] - if 'build-depends' in morphology and morphology['build-depends']: - strata += morphology['build-depends'] - if 'strata' in morphology and morphology['strata']: - strata += morphology['strata'] - for info in strata: - stratum_repo_dir = self.make_available( - info, branch, branch_dir, repo, repo_dir) - - # Load the stratum morphology and petrify it recursively if - # that hasn't happened yet. - stratum = self.load_morphology(stratum_repo_dir, info['morph']) - if not stratum in petrified_morphologies: - self.petrify_morphology(branch, branch_dir, - branch_root, branch_root_dir, - info.get('repo') or branch_root, - stratum_repo_dir, tagref, - info['morph'], stratum, - petrified_morphologies, - resolved_refs, env, - update_working_tree) - - # If this morphology is a stratum, resolve the refs of all its - # chunks into SHA1s. - if morphology['kind'] == 'stratum': - for info in morphology['chunks']: - commit = self.resolve_info(info, resolved_refs) - if info['ref'] != commit: - info['unpetrify-ref'] = info['ref'] - info['ref'] = commit - - # Write the petrified morphology to a temporary file in the - # branch root repository for inclusion in the tag commit. - with tempfile.NamedTemporaryFile(suffix='.morph') as f: - self.update_morphology( - repo_dir, name, morphology, output_fd=f.file) - - # Hash the petrified morphology and add it to the index - # for the tag commit. - sha1 = self.app.runcmd( - ['git', 'hash-object', '-t', 'blob', '-w', f.name], - cwd=branch_root_dir, env=env) - self.app.runcmd( - ['git', 'update-index', '--add', '--cacheinfo', - '100644', sha1, '%s.morph' % name], - cwd=branch_root_dir, env=env) - - # Update the working tree if requested. This can be done with - # git-checkout-index, but we still have the file, so use that - if update_working_tree: - shutil.copy(f.name, - os.path.join(branch_root_dir, '%s.morph' % name)) - - def resolve_info(self, info, resolved_refs): - '''Takes a morphology info and resolves its ref with cache support.''' - - key = (info.get('repo'), info.get('ref')) - if not key in resolved_refs: - commit_sha1, tree_sha1 = self.app.resolve_ref( - self.lrc, self.rrc, info['repo'], info['ref'], - update=not self.app.settings['no-git-update']) - resolved_refs[key] = commit_sha1 - return resolved_refs[key] - - def create_tag_commit(self, repo_dir, tagname, msg, env): - self.app.status(msg='%(repo)s: Creating commit for the tag', - repo=repo_dir) - - # Write and commit the tree. - tree = self.app.runcmd( - ['git', 'write-tree'], cwd=repo_dir, env=env).strip() - commit = self.app.runcmd( - ['git', 'commit-tree', tree, '-p', 'HEAD'], - feed_stdin=msg, cwd=repo_dir, env=env).strip() - return commit - - def create_annotated_tag(self, repo_dir, commit, env, args=[]): - self.app.status(msg='%(repo)s: Creating annotated tag for ' - 'commit %(commit)s', - repo=repo_dir, commit=commit) - - # Create an annotated tag for the commit - self.app.runcmd(['git', 'tag', '-a'] + args + [commit], - cwd=repo_dir, env=env) - - # When 'merge' is unset, git doesn't try to resolve conflicts itself in - # those files. - MERGE_ATTRIBUTE = '*.morph\t-merge\n' - - def disable_morph_merging(self, repo_dir): - attributes_file = os.path.join(repo_dir, ".git", "info", "attributes") - with open(attributes_file, 'a') as f: - f.write(self.MERGE_ATTRIBUTE) - - def enable_morph_merging(self, repo_dir): - attributes_file = os.path.join(repo_dir, ".git", "info", "attributes") - with open(attributes_file, 'r') as f: - attributes = f.read() - if attributes == self.MERGE_ATTRIBUTE: - os.unlink(attributes_file) - elif attributes.endswith(self.MERGE_ATTRIBUTE): - with morphlib.savefile.SaveFile(attributes_file, 'w') as f: - f.write(attributes[:-len(self.MERGE_ATTRIBUTE)]) - - def get_merge_files(self, repo_dir, from_sha1, to_ref, name): - '''Returns merge base, remote and local versions of a morphology - - We already ran 'git fetch', so the remote branch is available within - the target repository. + This undoes the changes `morph petrify` did. ''' - base_sha1 = self.app.runcmd(['git', 'merge-base', from_sha1, to_ref], - cwd=repo_dir).strip() - base_morph = self.load_morphology(repo_dir, name, ref=base_sha1) - from_morph = self.load_morphology(repo_dir, name, ref=from_sha1) - to_morph = self.load_morphology(repo_dir, name, ref=to_ref) - return base_morph, from_morph, to_morph - - def check_component(self, parent_kind, parent_path, from_info, to_info): - assert (parent_kind in ['system', 'stratum']) - - kind = 'chunk' if parent_kind == 'stratum' else 'stratum' - name = to_info.get('alias', to_info.get('name', to_info.get('morph'))) - path = parent_path + '.' + name - - if kind == 'chunk': - # Only chunks can be petrified - from_unpetrify_ref = from_info.get('unpetrify-ref', None) - to_unpetrify_ref = to_info.get('unpetrify-ref', None) - if from_unpetrify_ref is not None and to_unpetrify_ref is None: - self.app.output.write( - 'WARNING: chunk "%s" is now petrified\n' % path) - elif from_unpetrify_ref is None and to_unpetrify_ref is not None: - self.app.output.write( - 'WARNING: chunk "%s" is no longer petrified\n' % path) - elif from_unpetrify_ref != to_unpetrify_ref: - raise cliapp.AppException( - 'merge conflict: chunk "%s" is petrified to a different ' - 'ref in each branch' % path) - - def diff_morphologies(self, path, from_morph, to_morph): - '''Component-level diff between two versions of a morphology''' + if args: + raise cliapp.AppException('morph petrify takes no arguments') - def component_key(info): - # This function needs only to be stable and reproducible - if 'name' in info: - return (info.get('repo'), info['morph'], info['name']) - else: - return (info.get('repo'), info['morph']) + ws = morphlib.workspace.open('.') + sb = morphlib.sysbranchdir.open_from_within('.') + loader = morphlib.morphloader.MorphologyLoader() - if from_morph['name'] != to_morph['name']: - # We should enforce name == filename in load_morphology() - raise cliapp.AppException( - 'merge conflict: "name" of morphology %s (name should always ' - 'match filename)' % path) - if from_morph['kind'] != to_morph['kind']: - raise cliapp.AppException( - 'merge conflict: "kind" of morphology %s changed from %s to %s' - % (path, to_morph['kind'], from_morph['kind'])) - - kind = to_morph['kind'] - - # copy() makes a shallow copy, so editing the list elements will - # change the actual morphologies. - if kind == 'system': - from_components = copy.copy(from_morph['strata']) - to_components = copy.copy(to_morph['strata']) - elif kind == 'stratum': - from_components = copy.copy(from_morph['chunks']) - to_components = copy.copy(to_morph['chunks']) - from_components.sort(key=component_key) - to_components.sort(key=component_key) - - # These are not set() purely because a set requires a hashable type - intersection = [] # TO n FROM - from_diff = [] # FROM \ TO - to_diff = [] # TO \ FROM - while len(from_components) > 0 and len(to_components) > 0: - from_info = from_components.pop(0) - to_info = to_components.pop(0) - match = cmp(component_key(from_info), component_key(to_info)) - if match < 0: - from_diff.append(from_info) - elif match > 0: - to_diff.append(to_info) - elif match == 0: - intersection.append((from_info, to_info)) - if len(from_components) != 0: - from_diff.append(from_components.pop(0)) - if len(to_components) != 0: - to_diff.append(to_components.pop(0)) - return intersection, from_diff, to_diff - - def merge_repo(self, merged_repos, from_branch_dir, from_repo, from_ref, - to_branch_dir, to_repo, to_ref): - '''Merge changes for a system branch in a specific repository - - We disable merging for morphologies and do this manually later on. + morphs = self._load_all_sysbranch_morphologies(sb, loader) - ''' + # Restore the ref for each stratum and chunk + morphs.unpetrify_all() - if to_repo in merged_repos: - return merged_repos[to_repo] - - from_repo_dir = self.find_repository(from_branch_dir, from_repo) - to_repo_dir = self.checkout_repository(to_branch_dir, to_repo, to_ref) - - if self.get_uncommitted_changes(from_repo_dir) != []: - raise cliapp.AppException('repository %s has uncommitted ' - 'changes' % from_repo) - if self.get_uncommitted_changes(to_repo_dir) != []: - raise cliapp.AppException('repository %s has uncommitted ' - 'changes' % to_repo) - - # Fetch the local FROM branch; its sha1 will be stored in FETCH_HEAD. - # ':' in pathnames confuses git, so we have to pass it a URL. - from_repo_url = urlparse.urljoin('file://', from_repo_dir) - self.app.runcmd(['git', 'fetch', from_repo_url, from_ref], - cwd=to_repo_dir) - - # Merge everything but the morphologies; error output is ignored (it's - # not very good) and instead we report conflicts manually later on. - self.disable_morph_merging(to_repo_dir) - with open(os.path.join(to_repo_dir, '.git', 'FETCH_HEAD')) as f: - from_sha1 = f.read(40) - status, output, error = self.app.runcmd_unchecked( - ['git', 'merge', '--no-commit', '--no-ff', from_sha1], - cwd=to_repo_dir) - self.enable_morph_merging(to_repo_dir) - - merged_repos[to_repo] = (to_repo_dir, from_sha1) - return (to_repo_dir, from_sha1) - - def merge(self, args): - '''Pull and merge changes from a system branch into the current one. + # Write morphologies back out again. + self._save_dirty_morphologies(loader, sb, morphs.morphologies) - Command line arguments: - - * `BRANCH` is the name of the system branch to merge _from_. + def status(self, args): + '''Show information about the current system branch or workspace - This merges another system branch into the current one. Morph - will do a `git merge` for each component that has been edited, - and undo any changes to `ref` fields in system and stratum - morphologies that `morph edit` has made. + This shows the status of every local git repository of the + current system branch. This is similar to running `git status` + in each repository separately. - You need to be in the _target_ system branch when merging. If - you have two system branches, `TROVE_ID/release/1.2` and - `TROVE_ID/bugfixes/12765`, and want to merge the bug fix branch - into the release branch, you need to first checkout the release - system branch, and then merge the bugfix branch into that. + If run in a Morph workspace, but not in a system branch checkout, + it lists all checked out system branches in the workspace. ''' - if len(args) != 1: - raise cliapp.AppException('morph merge requires a system branch ' - 'name as its argument') - - self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app) - workspace = self.deduce_workspace() - from_branch = args[0] - from_branch_dir = self.find_system_branch(workspace, from_branch) - to_branch, to_branch_dir = self.deduce_system_branch() - if from_branch_dir is None: - raise cliapp.AppException('branch %s must be checked out before ' - 'it can be merged' % from_branch) - - root_repo = self.get_branch_config(from_branch_dir, 'branch.root') - other_root_repo = self.get_branch_config(to_branch_dir, 'branch.root') - if root_repo != other_root_repo: - raise cliapp.AppException('branches do not share a root ' - 'repository : %s vs %s' % - (root_repo, other_root_repo)) - - def merge_chunk(parent_path, old_ci, ci): - self.merge_repo(merged_repos, - from_branch_dir, old_ci['repo'], from_branch, - to_branch_dir, ci['repo'], ci['ref']) - - def merge_stratum(parent_path, old_si, si): - path = parent_path + '.' + si['morph'] - - to_repo_dir, from_sha1 = self.merge_repo(merged_repos, - from_branch_dir, old_si['repo'], from_branch, - to_branch_dir, si['repo'], si['ref']) - base_morph, from_morph, to_morph = self.get_merge_files( - to_repo_dir, from_sha1, si['ref'], si['morph']) - intersection, from_diff, to_diff = self.diff_morphologies( - path, from_morph, to_morph) - for from_ci, to_ci in intersection: - self.check_component('stratum', path, from_ci, to_ci) - - changed = False - edited_chunks = [ci for ci in from_morph['chunks'] - if ci['ref'] == from_branch] - for ci in edited_chunks: - for old_ci in to_morph['chunks']: - if old_ci['repo'] == ci['repo']: - break - else: - raise cliapp.AppException( - 'chunk %s was added within this branch and ' - 'subsequently edited. This is not yet supported: ' - 'refusing to merge.' % ci['name']) - changed = True - ci['ref'] = old_ci['ref'] - merge_chunk(path, old_ci, ci) - if changed: - self.update_morphology(to_repo_dir, si['morph'], to_morph) - self.app.runcmd(['git', 'add', si['morph'] + '.morph'], - cwd=to_repo_dir) - - def merge_system(name): - base_morph, from_morph, to_morph = self.get_merge_files( - to_root_dir, from_sha1, to_branch, name) - if to_morph['kind'] != 'system': - return - - intersection, from_diff, to_diff = self.diff_morphologies( - name, from_morph, to_morph) - for from_si, to_si in intersection: - self.check_component('system', name, from_si, to_si) - - changed = False - edited_strata = [si for si in from_morph['strata'] - if si.get('ref') == from_branch] - for si in edited_strata: - for old_si in to_morph['strata']: - # We make no attempt at rename / move detection - if (old_si['morph'] == si['morph'] - and old_si.get('repo') == si.get('repo')): - break - else: - raise cliapp.AppException( - 'stratum %s was added within this branch and ' - 'subsequently edited. This is not yet supported: ' - 'refusing to merge.' % si['morph']) - changed = True - si['ref'] = old_si.get('ref') - merge_stratum(name, old_si, si) - if changed: - self.update_morphology(to_root_dir, name, to_morph) - self.app.runcmd(['git', 'add', f], cwd=to_root_dir) - - merged_repos = {} - try: - to_root_dir, from_sha1 = self.merge_repo(merged_repos, - from_branch_dir, root_repo, from_branch, - to_branch_dir, root_repo, to_branch) - - for f in sorted(glob.iglob(os.path.join(to_root_dir, '*.morph'))): - name = os.path.basename(f)[:-len('.morph')] - merge_system(name) - - success = True - for repo_name, repo_info in merged_repos.iteritems(): - repo_dir = repo_info[0] - conflicts = self.get_unmerged_changes(repo_dir) - if len(conflicts) > 0: - self.app.output.write("Merge conflicts in %s:\n\t%s\n" % - (repo_name, '\n\t'.join(conflicts))) - success = False - elif morphlib.git.index_has_changes(self.app.runcmd, repo_dir): - # Repo may not be dirty if the changes only touched refs, - # because they may now match the previous state. - msg = "Merge system branch '%s'" % from_branch - self.app.runcmd(['git', 'commit', '--all', '-m%s' % msg], - cwd=repo_dir) - if not success: - raise cliapp.AppException( - "merge errors were encountered. Please manually merge the " - "target ref into %s in the remote system branch in each " - "case, and then repeat the 'morph merge' operation." % - from_branch) - self.app.status(msg="Merge successful") - except BaseException, e: - logging.error('Caught exception: %s' % str(e)) - logging.info('Resetting half-finished merge') - for repo_dir, sha1 in merged_repos.itervalues(): - self.reset_work_tree_safe(repo_dir) - raise - - def build(self, args): - '''Build a system image in the current system branch - - Command line arguments: - - * `SYSTEM` is the name of the system to build. - - This builds a system image, and any of its components that - need building. The system name is the basename of the system - morphology, in the root repository of the current system branch, - without the `.morph` suffix in the filename. - - The location of the resulting system image artifact is printed - at the end of the build output. - - You do not need to commit your changes before building, Morph - does that for you, in a temporary branch for each build. However, - note that Morph does not untracked files to the temporary branch, - only uncommitted changes to files git already knows about. You - need to `git add` and commit each new file yourself. - - Example: - - morph build devel-system-x86_64-generic - - ''' + if args: + raise cliapp.AppException('morph status takes no arguments') - if len(args) != 1: - raise cliapp.AppException('morph build expects exactly one ' - 'parameter: the system to build') - - # Raise an exception if there is not enough space - morphlib.util.check_disk_available( - self.app.settings['tempdir'], - self.app.settings['tempdir-min-space'], - self.app.settings['cachedir'], - self.app.settings['cachedir-min-space']) - - system_name = args[0] - - # Deduce workspace and system branch and branch root repository. - workspace = self.deduce_workspace() - branch, branch_dir = self.deduce_system_branch() - branch_root = self.get_branch_config(branch_dir, 'branch.root') - branch_uuid = self.get_branch_config(branch_dir, 'branch.uuid') - - # Generate a UUID for the build. - build_uuid = uuid.uuid4().hex - - build_command = morphlib.buildcommand.BuildCommand(self.app) - build_command = self.app.hookmgr.call('new-build-command', - build_command) - push = self.app.settings['push-build-branches'] - - self.app.status(msg='Starting build %(uuid)s', uuid=build_uuid) - - self.app.status(msg='Collecting morphologies involved in ' - 'building %(system)s from %(branch)s', - system=system_name, branch=branch) - - # Get repositories of morphologies involved in building this system - # from the current system branch. - build_repos = self.get_system_build_repos( - branch, branch_dir, branch_root, system_name) - - # Generate temporary build ref names for all these repositories. - self.generate_build_ref_names(build_repos, branch_uuid) - - # Create the build refs for all these repositories and commit - # all uncommitted changes to them, updating all references - # to system branch refs to point to the build refs instead. - self.update_build_refs(build_repos, branch, build_uuid, push) - - if push: - self.push_build_refs(build_repos) - build_branch_root = branch_root + ws = morphlib.workspace.open('.') + try: + sb = morphlib.sysbranchdir.open_from_within('.') + except morphlib.sysbranchdir.NotInSystemBranch: + self._workspace_status(ws) else: - dirname = build_repos[branch_root]['dirname'] - build_branch_root = urlparse.urljoin('file://', dirname) - - # Run the build. - build_command.build([build_branch_root, - build_repos[branch_root]['build-ref'], - system_name]) + self._branch_status(ws, sb) - if push: - self.delete_remote_build_refs(build_repos) + def _workspace_status(self, ws): + '''Show information about the current workspace - self.app.status(msg='Finished build %(uuid)s', uuid=build_uuid) - - def get_system_build_repos(self, system_branch, branch_dir, - branch_root, system_name): - '''Map upstream repository URLs to their checkouts in the system branch - - Also provides the list of morphologies stored in each repository, - grouped by kind. + This lists all checked out system branches in the workspace. ''' + self.app.output.write("System branches in current workspace:\n") + branches = sorted(ws.list_system_branches(), + key=lambda x: x.root_directory) + for sb in branches: + self.app.output.write(" %s\n" % sb.get_config('branch.name')) - build_repos = {} - - def prepare_repo_info(repo, dirname): - build_repos[repo] = { - 'dirname': dirname, - 'systems': [], - 'strata': [], - 'chunks': [] - } - - def add_morphology_info(info, category): - repo = info['repo'] or branch_root - if repo in build_repos: - repo_dir = build_repos[repo]['dirname'] - else: - repo_dir = self.find_repository(branch_dir, repo) - if repo_dir: - if not repo in build_repos: - prepare_repo_info(repo, repo_dir) - build_repos[repo][category].append(info['morph']) - return repo_dir - - # Add repository and morphology of the system. - branch_root_dir = self.find_repository(branch_dir, branch_root) - prepare_repo_info(branch_root, branch_root_dir) - build_repos[branch_root]['systems'].append(system_name) - - # Traverse and add repositories and morphologies involved in - # building this system from the system branch. - system_morphology = self.load_morphology(branch_root_dir, system_name) - for info in system_morphology['strata']: - if info['ref'] == system_branch or info['ref'] is None: - repo_dir = add_morphology_info(info, 'strata') - if repo_dir: - stratum_morphology = self.load_morphology( - repo_dir, info['morph']) - for info in stratum_morphology['chunks']: - if info['ref'] == system_branch: - add_morphology_info(info, 'chunks') - - return build_repos - - def inject_build_refs(self, morphology, build_repos, will_push): - # Starting from a system or stratum morphology, update all ref - # pointers of strata or chunks involved in a system build (represented - # by build_repos) to point to temporary build refs of the repos - # involved in the system build. - def inject_build_ref(info): - if info['repo'] in build_repos and ( - info['morph'] in build_repos[info['repo']]['strata'] or - info['morph'] in build_repos[info['repo']]['chunks']): - info['ref'] = build_repos[info['repo']]['build-ref'] - if not will_push: - dirname = build_repos[info['repo']]['dirname'] - info['repo'] = urlparse.urljoin('file://', dirname) - if morphology['kind'] == 'system': - for info in morphology['strata']: - inject_build_ref(info) - elif morphology['kind'] == 'stratum': - if morphology['build-depends'] is not None: - for info in morphology['build-depends']: - inject_build_ref(info) - for info in morphology['chunks']: - inject_build_ref(info) - - def generate_build_ref_names(self, build_repos, branch_uuid): - for repo, info in build_repos.iteritems(): - repo_dir = info['dirname'] - repo_uuid = self.get_repo_config(repo_dir, 'morph.uuid') - build_ref = os.path.join(self.app.settings['build-ref-prefix'], - branch_uuid, repo_uuid) - info['build-ref'] = build_ref - - def update_build_refs(self, build_repos, system_branch, build_uuid, - will_push): - '''Update build branches for each repository with any local changes ''' - - # Define the committer. - committer_name = 'Morph (on behalf of %s)' % \ - (morphlib.git.get_user_name(self.app.runcmd)) - committer_email = morphlib.git.get_user_email(self.app.runcmd) - - for repo, info in build_repos.iteritems(): - repo_dir = info['dirname'] - build_ref = info['build-ref'] - - self.app.status(msg='%(repo)s: Creating build branch', repo=repo) - - # Obtain parent SHA1 for the temporary ref tree to be committed. - # This will either be the current commit of the temporary ref or - # HEAD in case the temporary ref does not exist yet. - system_branch_sha1 = self.resolve_ref( - repo_dir, '%s^{commit}' % system_branch) - parent_sha1 = self.resolve_ref(repo_dir, '%s^{commit}' % build_ref) - if not parent_sha1: - parent_sha1 = system_branch_sha1 - - # Prepare an environment with our internal index file. - # This index file allows us to commit changes to a tree without - # git noticing any change in working tree or its own index. - env = dict(os.environ) - env['GIT_INDEX_FILE'] = os.path.join( - repo_dir, '.git', 'morph-index') - env['GIT_COMMITTER_NAME'] = committer_name - env['GIT_COMMITTER_EMAIL'] = committer_email - - # Read tree from current HEAD into the morph index. - self.app.runcmd(['git', 'read-tree', system_branch_sha1], - cwd=repo_dir, env=env) - - self.app.status(msg='%(repo)s: Adding uncommitted changes to ' - 'build branch', repo=repo) - - # Add all local, uncommitted changes to our internal index. - changed_files = self.get_uncommitted_changes(repo_dir, env) - self.app.runcmd(['git', 'add'] + changed_files, - cwd=repo_dir, env=env) - - self.app.status(msg='%(repo)s: Updating morphologies to use ' - 'build branch instead of "%(branch)s"', - repo=repo, branch=system_branch) - - # Update all references to the system branches of strata - # and chunks to point to the temporary refs, which is needed - # for building. - filenames = info['systems'] + info['strata'] - for filename in filenames: - # Inject temporary refs in the right places in each morphology. - morphology = self.load_morphology(repo_dir, filename) - self.inject_build_refs(morphology, build_repos, will_push) - with tempfile.NamedTemporaryFile(suffix='.morph') as f: - self.update_morphology( - repo_dir, filename, morphology, output_fd=f.file) - - morphology_sha1 = self.app.runcmd( - ['git', 'hash-object', '-t', 'blob', '-w', f.name], - cwd=repo_dir, env=env) - - try: - self.app.runcmd( - ['git', 'update-index', '--cacheinfo', - '100644', morphology_sha1, '%s.morph' % filename], - cwd=repo_dir, env=env) - except cliapp.AppException, e: - raise cliapp.AppException( - "You seem to want to build %s, but '%s.morph' " - "doesn't exist in the morphologies repository. " - "Did you forget to commit it?" % - (filename, filename)) - - # Create a commit message including the build UUID. This allows us - # to collect all commits of a build across repositories and thereby - # see the changes made to the entire system between any two builds. - message = 'Morph build %s\n\nSystem branch: %s\n' % \ - (build_uuid, system_branch) - - # Write and commit the tree and update the temporary build ref. - tree = self.app.runcmd( - ['git', 'write-tree'], cwd=repo_dir, env=env).strip() - commit = self.app.runcmd( - ['git', 'commit-tree', tree, '-p', parent_sha1], - feed_stdin=message, cwd=repo_dir, env=env).strip() - self.app.runcmd( - ['git', 'update-ref', '-m', message, - 'refs/heads/%s' % build_ref, commit], - cwd=repo_dir, env=env) - - def push_build_refs(self, build_repos): - for repo, info in build_repos.iteritems(): - self.app.status(msg='%(repo)s: Pushing build branch', repo=repo) - self.app.runcmd(['git', 'push', 'origin', '%s:%s' % - (info['build-ref'], info['build-ref'])], - cwd=info['dirname']) - - def delete_remote_build_refs(self, build_repos): - for repo, info in build_repos.iteritems(): - self.app.status(msg='%(repo)s: Deleting remote build branch', - repo=repo) - self.app.runcmd(['git', 'push', 'origin', - ':%s' % info['build-ref']], cwd=info['dirname']) - - def status(self, args): - '''Show information about the current system branch or workspace + def _branch_status(self, ws, sb): + '''Show information about the current branch This shows the status of every local git repository of the current system branch. This is similar to running `git status` - in each repository separately, but the output is nicer. - - If run in a Morph workspace, but not in a system branch checkout, - it lists all checked out system branches in the workspace. + in each repository separately. ''' + branch = sb.get_config('branch.name') + root = sb.get_config('branch.root') - if len(args) != 0: - raise cliapp.AppException('morph status takes no arguments') - - workspace = self.deduce_workspace() - try: - branch, branch_path = self.deduce_system_branch() - except cliapp.AppException: - branch = None - - if branch is None: - self.app.output.write("System branches in current workspace:\n") - branch_dirs = sorted(self.walk_special_directories( - workspace, special_subdir='.morph-system-branch')) - for dirname in branch_dirs: - branch = self.get_branch_config(dirname, 'branch.name') - self.app.output.write(" %s\n" % branch) - return - - root_repo = self.get_branch_config(branch_path, 'branch.root') - root_repo_path = self.find_repository(branch_path, root_repo) - - self.app.output.write("On branch %s, root %s\n" % (branch, root_repo)) + self.app.output.write("On branch %s, root %s\n" % (branch, root)) has_uncommitted_changes = False - for d in self.iterate_branch_repos(branch_path, root_repo_path): + for gd in sorted(sb.list_git_directories(), key=lambda x: x.dirname): try: - repo = self.get_repo_config(d, 'morph.repository') + repo = gd.get_config('morph.repository') except cliapp.AppException: self.app.output.write( - ' %s: not part of system branch\n' % d) - continue - head = self.get_head(d) + ' %s: not part of system branch\n' % gd.dirname) + # TODO: make this less vulnerable to a branch using + # refs/heads/foo instead of foo + head = gd.HEAD if head != branch: self.app.output.write( - ' %s: unexpected ref checked out "%s"\n' % (repo, head)) - if len(self.get_uncommitted_changes(d)) > 0: + ' %s: unexpected ref checked out %r\n' % (repo, head)) + if any(gd.get_index().get_uncommitted_changes()): has_uncommitted_changes = True self.app.output.write(' %s: uncommitted changes\n' % repo) if not has_uncommitted_changes: self.app.output.write("\nNo repos have outstanding changes.\n") - def foreach(self, args): - '''Run a command in each repository checked out in a system branch. + def branch_from_image(self, args): + '''Produce a branch of an existing system image. - Use -- before specifying the command to separate its arguments from - Morph's own arguments. + Given the metadata specified by --metadata-dir, create a new + branch then petrify it to the state of the commits at the time + the system was built. - Command line arguments: + If --metadata-dir is not specified, it defaults to your currently + running system. - * `--` indicates the end of option processing for Morph. - * `COMMAND` is a command to run. - * `ARGS` is a list of arguments or options to be passed onto - `COMMAND`. + ''' + if len(args) != 1: + raise cliapp.AppException( + "branch-from-image needs exactly 1 argument " + "of the new system branch's name") + system_branch = args[0] + metadata_path = self.app.settings['metadata-dir'] + alias_resolver = morphlib.repoaliasresolver.RepoAliasResolver( + self.app.settings['repo-alias']) - This runs the given `COMMAND` in each git repository belonging - to the current system branch that exists locally in the current - workspace. This can be a handy way to do the same thing in all - the local git repositories. + self._require_git_user_config() - For example: + ws = morphlib.workspace.open('.') - morph foreach -- git push + system, metadata = self._load_system_metadata(metadata_path) + resolved_refs = dict(self._resolve_refs_from_metadata(alias_resolver, + metadata)) + self.app.status(msg='Resolved refs: %r' % resolved_refs) + base_ref = system['sha1'] + # The previous version would fall back to deducing this from the repo + # url and the repo alias resolver, but this does not always work, and + # new systems always have repo-alias in the metadata + root_url = system['repo-alias'] - The above command would push any committed changes in each - repository to the git server. + lrc, rrc = morphlib.util.new_repo_caches(self.app) + cached_repo = lrc.get_updated_repo(root_url) - ''' - # For simplicity, this simply iterates repositories in the directory - # rather than walking through the morphologies as 'morph merge' does. + with self._initializing_system_branch( + ws, root_url, system_branch, cached_repo, base_ref) as (sb, gd): - if len(args) == 0: - raise cliapp.AppException('morph foreach expects a command to run') + # TODO: It's nasty to clone to a sha1 then create a branch + # of that sha1 then check it out, a nicer API may be the + # initial clone not checking out a branch at all, then + # the user creates and checks out their own branches + gd.branch(system_branch, base_ref) + gd.checkout(system_branch) - workspace = self.deduce_workspace() - branch, branch_path = self.deduce_system_branch() + loader = morphlib.morphloader.MorphologyLoader() + morphs = self._load_all_sysbranch_morphologies(sb, loader) - root_repo = self.get_branch_config(branch_path, 'branch.root') - root_repo_path = self.find_repository(branch_path, root_repo) + morphs.repoint_refs(sb.root_repository_url, + sb.system_branch_name) - for d in self.iterate_branch_repos(branch_path, root_repo_path): - try: - repo = self.get_repo_config(d, 'morph.repository') - except cliapp.AppException: - continue + morphs.petrify_chunks(resolved_refs) - if d != root_repo_path: - self.app.output.write('\n') - self.app.output.write('%s\n' % repo) + self._save_dirty_morphologies(loader, sb, morphs.morphologies) - status, output, error = self.app.runcmd_unchecked(args, cwd=d) - self.app.output.write(output) - if status != 0: - self.app.output.write(error) - raise cliapp.AppException( - 'Command failed at repo %s: %s' % (repo, ' '.join(args))) + @staticmethod + def _load_system_metadata(path): + '''Load all metadata in `path` corresponding to a single System. + ''' + + smd = morphlib.systemmetadatadir.SystemMetadataDir(path) + metadata = smd.values() + systems = [md for md in metadata + if 'kind' in md and md['kind'] == 'system'] + + if not systems: + raise cliapp.AppException( + 'Metadata directory does not contain any systems.') + if len(systems) > 1: + raise cliapp.AppException( + 'Metadata directory contains multiple systems.') + system_metadatum = systems[0] + + metadata_cache_id_lookup = dict((md['cache-key'], md) + for md in metadata) + + return system_metadatum, metadata_cache_id_lookup + + @staticmethod + def _resolve_refs_from_metadata(alias_resolver, metadata): + '''Pre-resolve a set of refs from existing metadata. + + Given the metadata, generate a mapping of all the (repo, ref) + pairs defined in the metadata and the commit id they resolved to. + ''' + for md in metadata.itervalues(): + repourls = set((md['repo-alias'], md['repo'])) + repourls.update(alias_resolver.aliases_from_url(md['repo'])) + for repourl in repourls: + yield ((repourl, md['original_ref']), md['sha1']) diff --git a/morphlib/plugins/build_plugin.py b/morphlib/plugins/build_plugin.py index fb7efa5b..1a4fb573 100644 --- a/morphlib/plugins/build_plugin.py +++ b/morphlib/plugins/build_plugin.py @@ -76,7 +76,7 @@ class BuildPlugin(cliapp.Plugin): Example: - morph distbuild devel-system-x86_64-generic + morph distbuild devel-system-x86_64-generic.morph ''' @@ -103,8 +103,8 @@ class BuildPlugin(cliapp.Plugin): Example: - morph build-morphology baserock:baserock/morphs \ - master devel-system-x86_64-generic + morph build-morphology baserock:baserock/definitions \ + master devel-system-x86_64-generic.morph ''' @@ -141,7 +141,7 @@ class BuildPlugin(cliapp.Plugin): Example: - morph build devel-system-x86_64-generic + morph build devel-system-x86_64-generic.morph ''' @@ -156,7 +156,7 @@ class BuildPlugin(cliapp.Plugin): self.app.settings['cachedir'], self.app.settings['cachedir-min-space']) - system_name = morphlib.util.strip_morph_extension(args[0]) + system_filename = morphlib.util.sanitise_morphology_path(args[0]) ws = morphlib.workspace.open('.') sb = morphlib.sysbranchdir.open_from_within('.') @@ -181,7 +181,8 @@ class BuildPlugin(cliapp.Plugin): self.app.status(msg='Starting build %(uuid)s', uuid=build_uuid) self.app.status(msg='Collecting morphologies involved in ' 'building %(system)s from %(branch)s', - system=system_name, branch=sb.system_branch_name) + system=system_filename, + branch=sb.system_branch_name) bb = morphlib.buildbranch.BuildBranch(sb, build_ref_prefix, push_temporary=push) @@ -210,4 +211,4 @@ class BuildPlugin(cliapp.Plugin): build_command.build([bb.root_repo_url, bb.root_ref, - system_name]) + system_filename]) diff --git a/morphlib/plugins/cross-bootstrap_plugin.py b/morphlib/plugins/cross-bootstrap_plugin.py index bfd0d047..cd8e355e 100644 --- a/morphlib/plugins/cross-bootstrap_plugin.py +++ b/morphlib/plugins/cross-bootstrap_plugin.py @@ -260,7 +260,7 @@ class CrossBootstrapPlugin(cliapp.Plugin): self.app.settings, arch) build_command = morphlib.buildcommand.BuildCommand(self.app, build_env) - morph_name = system_name + '.morph' + morph_name = morphlib.util.sanitise_morphology_path(system_name) builds_artifacts = [system_name + '-bootstrap-rootfs'] srcpool = build_command.create_source_pool(root_repo, ref, morph_name) diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index 6fc0998c..9384c422 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -99,7 +99,7 @@ class DeployPlugin(cliapp.Plugin): name: cluster-foo kind: cluster systems: - - morph: devel-system-x86_64-generic + - morph: devel-system-x86_64-generic.morph deploy: cluster-foo-x86_64-1: type: kvm @@ -278,7 +278,7 @@ class DeployPlugin(cliapp.Plugin): '/', 0) self.app.settings['no-git-update'] = True - cluster_name = morphlib.util.strip_morph_extension(args[0]) + cluster_filename = morphlib.util.sanitise_morphology_path(args[0]) ws = morphlib.workspace.open('.') sb = morphlib.sysbranchdir.open_from_within('.') @@ -292,11 +292,9 @@ class DeployPlugin(cliapp.Plugin): name = morphlib.git.get_user_name(self.app.runcmd) email = morphlib.git.get_user_email(self.app.runcmd) build_ref_prefix = self.app.settings['build-ref-prefix'] - root_repo_dir = morphlib.gitdir.GitDirectory( sb.get_git_directory_name(sb.root_repository_url)) - mf = morphlib.morphologyfinder.MorphologyFinder(root_repo_dir) - cluster_text, cluster_filename = mf.read_morphology(cluster_name) + cluster_text = root_repo_dir.read_file(cluster_filename) cluster_morphology = loader.load_from_string(cluster_text, filename=cluster_filename) @@ -388,9 +386,8 @@ class DeployPlugin(cliapp.Plugin): self.app.status_prefix = system_status_prefix try: # Find the artifact to build - morph = system['morph'] - srcpool = build_command.create_source_pool(build_repo, ref, - morph + '.morph') + morph = morphlib.util.sanitise_morphology_path(system['morph']) + srcpool = build_command.create_source_pool(build_repo, ref, morph) artifact = build_command.resolve_artifacts(srcpool) diff --git a/morphlib/plugins/list_artifacts_plugin.py b/morphlib/plugins/list_artifacts_plugin.py index 5e64f708..ad6bc772 100644 --- a/morphlib/plugins/list_artifacts_plugin.py +++ b/morphlib/plugins/list_artifacts_plugin.py @@ -55,21 +55,22 @@ class ListArtifactsPlugin(cliapp.Plugin): '(see help)') repo, ref = args[0], args[1] - system_names = map(morphlib.util.strip_morph_extension, args[2:]) + system_filenames = map(morphlib.util.sanitise_morphology_path, + args[2:]) self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app) self.resolver = morphlib.artifactresolver.ArtifactResolver() artifact_files = set() - for system_name in system_names: + for system_filename in system_filenames: system_artifact_files = self.list_artifacts_for_system( - repo, ref, system_name) + repo, ref, system_filename) artifact_files.update(system_artifact_files) for artifact_file in sorted(artifact_files): print artifact_file - def list_artifacts_for_system(self, repo, ref, system_name): + def list_artifacts_for_system(self, repo, ref, system_filename): '''List all artifact files in the build graph of a single system.''' # Sadly, we must use a fresh source pool and a fresh list of artifacts @@ -82,24 +83,24 @@ class ListArtifactsPlugin(cliapp.Plugin): # different architectures right now. self.app.status( - msg='Creating source pool for %s' % system_name, chatty=True) + msg='Creating source pool for %s' % system_filename, chatty=True) source_pool = self.app.create_source_pool( - self.lrc, self.rrc, (repo, ref, system_name + '.morph')) + self.lrc, self.rrc, (repo, ref, system_filename)) self.app.status( - msg='Resolving artifacts for %s' % system_name, chatty=True) + msg='Resolving artifacts for %s' % system_filename, chatty=True) artifacts = self.resolver.resolve_artifacts(source_pool) - def find_artifact_by_name(artifacts_list, name): + def find_artifact_by_name(artifacts_list, filename): for a in artifacts_list: - if a.source.filename == name + '.morph': + if a.source.filename == name: return a raise ValueError - system_artifact = find_artifact_by_name(artifacts, system_name) + system_artifact = find_artifact_by_name(artifacts, system_filename) self.app.status( - msg='Computing cache keys for %s' % system_name, chatty=True) + msg='Computing cache keys for %s' % system_filename, chatty=True) build_env = morphlib.buildenvironment.BuildEnvironment( self.app.settings, system_artifact.source.morphology['arch']) ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env) diff --git a/morphlib/source.py b/morphlib/source.py index 75a2e4de..2dbabad1 100644 --- a/morphlib/source.py +++ b/morphlib/source.py @@ -55,4 +55,4 @@ class Source(object): def __str__(self): # pragma: no cover return '%s|%s|%s' % (self.repo_name, self.original_ref, - self.filename[:-len('.morph')]) + self.filename) diff --git a/morphlib/sysbranchdir.py b/morphlib/sysbranchdir.py index 9d96e974..b8953c2f 100644 --- a/morphlib/sysbranchdir.py +++ b/morphlib/sysbranchdir.py @@ -176,8 +176,9 @@ class SystemBranchDirectory(object): gd_name = self.get_git_directory_name(self.root_repository_url) gd = morphlib.gitdir.GitDirectory(gd_name) mf = morphlib.morphologyfinder.MorphologyFinder(gd) - for morph in mf.list_morphologies(): - text, filename = mf.read_morphology(morph) + for filename in (f for f in mf.list_morphologies() + if not gd.is_symlink(f)): + text = mf.read_morphology(filename) m = loader.load_from_string(text, filename=filename) m.repo_url = self.root_repository_url m.ref = self.system_branch_name diff --git a/morphlib/util.py b/morphlib/util.py index 024de495..0c551296 100644 --- a/morphlib/util.py +++ b/morphlib/util.py @@ -61,13 +61,22 @@ def indent(string, spaces=4): return '\n'.join(lines) -def strip_morph_extension(morph_name): - if morph_name.startswith('.'): - raise morphlib.Error( - 'Invalid morphology name: %s' % morph_name) - elif morph_name.endswith('.morph'): - return morph_name[:-len('.morph')] - return morph_name +def sanitise_morphology_path(morph_name): + '''Turn morph_name into a file path to a morphology. + + We support both a file path being provided, and just the morphology + name for backwards compatibility. + + ''' + # If it has a / it must be a path, so return it unmolested + if '/' in morph_name: + return morph_name + # Must be an old format, which is always name + .morph + elif not morph_name.endswith('.morph'): + return morph_name + '.morph' + # morphology already ends with .morph + else: + return morph_name def make_concurrency(cores=None): diff --git a/morphlib/util_tests.py b/morphlib/util_tests.py index 5a8ae797..715892b6 100644 --- a/morphlib/util_tests.py +++ b/morphlib/util_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2013 Codethink Limited +# Copyright (C) 2011-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -38,26 +38,19 @@ class IndentTests(unittest.TestCase): ' foo\n bar') -class StripMorphExtensionTests(unittest.TestCase): +class SanitiseMorphologyPathTests(unittest.TestCase): - def test_raises_error_when_string_starts_with_period(self): - with self.assertRaises(morphlib.Error): - morphlib.util.strip_morph_extension('.morph') - - def test_strips_morph_extension_from_string(self): - self.assertEqual(morphlib.util.strip_morph_extension('a.morph'), 'a') - - def test_returns_morph_when_not_given_as_extension(self): - self.assertEqual(morphlib.util.strip_morph_extension('morph'), 'morph') - - def test_strips_extension_only_once_from_string(self): - self.assertEqual(morphlib.util.strip_morph_extension('a.morph.morph'), + def test_appends_morph_to_string(self): + self.assertEqual(morphlib.util.sanitise_morphology_path('a'), 'a.morph') - def test_returns_input_without_modification_if_no_extension(self): - self.assertEqual( - morphlib.util.strip_morph_extension('completely not a path'), - 'completely not a path') + def test_returns_morph_when_given_a_filename(self): + self.assertEqual(morphlib.util.sanitise_morphology_path('a.morph'), + 'a.morph') + + def test_returns_morph_when_given_a_path(self): + self.assertEqual('stratum/a.morph', + morphlib.util.sanitise_morphology_path('stratum/a.morph')) class MakeConcurrencyTests(unittest.TestCase): |