summaryrefslogtreecommitdiff
path: root/morphlib
diff options
context:
space:
mode:
Diffstat (limited to 'morphlib')
-rw-r--r--morphlib/app.py28
-rw-r--r--morphlib/artifactresolver.py6
-rw-r--r--morphlib/buildcommand.py2
-rw-r--r--morphlib/gitdir.py10
-rw-r--r--morphlib/gitdir_tests.py37
-rw-r--r--morphlib/morphloader.py64
-rw-r--r--morphlib/morphologyfactory.py56
-rw-r--r--morphlib/morphologyfinder.py21
-rw-r--r--morphlib/morphologyfinder_tests.py16
-rw-r--r--morphlib/morphset.py49
-rw-r--r--morphlib/morphset_tests.py29
-rw-r--r--morphlib/plugins/branch_and_merge_new_plugin.py829
-rw-r--r--morphlib/plugins/branch_and_merge_plugin.py2242
-rw-r--r--morphlib/plugins/build_plugin.py15
-rw-r--r--morphlib/plugins/cross-bootstrap_plugin.py2
-rw-r--r--morphlib/plugins/deploy_plugin.py13
-rw-r--r--morphlib/plugins/list_artifacts_plugin.py23
-rw-r--r--morphlib/source.py2
-rw-r--r--morphlib/sysbranchdir.py5
-rw-r--r--morphlib/util.py23
-rw-r--r--morphlib/util_tests.py29
21 files changed, 756 insertions, 2745 deletions
diff --git a/morphlib/app.py b/morphlib/app.py
index df8c360a..e0874317 100644
--- a/morphlib/app.py
+++ b/morphlib/app.py
@@ -282,7 +282,8 @@ class Morph(cliapp.Application):
while args:
assert len(args) >= 2, args
- yield args[0], args[1], args[2] + ".morph"
+ yield (args[0], args[1],
+ morphlib.util.sanitise_morphology_path(args[2]))
args = args[3:]
def create_source_pool(self, lrc, rrc, triplet):
@@ -367,18 +368,23 @@ class Morph(cliapp.Application):
raise cliapp.AppException(
"Cannot build a morphology of type 'cluster'.")
elif morphology['kind'] == 'system':
- queue.extend((s.get('repo') or reponame,
- s.get('ref') or ref,
- '%s.morph' % s['morph'])
- for s in morphology['strata'])
+ queue.extend(
+ (s.get('repo') or reponame,
+ s.get('ref') or ref,
+ morphlib.util.sanitise_morphology_path(s['morph']))
+ for s in morphology['strata'])
elif morphology['kind'] == 'stratum':
if morphology['build-depends']:
- queue.extend((s.get('repo') or reponame,
- s.get('ref') or ref,
- '%s.morph' % s['morph'])
- for s in morphology['build-depends'])
- queue.extend((c['repo'], c['ref'], '%s.morph' % c['morph'])
- for c in morphology['chunks'])
+ queue.extend(
+ (s.get('repo') or reponame,
+ s.get('ref') or ref,
+ morphlib.util.sanitise_morphology_path(s['morph']))
+ for s in morphology['build-depends'])
+ queue.extend(
+ (c['repo'],
+ c['ref'],
+ morphlib.util.sanitise_morphology_path(c['morph']))
+ for c in morphology['chunks'])
def cache_repo_and_submodules(self, cache, url, ref, done):
subs_to_process = set()
diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py
index 00976eb7..c18042a3 100644
--- a/morphlib/artifactresolver.py
+++ b/morphlib/artifactresolver.py
@@ -142,7 +142,7 @@ class ArtifactResolver(object):
stratum_source = self._source_pool.lookup(
info.get('repo') or source.repo_name,
info.get('ref') or source.original_ref,
- '%s.morph' % info['morph'])
+ morphlib.util.sanitise_morphology_path(info['morph']))
stratum_name = stratum_source.morphology['name']
matches, overlaps, unmatched = source.split_rules.partition(
@@ -167,7 +167,7 @@ class ArtifactResolver(object):
other_source = self._source_pool.lookup(
stratum_info.get('repo') or source.repo_name,
stratum_info.get('ref') or source.original_ref,
- '%s.morph' % stratum_info['morph'])
+ morphlib.util.sanitise_morphology_path(stratum_info['morph']))
# Make every stratum artifact this stratum source produces
# depend on every stratum artifact the other stratum source
@@ -194,7 +194,7 @@ class ArtifactResolver(object):
chunk_source = self._source_pool.lookup(
info['repo'],
info['ref'],
- '%s.morph' % info['morph'])
+ morphlib.util.sanitise_morphology_path(info['morph']))
chunk_name = chunk_source.morphology['name']
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py
index f68046e3..45c6ef00 100644
--- a/morphlib/buildcommand.py
+++ b/morphlib/buildcommand.py
@@ -231,7 +231,7 @@ class BuildCommand(object):
for spec in specs:
repo_name = spec.get('repo') or src.repo_name
ref = spec.get('ref') or src.original_ref
- filename = '%s.morph' % spec['morph']
+ filename = morphlib.util.sanitise_morphology_path(spec['morph'])
logging.debug(
'Validating cross ref to %s:%s:%s' %
(repo_name, ref, filename))
diff --git a/morphlib/gitdir.py b/morphlib/gitdir.py
index 8f6d69d7..5b0693cb 100644
--- a/morphlib/gitdir.py
+++ b/morphlib/gitdir.py
@@ -515,6 +515,16 @@ class GitDirectory(object):
tree = self.resolve_ref_to_tree(ref)
return self.get_file_from_ref(tree, filename)
+ def is_symlink(self, filename, ref=None):
+ if ref is None and self.is_bare():
+ raise NoWorkingTreeError(self)
+ if ref is None:
+ filepath = os.path.join(self.dirname, filename.lstrip('/'))
+ return os.path.islink(filepath)
+ tree_entry = self._runcmd(['git', 'ls-tree', ref, filename])
+ file_mode = tree_entry.split(' ', 1)[0]
+ return file_mode == '120000'
+
@property
def HEAD(self):
output = self._runcmd(['git', 'rev-parse', '--abbrev-ref', 'HEAD'])
diff --git a/morphlib/gitdir_tests.py b/morphlib/gitdir_tests.py
index 14b2a57a..b3b4a8ab 100644
--- a/morphlib/gitdir_tests.py
+++ b/morphlib/gitdir_tests.py
@@ -216,6 +216,43 @@ class GitDirectoryContentsTests(unittest.TestCase):
self.assertEqual(gd.describe(), 'example')
+class GitDirectoryFileTypeTests(unittest.TestCase):
+
+ def setUp(self):
+ self.tempdir = tempfile.mkdtemp()
+ self.dirname = os.path.join(self.tempdir, 'foo')
+ os.mkdir(self.dirname)
+ gd = morphlib.gitdir.init(self.dirname)
+ with open(os.path.join(self.dirname, 'file'), "w") as f:
+ f.write('dummy morphology text')
+ os.symlink('file', os.path.join(self.dirname, 'link'))
+ os.symlink('no file', os.path.join(self.dirname, 'broken'))
+ gd._runcmd(['git', 'add', '.'])
+ gd._runcmd(['git', 'commit', '-m', 'Initial commit'])
+ self.mirror = os.path.join(self.tempdir, 'mirror')
+ gd._runcmd(['git', 'clone', '--mirror', self.dirname, self.mirror])
+
+ def tearDown(self):
+ shutil.rmtree(self.tempdir)
+
+ def test_working_tree_symlinks(self):
+ gd = morphlib.gitdir.GitDirectory(self.dirname)
+ self.assertTrue(gd.is_symlink('link'))
+ self.assertTrue(gd.is_symlink('broken'))
+ self.assertFalse(gd.is_symlink('file'))
+
+ def test_bare_symlinks(self):
+ gd = morphlib.gitdir.GitDirectory(self.mirror)
+ self.assertTrue(gd.is_symlink('link', 'HEAD'))
+ self.assertTrue(gd.is_symlink('broken', 'HEAD'))
+ self.assertFalse(gd.is_symlink('file', 'HEAD'))
+
+ def test_is_symlink_raises_no_ref_no_work_tree(self):
+ gd = morphlib.gitdir.GitDirectory(self.mirror)
+ self.assertRaises(morphlib.gitdir.NoWorkingTreeError,
+ gd.is_symlink, 'file')
+
+
class GitDirectoryRefTwiddlingTests(unittest.TestCase):
def setUp(self):
diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py
index 368e5477..45416a19 100644
--- a/morphlib/morphloader.py
+++ b/morphlib/morphloader.py
@@ -40,25 +40,33 @@ class MorphologyObsoleteFieldWarning(UserWarning):
class MorphologySyntaxError(morphlib.Error):
+ pass
+
+
+class MorphologyNotYamlError(MorphologySyntaxError):
def __init__(self, morphology, errmsg):
self.msg = 'Syntax error in morphology %s:\n%s' % (morphology, errmsg)
-class NotADictionaryError(morphlib.Error):
+class NotADictionaryError(MorphologySyntaxError):
def __init__(self, morph_filename):
self.msg = 'Not a dictionary: morphology %s' % morph_filename
-class UnknownKindError(morphlib.Error):
+class MorphologyValidationError(morphlib.Error):
+ pass
+
+
+class UnknownKindError(MorphologyValidationError):
def __init__(self, kind, morph_filename):
self.msg = (
'Unknown kind %s in morphology %s' % (kind, morph_filename))
-class MissingFieldError(morphlib.Error):
+class MissingFieldError(MorphologyValidationError):
def __init__(self, field, morphology_name):
self.field = field
@@ -67,7 +75,7 @@ class MissingFieldError(morphlib.Error):
'Missing field %s from morphology %s' % (field, morphology_name))
-class InvalidFieldError(morphlib.Error):
+class InvalidFieldError(MorphologyValidationError):
def __init__(self, field, morphology_name):
self.field = field
@@ -76,7 +84,7 @@ class InvalidFieldError(morphlib.Error):
'Field %s not allowed in morphology %s' % (field, morphology_name))
-class InvalidTypeError(morphlib.Error):
+class InvalidTypeError(MorphologyValidationError):
def __init__(self, field, expected, actual, morphology_name):
self.field = field
@@ -88,7 +96,7 @@ class InvalidTypeError(morphlib.Error):
(field, expected, actual, morphology_name))
-class ObsoleteFieldsError(morphlib.Error):
+class ObsoleteFieldsError(MorphologyValidationError):
def __init__(self, fields, morph_filename):
self.msg = (
@@ -96,14 +104,14 @@ class ObsoleteFieldsError(morphlib.Error):
(morph_filename, ' '.join(fields)))
-class UnknownArchitectureError(morphlib.Error):
+class UnknownArchitectureError(MorphologyValidationError):
def __init__(self, arch, morph_filename):
self.msg = ('Unknown architecture %s in morphology %s'
% (arch, morph_filename))
-class NoBuildDependenciesError(morphlib.Error):
+class NoBuildDependenciesError(MorphologyValidationError):
def __init__(self, stratum_name, chunk_name, morph_filename):
self.msg = (
@@ -111,7 +119,7 @@ class NoBuildDependenciesError(morphlib.Error):
(stratum_name, chunk_name, morph_filename))
-class NoStratumBuildDependenciesError(morphlib.Error):
+class NoStratumBuildDependenciesError(MorphologyValidationError):
def __init__(self, stratum_name, morph_filename):
self.msg = (
@@ -119,7 +127,7 @@ class NoStratumBuildDependenciesError(morphlib.Error):
(stratum_name, morph_filename))
-class EmptyStratumError(morphlib.Error):
+class EmptyStratumError(MorphologyValidationError):
def __init__(self, stratum_name, morph_filename):
self.msg = (
@@ -127,86 +135,86 @@ class EmptyStratumError(morphlib.Error):
(stratum_name, morph_filename))
-class DuplicateChunkError(morphlib.Error):
+class DuplicateChunkError(MorphologyValidationError):
def __init__(self, stratum_name, chunk_name):
self.stratum_name = stratum_name
self.chunk_name = chunk_name
- morphlib.Error.__init__(
+ MorphologyValidationError.__init__(
self, 'Duplicate chunk %(chunk_name)s '\
'in stratum %(stratum_name)s' % locals())
-class EmptyRefError(morphlib.Error):
+class EmptyRefError(MorphologyValidationError):
def __init__(self, ref_location, morph_filename):
self.ref_location = ref_location
self.morph_filename = morph_filename
- morphlib.Error.__init__(
+ MorphologyValidationError.__init__(
self, 'Empty ref found for %(ref_location)s '\
'in %(morph_filename)s' % locals())
-class ChunkSpecRefNotStringError(morphlib.Error):
+class ChunkSpecRefNotStringError(MorphologyValidationError):
def __init__(self, ref_value, chunk_name, stratum_name):
self.ref_value = ref_value
self.chunk_name = chunk_name
self.stratum_name = stratum_name
- morphlib.Error.__init__(
+ MorphologyValidationError.__init__(
self, 'Ref %(ref_value)s for %(chunk_name)s '\
'in stratum %(stratum_name)s is not a string' % locals())
-class SystemStrataNotListError(morphlib.Error):
+class SystemStrataNotListError(MorphologyValidationError):
def __init__(self, system_name, strata_type):
self.system_name = system_name
self.strata_type = strata_type
typename = strata_type.__name__
- morphlib.Error.__init__(
+ MorphologyValidationError.__init__(
self, 'System %(system_name)s has the wrong type for its strata: '\
'%(typename)s, expected list' % locals())
-class DuplicateStratumError(morphlib.Error):
+class DuplicateStratumError(MorphologyValidationError):
def __init__(self, system_name, stratum_name):
self.system_name = system_name
self.stratum_name = stratum_name
- morphlib.Error.__init__(
+ MorphologyValidationError.__init__(
self, 'Duplicate stratum %(stratum_name)s '\
'in system %(system_name)s' % locals())
-class SystemStratumSpecsNotMappingError(morphlib.Error):
+class SystemStratumSpecsNotMappingError(MorphologyValidationError):
def __init__(self, system_name, strata):
self.system_name = system_name
self.strata = strata
- morphlib.Error.__init__(
+ MorphologyValidationError.__init__(
self, 'System %(system_name)s has stratum specs '\
'that are not mappings.' % locals())
-class EmptySystemError(morphlib.Error):
+class EmptySystemError(MorphologyValidationError):
def __init__(self, system_name):
- morphlib.Error.__init__(
+ MorphologyValidationError.__init__(
self, 'System %(system_name)s has no strata.' % locals())
-class MultipleValidationErrors(morphlib.Error):
+class MultipleValidationErrors(MorphologyValidationError):
def __init__(self, name, errors):
self.name = name
self.errors = errors
self.msg = 'Multiple errors when validating %(name)s:'
for error in errors:
- self.msg += ('\t' + str(error))
+ self.msg += ('\n' + str(error))
-class DuplicateDeploymentNameError(morphlib.Error):
+class DuplicateDeploymentNameError(MorphologyValidationError):
def __init__(self, cluster_filename, duplicates):
self.duplicates = duplicates
@@ -352,7 +360,7 @@ class MorphologyLoader(object):
try:
obj = yaml.safe_load(text)
except yaml.error.YAMLError as e:
- raise MorphologySyntaxError(morph_filename, e)
+ raise MorphologyNotYamlError(morph_filename, e)
if not isinstance(obj, dict):
raise NotADictionaryError(morph_filename)
diff --git a/morphlib/morphologyfactory.py b/morphlib/morphologyfactory.py
index cd59d4ec..1cde2c77 100644
--- a/morphlib/morphologyfactory.py
+++ b/morphlib/morphologyfactory.py
@@ -14,6 +14,8 @@
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+import os
+
import morphlib
import cliapp
@@ -74,43 +76,41 @@ class MorphologyFactory(object):
self._app.status(*args, **kwargs)
def _get_morphology_text(self, reponame, sha1, filename):
+ morph_name = os.path.splitext(os.path.basename(filename))[0]
if self._lrc.has_repo(reponame):
self.status(msg="Looking for %s in local repo cache" % filename,
chatty=True)
- repo = self._lrc.get_repo(reponame)
- file_list = repo.ls_tree(sha1)
-
- if filename in file_list:
- return repo.cat(sha1, filename)
+ try:
+ repo = self._lrc.get_repo(reponame)
+ text = repo.cat(sha1, filename)
+ except IOError:
+ text = None
+ file_list = repo.ls_tree(sha1)
elif self._rrc is not None:
- self.status(msg="Looking for %s in remote repo cache" % filename,
+ self.status(msg="Retrieving %(reponame)s %(sha1)s %(filename)s"
+ " from the remote artifact cache.",
+ reponame=reponame, sha1=sha1, filename=filename,
chatty=True)
- file_list = self._rrc.ls_tree(reponame, sha1)
-
- if filename in file_list:
- self.status(msg='Retrieving %s %s %s'
- 'from the remote artifact cache.'
- % (reponame, sha1, filename), chatty=True)
- return self._rrc.cat_file(reponame, sha1, filename)
+ try:
+ text = self._rrc.cat_file(reponame, sha1, filename)
+ except morphlib.remoterepocache.CatFileError:
+ text = None
+ file_list = self._rrc.ls_tree(reponame, sha1)
else:
raise NotcachedError(reponame)
- self.status(msg="File %s doesn't exist: "
- "attempting to infer chunk morph from repo's build system"
- % filename, chatty=True)
- bs = morphlib.buildsystem.detect_build_system(file_list)
- if bs is None:
- raise MorphologyNotFoundError(filename)
- # TODO consider changing how morphs are located to be by morph
- # name rather than filename, it would save creating a
- # filename only to strip it back to its morph name again
- # and would allow future changes like morphologies being
- # stored as git metadata instead of as a file in the repo
- morph_name = filename[:-len('.morph')]
- return bs.get_morphology_text(morph_name)
+ if text is None:
+ self.status(msg="File %s doesn't exist: attempting to infer "
+ "chunk morph from repo's build system"
+ % filename, chatty=True)
+ bs = morphlib.buildsystem.detect_build_system(file_list)
+ if bs is None:
+ raise MorphologyNotFoundError(filename)
+ text = bs.get_morphology_text(morph_name)
+ return morph_name, text
def get_morphology(self, reponame, sha1, filename):
- text = self._get_morphology_text(reponame, sha1, filename)
+ morph_name, text = self._get_morphology_text(reponame, sha1, filename)
try:
morphology = morphlib.morph2.Morphology(text)
@@ -118,7 +118,7 @@ class MorphologyFactory(object):
raise morphlib.Error("Error parsing %s: %s" %
(filename, str(e)))
- if filename != morphology['name'] + '.morph':
+ if morph_name != morphology['name']:
raise morphlib.Error(
"Name %s does not match basename of morphology file %s" %
(morphology['name'], filename))
diff --git a/morphlib/morphologyfinder.py b/morphlib/morphologyfinder.py
index affa0e97..87c0de1a 100644
--- a/morphlib/morphologyfinder.py
+++ b/morphlib/morphologyfinder.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2013 Codethink Limited
+# Copyright (C) 2013-2014 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -34,38 +34,29 @@ class MorphologyFinder(object):
self.gitdir = gitdir
self.ref = ref
- def read_morphology(self, name):
+ def read_morphology(self, filename):
'''Return the un-parsed text of a morphology.
For the given morphology name, locate and return the contents
of the morphology as a string.
- Also returns a string describing where in the repository the
- morphology is located.
-
Parsing of this morphology into a form useful for manipulating
is handled by the MorphologyLoader class.
'''
- filename = '%s.morph' % name
- return self.gitdir.read_file(filename, self.ref), filename
+ return self.gitdir.read_file(filename, self.ref)
def list_morphologies(self):
- '''Return the names of all morphologies in the (repo, ref).
+ '''Return the filenames of all morphologies in the (repo, ref).
Finds all morphologies in the git directory at the specified
- ref. Morphology names are returned instead of filenames,
- so the implementation may change how morphologies are stored
- in git repositories.
+ ref.
'''
def is_morphology_path(path):
return path.endswith('.morph')
- def transform_path_to_name(path):
- return path[:-len('.morph')]
-
- return (transform_path_to_name(path)
+ return (path
for path in self.gitdir.list_files(self.ref)
if is_morphology_path(path))
diff --git a/morphlib/morphologyfinder_tests.py b/morphlib/morphologyfinder_tests.py
index ff83ccff..b07b2613 100644
--- a/morphlib/morphologyfinder_tests.py
+++ b/morphlib/morphologyfinder_tests.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2013 Codethink Limited
+# Copyright (C) 2013-2014 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -54,13 +54,13 @@ class MorphologyFinderTests(unittest.TestCase):
gd = morphlib.gitdir.GitDirectory(self.dirname)
mf = morphlib.morphologyfinder.MorphologyFinder(gd, 'HEAD')
self.assertEqual(sorted(mf.list_morphologies()),
- ['bar', 'baz'])
+ ['bar.morph', 'baz.morph'])
def test_list_morphs_in_master(self):
gd = morphlib.gitdir.GitDirectory(self.dirname)
mf = morphlib.morphologyfinder.MorphologyFinder(gd, 'master')
self.assertEqual(sorted(mf.list_morphologies()),
- ['bar', 'baz'])
+ ['bar.morph', 'baz.morph'])
def test_list_morphs_raises_with_invalid_ref(self):
gd = morphlib.gitdir.GitDirectory(self.dirname)
@@ -72,7 +72,7 @@ class MorphologyFinderTests(unittest.TestCase):
gd = morphlib.gitdir.GitDirectory(self.dirname)
mf = morphlib.morphologyfinder.MorphologyFinder(gd)
self.assertEqual(sorted(mf.list_morphologies()),
- ['bar', 'baz', 'foo'])
+ ['bar.morph', 'baz.morph', 'foo.morph'])
def test_list_morphs_raises_no_worktree_no_ref(self):
gd = morphlib.gitdir.GitDirectory(self.mirror)
@@ -83,13 +83,13 @@ class MorphologyFinderTests(unittest.TestCase):
def test_read_morph_in_HEAD(self):
gd = morphlib.gitdir.GitDirectory(self.dirname)
mf = morphlib.morphologyfinder.MorphologyFinder(gd, 'HEAD')
- self.assertEqual(mf.read_morphology('bar')[0],
+ self.assertEqual(mf.read_morphology('bar.morph'),
"dummy morphology text")
def test_read_morph_in_master(self):
gd = morphlib.gitdir.GitDirectory(self.dirname)
mf = morphlib.morphologyfinder.MorphologyFinder(gd, 'master')
- self.assertEqual(mf.read_morphology('bar')[0],
+ self.assertEqual(mf.read_morphology('bar.morph'),
"dummy morphology text")
def test_read_morph_raises_with_invalid_ref(self):
@@ -101,11 +101,11 @@ class MorphologyFinderTests(unittest.TestCase):
def test_read_morph_in_work_tree(self):
gd = morphlib.gitdir.GitDirectory(self.dirname)
mf = morphlib.morphologyfinder.MorphologyFinder(gd)
- self.assertEqual(mf.read_morphology('foo')[0],
+ self.assertEqual(mf.read_morphology('foo.morph'),
"altered morphology text")
def test_read_morph_raises_no_worktree_no_ref(self):
gd = morphlib.gitdir.GitDirectory(self.mirror)
mf = morphlib.morphologyfinder.MorphologyFinder(gd)
self.assertRaises(morphlib.gitdir.NoWorkingTreeError,
- mf.read_morphology, 'bar')
+ mf.read_morphology, 'bar.morph')
diff --git a/morphlib/morphset.py b/morphlib/morphset.py
index dedbabd5..590ac51e 100644
--- a/morphlib/morphset.py
+++ b/morphlib/morphset.py
@@ -19,19 +19,6 @@
import morphlib
-class StratumNotInSystemError(morphlib.Error):
-
- def __init__(self, system_name, stratum_name):
- self.msg = (
- 'System %s does not contain %s' % (system_name, stratum_name))
-
-
-class StratumNotInSetError(morphlib.Error):
-
- def __init__(self, stratum_name):
- self.msg = 'Stratum %s is not in MorphologySet' % stratum_name
-
-
class ChunkNotInStratumError(morphlib.Error):
def __init__(self, stratum_name, chunk_name):
@@ -79,31 +66,11 @@ class MorphologySet(object):
def _find_spec(self, specs, wanted_name):
for spec in specs:
- name = spec.get('morph', spec.get('name'))
+ name = spec.get('name', spec.get('morph'))
if name == wanted_name:
return spec.get('repo'), spec.get('ref'), name
return None, None, None
- def get_stratum_in_system(self, system_morph, stratum_name):
- '''Return morphology for a stratum that is in a system.
-
- If the stratum is not in the system, raise StratumNotInSystemError.
- If the stratum morphology has not been added to the set,
- raise StratumNotInSetError.
-
- '''
-
- repo_url, ref, morph = self._find_spec(
- system_morph['strata'], stratum_name)
- if (repo_url, ref, morph) == (None, None, None):
- raise StratumNotInSystemError(system_morph['name'], stratum_name)
- m = self._get_morphology(repo_url or system_morph.repo_url,
- ref or system_morph.ref,
- '%s.morph' % morph)
- if m is None:
- raise StratumNotInSetError(stratum_name)
- return m
-
def get_chunk_triplet(self, stratum_morph, chunk_name):
'''Return the repo url, ref, morph name triplet for a chunk.
@@ -160,8 +127,8 @@ class MorphologySet(object):
specs = m[kind]
for spec in specs:
if cb_filter(m, kind, spec):
- orig_spec = (spec.get('repo'), spec.get('ref'),
- spec['morph'])
+ fn = morphlib.util.sanitise_morphology_path(spec['morph'])
+ orig_spec = (spec.get('repo'), spec.get('ref'), fn)
dirtied = cb_process(m, kind, spec)
if dirtied:
m.dirty = True
@@ -175,17 +142,18 @@ class MorphologySet(object):
process_spec_list(m, 'chunks')
for m in self.morphologies:
- tup = (m.repo_url, m.ref, m.filename[:-len('.morph')])
+ tup = (m.repo_url, m.ref, m.filename)
if tup in altered_references:
spec = altered_references[tup]
if m.ref != spec.get('ref'):
m.ref = spec.get('ref')
m.dirty = True
- assert (m.filename == spec['morph'] + '.morph'
+ file = morphlib.util.sanitise_morphology_path(spec['morph'])
+ assert (m.filename == file
or m.repo_url == spec.get('repo')), \
'Moving morphologies is not supported.'
- def change_ref(self, repo_url, orig_ref, morph_filename, new_ref):
+ def change_ref(self, repo_url, orig_ref, morph_name, new_ref):
'''Change a triplet's ref to a new one in all morphologies in a ref.
Change orig_ref to new_ref in any morphology that references the
@@ -194,9 +162,10 @@ class MorphologySet(object):
'''
def wanted_spec(m, kind, spec):
+ spec_name = spec.get('name', spec['morph'])
return (spec.get('repo') == repo_url and
spec.get('ref') == orig_ref and
- spec['morph'] + '.morph' == morph_filename)
+ spec_name == morph_name)
def process_spec(m, kind, spec):
spec['unpetrify-ref'] = spec.get('ref')
diff --git a/morphlib/morphset_tests.py b/morphlib/morphset_tests.py
index d6908844..8679c64a 100644
--- a/morphlib/morphset_tests.py
+++ b/morphlib/morphset_tests.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2013 Codethink Limited
+# Copyright (C) 2013, 2014 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -80,27 +80,6 @@ class MorphologySetTests(unittest.TestCase):
self.morphs.add_morphology(self.system)
self.assertEqual(self.morphs.morphologies, [self.system])
- def test_get_stratum_in_system(self):
- self.morphs.add_morphology(self.system)
- self.morphs.add_morphology(self.stratum)
- self.assertEqual(
- self.morphs.get_stratum_in_system(
- self.system, self.stratum['name']),
- self.stratum)
-
- def test_raises_stratum_not_in_system_error(self):
- self.morphs.add_morphology(self.system)
- self.morphs.add_morphology(self.stratum)
- self.assertRaises(
- morphlib.morphset.StratumNotInSystemError,
- self.morphs.get_stratum_in_system, self.system, 'unknown-stratum')
-
- def test_raises_stratum_not_in_set_error(self):
- self.morphs.add_morphology(self.system)
- self.assertRaises(
- morphlib.morphset.StratumNotInSetError,
- self.morphs.get_stratum_in_system, self.system, 'foo-stratum')
-
def test_get_chunk_triplet(self):
self.morphs.add_morphology(self.system)
self.morphs.add_morphology(self.stratum)
@@ -119,7 +98,7 @@ class MorphologySetTests(unittest.TestCase):
self.morphs.change_ref(
self.stratum.repo_url,
self.stratum.ref,
- self.stratum.filename,
+ self.stratum['name'],
'new-ref')
self.assertEqual(self.stratum.ref, 'new-ref')
self.assertEqual(
@@ -155,7 +134,7 @@ class MorphologySetTests(unittest.TestCase):
self.morphs.change_ref(
self.stratum.repo_url,
self.stratum.ref,
- self.stratum.filename,
+ self.stratum['name'],
'new-ref')
self.assertEqual(
other_stratum['build-depends'][0],
@@ -172,7 +151,7 @@ class MorphologySetTests(unittest.TestCase):
self.morphs.change_ref(
'test:foo-chunk',
'master',
- 'foo-chunk.morph',
+ 'foo-chunk',
'new-ref')
self.assertEqual(
self.stratum['chunks'],
diff --git a/morphlib/plugins/branch_and_merge_new_plugin.py b/morphlib/plugins/branch_and_merge_new_plugin.py
deleted file mode 100644
index 5ac8353a..00000000
--- a/morphlib/plugins/branch_and_merge_new_plugin.py
+++ /dev/null
@@ -1,829 +0,0 @@
-# Copyright (C) 2012,2013,2014 Codethink Limited
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-import cliapp
-import contextlib
-import glob
-import logging
-import os
-import shutil
-
-import morphlib
-
-
-class BranchRootHasNoSystemsError(cliapp.AppException):
- def __init__(self, repo, ref):
- cliapp.AppException.__init__(
- self, 'System branch root repository %s '
- 'has no system morphologies at ref %s' % (repo, ref))
-
-
-class SimpleBranchAndMergePlugin(cliapp.Plugin):
-
- '''Add subcommands for handling workspaces and system branches.'''
-
- def enable(self):
- self.app.add_subcommand('init', self.init, arg_synopsis='[DIR]')
- self.app.add_subcommand('workspace', self.workspace, arg_synopsis='')
- self.app.add_subcommand(
- 'checkout', self.checkout, arg_synopsis='REPO BRANCH')
- self.app.add_subcommand(
- 'branch', self.branch, arg_synopsis='REPO NEW [OLD]')
- self.app.add_subcommand(
- 'edit', self.edit, arg_synopsis='SYSTEM STRATUM [CHUNK]')
- self.app.add_subcommand(
- 'petrify', self.petrify, arg_synopsis='')
- self.app.add_subcommand(
- 'unpetrify', self.unpetrify, arg_synopsis='')
- self.app.add_subcommand(
- 'show-system-branch', self.show_system_branch, arg_synopsis='')
- self.app.add_subcommand(
- 'show-branch-root', self.show_branch_root, arg_synopsis='')
- self.app.add_subcommand('foreach', self.foreach,
- arg_synopsis='-- COMMAND [ARGS...]')
- self.app.add_subcommand('status', self.status,
- arg_synopsis='')
- self.app.add_subcommand('branch-from-image', self.branch_from_image,
- arg_synopsis='BRANCH')
- group_branch = 'Branching Options'
- self.app.settings.string(['metadata-dir'],
- 'Set metadata location for branch-from-image'
- ' (default: /baserock)',
- metavar='DIR',
- default='/baserock',
- group=group_branch)
-
- def disable(self):
- pass
-
- def init(self, args):
- '''Initialize a workspace directory.
-
- Command line argument:
-
- * `DIR` is the directory to use as a workspace, and defaults to
- the current directory.
-
- This creates a workspace, either in the current working directory,
- or if `DIR` is given, in that directory. If the directory doesn't
- exist, it is created. If it does exist, it must be empty.
-
- You need to run `morph init` to initialise a workspace, or none
- of the other system branching tools will work: they all assume
- an existing workspace. Note that a workspace only exists on your
- machine, not on the git server.
-
- Example:
-
- morph init /src/workspace
- cd /src/workspace
-
- '''
-
- if not args:
- args = ['.']
- elif len(args) > 1:
- raise morphlib.Error('init must get at most one argument')
-
- ws = morphlib.workspace.create(args[0])
- self.app.status(msg='Initialized morph workspace', chatty=True)
-
- def workspace(self, args):
- '''Show the toplevel directory of the current workspace.'''
-
- ws = morphlib.workspace.open('.')
- self.app.output.write('%s\n' % ws.root)
-
- # TODO: Move this somewhere nicer
- @contextlib.contextmanager
- def _initializing_system_branch(self, ws, root_url, system_branch,
- cached_repo, base_ref):
- '''A context manager for system branches under construction.
-
- The purpose of this context manager is to factor out the branch
- cleanup code for if an exception occurs while a branch is being
- constructed.
-
- This could be handled by a higher order function which takes
- a function to initialize the branch as a parameter, but with
- statements look nicer and are more obviously about resource
- cleanup.
-
- '''
- root_dir = ws.get_default_system_branch_directory_name(system_branch)
- try:
- sb = morphlib.sysbranchdir.create(
- root_dir, root_url, system_branch)
- gd = sb.clone_cached_repo(cached_repo, base_ref)
-
- yield (sb, gd)
-
- gd.update_submodules(self.app)
- gd.update_remotes()
-
- except morphlib.sysbranchdir.SystemBranchDirectoryAlreadyExists as e:
- logging.error('Caught exception: %s' % str(e))
- raise
- except BaseException as e:
- # Oops. Clean up.
- logging.error('Caught exception: %s' % str(e))
- logging.info('Removing half-finished branch %s' % system_branch)
- self._remove_branch_dir_safe(ws.root, root_dir)
- raise
-
- def checkout(self, args):
- '''Check out an existing system branch.
-
- Command line arguments:
-
- * `REPO` is the URL to the repository to the root repository of
- a system branch.
- * `BRANCH` is the name of the system branch.
-
- This will check out an existing system branch to an existing
- workspace. You must create the workspace first. This only checks
- out the root repository, not the repositories for individual
- components. You need to use `morph edit` to check out those.
-
- Example:
-
- cd /src/workspace
- morph checkout baserock:baserock/morphs master
-
- '''
-
- if len(args) != 2:
- raise cliapp.AppException('morph checkout needs a repo and the '
- 'name of a branch as parameters')
-
- root_url = args[0]
- system_branch = args[1]
- base_ref = system_branch
-
- self._require_git_user_config()
-
- # Open the workspace first thing, so user gets a quick error if
- # we're not inside a workspace.
- ws = morphlib.workspace.open('.')
-
- # Make sure the root repository is in the local git repository
- # cache, and is up to date.
- lrc, rrc = morphlib.util.new_repo_caches(self.app)
- cached_repo = lrc.get_updated_repo(root_url)
-
- # Check the git branch exists.
- cached_repo.resolve_ref(system_branch)
-
- with self._initializing_system_branch(
- ws, root_url, system_branch, cached_repo, base_ref) as (sb, gd):
-
- if gd.has_fat():
- gd.fat_init()
- gd.fat_pull()
-
- if not self._checkout_has_systems(gd):
- raise BranchRootHasNoSystemsError(root_url, base_ref)
-
-
- def branch(self, args):
- '''Create a new system branch.
-
- Command line arguments:
-
- * `REPO` is a repository URL.
- * `NEW` is the name of the new system branch.
- * `OLD` is the point from which to branch, and defaults to `master`.
-
- This creates a new system branch. It needs to be run in an
- existing workspace (see `morph workspace`). It creates a new
- git branch in the clone of the repository in the workspace. The
- system branch will not be visible on the git server until you
- push your changes to the repository.
-
- Example:
-
- cd /src/workspace
- morph branch baserock:baserock/morphs jrandom/new-feature
-
- '''
-
- if len(args) not in [2, 3]:
- raise cliapp.AppException(
- 'morph branch needs name of branch as parameter')
-
- root_url = args[0]
- system_branch = args[1]
- base_ref = 'master' if len(args) == 2 else args[2]
- origin_base_ref = 'origin/%s' % base_ref
-
- self._require_git_user_config()
-
- # Open the workspace first thing, so user gets a quick error if
- # we're not inside a workspace.
- ws = morphlib.workspace.open('.')
-
- # Make sure the root repository is in the local git repository
- # cache, and is up to date.
- lrc, rrc = morphlib.util.new_repo_caches(self.app)
- cached_repo = lrc.get_updated_repo(root_url)
-
- # Make sure the system branch doesn't exist yet.
- if cached_repo.ref_exists(system_branch):
- raise cliapp.AppException(
- 'branch %s already exists in repository %s' %
- (system_branch, root_url))
-
- # Make sure the base_ref exists.
- cached_repo.resolve_ref(base_ref)
-
- with self._initializing_system_branch(
- ws, root_url, system_branch, cached_repo, base_ref) as (sb, gd):
-
- gd.branch(system_branch, base_ref)
- gd.checkout(system_branch)
- if gd.has_fat():
- gd.fat_init()
- gd.fat_pull()
-
- if not self._checkout_has_systems(gd):
- raise BranchRootHasNoSystemsError(root_url, base_ref)
-
- def _save_dirty_morphologies(self, loader, sb, morphs):
- logging.debug('Saving dirty morphologies: start')
- for morph in morphs:
- if morph.dirty:
- logging.debug(
- 'Saving morphology: %s %s %s' %
- (morph.repo_url, morph.ref, morph.filename))
- loader.unset_defaults(morph)
- loader.save_to_file(
- sb.get_filename(morph.repo_url, morph.filename), morph)
- morph.dirty = False
- logging.debug('Saving dirty morphologies: done')
-
- def _get_stratum_triplets(self, morph):
- # Gather all references to other strata from a morphology. The
- # morphology must be either a system or a stratum one. In a
- # stratum one, the refs are all for build dependencies of the
- # stratum. In a system one, they're the list of strata in the
- # system.
-
- assert morph['kind'] in ('system', 'stratum')
- if morph['kind'] == 'system':
- specs = morph.get('strata', [])
- elif morph['kind'] == 'stratum':
- specs = morph.get('build-depends', [])
-
- # Given a list of dicts that reference strata, return a list
- # of triplets (repo url, ref, filename).
-
- return [
- (spec.get('repo') or morph.repo_url,
- spec.get('ref') or morph.ref,
- '%s.morph' % spec['morph'])
- for spec in specs
- ]
-
- def _checkout(self, lrc, sb, repo_url, ref):
- logging.debug(
- 'Checking out %s (%s) into %s' %
- (repo_url, ref, sb.root_directory))
- cached_repo = lrc.get_updated_repo(repo_url)
- gd = sb.clone_cached_repo(cached_repo, ref)
- gd.update_submodules(self.app)
- gd.update_remotes()
-
- def _load_morphology_from_file(self, loader, dirname, filename):
- full_filename = os.path.join(dirname, filename)
- return loader.load_from_file(full_filename)
-
- def _load_morphology_from_git(self, loader, gd, ref, filename):
- try:
- text = gd.get_file_from_ref(ref, filename)
- except cliapp.AppException:
- text = gd.get_file_from_ref('origin/%s' % ref, filename)
- return loader.load_from_string(text, filename)
-
- def _load_stratum_morphologies(self, loader, sb, system_morph):
- logging.debug('Starting to load strata for %s' % system_morph.filename)
- lrc, rrc = morphlib.util.new_repo_caches(self.app)
- morphset = morphlib.morphset.MorphologySet()
- queue = self._get_stratum_triplets(system_morph)
- while queue:
- repo_url, ref, filename = queue.pop()
- if not morphset.has(repo_url, ref, filename):
- logging.debug('Loading: %s %s %s' % (repo_url, ref, filename))
- dirname = sb.get_git_directory_name(repo_url)
-
- # Get the right morphology. The right ref might not be
- # checked out, in which case we get the file from git.
- # However, if it is checked out, we get it from the
- # filesystem directly, in case the user has made any
- # changes to it. If the entire repo hasn't been checked
- # out yet, do that first.
-
- if not os.path.exists(dirname):
- self._checkout(lrc, sb, repo_url, ref)
- m = self._load_morphology_from_file(
- loader, dirname, filename)
- else:
- gd = morphlib.gitdir.GitDirectory(dirname)
- if gd.is_currently_checked_out(ref):
- m = self._load_morphology_from_file(
- loader, dirname, filename)
- else:
- m = self._load_morphology_from_git(
- loader, gd, ref, filename)
-
- m.repo_url = repo_url
- m.ref = ref
- m.filename = filename
-
- morphset.add_morphology(m)
- queue.extend(self._get_stratum_triplets(m))
-
- logging.debug('All strata loaded')
- return morphset
-
- def edit(self, args):
- '''Edit or checkout a component in a system branch.
-
- Command line arguments:
-
- * `CHUNK` is the name of a chunk
-
- This makes a local checkout of CHUNK in the current system branch
- and edits any stratum morphology file(s) containing the chunk
-
- '''
-
- if len(args) != 1:
- raise cliapp.AppException('morph edit needs a chunk '
- 'as parameter')
-
- ws = morphlib.workspace.open('.')
- sb = morphlib.sysbranchdir.open_from_within('.')
- loader = morphlib.morphloader.MorphologyLoader()
- morphs = self._load_all_sysbranch_morphologies(sb, loader)
-
- def edit_chunk(morph, chunk_name):
- chunk_url, chunk_ref, chunk_morph = (
- morphs.get_chunk_triplet(morph, chunk_name))
-
- chunk_dirname = sb.get_git_directory_name(chunk_url)
-
- if not os.path.exists(chunk_dirname):
- lrc, rrc = morphlib.util.new_repo_caches(self.app)
- cached_repo = lrc.get_updated_repo(chunk_url)
-
- gd = sb.clone_cached_repo(cached_repo, chunk_ref)
- if chunk_ref != sb.system_branch_name:
- gd.branch(sb.system_branch_name, chunk_ref)
- gd.checkout(sb.system_branch_name)
- gd.update_submodules(self.app)
- gd.update_remotes()
- if gd.has_fat():
- gd.fat_init()
- gd.fat_pull()
-
- # Change the refs to the chunk.
- if chunk_ref != sb.system_branch_name:
- morphs.change_ref(
- chunk_url, chunk_ref, chunk_morph + '.morph',
- sb.system_branch_name)
-
- return chunk_dirname
-
- chunk_name = morphlib.util.strip_morph_extension(args[0])
- dirs = set()
- found = 0
-
- for morph in morphs.morphologies:
- if morph['kind'] == 'stratum':
- for chunk in morph['chunks']:
- if chunk['name'] == chunk_name:
- self.app.status(
- msg='Editing %(chunk)s in %(stratum)s stratum',
- chunk=chunk_name, stratum=morph['name'])
- chunk_dirname = edit_chunk(morph, chunk_name)
- dirs.add(chunk_dirname)
- found = found + 1
-
- # Save any modified strata.
-
- self._save_dirty_morphologies(loader, sb, morphs.morphologies)
-
- if found == 0:
- self.app.status(
- msg="No chunk %(chunk)s found. If you want to create one, add "
- "an entry to a stratum morph file.", chunk=chunk_name)
-
- if found >= 1:
- dirs_list = ', '.join(sorted(dirs))
- self.app.status(
- msg="Chunk %(chunk)s source is available at %(dirs)s",
- chunk=chunk_name, dirs=dirs_list)
-
- if found > 1:
- self.app.status(
- msg="Notice that this chunk appears in more than one stratum")
-
- def show_system_branch(self, args):
- '''Show the name of the current system branch.'''
-
- ws = morphlib.workspace.open('.')
- sb = morphlib.sysbranchdir.open_from_within('.')
- self.app.output.write('%s\n' % sb.system_branch_name)
-
- def show_branch_root(self, args):
- '''Show the name of the repository holding the system morphologies.
-
- This would, for example, write out something like:
-
- /src/ws/master/baserock:baserock/morphs
-
- when the master branch of the `baserock:baserock/morphs`
- repository is checked out.
-
- '''
-
- ws = morphlib.workspace.open('.')
- sb = morphlib.sysbranchdir.open_from_within('.')
- self.app.output.write('%s\n' % sb.get_config('branch.root'))
-
- def _remove_branch_dir_safe(self, workspace_root, system_branch_root):
- # This function avoids throwing any exceptions, so it is safe to call
- # inside an 'except' block without altering the backtrace.
-
- def handle_error(function, path, excinfo):
- logging.warning ("Error while trying to clean up %s: %s" %
- (path, excinfo))
-
- shutil.rmtree(system_branch_root, onerror=handle_error)
-
- # Remove parent directories that are empty too, avoiding exceptions
- parent = os.path.dirname(system_branch_root)
- while parent != os.path.abspath(workspace_root):
- if len(os.listdir(parent)) > 0 or os.path.islink(parent):
- break
- os.rmdir(parent)
- parent = os.path.dirname(parent)
-
- def _require_git_user_config(self):
- '''Warn if the git user.name and user.email variables are not set.'''
-
- keys = {
- 'user.name': 'My Name',
- 'user.email': 'me@example.com',
- }
-
- try:
- morphlib.git.check_config_set(self.app.runcmd, keys)
- except morphlib.git.ConfigNotSetException as e:
- self.app.status(
- msg="WARNING: %(message)s",
- message=str(e), error=True)
-
- @staticmethod
- def _checkout_has_systems(gd):
- loader = morphlib.morphloader.MorphologyLoader()
- for filename in glob.iglob(os.path.join(gd.dirname, '*.morph')):
- m = loader.load_from_file(filename)
- if m['kind'] == 'system':
- return True
- return False
-
- def foreach(self, args):
- '''Run a command in each repository checked out in a system branch.
-
- Use -- before specifying the command to separate its arguments from
- Morph's own arguments.
-
- Command line arguments:
-
- * `--` indicates the end of option processing for Morph.
- * `COMMAND` is a command to run.
- * `ARGS` is a list of arguments or options to be passed onto
- `COMMAND`.
-
- This runs the given `COMMAND` in each git repository belonging
- to the current system branch that exists locally in the current
- workspace. This can be a handy way to do the same thing in all
- the local git repositories.
-
- For example:
-
- morph foreach -- git push
-
- The above command would push any committed changes in each
- repository to the git server.
-
- '''
-
- if not args:
- raise cliapp.AppException('morph foreach expects a command to run')
-
- ws = morphlib.workspace.open('.')
- sb = morphlib.sysbranchdir.open_from_within('.')
-
- for gd in sorted(sb.list_git_directories(), key=lambda gd: gd.dirname):
- # Get the repository's original name
- # Continue in the case of error, since the previous iteration
- # worked in the case of the user cloning a repository in the
- # system branch's directory.
- try:
- repo = gd.get_config('morph.repository')
- except cliapp.AppException:
- continue
-
- self.app.output.write('%s\n' % repo)
- status, output, error = self.app.runcmd_unchecked(
- args, cwd=gd.dirname)
- self.app.output.write(output)
- if status != 0:
- self.app.output.write(error)
- pretty_command = ' '.join(cliapp.shell_quote(arg)
- for arg in args)
- raise cliapp.AppException(
- 'Command failed at repo %s: %s'
- % (repo, pretty_command))
- self.app.output.write('\n')
- self.app.output.flush()
-
- def _load_all_sysbranch_morphologies(self, sb, loader):
- '''Read in all the morphologies in the root repository.'''
- self.app.status(msg='Loading in all morphologies')
- morphs = morphlib.morphset.MorphologySet()
- for morph in sb.load_all_morphologies(loader):
- morphs.add_morphology(morph)
- return morphs
-
- def petrify(self, args):
- '''Convert all chunk refs in a system branch to be fixed SHA1s.
-
- This modifies all git commit references in system and stratum
- morphologies, in the current system branch, to be fixed SHA
- commit identifiers, rather than symbolic branch or tag names.
- This is useful for making sure none of the components in a system
- branch change accidentally.
-
- Consider the following scenario:
-
- * The `master` system branch refers to `gcc` using the
- `baserock/morph` ref. This is appropriate, since the main line
- of development should use the latest curated code.
-
- * You create a system branch to prepare for a release, called
- `TROVE_ID/release/2.0`. The reference to `gcc` is still
- `baserock/morph`.
-
- * You test everything, and make a release. You deploy the release
- images onto devices, which get shipped to your customers.
-
- * A new version GCC is committed to the `baserock/morph` branch.
-
- * Your release branch suddenly uses a new compiler, which may
- or may not work for your particular system at that release.
-
- To avoid this, you need to _petrify_ all git references
- so that they do not change accidentally. If you've tested
- your release with the GCC release that is stored in commit
- `94c50665324a7aeb32f3096393ec54b2e63bfb28`, then you should
- continue to use that version of GCC, regardless of what might
- happen in the master system branch. If, and only if, you decide
- that a new compiler would be good for your release should you
- include it in your release branch. This way, only the things
- that you change intentionally change in your release branch.
-
- '''
-
- if args:
- raise cliapp.AppException('morph petrify takes no arguments')
-
- ws = morphlib.workspace.open('.')
- sb = morphlib.sysbranchdir.open_from_within('.')
- loader = morphlib.morphloader.MorphologyLoader()
- lrc, rrc = morphlib.util.new_repo_caches(self.app)
- update_repos = not self.app.settings['no-git-update']
-
- morphs = self._load_all_sysbranch_morphologies(sb, loader)
-
- #TODO: Stop using app.resolve_ref
- def resolve_refs(morphs):
- for repo, ref in morphs.list_refs():
- # You can't resolve null refs, so don't attempt to.
- if repo is None or ref is None:
- continue
- # TODO: Handle refs that are only in workspace in general
- if (repo == sb.root_repository_url
- and ref == sb.system_branch_name):
- continue
- commit_sha1, tree_sha1 = self.app.resolve_ref(
- lrc, rrc, repo, ref, update=update_repos)
- yield ((repo, ref), commit_sha1)
-
- morphs.repoint_refs(sb.root_repository_url,
- sb.system_branch_name)
-
- morphs.petrify_chunks(dict(resolve_refs(morphs)))
-
- # Write morphologies back out again.
- self._save_dirty_morphologies(loader, sb, morphs.morphologies)
-
- def unpetrify(self, args):
- '''Reverse the process of petrification.
-
- This undoes the changes `morph petrify` did.
-
- '''
-
- if args:
- raise cliapp.AppException('morph petrify takes no arguments')
-
- ws = morphlib.workspace.open('.')
- sb = morphlib.sysbranchdir.open_from_within('.')
- loader = morphlib.morphloader.MorphologyLoader()
-
- morphs = self._load_all_sysbranch_morphologies(sb, loader)
-
- # Restore the ref for each stratum and chunk
- morphs.unpetrify_all()
-
- # Write morphologies back out again.
- self._save_dirty_morphologies(loader, sb, morphs.morphologies)
-
- def status(self, args):
- '''Show information about the current system branch or workspace
-
- This shows the status of every local git repository of the
- current system branch. This is similar to running `git status`
- in each repository separately.
-
- If run in a Morph workspace, but not in a system branch checkout,
- it lists all checked out system branches in the workspace.
-
- '''
-
- if args:
- raise cliapp.AppException('morph status takes no arguments')
-
- ws = morphlib.workspace.open('.')
- try:
- sb = morphlib.sysbranchdir.open_from_within('.')
- except morphlib.sysbranchdir.NotInSystemBranch:
- self._workspace_status(ws)
- else:
- self._branch_status(ws, sb)
-
- def _workspace_status(self, ws):
- '''Show information about the current workspace
-
- This lists all checked out system branches in the workspace.
-
- '''
- self.app.output.write("System branches in current workspace:\n")
- branches = sorted(ws.list_system_branches(),
- key=lambda x: x.root_directory)
- for sb in branches:
- self.app.output.write(" %s\n" % sb.get_config('branch.name'))
-
- def _branch_status(self, ws, sb):
- '''Show information about the current branch
-
- This shows the status of every local git repository of the
- current system branch. This is similar to running `git status`
- in each repository separately.
-
- '''
- branch = sb.get_config('branch.name')
- root = sb.get_config('branch.root')
-
- self.app.output.write("On branch %s, root %s\n" % (branch, root))
-
- has_uncommitted_changes = False
- for gd in sorted(sb.list_git_directories(), key=lambda x: x.dirname):
- try:
- repo = gd.get_config('morph.repository')
- except cliapp.AppException:
- self.app.output.write(
- ' %s: not part of system branch\n' % gd.dirname)
- # TODO: make this less vulnerable to a branch using
- # refs/heads/foo instead of foo
- head = gd.HEAD
- if head != branch:
- self.app.output.write(
- ' %s: unexpected ref checked out %r\n' % (repo, head))
- if any(gd.get_index().get_uncommitted_changes()):
- has_uncommitted_changes = True
- self.app.output.write(' %s: uncommitted changes\n' % repo)
-
- if not has_uncommitted_changes:
- self.app.output.write("\nNo repos have outstanding changes.\n")
-
- def branch_from_image(self, args):
- '''Produce a branch of an existing system image.
-
- Given the metadata specified by --metadata-dir, create a new
- branch then petrify it to the state of the commits at the time
- the system was built.
-
- If --metadata-dir is not specified, it defaults to your currently
- running system.
-
- '''
- if len(args) != 1:
- raise cliapp.AppException(
- "branch-from-image needs exactly 1 argument "
- "of the new system branch's name")
- system_branch = args[0]
- metadata_path = self.app.settings['metadata-dir']
- alias_resolver = morphlib.repoaliasresolver.RepoAliasResolver(
- self.app.settings['repo-alias'])
-
- self._require_git_user_config()
-
- ws = morphlib.workspace.open('.')
-
- system, metadata = self._load_system_metadata(metadata_path)
- resolved_refs = dict(self._resolve_refs_from_metadata(alias_resolver,
- metadata))
- self.app.status(msg='Resolved refs: %r' % resolved_refs)
- base_ref = system['sha1']
- # The previous version would fall back to deducing this from the repo
- # url and the repo alias resolver, but this does not always work, and
- # new systems always have repo-alias in the metadata
- root_url = system['repo-alias']
-
- lrc, rrc = morphlib.util.new_repo_caches(self.app)
- cached_repo = lrc.get_updated_repo(root_url)
-
-
- with self._initializing_system_branch(
- ws, root_url, system_branch, cached_repo, base_ref) as (sb, gd):
-
- # TODO: It's nasty to clone to a sha1 then create a branch
- # of that sha1 then check it out, a nicer API may be the
- # initial clone not checking out a branch at all, then
- # the user creates and checks out their own branches
- gd.branch(system_branch, base_ref)
- gd.checkout(system_branch)
-
- loader = morphlib.morphloader.MorphologyLoader()
- morphs = self._load_all_sysbranch_morphologies(sb, loader)
-
- morphs.repoint_refs(sb.root_repository_url,
- sb.system_branch_name)
-
- morphs.petrify_chunks(resolved_refs)
-
- self._save_dirty_morphologies(loader, sb, morphs.morphologies)
-
- @staticmethod
- def _load_system_metadata(path):
- '''Load all metadata in `path` corresponding to a single System.
- '''
-
- smd = morphlib.systemmetadatadir.SystemMetadataDir(path)
- metadata = smd.values()
- systems = [md for md in metadata
- if 'kind' in md and md['kind'] == 'system']
-
- if not systems:
- raise cliapp.AppException(
- 'Metadata directory does not contain any systems.')
- if len(systems) > 1:
- raise cliapp.AppException(
- 'Metadata directory contains multiple systems.')
- system_metadatum = systems[0]
-
- metadata_cache_id_lookup = dict((md['cache-key'], md)
- for md in metadata)
-
- return system_metadatum, metadata_cache_id_lookup
-
- @staticmethod
- def _resolve_refs_from_metadata(alias_resolver, metadata):
- '''Pre-resolve a set of refs from existing metadata.
-
- Given the metadata, generate a mapping of all the (repo, ref)
- pairs defined in the metadata and the commit id they resolved to.
-
- '''
- for md in metadata.itervalues():
- repourls = set((md['repo-alias'], md['repo']))
- repourls.update(alias_resolver.aliases_from_url(md['repo']))
- for repourl in repourls:
- yield ((repourl, md['original_ref']), md['sha1'])
diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py
index d268decf..a66098b8 100644
--- a/morphlib/plugins/branch_and_merge_plugin.py
+++ b/morphlib/plugins/branch_and_merge_plugin.py
@@ -15,882 +15,476 @@
import cliapp
-import copy
-import functools
+import contextlib
import glob
import logging
import os
import shutil
-import socket
-import tempfile
-import time
-import urlparse
-import uuid
import morphlib
-def warns_git_config(keys):
- def decorator(func):
- @functools.wraps(func)
- def check_config(self, *args, **kwargs):
- try:
- morphlib.git.check_config_set(self.app.runcmd, keys)
- except cliapp.AppException, e:
- self.app.status(msg="WARNING: %(message)s",
- message=str(e), error=True)
- return func(self, *args, **kwargs)
- return check_config
-
- return decorator
-
-
-warns_git_identity = warns_git_config({'user.name': 'My Name',
- 'user.email': 'me@example.com'})
-
-
class BranchAndMergePlugin(cliapp.Plugin):
- def __init__(self):
- # Start recording changes.
- self.init_changelog()
+ '''Add subcommands for handling workspaces and system branches.'''
def enable(self):
- # User-facing commands
- self.app.add_subcommand('merge', self.merge,
- arg_synopsis='BRANCH')
-# self.app.add_subcommand('edit', self.edit,
-# arg_synopsis='SYSTEM STRATUM [CHUNK]')
- self.app.add_subcommand('old-petrify', self.petrify)
- self.app.add_subcommand('old-unpetrify', self.unpetrify)
+ self.app.add_subcommand('init', self.init, arg_synopsis='[DIR]')
+ self.app.add_subcommand('workspace', self.workspace, arg_synopsis='')
+ self.app.add_subcommand(
+ 'checkout', self.checkout, arg_synopsis='REPO BRANCH')
+ self.app.add_subcommand(
+ 'branch', self.branch, arg_synopsis='REPO NEW [OLD]')
+ self.app.add_subcommand(
+ 'edit', self.edit, arg_synopsis='SYSTEM STRATUM [CHUNK]')
self.app.add_subcommand(
- 'tag', self.tag, arg_synopsis='TAG-NAME -- [GIT-COMMIT-ARG...]')
- self.app.add_subcommand('old-build', self.build,
- arg_synopsis='SYSTEM')
- self.app.add_subcommand('old-status', self.status)
- self.app.add_subcommand('old-branch-from-image',
- self.branch_from_image,
- arg_synopsis='REPO BRANCH')
-
- # Advanced commands
- self.app.add_subcommand('old-foreach', self.foreach,
+ 'petrify', self.petrify, arg_synopsis='')
+ self.app.add_subcommand(
+ 'unpetrify', self.unpetrify, arg_synopsis='')
+ self.app.add_subcommand(
+ 'show-system-branch', self.show_system_branch, arg_synopsis='')
+ self.app.add_subcommand(
+ 'show-branch-root', self.show_branch_root, arg_synopsis='')
+ self.app.add_subcommand('foreach', self.foreach,
arg_synopsis='-- COMMAND [ARGS...]')
+ self.app.add_subcommand('status', self.status,
+ arg_synopsis='')
+ self.app.add_subcommand('branch-from-image', self.branch_from_image,
+ arg_synopsis='BRANCH')
+ group_branch = 'Branching Options'
+ self.app.settings.string(['metadata-dir'],
+ 'Set metadata location for branch-from-image'
+ ' (default: /baserock)',
+ metavar='DIR',
+ default='/baserock',
+ group=group_branch)
def disable(self):
pass
- def init_changelog(self):
- self.changelog = {}
-
- def log_change(self, repo, text):
- if not repo in self.changelog:
- self.changelog[repo] = []
- self.changelog[repo].append(text)
-
- def print_changelog(self, title, early_keys=[]):
- if self.changelog and self.app.settings['verbose']:
- msg = '\n%s:\n\n' % title
- keys = [x for x in early_keys if x in self.changelog]
- keys.extend([x for x in self.changelog if x not in early_keys])
- for key in keys:
- messages = self.changelog[key]
- msg += ' %s:\n' % key
- msg += '\n'.join([' %s' % x for x in messages])
- msg += '\n\n'
- self.app.output.write(msg)
+ def init(self, args):
+ '''Initialize a workspace directory.
- @staticmethod
- def deduce_workspace():
- dirname = os.getcwd()
- while dirname != '/':
- dot_morph = os.path.join(dirname, '.morph')
- if os.path.isdir(dot_morph):
- return dirname
- dirname = os.path.dirname(dirname)
- raise cliapp.AppException("Can't find the workspace directory.\n"
- "Morph must be built and deployed within "
- "the system branch checkout within the "
- "workspace directory.")
-
- def deduce_system_branch(self):
- # 1. Deduce the workspace. If this fails, we're not inside a workspace.
- workspace = self.deduce_workspace()
-
- # 2. We're in a workspace. Check if we're inside a system branch.
- # If we are, return its name.
- dirname = os.getcwd()
- while dirname != workspace and dirname != '/':
- if os.path.isdir(os.path.join(dirname, '.morph-system-branch')):
- branch_name = self.get_branch_config(dirname, 'branch.name')
- return branch_name, dirname
- dirname = os.path.dirname(dirname)
-
- # 3. We're in a workspace but not inside a branch. Try to find a
- # branch directory in the directories below the current working
- # directory. Avoid ambiguity by only recursing deeper if there
- # is only one subdirectory.
- for dirname in self.walk_special_directories(
- os.getcwd(), special_subdir='.morph-system-branch',
- max_subdirs=1):
- branch_name = self.get_branch_config(dirname, 'branch.name')
- return branch_name, dirname
-
- raise cliapp.AppException("Can't find the system branch directory.\n"
- "Morph must be built and deployed within "
- "the system branch checkout.")
-
- def find_repository(self, branch_dir, repo):
- for dirname in self.walk_special_directories(branch_dir,
- special_subdir='.git'):
- try:
- original_repo = self.get_repo_config(
- dirname, 'morph.repository')
- except cliapp.AppException:
- # The user may have manually put a git repo in the branch
- continue
- if repo == original_repo:
- return dirname
- return None
-
- def find_system_branch(self, workspace, branch_name):
- for dirname in self.walk_special_directories(
- workspace, special_subdir='.morph-system-branch'):
- branch = self.get_branch_config(dirname, 'branch.name')
- if branch_name == branch:
- return dirname
- return None
-
- def set_branch_config(self, branch_dir, option, value):
- filename = os.path.join(branch_dir, '.morph-system-branch', 'config')
- self.app.runcmd(['git', 'config', '-f', filename, option, value],
- print_command=False)
-
- def get_branch_config(self, branch_dir, option):
- filename = os.path.join(branch_dir, '.morph-system-branch', 'config')
- value = self.app.runcmd(['git', 'config', '-f', filename, option],
- print_command=False)
- return value.strip()
-
- def set_repo_config(self, repo_dir, option, value):
- self.app.runcmd(['git', 'config', option, value], cwd=repo_dir,
- print_command=False)
-
- def get_repo_config(self, repo_dir, option):
- value = self.app.runcmd(['git', 'config', option], cwd=repo_dir,
- print_command=False)
- return value.strip()
-
- def get_head(self, repo_path):
- '''Return the ref that the working tree is on for a repo'''
-
- ref = self.app.runcmd(['git', 'rev-parse', '--abbrev-ref', 'HEAD'],
- cwd=repo_path).strip()
- if ref == 'HEAD':
- ref = 'detached HEAD'
- return ref
-
- def get_uncommitted_changes(self, repo_dir, env={}):
- status = self.app.runcmd(['git', 'status', '--porcelain'],
- cwd=repo_dir, env=env)
- changes = []
- for change in status.strip().splitlines():
- xy, paths = change.strip().split(' ', 1)
- if xy != '??':
- changes.append(paths.split()[0])
- return changes
-
- def get_unmerged_changes(self, repo_dir, env={}):
- '''Identifies files which have unresolved merge conflicts'''
-
- # The second column of the git command output is set either if the
- # file has changes in the working tree or if it has conflicts.
- status = self.app.runcmd(['git', 'status', '--porcelain'],
- cwd=repo_dir, env=env)
- changes = []
- for change in status.strip().splitlines():
- xy, paths = change[0:2], change[2:].strip()
- if xy[1] != ' ' and xy != '??':
- changes.append(paths.split()[0])
- return changes
-
- def resolve_ref(self, repodir, ref):
- try:
- return self.app.runcmd(['git', 'rev-parse', '--verify', ref],
- cwd=repodir)[0:40]
- except cliapp.AppException, e:
- logging.info(
- 'Ignoring error executing git rev-parse: %s' % str(e))
- return None
+ Command line argument:
- def resolve_reponame(self, reponame):
- '''Return the full pull URL of a reponame.'''
+ * `DIR` is the directory to use as a workspace, and defaults to
+ the current directory.
- resolver = morphlib.repoaliasresolver.RepoAliasResolver(
- self.app.settings['repo-alias'])
- return resolver.pull_url(reponame)
+ This creates a workspace, either in the current working directory,
+ or if `DIR` is given, in that directory. If the directory doesn't
+ exist, it is created. If it does exist, it must be empty.
+
+ You need to run `morph init` to initialise a workspace, or none
+ of the other system branching tools will work: they all assume
+ an existing workspace. Note that a workspace only exists on your
+ machine, not on the git server.
+
+ Example:
- def get_cached_repo(self, repo_name):
- '''Return CachedRepo object from the local repository cache
+ morph init /src/workspace
+ cd /src/workspace
- Repo is cached and updated if necessary. The cache itself has a
- mechanism in place to avoid multiple updates per Morph invocation.
'''
- self.app.status(msg='Updating git repository %s in cache' % repo_name)
- if not self.app.settings['no-git-update']:
- repo = self.lrc.cache_repo(repo_name)
- repo.update()
- else:
- repo = self.lrc.get_repo(repo_name)
- return repo
+ if not args:
+ args = ['.']
+ elif len(args) > 1:
+ raise morphlib.Error('init must get at most one argument')
- def clone_to_directory(self, dirname, reponame, ref):
- '''Clone a repository below a directory.
+ ws = morphlib.workspace.create(args[0])
+ self.app.status(msg='Initialized morph workspace', chatty=True)
- As a side effect, clone it into the local repo cache.
+ def workspace(self, args):
+ '''Show the toplevel directory of the current workspace.'''
- '''
+ ws = morphlib.workspace.open('.')
+ self.app.output.write('%s\n' % ws.root)
- # Setup.
- resolver = morphlib.repoaliasresolver.RepoAliasResolver(
- self.app.settings['repo-alias'])
- repo = self.get_cached_repo(reponame)
-
- # Make sure the parent directories needed for the repo dir exist.
- parent_dir = os.path.dirname(dirname)
- if not os.path.exists(parent_dir):
- os.makedirs(parent_dir)
-
- # Clone it from cache to target directory.
- target_path = os.path.abspath(dirname)
- repo.clone_checkout(ref, target_path)
-
- # Remember the repo name we cloned from in order to be able
- # to identify the repo again later using the same name, even
- # if the user happens to rename the directory.
- self.set_repo_config(dirname, 'morph.repository', reponame)
-
- # Create a UUID for the clone. We will use this for naming
- # temporary refs, e.g. for building.
- self.set_repo_config(dirname, 'morph.uuid', uuid.uuid4().hex)
-
- # URL configuration
- morphlib.git.set_remote(self.app.runcmd, dirname, 'origin', repo.url)
- self.set_repo_config(
- dirname, 'url.%s.pushInsteadOf' % resolver.push_url(reponame),
- resolver.pull_url(reponame))
- morphlib.git.update_submodules(self.app, target_path)
-
- self.app.runcmd(['git', 'remote', 'update'], cwd=dirname)
-
- def load_morphology(self, repo_dir, name, ref=None):
- '''Loads a morphology from a repo in a system branch
-
- If 'ref' is specified, the version is taken from there instead of the
- working tree. Note that you shouldn't use this to fetch files on
- branches other than the current system branch, because the remote in
- the system branch repo may be completely out of date. Use the local
- repository cache instead for this.
- '''
+ # TODO: Move this somewhere nicer
+ @contextlib.contextmanager
+ def _initializing_system_branch(self, ws, root_url, system_branch,
+ cached_repo, base_ref):
+ '''A context manager for system branches under construction.
- if ref is None:
- filename = os.path.join(repo_dir, '%s.morph' % name)
- with open(filename) as f:
- text = f.read()
- else:
- filename = '%s.morph at ref %s in %s' % (name, ref, repo_dir)
- if not morphlib.git.is_valid_sha1(ref):
- ref = morphlib.git.rev_parse(self.app.runcmd, repo_dir, ref)
- try:
- text = self.app.runcmd(['git', 'cat-file', 'blob',
- '%s:%s.morph' % (ref, name)],
- cwd=repo_dir)
- except cliapp.AppException as e:
- msg = '%s.morph was not found in %s' % (name, repo_dir)
- if ref is not None:
- msg += ' at ref %s' % ref
- raise cliapp.AppException(msg)
+ The purpose of this context manager is to factor out the branch
+ cleanup code for if an exception occurs while a branch is being
+ constructed.
+ This could be handled by a higher order function which takes
+ a function to initialize the branch as a parameter, but with
+ statements look nicer and are more obviously about resource
+ cleanup.
+
+ '''
+ root_dir = ws.get_default_system_branch_directory_name(system_branch)
try:
- morphology = morphlib.morph2.Morphology(text)
- except ValueError as e:
- raise morphlib.Error("Error parsing %s: %s" %
- (filename, str(e)))
-
- self._validate_morphology(morphology, '%s.morph' % name)
-
- return morphology
-
- def _validate_morphology(self, morphology, basename):
- # FIXME: This really should be in MorphologyFactory. Later.
-
- def require(field):
- if field not in morphology:
- raise morphlib.Error(
- 'Required field "%s" is missing from morphology %s' %
- (field, basename))
-
- required = {
- 'system': [
- 'name',
- 'arch',
- 'strata',
- ],
- 'stratum': [
- 'name',
- 'chunks',
- ],
- 'chunk': [
- 'name',
- ],
- 'cluster': [
- 'name',
- 'systems',
- ],
- }
-
- also_known = {
- 'system': [
- 'kind',
- 'description',
- 'configuration-extensions',
- ],
- 'stratum': [
- 'kind',
- 'description',
- 'build-depends',
- ],
- 'chunk': [
- 'kind',
- 'description',
- 'build-system',
- 'configure-commands',
- 'build-commands',
- 'test-commands',
- 'install-commands',
- 'max-jobs',
- 'chunks',
- 'devices',
- ],
- 'cluster': [
- 'kind'
- ]
- }
-
- require('kind')
- kind = morphology['kind']
- if kind not in required:
- raise morphlib.Error(
- 'Unknown morphology kind "%s" in %s' % (kind, basename))
- for field in required[kind]:
- require(field)
-
- known = required[kind] + also_known[kind]
- for field in morphology.keys():
- if field not in known and not field.startswith('_orig_'):
- msg = 'Unknown field "%s" in %s' % (field, basename)
- logging.warning(msg)
- self.app.status(msg=msg)
-
- def reset_work_tree_safe(self, repo_dir):
- # This function avoids throwing any exceptions, so it is safe to call
- # inside an 'except' block without altering the backtrace.
+ sb = morphlib.sysbranchdir.create(
+ root_dir, root_url, system_branch)
+ gd = sb.clone_cached_repo(cached_repo, base_ref)
- command = 'git', 'reset', '--hard'
- status, output, error = self.app.runcmd_unchecked(command,
- cwd=repo_dir)
- if status != 0:
- logging.warning ("Warning: error while trying to clean up %s: %s" %
- (repo_dir, error))
+ yield (sb, gd)
- @staticmethod
- def update_morphology(repo_dir, name, morphology, output_fd=None):
- if not name.endswith('.morph'):
- name = '%s.morph' % name
- filename = os.path.join(repo_dir, '%s' % name)
- morphology.update_file(filename, output_fd=output_fd)
+ gd.update_submodules(self.app)
+ gd.update_remotes()
- if name != morphology['name'] + '.morph':
- logging.warning('%s: morphology "name" should match filename' %
- filename)
+ except morphlib.sysbranchdir.SystemBranchDirectoryAlreadyExists as e:
+ logging.error('Caught exception: %s' % str(e))
+ raise
+ except BaseException as e:
+ # Oops. Clean up.
+ logging.error('Caught exception: %s' % str(e))
+ logging.info('Removing half-finished branch %s' % system_branch)
+ self._remove_branch_dir_safe(ws.root, root_dir)
+ raise
- @staticmethod
- def get_edit_info(morphology_name, morphology, name, collection='strata'):
- try:
- return morphology.lookup_child_by_name(name)
- except KeyError:
- if collection is 'strata':
- raise cliapp.AppException(
- 'Stratum "%s" not found in system "%s"' %
- (name, morphology_name))
- else:
- raise cliapp.AppException(
- 'Chunk "%s" not found in stratum "%s"' %
- (name, morphology_name))
+ def checkout(self, args):
+ '''Check out an existing system branch.
- @staticmethod
- def convert_uri_to_path(uri):
- parts = urlparse.urlparse(uri)
-
- # If the URI path is relative, assume it is an aliased repo (e.g.
- # baserock:morphs). Otherwise assume it is a full URI where we need
- # to strip off the scheme and .git suffix.
- if not os.path.isabs(parts.path):
- return uri
- else:
- path = parts.netloc
- if parts.path.endswith('.git'):
- path = os.path.join(path, parts.path[1:-len('.git')])
- else:
- path = os.path.join(path, parts.path[1:])
- return path
+ Command line arguments:
- @staticmethod
- def remove_branch_dir_safe(workspace, branch):
- # This function avoids throwing any exceptions, so it is safe to call
- # inside an 'except' block without altering the backtrace.
+ * `REPO` is the URL to the repository to the root repository of
+ a system branch.
+ * `BRANCH` is the name of the system branch.
- def handle_error(function, path, excinfo):
- logging.warning ("Warning: error while trying to clean up %s: %s" %
- (path, excinfo))
+ This will check out an existing system branch to an existing
+ workspace. You must create the workspace first. This only checks
+ out the root repository, not the repositories for individual
+ components. You need to use `morph edit` to check out those.
- branch_dir = os.path.join(workspace, branch)
- shutil.rmtree(branch_dir, onerror=handle_error)
+ Example:
- # Remove parent directories that are empty too, avoiding exceptions
- parent = os.path.dirname(branch_dir)
- while parent != os.path.abspath(workspace):
- if len(os.listdir(parent)) > 0 or os.path.islink(parent):
- break
- os.rmdir(parent)
- parent = os.path.dirname(parent)
+ cd /src/workspace
+ morph checkout baserock:baserock/morphs master
- @staticmethod
- def iterate_branch_repos(branch_path, root_repo_path):
- '''Produces a sorted list of component repos in a branch checkout'''
+ '''
- dirs = [d for d in BranchAndMergePlugin.walk_special_directories(
- branch_path, special_subdir='.git')
- if not os.path.samefile(d, root_repo_path)]
- dirs.sort()
+ if len(args) != 2:
+ raise cliapp.AppException('morph checkout needs a repo and the '
+ 'name of a branch as parameters')
- for d in [root_repo_path] + dirs:
- yield d
+ root_url = args[0]
+ system_branch = args[1]
+ base_ref = system_branch
- @staticmethod
- def walk_special_directories(root_dir, special_subdir=None, max_subdirs=0):
- assert(special_subdir is not None)
- assert(max_subdirs >= 0)
-
- visited = set()
- for dirname, subdirs, files in os.walk(root_dir, followlinks=True):
- # Avoid infinite recursion due to symlinks.
- if dirname in visited:
- subdirs[:] = []
- continue
- visited.add(dirname)
+ self._require_git_user_config()
- # Check if the current directory has the special subdirectory.
- if special_subdir in subdirs:
- yield dirname
+ # Open the workspace first thing, so user gets a quick error if
+ # we're not inside a workspace.
+ ws = morphlib.workspace.open('.')
- # Do not recurse into hidden directories.
- subdirs[:] = [x for x in subdirs if not x.startswith('.')]
+ # Make sure the root repository is in the local git repository
+ # cache, and is up to date.
+ lrc, rrc = morphlib.util.new_repo_caches(self.app)
+ cached_repo = lrc.get_updated_repo(root_url)
- # Do not recurse if there is more than the maximum number of
- # subdirectories allowed.
- if max_subdirs > 0 and len(subdirs) > max_subdirs:
- break
+ # Check the git branch exists.
+ cached_repo.resolve_ref(system_branch)
- def read_metadata(self, metadata_path):
- '''Load every metadata file in `metadata_path`.
-
- Given a directory containing metadata, load them into memory
- and retain the id of the system.
+ with self._initializing_system_branch(
+ ws, root_url, system_branch, cached_repo, base_ref) as (sb, gd):
- Returns the cache_key of the system and a mapping of cache_key
- to metadata.
- '''
- self.app.status(msg='Reading metadata', chatty=True)
- metadata_cache_id_lookup = {}
- system_key = None
- for path in sorted(glob.iglob(os.path.join(metadata_path, '*.meta'))):
- with open(path) as f:
- metadata = morphlib.util.json.load(f)
- cache_key = metadata['cache-key']
- metadata_cache_id_lookup[cache_key] = metadata
-
- if metadata['kind'] == 'system':
- if system_key is not None:
- raise morphlib.Error(
- "Metadata directory contains multiple systems.")
- system_key = cache_key
-
- if system_key is None:
- raise morphlib.Error(
- "Metadata directory does not contain any systems.")
-
- return system_key, metadata_cache_id_lookup
-
- def _create_branch(self, workspace, branch_name, repo, original_ref):
- '''Create a branch called branch_name based off original_ref.
-
- NOTE: self.lrc and self.rrc need to be initialized before
- calling since clone_to_directory uses them indirectly via
- get_cached_repo
- '''
- branch_dir = os.path.join(workspace, branch_name)
- os.makedirs(branch_dir)
- try:
- # Create a .morph-system-branch directory to clearly identify
- # this directory as a morph system branch.
- os.mkdir(os.path.join(branch_dir, '.morph-system-branch'))
-
- # Remember the system branch name and the repository we branched
- # off from initially.
- self.set_branch_config(branch_dir, 'branch.name', branch_name)
- self.set_branch_config(branch_dir, 'branch.root', repo)
-
- # Generate a UUID for the branch. We will use this for naming
- # temporary refs, e.g. building.
- self.set_branch_config(branch_dir, 'branch.uuid', uuid.uuid4().hex)
-
- # Clone into system branch directory.
- repo_dir = os.path.join(branch_dir, self.convert_uri_to_path(repo))
- self.clone_to_directory(repo_dir, repo, original_ref)
-
- # Create a new branch in the local morphs repository.
- if original_ref != branch_name:
- self.app.runcmd(['git', 'checkout', '-b', branch_name,
- original_ref], cwd=repo_dir)
-
- return branch_dir
- except BaseException, e:
- logging.error('Caught exception: %s' % str(e))
- logging.info('Removing half-finished branch %s' % branch_name)
- self.remove_branch_dir_safe(workspace, branch_name)
- raise
+ if gd.has_fat():
+ gd.fat_init()
+ gd.fat_pull()
- def checkout_repository(self, branch_dir, repo, ref, parent_ref=None):
- '''Make a chunk or stratum repository available for a system branch
- We ensure the 'system_branch' ref within 'repo' is checked out,
- creating it from 'parent_ref' if required.
+ def branch(self, args):
+ '''Create a new system branch.
- The function aims for permissiveness, so users can try to fix any
- weirdness they have caused in the repos with another call to 'morph
- edit'.
+ Command line arguments:
+
+ * `REPO` is a repository URL.
+ * `NEW` is the name of the new system branch.
+ * `OLD` is the point from which to branch, and defaults to `master`.
+
+ This creates a new system branch. It needs to be run in an
+ existing workspace (see `morph workspace`). It creates a new
+ git branch in the clone of the repository in the workspace. The
+ system branch will not be visible on the git server until you
+ push your changes to the repository.
+
+ Example:
+
+ cd /src/workspace
+ morph branch baserock:baserock/morphs jrandom/new-feature
'''
- parent_ref = parent_ref or ref
+ if len(args) not in [2, 3]:
+ raise cliapp.AppException(
+ 'morph branch needs name of branch as parameter')
- repo_dir = self.find_repository(branch_dir, repo)
- if repo_dir is None:
- repo_url = self.resolve_reponame(repo)
- repo_dir = os.path.join(branch_dir, self.convert_uri_to_path(repo))
- self.clone_to_directory(repo_dir, repo, parent_ref)
+ root_url = args[0]
+ system_branch = args[1]
+ base_ref = 'master' if len(args) == 2 else args[2]
+ origin_base_ref = 'origin/%s' % base_ref
- if self.resolve_ref(repo_dir, ref) is None:
- self.log_change(repo, 'branch "%s" created from "%s"' %
- (ref, parent_ref))
- command = ['git', 'checkout', '-b', ref]
- else:
- # git copes even if the system_branch ref is already checked out
- command = ['git', 'checkout', ref]
-
- status, output, error = self.app.runcmd_unchecked(
- command, cwd=repo_dir)
- if status != 0:
- raise cliapp.AppException('Command failed: %s in repo %s\n%s' %
- (' '.join(command), repo, error))
- return repo_dir
-
- def make_available(self, spec, branch, branch_dir, root_repo,
- root_repo_dir):
- '''Check out the morphology that 'spec' refers to, for editing'''
-
- if spec.get('repo') in (None, root_repo):
- # This is only possible for stratum morphologies
- repo_dir = root_repo_dir
- if spec.get('ref') not in (None, root_repo):
- # Bring the morphology forward from its ref to the current HEAD
- repo = self.lrc.get_repo(root_repo)
- m = repo.load_morphology(spec['ref'], spec['morph'])
- self.update_morphology(root_repo_dir, spec['morph'], m)
- self.log_change(spec['repo'],
- '"%s" copied from "%s" to "%s"' %
- (spec['morph'], spec['ref'], branch))
- else:
- repo_dir = self.checkout_repository(
- branch_dir, spec['repo'], branch, parent_ref=spec['ref'])
- return repo_dir
+ self._require_git_user_config()
+
+ # Open the workspace first thing, so user gets a quick error if
+ # we're not inside a workspace.
+ ws = morphlib.workspace.open('.')
+
+ # Make sure the root repository is in the local git repository
+ # cache, and is up to date.
+ lrc, rrc = morphlib.util.new_repo_caches(self.app)
+ cached_repo = lrc.get_updated_repo(root_url)
+
+ # Make sure the system branch doesn't exist yet.
+ if cached_repo.ref_exists(system_branch):
+ raise cliapp.AppException(
+ 'branch %s already exists in repository %s' %
+ (system_branch, root_url))
+
+ # Make sure the base_ref exists.
+ cached_repo.resolve_ref(base_ref)
+
+ with self._initializing_system_branch(
+ ws, root_url, system_branch, cached_repo, base_ref) as (sb, gd):
+
+ gd.branch(system_branch, base_ref)
+ gd.checkout(system_branch)
+ if gd.has_fat():
+ gd.fat_init()
+ gd.fat_pull()
+
+ def _save_dirty_morphologies(self, loader, sb, morphs):
+ logging.debug('Saving dirty morphologies: start')
+ for morph in morphs:
+ if morph.dirty:
+ logging.debug(
+ 'Saving morphology: %s %s %s' %
+ (morph.repo_url, morph.ref, morph.filename))
+ loader.unset_defaults(morph)
+ loader.save_to_file(
+ sb.get_filename(morph.repo_url, morph.filename), morph)
+ morph.dirty = False
+ logging.debug('Saving dirty morphologies: done')
+
+ def _checkout(self, lrc, sb, repo_url, ref):
+ logging.debug(
+ 'Checking out %s (%s) into %s' %
+ (repo_url, ref, sb.root_directory))
+ cached_repo = lrc.get_updated_repo(repo_url)
+ gd = sb.clone_cached_repo(cached_repo, ref)
+ gd.update_submodules(self.app)
+ gd.update_remotes()
+
+ def _load_morphology_from_file(self, loader, dirname, filename):
+ full_filename = os.path.join(dirname, filename)
+ return loader.load_from_file(full_filename)
+
+ def _load_morphology_from_git(self, loader, gd, ref, filename):
+ try:
+ text = gd.get_file_from_ref(ref, filename)
+ except cliapp.AppException:
+ text = gd.get_file_from_ref('origin/%s' % ref, filename)
+ return loader.load_from_string(text, filename)
- @warns_git_identity
def edit(self, args):
'''Edit or checkout a component in a system branch.
Command line arguments:
- * `SYSTEM` is the name of a system morphology in the root repository
- of the current system branch.
- * `STRATUM` is the name of a stratum inside the system.
- * `CHUNK` is the name of a chunk inside the stratum.
+ * `CHUNK` is the name of a chunk
- This marks the specified stratum or chunk (if given) as being
- changed within the system branch, by creating the git branches in
- the affected repositories, and changing the relevant morphologies
- to point at those branches. It also creates a local clone of
- the git repository of the stratum or chunk.
+ This makes a local checkout of CHUNK in the current system branch
+ and edits any stratum morphology file(s) containing the chunk
- For example:
+ '''
- morph edit devel-system-x86-64-generic devel
+ if len(args) != 1:
+ raise cliapp.AppException('morph edit needs a chunk '
+ 'as parameter')
+
+ ws = morphlib.workspace.open('.')
+ sb = morphlib.sysbranchdir.open_from_within('.')
+ loader = morphlib.morphloader.MorphologyLoader()
+ morphs = self._load_all_sysbranch_morphologies(sb, loader)
+
+ def edit_chunk(morph, chunk_name):
+ chunk_url, chunk_ref, chunk_morph = (
+ morphs.get_chunk_triplet(morph, chunk_name))
+
+ chunk_dirname = sb.get_git_directory_name(chunk_url)
+
+ if not os.path.exists(chunk_dirname):
+ lrc, rrc = morphlib.util.new_repo_caches(self.app)
+ cached_repo = lrc.get_updated_repo(chunk_url)
+
+ gd = sb.clone_cached_repo(cached_repo, chunk_ref)
+ if chunk_ref != sb.system_branch_name:
+ gd.branch(sb.system_branch_name, chunk_ref)
+ gd.checkout(sb.system_branch_name)
+ gd.update_submodules(self.app)
+ gd.update_remotes()
+ if gd.has_fat():
+ gd.fat_init()
+ gd.fat_pull()
+
+ # Change the refs to the chunk.
+ if chunk_ref != sb.system_branch_name:
+ morphs.change_ref(
+ chunk_url, chunk_ref,
+ chunk_morph,
+ sb.system_branch_name)
+
+ return chunk_dirname
+
+ chunk_name = args[0]
+ dirs = set()
+ found = 0
+
+ for morph in morphs.morphologies:
+ if morph['kind'] == 'stratum':
+ for chunk in morph['chunks']:
+ if chunk['name'] == chunk_name:
+ self.app.status(
+ msg='Editing %(chunk)s in %(stratum)s stratum',
+ chunk=chunk_name, stratum=morph['name'])
+ chunk_dirname = edit_chunk(morph, chunk_name)
+ dirs.add(chunk_dirname)
+ found = found + 1
+
+ # Save any modified strata.
+
+ self._save_dirty_morphologies(loader, sb, morphs.morphologies)
+
+ if found == 0:
+ self.app.status(
+ msg="No chunk %(chunk)s found. If you want to create one, add "
+ "an entry to a stratum morph file.", chunk=chunk_name)
+
+ if found >= 1:
+ dirs_list = ', '.join(sorted(dirs))
+ self.app.status(
+ msg="Chunk %(chunk)s source is available at %(dirs)s",
+ chunk=chunk_name, dirs=dirs_list)
+
+ if found > 1:
+ self.app.status(
+ msg="Notice that this chunk appears in more than one stratum")
+
+ def show_system_branch(self, args):
+ '''Show the name of the current system branch.'''
+
+ ws = morphlib.workspace.open('.')
+ sb = morphlib.sysbranchdir.open_from_within('.')
+ self.app.output.write('%s\n' % sb.system_branch_name)
+
+ def show_branch_root(self, args):
+ '''Show the name of the repository holding the system morphologies.
+
+ This would, for example, write out something like:
+
+ /src/ws/master/baserock:baserock/morphs
+
+ when the master branch of the `baserock:baserock/morphs`
+ repository is checked out.
- The above command will mark the `devel` stratum as being
- modified in the current system branch. In this case, the stratum's
- morphology is in the same git repository as the system morphology,
- so there is no need to create a new git branch. However, the
- system morphology is modified to point at the stratum morphology
- in the same git branch, rather than the original branch.
+ '''
- In other words, where the system morphology used to say this:
+ ws = morphlib.workspace.open('.')
+ sb = morphlib.sysbranchdir.open_from_within('.')
+ self.app.output.write('%s\n' % sb.get_config('branch.root'))
- morph: devel
- repo: baserock:baserock/morphs
- ref: master
+ def _remove_branch_dir_safe(self, workspace_root, system_branch_root):
+ # This function avoids throwing any exceptions, so it is safe to call
+ # inside an 'except' block without altering the backtrace.
- The updated system morphology will now say this instead:
+ def handle_error(function, path, excinfo):
+ logging.warning ("Error while trying to clean up %s: %s" %
+ (path, excinfo))
- morph: devel
- repo: baserock:baserock/morphs
- ref: jrandom/new-feature
+ shutil.rmtree(system_branch_root, onerror=handle_error)
- (Assuming the system branch is called `jrandom/new-feature`.)
+ # Remove parent directories that are empty too, avoiding exceptions
+ parent = os.path.dirname(system_branch_root)
+ while parent != os.path.abspath(workspace_root):
+ if len(os.listdir(parent)) > 0 or os.path.islink(parent):
+ break
+ os.rmdir(parent)
+ parent = os.path.dirname(parent)
- Another example:
+ def _require_git_user_config(self):
+ '''Warn if the git user.name and user.email variables are not set.'''
- morph edit devel-system-x86_64-generic devel gcc
+ keys = {
+ 'user.name': 'My Name',
+ 'user.email': 'me@example.com',
+ }
- The above command will mark the `gcc` chunk as being edited in
- the current system branch. Morph will clone the `gcc` repository
- locally, into the current workspace, and create a new (local)
- branch named after the system branch. It will also change the
- stratum morphology to refer to the new git branch, instead of
- whatever branch it was referring to originally.
+ try:
+ morphlib.git.check_config_set(self.app.runcmd, keys)
+ except morphlib.git.ConfigNotSetException as e:
+ self.app.status(
+ msg="WARNING: %(message)s",
+ message=str(e), error=True)
- If the `gcc` repository already had a git branch named after
- the system branch, that is reused. Similarly, if the stratum
- morphology was already pointing at that branch, it doesn't
- need to be changed again. In that case, the only action Morph
- does is to clone the chunk repository locally, and if that was
- also done already, Morph does nothing.
+ def foreach(self, args):
+ '''Run a command in each repository checked out in a system branch.
- '''
+ Use -- before specifying the command to separate its arguments from
+ Morph's own arguments.
- if len(args) not in (2, 3):
- raise cliapp.AppException(
- 'morph edit must either get a system and a stratum '
- 'or a system, a stratum and a chunk as arguments')
-
- workspace = self.deduce_workspace()
- branch, branch_dir = self.deduce_system_branch()
-
- # Find out which repository we branched off from.
- root_repo = self.get_branch_config(branch_dir, 'branch.root')
- root_repo_dir = self.find_repository(branch_dir, root_repo)
-
- system_name = args[0]
- stratum_name = args[1]
- chunk_name = args[2] if len(args) > 2 else None
-
- self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app)
-
- # We need to touch every stratum in the system, not just the target
- # the user specified, because others may have build-depends that
- # point to the edited stratum.
- system_morphology = self.load_morphology(root_repo_dir, system_name)
-
- # Test that the specified stratum exists in the system
- self.get_edit_info(system_name, system_morphology, stratum_name)
-
- all_strata = system_morphology['strata']
- stratum_refs_to_correct = set()
-
- # Petrify the chunk
- for stratum_spec in (s for s in all_strata
- if s['morph'] == stratum_name):
- stratum_repo_dir = self.make_available(
- stratum_spec, branch, branch_dir, root_repo, root_repo_dir)
- stratum_morphology = self.load_morphology(
- stratum_repo_dir, stratum_spec['morph'])
-
- if chunk_name is not None:
- # Change the stratum's ref to the chunk
- chunk_spec = self.get_edit_info(
- stratum_name, stratum_morphology, chunk_name,
- collection='chunks')
-
- if 'unpetrify-ref' in chunk_spec:
- chunk_spec['ref'] = chunk_spec['unpetrify-ref']
- del chunk_spec['unpetrify-ref']
-
- self.make_available(
- chunk_spec, branch, branch_dir, root_repo,
- root_repo_dir)
-
- if chunk_spec['ref'] != branch:
- chunk_spec['ref'] = branch
-
- self.log_change(stratum_spec['repo'],
- '"%s" now includes "%s" from "%s"' %
- (stratum_name, chunk_name, branch))
- stratum_refs_to_correct.add((stratum_spec['repo'],
- stratum_spec['ref'],
- stratum_spec['morph']))
- # Correct the System Morphology's reference
- stratum_spec['ref'] = branch
- self.update_morphology(stratum_repo_dir, stratum_spec['morph'],
- stratum_morphology)
- self.log_change(root_repo,
- '"%s" now includes "%s" from "%s"' %
- (system_name, stratum_name, branch))
-
- # Correct all references to altered strata
- while stratum_refs_to_correct:
- repo, ref, morph = stratum_refs_to_correct.pop()
- spec = {"repo": repo, "ref": ref, "morph": morph}
- for stratum_spec in all_strata:
- changed = False
- if repo == root_repo:
- stratum_repo_dir = root_repo_dir
- else:
- stratum_repo_dir = self.checkout_repository(
- branch_dir, stratum_spec['repo'],
- branch, stratum_spec['ref'])
- stratum_morphology = self.load_morphology(
- stratum_repo_dir, stratum_spec['morph'])
- if ('build-depends' in stratum_morphology
- and stratum_morphology['build-depends'] is not None):
- for bd_spec in stratum_morphology['build-depends']:
- bd_triplet = (bd_spec['repo'],
- bd_spec['ref'],
- bd_spec['morph'])
- if (bd_triplet == (repo, ref, morph)):
- bd_spec['ref'] = branch
- changed = True
- if changed:
- stratum_refs_to_correct.add((stratum_spec['repo'],
- stratum_spec['ref'],
- stratum_spec['morph']))
- # Update the System morphology to use
- # the modified version of the Stratum
- stratum_spec['ref'] = branch
- self.update_morphology(stratum_repo_dir,
- stratum_spec['morph'],
- stratum_morphology)
- self.log_change(root_repo,
- '"%s" now includes "%s" from "%s"' %
- (system_name, stratum_name, branch))
-
- self.update_morphology(root_repo_dir, system_name, system_morphology)
-
- self.print_changelog('The following changes were made but have not '
- 'been committed')
-
- def _get_repo_name(self, alias_resolver, metadata):
- '''Attempt to find the best name for the repository.
-
- A defined repo-alias is preferred, but older builds may
- not have it.
-
- A guessed repo-alias is the next best thing, but there may
- be none or more alilases that would resolve to that URL, so
- if there are any, use the shortest as it is likely to be
- the most specific.
-
- If all else fails just use the URL.
- '''
- if 'repo-alias' in metadata:
- return metadata['repo-alias']
-
- repo_url = metadata['repo']
- aliases = alias_resolver.aliases_from_url(repo_url)
-
- if len(aliases) >= 1:
- # If there are multiple valid aliases, use the shortest
- return min(aliases, key=len)
-
- # If there are no aliases, just return the url
- return repo_url
-
- def _resolve_refs_from_metadata(self, alias_resolver,
- metadata_cache_id_lookup):
- '''Pre-resolve a set of refs from metadata.
-
- Resolved refs are a dict as {(repo, ref): sha1}.
-
- If the metadata contains the repo-alias then every
- metadata item adds the mapping of its repo-alias and ref
- to the commit it was built with.
-
- If the repo-alias does not exist, such as if the image was
- built before that field was added, then mappings of every
- possible repo url are added.
- '''
- resolved_refs = {}
- for md in metadata_cache_id_lookup.itervalues():
- if 'repo-alias' in md:
- repourls = [md['repo-alias']]
- else:
- repourls = [md['repo']]
- repourls.extend(alias_resolver.aliases_from_url(md['repo']))
- for repourl in repourls:
- resolved_refs[repourl, md['original_ref']] = md['sha1']
- return resolved_refs
+ Command line arguments:
+
+ * `--` indicates the end of option processing for Morph.
+ * `COMMAND` is a command to run.
+ * `ARGS` is a list of arguments or options to be passed onto
+ `COMMAND`.
+
+ This runs the given `COMMAND` in each git repository belonging
+ to the current system branch that exists locally in the current
+ workspace. This can be a handy way to do the same thing in all
+ the local git repositories.
+
+ For example:
+
+ morph foreach -- git push
+
+ The above command would push any committed changes in each
+ repository to the git server.
- def branch_from_image(self, args):
- '''Given the contents of a /baserock directory, produce a branch
- of the System, petrified to when the System was made.
'''
- if len(args) not in (2, 3):
- raise cliapp.AppException(
- 'branch-from-image needs repository, ref and path to metadata')
- root_repo = args[0]
- branch = args[1]
- metadata_path = self.app.settings['metadata-dir']
- workspace = self.deduce_workspace()
- self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app)
- alias_resolver = morphlib.repoaliasresolver.RepoAliasResolver(
- self.app.settings['repo-alias'])
+ if not args:
+ raise cliapp.AppException('morph foreach expects a command to run')
+
+ ws = morphlib.workspace.open('.')
+ sb = morphlib.sysbranchdir.open_from_within('.')
- system_key, metadata_cache_id_lookup = self.read_metadata(
- metadata_path)
-
- system_metadata = metadata_cache_id_lookup[system_key]
- repo = self._get_repo_name(alias_resolver, system_metadata)
-
- # Which repo to use? Specified or deduced?
- branch_dir = self._create_branch(workspace, branch, repo,
- system_metadata['sha1'])
-
- # Resolve refs from metadata so petrify substitutes these refs
- # into morphologies instead of the current state of the branches
- resolved_refs = self._resolve_refs_from_metadata(
- alias_resolver,
- metadata_cache_id_lookup)
-
- branch_root_dir = self.find_repository(branch_dir, repo)
- name = system_metadata['morphology'][:-len('.morph')]
- morphology = self.load_morphology(branch_root_dir, name)
- self.petrify_morphology(branch, branch_dir,
- repo, branch_root_dir,
- repo, branch_root_dir, # not a typo
- branch, name, morphology,
- petrified_morphologies=set(),
- resolved_refs=resolved_refs,
- update_working_tree=True)
+ for gd in sorted(sb.list_git_directories(), key=lambda gd: gd.dirname):
+ # Get the repository's original name
+ # Continue in the case of error, since the previous iteration
+ # worked in the case of the user cloning a repository in the
+ # system branch's directory.
+ try:
+ repo = gd.get_config('morph.repository')
+ except cliapp.AppException:
+ continue
+
+ self.app.output.write('%s\n' % repo)
+ status, output, error = self.app.runcmd_unchecked(
+ args, cwd=gd.dirname)
+ self.app.output.write(output)
+ if status != 0:
+ self.app.output.write(error)
+ pretty_command = ' '.join(cliapp.shell_quote(arg)
+ for arg in args)
+ raise cliapp.AppException(
+ 'Command failed at repo %s: %s'
+ % (repo, pretty_command))
+ self.app.output.write('\n')
+ self.app.output.flush()
+
+ def _load_all_sysbranch_morphologies(self, sb, loader):
+ '''Read in all the morphologies in the root repository.'''
+ self.app.status(msg='Loading in all morphologies')
+ morphs = morphlib.morphset.MorphologySet()
+ for morph in sb.load_all_morphologies(loader):
+ morphs.add_morphology(morph)
+ return morphs
def petrify(self, args):
'''Convert all chunk refs in a system branch to be fixed SHA1s.
@@ -931,976 +525,220 @@ class BranchAndMergePlugin(cliapp.Plugin):
'''
- # Stratum refs are not petrified, because they must all be edited to
- # set the new chunk refs, which requires branching them all for the
- # current branch - so they will not be updated outside of the user's
- # control in any case. Chunks that have already been edited on the
- # current branch are also not petrified.
-
- if len(args) != 0:
+ if args:
raise cliapp.AppException('morph petrify takes no arguments')
- branch, branch_path = self.deduce_system_branch()
- root_repo = self.get_branch_config(branch_path, 'branch.root')
- root_repo_dir = self.find_repository(branch_path, root_repo)
- self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app)
-
- self.petrify_everything(branch, branch_path, root_repo, root_repo_dir,
- branch, os.environ, None, True)
-
- def unpetrify(self, args):
- '''Reverse the process of petrification.
-
- This undoes the changes `morph petrify` did.
-
- '''
-
- # This function makes no attempt to 'unedit' strata that were branched
- # solely so they could be petrified.
-
- if len(args) != 0:
- raise cliapp.AppException('morph unpetrify takes no arguments')
-
- self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app)
+ ws = morphlib.workspace.open('.')
+ sb = morphlib.sysbranchdir.open_from_within('.')
+ loader = morphlib.morphloader.MorphologyLoader()
+ lrc, rrc = morphlib.util.new_repo_caches(self.app)
+ update_repos = not self.app.settings['no-git-update']
- workspace = self.deduce_workspace()
- branch, branch_path = self.deduce_system_branch()
- root_repo = self.get_branch_config(branch_path, 'branch.root')
- root_repo_dir = self.find_repository(branch_path, root_repo)
+ morphs = self._load_all_sysbranch_morphologies(sb, loader)
- for f in sorted(glob.iglob(os.path.join(root_repo_dir, '*.morph'))):
- name = os.path.basename(f)[:-len('.morph')]
- morphology = self.load_morphology(root_repo_dir, name)
- if morphology['kind'] != 'system':
- continue
-
- for stratum_info in morphology['strata']:
- repo_dir = self.make_available(
- stratum_info, branch, branch_path, root_repo,
- root_repo_dir)
- stratum_info['ref'] = branch
-
- stratum = self.load_morphology(repo_dir, stratum_info['morph'])
+ #TODO: Stop using app.resolve_ref
+ def resolve_refs(morphs):
+ for repo, ref in morphs.list_refs():
+ # You can't resolve null refs, so don't attempt to.
+ if repo is None or ref is None:
+ continue
+ # TODO: Handle refs that are only in workspace in general
+ if (repo == sb.root_repository_url
+ and ref == sb.system_branch_name):
+ continue
+ commit_sha1, tree_sha1 = self.app.resolve_ref(
+ lrc, rrc, repo, ref, update=update_repos)
+ yield ((repo, ref), commit_sha1)
- for chunk_info in stratum['chunks']:
- if 'unpetrify-ref' in chunk_info:
- chunk_info['ref'] = chunk_info['unpetrify-ref']
- del chunk_info['unpetrify-ref']
- self.update_morphology(repo_dir, stratum_info['morph'],
- stratum)
+ morphs.repoint_refs(sb.root_repository_url,
+ sb.system_branch_name)
- self.update_morphology(root_repo_dir, name, morphology)
+ morphs.petrify_chunks(dict(resolve_refs(morphs)))
- self.print_changelog('The following changes were made but have not '
- 'been committed')
+ # Write morphologies back out again.
+ self._save_dirty_morphologies(loader, sb, morphs.morphologies)
- @warns_git_identity
- def tag(self, args):
- '''Create an annotated Git tag of a petrified system branch.
-
- Command line arguments:
-
- * `TAG-NAME` is the name of the Git tag to be created.
- * `--` separates the Git arguments and options from the ones
- Morph parses for itself.
- * `GIT-COMMIT-ARG` is a `git commit` option or argument,
- e.g., '-m' or '-F'. These should provide the commit message.
-
- This command creates an annotated Git tag that points at a commit
- where all system and stratum morphologies have been petrified.
- The working tree won't be petrified, only the commit.
-
- Example:
-
- morph tag release-12.765 -- -m "Release 12.765"
-
- '''
+ def unpetrify(self, args):
+ '''Reverse the process of petrification.
- if len(args) < 1:
- raise cliapp.AppException('morph tag expects a tag name')
-
- tagname = args[0]
-
- # Deduce workspace, system branch and branch root repository.
- workspace = self.deduce_workspace()
- branch, branch_dir = self.deduce_system_branch()
- branch_root = self.get_branch_config(branch_dir, 'branch.root')
- branch_root_dir = self.find_repository(branch_dir, branch_root)
-
- # Prepare an environment for our internal index file.
- # This index file allows us to commit changes to a tree without
- # git noticing any change in the working tree or its own index.
- env = dict(os.environ)
- env['GIT_INDEX_FILE'] = os.path.join(
- branch_root_dir, '.git', 'morph-tag-index')
-
- # Extract git arguments that deal with the commit message.
- # This is so that we can use them for creating the tag commit.
- msg = None
- msg_args = []
- for i in xrange(0, len(args)):
- if args[i] == '-m' or args[i] == '-F':
- if i < len(args)-1:
- msg_args.append(args[i])
- msg_args.append(args[i+1])
- if args[i] == '-m':
- msg = args[i+1]
- else:
- msg = open(args[i+1]).read()
- elif args[i].startswith('--message='):
- msg_args.append(args[i])
- msg = args[i][len('--message='):]
-
- # Fail if no commit message was provided.
- if not msg or not msg_args:
- raise cliapp.AppException(
- 'Commit message expected. Please run one of '
- 'the following commands to provide one:\n'
- ' morph tag NAME -- -m "Message"\n'
- ' morph tag NAME -- --message="Message"\n'
- ' morph tag NAME -- -F <message file>')
-
- # Abort if the tag already exists.
- # FIXME At the moment this only checks the local repo in the
- # workspace, not the remote repo cache or the local repo cache.
- if self.ref_exists_locally(branch_root_dir, 'refs/tags/%s' % tagname):
- raise cliapp.AppException('%s: Tag "%s" already exists' %
- (branch_root, tagname))
-
- self.app.status(msg='%(repo)s: Preparing tag commit',
- repo=branch_root)
-
- # Read current tree into the internal index.
- parent_sha1 = self.resolve_ref(branch_root_dir, branch)
- self.app.runcmd(['git', 'read-tree', parent_sha1],
- cwd=branch_root_dir, env=env)
-
- self.app.status(msg='%(repo)s: Petrifying everything',
- repo=branch_root)
-
- # Petrify everything.
- self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app)
- self.petrify_everything(branch, branch_dir,
- branch_root, branch_root_dir,
- tagname, env)
-
- self.app.status(msg='%(repo)s: Creating tag commit',
- repo=branch_root)
-
- # Create a dangling commit.
- commit = self.create_tag_commit(
- branch_root_dir, tagname, msg, env)
-
- self.app.status(msg='%(repo)s: Creating annotated tag "%(tag)s"',
- repo=branch_root, tag=tagname)
-
- # Create an annotated tag for this commit.
- self.create_annotated_tag(branch_root_dir, commit, env, args)
-
- def ref_exists_locally(self, repo_dir, ref):
- try:
- morphlib.git.rev_parse(self.app.runcmd, repo_dir, ref)
- return True
- except cliapp.AppException:
- return False
-
- def petrify_everything(self, branch, branch_dir,
- branch_root, branch_root_dir, tagref, env=os.environ,
- resolved_refs=None, update_working_tree=False):
- petrified_morphologies = set()
- resolved_refs = resolved_refs or {}
- for f in sorted(glob.iglob(os.path.join(branch_root_dir, '*.morph'))):
- name = os.path.basename(f)[:-len('.morph')]
- morphology = self.load_morphology(branch_root_dir, name)
- self.petrify_morphology(branch, branch_dir,
- branch_root, branch_root_dir,
- branch_root, branch_root_dir,
- tagref, name, morphology,
- petrified_morphologies, resolved_refs,
- env, update_working_tree)
-
- def petrify_morphology(self, branch, branch_dir,
- branch_root, branch_root_dir, repo, repo_dir,
- tagref, name, morphology,
- petrified_morphologies, resolved_refs,
- env=os.environ, update_working_tree=False):
- self.app.status(msg='%(repo)s: Petrifying morphology \"%(morph)s\"',
- repo=repo, morph=name)
-
- # Mark morphology as petrified so we don't petrify it twice.
- petrified_morphologies.add(morphology)
-
- # Resolve the refs of all build dependencies (strata) and strata
- # in the morphology into commit SHA1s.
- strata = []
- if 'build-depends' in morphology and morphology['build-depends']:
- strata += morphology['build-depends']
- if 'strata' in morphology and morphology['strata']:
- strata += morphology['strata']
- for info in strata:
- stratum_repo_dir = self.make_available(
- info, branch, branch_dir, repo, repo_dir)
-
- # Load the stratum morphology and petrify it recursively if
- # that hasn't happened yet.
- stratum = self.load_morphology(stratum_repo_dir, info['morph'])
- if not stratum in petrified_morphologies:
- self.petrify_morphology(branch, branch_dir,
- branch_root, branch_root_dir,
- info.get('repo') or branch_root,
- stratum_repo_dir, tagref,
- info['morph'], stratum,
- petrified_morphologies,
- resolved_refs, env,
- update_working_tree)
-
- # If this morphology is a stratum, resolve the refs of all its
- # chunks into SHA1s.
- if morphology['kind'] == 'stratum':
- for info in morphology['chunks']:
- commit = self.resolve_info(info, resolved_refs)
- if info['ref'] != commit:
- info['unpetrify-ref'] = info['ref']
- info['ref'] = commit
-
- # Write the petrified morphology to a temporary file in the
- # branch root repository for inclusion in the tag commit.
- with tempfile.NamedTemporaryFile(suffix='.morph') as f:
- self.update_morphology(
- repo_dir, name, morphology, output_fd=f.file)
-
- # Hash the petrified morphology and add it to the index
- # for the tag commit.
- sha1 = self.app.runcmd(
- ['git', 'hash-object', '-t', 'blob', '-w', f.name],
- cwd=branch_root_dir, env=env)
- self.app.runcmd(
- ['git', 'update-index', '--add', '--cacheinfo',
- '100644', sha1, '%s.morph' % name],
- cwd=branch_root_dir, env=env)
-
- # Update the working tree if requested. This can be done with
- # git-checkout-index, but we still have the file, so use that
- if update_working_tree:
- shutil.copy(f.name,
- os.path.join(branch_root_dir, '%s.morph' % name))
-
- def resolve_info(self, info, resolved_refs):
- '''Takes a morphology info and resolves its ref with cache support.'''
-
- key = (info.get('repo'), info.get('ref'))
- if not key in resolved_refs:
- commit_sha1, tree_sha1 = self.app.resolve_ref(
- self.lrc, self.rrc, info['repo'], info['ref'],
- update=not self.app.settings['no-git-update'])
- resolved_refs[key] = commit_sha1
- return resolved_refs[key]
-
- def create_tag_commit(self, repo_dir, tagname, msg, env):
- self.app.status(msg='%(repo)s: Creating commit for the tag',
- repo=repo_dir)
-
- # Write and commit the tree.
- tree = self.app.runcmd(
- ['git', 'write-tree'], cwd=repo_dir, env=env).strip()
- commit = self.app.runcmd(
- ['git', 'commit-tree', tree, '-p', 'HEAD'],
- feed_stdin=msg, cwd=repo_dir, env=env).strip()
- return commit
-
- def create_annotated_tag(self, repo_dir, commit, env, args=[]):
- self.app.status(msg='%(repo)s: Creating annotated tag for '
- 'commit %(commit)s',
- repo=repo_dir, commit=commit)
-
- # Create an annotated tag for the commit
- self.app.runcmd(['git', 'tag', '-a'] + args + [commit],
- cwd=repo_dir, env=env)
-
- # When 'merge' is unset, git doesn't try to resolve conflicts itself in
- # those files.
- MERGE_ATTRIBUTE = '*.morph\t-merge\n'
-
- def disable_morph_merging(self, repo_dir):
- attributes_file = os.path.join(repo_dir, ".git", "info", "attributes")
- with open(attributes_file, 'a') as f:
- f.write(self.MERGE_ATTRIBUTE)
-
- def enable_morph_merging(self, repo_dir):
- attributes_file = os.path.join(repo_dir, ".git", "info", "attributes")
- with open(attributes_file, 'r') as f:
- attributes = f.read()
- if attributes == self.MERGE_ATTRIBUTE:
- os.unlink(attributes_file)
- elif attributes.endswith(self.MERGE_ATTRIBUTE):
- with morphlib.savefile.SaveFile(attributes_file, 'w') as f:
- f.write(attributes[:-len(self.MERGE_ATTRIBUTE)])
-
- def get_merge_files(self, repo_dir, from_sha1, to_ref, name):
- '''Returns merge base, remote and local versions of a morphology
-
- We already ran 'git fetch', so the remote branch is available within
- the target repository.
+ This undoes the changes `morph petrify` did.
'''
- base_sha1 = self.app.runcmd(['git', 'merge-base', from_sha1, to_ref],
- cwd=repo_dir).strip()
- base_morph = self.load_morphology(repo_dir, name, ref=base_sha1)
- from_morph = self.load_morphology(repo_dir, name, ref=from_sha1)
- to_morph = self.load_morphology(repo_dir, name, ref=to_ref)
- return base_morph, from_morph, to_morph
-
- def check_component(self, parent_kind, parent_path, from_info, to_info):
- assert (parent_kind in ['system', 'stratum'])
-
- kind = 'chunk' if parent_kind == 'stratum' else 'stratum'
- name = to_info.get('alias', to_info.get('name', to_info.get('morph')))
- path = parent_path + '.' + name
-
- if kind == 'chunk':
- # Only chunks can be petrified
- from_unpetrify_ref = from_info.get('unpetrify-ref', None)
- to_unpetrify_ref = to_info.get('unpetrify-ref', None)
- if from_unpetrify_ref is not None and to_unpetrify_ref is None:
- self.app.output.write(
- 'WARNING: chunk "%s" is now petrified\n' % path)
- elif from_unpetrify_ref is None and to_unpetrify_ref is not None:
- self.app.output.write(
- 'WARNING: chunk "%s" is no longer petrified\n' % path)
- elif from_unpetrify_ref != to_unpetrify_ref:
- raise cliapp.AppException(
- 'merge conflict: chunk "%s" is petrified to a different '
- 'ref in each branch' % path)
-
- def diff_morphologies(self, path, from_morph, to_morph):
- '''Component-level diff between two versions of a morphology'''
+ if args:
+ raise cliapp.AppException('morph petrify takes no arguments')
- def component_key(info):
- # This function needs only to be stable and reproducible
- if 'name' in info:
- return (info.get('repo'), info['morph'], info['name'])
- else:
- return (info.get('repo'), info['morph'])
+ ws = morphlib.workspace.open('.')
+ sb = morphlib.sysbranchdir.open_from_within('.')
+ loader = morphlib.morphloader.MorphologyLoader()
- if from_morph['name'] != to_morph['name']:
- # We should enforce name == filename in load_morphology()
- raise cliapp.AppException(
- 'merge conflict: "name" of morphology %s (name should always '
- 'match filename)' % path)
- if from_morph['kind'] != to_morph['kind']:
- raise cliapp.AppException(
- 'merge conflict: "kind" of morphology %s changed from %s to %s'
- % (path, to_morph['kind'], from_morph['kind']))
-
- kind = to_morph['kind']
-
- # copy() makes a shallow copy, so editing the list elements will
- # change the actual morphologies.
- if kind == 'system':
- from_components = copy.copy(from_morph['strata'])
- to_components = copy.copy(to_morph['strata'])
- elif kind == 'stratum':
- from_components = copy.copy(from_morph['chunks'])
- to_components = copy.copy(to_morph['chunks'])
- from_components.sort(key=component_key)
- to_components.sort(key=component_key)
-
- # These are not set() purely because a set requires a hashable type
- intersection = [] # TO n FROM
- from_diff = [] # FROM \ TO
- to_diff = [] # TO \ FROM
- while len(from_components) > 0 and len(to_components) > 0:
- from_info = from_components.pop(0)
- to_info = to_components.pop(0)
- match = cmp(component_key(from_info), component_key(to_info))
- if match < 0:
- from_diff.append(from_info)
- elif match > 0:
- to_diff.append(to_info)
- elif match == 0:
- intersection.append((from_info, to_info))
- if len(from_components) != 0:
- from_diff.append(from_components.pop(0))
- if len(to_components) != 0:
- to_diff.append(to_components.pop(0))
- return intersection, from_diff, to_diff
-
- def merge_repo(self, merged_repos, from_branch_dir, from_repo, from_ref,
- to_branch_dir, to_repo, to_ref):
- '''Merge changes for a system branch in a specific repository
-
- We disable merging for morphologies and do this manually later on.
+ morphs = self._load_all_sysbranch_morphologies(sb, loader)
- '''
+ # Restore the ref for each stratum and chunk
+ morphs.unpetrify_all()
- if to_repo in merged_repos:
- return merged_repos[to_repo]
-
- from_repo_dir = self.find_repository(from_branch_dir, from_repo)
- to_repo_dir = self.checkout_repository(to_branch_dir, to_repo, to_ref)
-
- if self.get_uncommitted_changes(from_repo_dir) != []:
- raise cliapp.AppException('repository %s has uncommitted '
- 'changes' % from_repo)
- if self.get_uncommitted_changes(to_repo_dir) != []:
- raise cliapp.AppException('repository %s has uncommitted '
- 'changes' % to_repo)
-
- # Fetch the local FROM branch; its sha1 will be stored in FETCH_HEAD.
- # ':' in pathnames confuses git, so we have to pass it a URL.
- from_repo_url = urlparse.urljoin('file://', from_repo_dir)
- self.app.runcmd(['git', 'fetch', from_repo_url, from_ref],
- cwd=to_repo_dir)
-
- # Merge everything but the morphologies; error output is ignored (it's
- # not very good) and instead we report conflicts manually later on.
- self.disable_morph_merging(to_repo_dir)
- with open(os.path.join(to_repo_dir, '.git', 'FETCH_HEAD')) as f:
- from_sha1 = f.read(40)
- status, output, error = self.app.runcmd_unchecked(
- ['git', 'merge', '--no-commit', '--no-ff', from_sha1],
- cwd=to_repo_dir)
- self.enable_morph_merging(to_repo_dir)
-
- merged_repos[to_repo] = (to_repo_dir, from_sha1)
- return (to_repo_dir, from_sha1)
-
- def merge(self, args):
- '''Pull and merge changes from a system branch into the current one.
+ # Write morphologies back out again.
+ self._save_dirty_morphologies(loader, sb, morphs.morphologies)
- Command line arguments:
-
- * `BRANCH` is the name of the system branch to merge _from_.
+ def status(self, args):
+ '''Show information about the current system branch or workspace
- This merges another system branch into the current one. Morph
- will do a `git merge` for each component that has been edited,
- and undo any changes to `ref` fields in system and stratum
- morphologies that `morph edit` has made.
+ This shows the status of every local git repository of the
+ current system branch. This is similar to running `git status`
+ in each repository separately.
- You need to be in the _target_ system branch when merging. If
- you have two system branches, `TROVE_ID/release/1.2` and
- `TROVE_ID/bugfixes/12765`, and want to merge the bug fix branch
- into the release branch, you need to first checkout the release
- system branch, and then merge the bugfix branch into that.
+ If run in a Morph workspace, but not in a system branch checkout,
+ it lists all checked out system branches in the workspace.
'''
- if len(args) != 1:
- raise cliapp.AppException('morph merge requires a system branch '
- 'name as its argument')
-
- self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app)
- workspace = self.deduce_workspace()
- from_branch = args[0]
- from_branch_dir = self.find_system_branch(workspace, from_branch)
- to_branch, to_branch_dir = self.deduce_system_branch()
- if from_branch_dir is None:
- raise cliapp.AppException('branch %s must be checked out before '
- 'it can be merged' % from_branch)
-
- root_repo = self.get_branch_config(from_branch_dir, 'branch.root')
- other_root_repo = self.get_branch_config(to_branch_dir, 'branch.root')
- if root_repo != other_root_repo:
- raise cliapp.AppException('branches do not share a root '
- 'repository : %s vs %s' %
- (root_repo, other_root_repo))
-
- def merge_chunk(parent_path, old_ci, ci):
- self.merge_repo(merged_repos,
- from_branch_dir, old_ci['repo'], from_branch,
- to_branch_dir, ci['repo'], ci['ref'])
-
- def merge_stratum(parent_path, old_si, si):
- path = parent_path + '.' + si['morph']
-
- to_repo_dir, from_sha1 = self.merge_repo(merged_repos,
- from_branch_dir, old_si['repo'], from_branch,
- to_branch_dir, si['repo'], si['ref'])
- base_morph, from_morph, to_morph = self.get_merge_files(
- to_repo_dir, from_sha1, si['ref'], si['morph'])
- intersection, from_diff, to_diff = self.diff_morphologies(
- path, from_morph, to_morph)
- for from_ci, to_ci in intersection:
- self.check_component('stratum', path, from_ci, to_ci)
-
- changed = False
- edited_chunks = [ci for ci in from_morph['chunks']
- if ci['ref'] == from_branch]
- for ci in edited_chunks:
- for old_ci in to_morph['chunks']:
- if old_ci['repo'] == ci['repo']:
- break
- else:
- raise cliapp.AppException(
- 'chunk %s was added within this branch and '
- 'subsequently edited. This is not yet supported: '
- 'refusing to merge.' % ci['name'])
- changed = True
- ci['ref'] = old_ci['ref']
- merge_chunk(path, old_ci, ci)
- if changed:
- self.update_morphology(to_repo_dir, si['morph'], to_morph)
- self.app.runcmd(['git', 'add', si['morph'] + '.morph'],
- cwd=to_repo_dir)
-
- def merge_system(name):
- base_morph, from_morph, to_morph = self.get_merge_files(
- to_root_dir, from_sha1, to_branch, name)
- if to_morph['kind'] != 'system':
- return
-
- intersection, from_diff, to_diff = self.diff_morphologies(
- name, from_morph, to_morph)
- for from_si, to_si in intersection:
- self.check_component('system', name, from_si, to_si)
-
- changed = False
- edited_strata = [si for si in from_morph['strata']
- if si.get('ref') == from_branch]
- for si in edited_strata:
- for old_si in to_morph['strata']:
- # We make no attempt at rename / move detection
- if (old_si['morph'] == si['morph']
- and old_si.get('repo') == si.get('repo')):
- break
- else:
- raise cliapp.AppException(
- 'stratum %s was added within this branch and '
- 'subsequently edited. This is not yet supported: '
- 'refusing to merge.' % si['morph'])
- changed = True
- si['ref'] = old_si.get('ref')
- merge_stratum(name, old_si, si)
- if changed:
- self.update_morphology(to_root_dir, name, to_morph)
- self.app.runcmd(['git', 'add', f], cwd=to_root_dir)
-
- merged_repos = {}
- try:
- to_root_dir, from_sha1 = self.merge_repo(merged_repos,
- from_branch_dir, root_repo, from_branch,
- to_branch_dir, root_repo, to_branch)
-
- for f in sorted(glob.iglob(os.path.join(to_root_dir, '*.morph'))):
- name = os.path.basename(f)[:-len('.morph')]
- merge_system(name)
-
- success = True
- for repo_name, repo_info in merged_repos.iteritems():
- repo_dir = repo_info[0]
- conflicts = self.get_unmerged_changes(repo_dir)
- if len(conflicts) > 0:
- self.app.output.write("Merge conflicts in %s:\n\t%s\n" %
- (repo_name, '\n\t'.join(conflicts)))
- success = False
- elif morphlib.git.index_has_changes(self.app.runcmd, repo_dir):
- # Repo may not be dirty if the changes only touched refs,
- # because they may now match the previous state.
- msg = "Merge system branch '%s'" % from_branch
- self.app.runcmd(['git', 'commit', '--all', '-m%s' % msg],
- cwd=repo_dir)
- if not success:
- raise cliapp.AppException(
- "merge errors were encountered. Please manually merge the "
- "target ref into %s in the remote system branch in each "
- "case, and then repeat the 'morph merge' operation." %
- from_branch)
- self.app.status(msg="Merge successful")
- except BaseException, e:
- logging.error('Caught exception: %s' % str(e))
- logging.info('Resetting half-finished merge')
- for repo_dir, sha1 in merged_repos.itervalues():
- self.reset_work_tree_safe(repo_dir)
- raise
-
- def build(self, args):
- '''Build a system image in the current system branch
-
- Command line arguments:
-
- * `SYSTEM` is the name of the system to build.
-
- This builds a system image, and any of its components that
- need building. The system name is the basename of the system
- morphology, in the root repository of the current system branch,
- without the `.morph` suffix in the filename.
-
- The location of the resulting system image artifact is printed
- at the end of the build output.
-
- You do not need to commit your changes before building, Morph
- does that for you, in a temporary branch for each build. However,
- note that Morph does not untracked files to the temporary branch,
- only uncommitted changes to files git already knows about. You
- need to `git add` and commit each new file yourself.
-
- Example:
-
- morph build devel-system-x86_64-generic
-
- '''
+ if args:
+ raise cliapp.AppException('morph status takes no arguments')
- if len(args) != 1:
- raise cliapp.AppException('morph build expects exactly one '
- 'parameter: the system to build')
-
- # Raise an exception if there is not enough space
- morphlib.util.check_disk_available(
- self.app.settings['tempdir'],
- self.app.settings['tempdir-min-space'],
- self.app.settings['cachedir'],
- self.app.settings['cachedir-min-space'])
-
- system_name = args[0]
-
- # Deduce workspace and system branch and branch root repository.
- workspace = self.deduce_workspace()
- branch, branch_dir = self.deduce_system_branch()
- branch_root = self.get_branch_config(branch_dir, 'branch.root')
- branch_uuid = self.get_branch_config(branch_dir, 'branch.uuid')
-
- # Generate a UUID for the build.
- build_uuid = uuid.uuid4().hex
-
- build_command = morphlib.buildcommand.BuildCommand(self.app)
- build_command = self.app.hookmgr.call('new-build-command',
- build_command)
- push = self.app.settings['push-build-branches']
-
- self.app.status(msg='Starting build %(uuid)s', uuid=build_uuid)
-
- self.app.status(msg='Collecting morphologies involved in '
- 'building %(system)s from %(branch)s',
- system=system_name, branch=branch)
-
- # Get repositories of morphologies involved in building this system
- # from the current system branch.
- build_repos = self.get_system_build_repos(
- branch, branch_dir, branch_root, system_name)
-
- # Generate temporary build ref names for all these repositories.
- self.generate_build_ref_names(build_repos, branch_uuid)
-
- # Create the build refs for all these repositories and commit
- # all uncommitted changes to them, updating all references
- # to system branch refs to point to the build refs instead.
- self.update_build_refs(build_repos, branch, build_uuid, push)
-
- if push:
- self.push_build_refs(build_repos)
- build_branch_root = branch_root
+ ws = morphlib.workspace.open('.')
+ try:
+ sb = morphlib.sysbranchdir.open_from_within('.')
+ except morphlib.sysbranchdir.NotInSystemBranch:
+ self._workspace_status(ws)
else:
- dirname = build_repos[branch_root]['dirname']
- build_branch_root = urlparse.urljoin('file://', dirname)
-
- # Run the build.
- build_command.build([build_branch_root,
- build_repos[branch_root]['build-ref'],
- system_name])
+ self._branch_status(ws, sb)
- if push:
- self.delete_remote_build_refs(build_repos)
+ def _workspace_status(self, ws):
+ '''Show information about the current workspace
- self.app.status(msg='Finished build %(uuid)s', uuid=build_uuid)
-
- def get_system_build_repos(self, system_branch, branch_dir,
- branch_root, system_name):
- '''Map upstream repository URLs to their checkouts in the system branch
-
- Also provides the list of morphologies stored in each repository,
- grouped by kind.
+ This lists all checked out system branches in the workspace.
'''
+ self.app.output.write("System branches in current workspace:\n")
+ branches = sorted(ws.list_system_branches(),
+ key=lambda x: x.root_directory)
+ for sb in branches:
+ self.app.output.write(" %s\n" % sb.get_config('branch.name'))
- build_repos = {}
-
- def prepare_repo_info(repo, dirname):
- build_repos[repo] = {
- 'dirname': dirname,
- 'systems': [],
- 'strata': [],
- 'chunks': []
- }
-
- def add_morphology_info(info, category):
- repo = info['repo'] or branch_root
- if repo in build_repos:
- repo_dir = build_repos[repo]['dirname']
- else:
- repo_dir = self.find_repository(branch_dir, repo)
- if repo_dir:
- if not repo in build_repos:
- prepare_repo_info(repo, repo_dir)
- build_repos[repo][category].append(info['morph'])
- return repo_dir
-
- # Add repository and morphology of the system.
- branch_root_dir = self.find_repository(branch_dir, branch_root)
- prepare_repo_info(branch_root, branch_root_dir)
- build_repos[branch_root]['systems'].append(system_name)
-
- # Traverse and add repositories and morphologies involved in
- # building this system from the system branch.
- system_morphology = self.load_morphology(branch_root_dir, system_name)
- for info in system_morphology['strata']:
- if info['ref'] == system_branch or info['ref'] is None:
- repo_dir = add_morphology_info(info, 'strata')
- if repo_dir:
- stratum_morphology = self.load_morphology(
- repo_dir, info['morph'])
- for info in stratum_morphology['chunks']:
- if info['ref'] == system_branch:
- add_morphology_info(info, 'chunks')
-
- return build_repos
-
- def inject_build_refs(self, morphology, build_repos, will_push):
- # Starting from a system or stratum morphology, update all ref
- # pointers of strata or chunks involved in a system build (represented
- # by build_repos) to point to temporary build refs of the repos
- # involved in the system build.
- def inject_build_ref(info):
- if info['repo'] in build_repos and (
- info['morph'] in build_repos[info['repo']]['strata'] or
- info['morph'] in build_repos[info['repo']]['chunks']):
- info['ref'] = build_repos[info['repo']]['build-ref']
- if not will_push:
- dirname = build_repos[info['repo']]['dirname']
- info['repo'] = urlparse.urljoin('file://', dirname)
- if morphology['kind'] == 'system':
- for info in morphology['strata']:
- inject_build_ref(info)
- elif morphology['kind'] == 'stratum':
- if morphology['build-depends'] is not None:
- for info in morphology['build-depends']:
- inject_build_ref(info)
- for info in morphology['chunks']:
- inject_build_ref(info)
-
- def generate_build_ref_names(self, build_repos, branch_uuid):
- for repo, info in build_repos.iteritems():
- repo_dir = info['dirname']
- repo_uuid = self.get_repo_config(repo_dir, 'morph.uuid')
- build_ref = os.path.join(self.app.settings['build-ref-prefix'],
- branch_uuid, repo_uuid)
- info['build-ref'] = build_ref
-
- def update_build_refs(self, build_repos, system_branch, build_uuid,
- will_push):
- '''Update build branches for each repository with any local changes '''
-
- # Define the committer.
- committer_name = 'Morph (on behalf of %s)' % \
- (morphlib.git.get_user_name(self.app.runcmd))
- committer_email = morphlib.git.get_user_email(self.app.runcmd)
-
- for repo, info in build_repos.iteritems():
- repo_dir = info['dirname']
- build_ref = info['build-ref']
-
- self.app.status(msg='%(repo)s: Creating build branch', repo=repo)
-
- # Obtain parent SHA1 for the temporary ref tree to be committed.
- # This will either be the current commit of the temporary ref or
- # HEAD in case the temporary ref does not exist yet.
- system_branch_sha1 = self.resolve_ref(
- repo_dir, '%s^{commit}' % system_branch)
- parent_sha1 = self.resolve_ref(repo_dir, '%s^{commit}' % build_ref)
- if not parent_sha1:
- parent_sha1 = system_branch_sha1
-
- # Prepare an environment with our internal index file.
- # This index file allows us to commit changes to a tree without
- # git noticing any change in working tree or its own index.
- env = dict(os.environ)
- env['GIT_INDEX_FILE'] = os.path.join(
- repo_dir, '.git', 'morph-index')
- env['GIT_COMMITTER_NAME'] = committer_name
- env['GIT_COMMITTER_EMAIL'] = committer_email
-
- # Read tree from current HEAD into the morph index.
- self.app.runcmd(['git', 'read-tree', system_branch_sha1],
- cwd=repo_dir, env=env)
-
- self.app.status(msg='%(repo)s: Adding uncommitted changes to '
- 'build branch', repo=repo)
-
- # Add all local, uncommitted changes to our internal index.
- changed_files = self.get_uncommitted_changes(repo_dir, env)
- self.app.runcmd(['git', 'add'] + changed_files,
- cwd=repo_dir, env=env)
-
- self.app.status(msg='%(repo)s: Updating morphologies to use '
- 'build branch instead of "%(branch)s"',
- repo=repo, branch=system_branch)
-
- # Update all references to the system branches of strata
- # and chunks to point to the temporary refs, which is needed
- # for building.
- filenames = info['systems'] + info['strata']
- for filename in filenames:
- # Inject temporary refs in the right places in each morphology.
- morphology = self.load_morphology(repo_dir, filename)
- self.inject_build_refs(morphology, build_repos, will_push)
- with tempfile.NamedTemporaryFile(suffix='.morph') as f:
- self.update_morphology(
- repo_dir, filename, morphology, output_fd=f.file)
-
- morphology_sha1 = self.app.runcmd(
- ['git', 'hash-object', '-t', 'blob', '-w', f.name],
- cwd=repo_dir, env=env)
-
- try:
- self.app.runcmd(
- ['git', 'update-index', '--cacheinfo',
- '100644', morphology_sha1, '%s.morph' % filename],
- cwd=repo_dir, env=env)
- except cliapp.AppException, e:
- raise cliapp.AppException(
- "You seem to want to build %s, but '%s.morph' "
- "doesn't exist in the morphologies repository. "
- "Did you forget to commit it?" %
- (filename, filename))
-
- # Create a commit message including the build UUID. This allows us
- # to collect all commits of a build across repositories and thereby
- # see the changes made to the entire system between any two builds.
- message = 'Morph build %s\n\nSystem branch: %s\n' % \
- (build_uuid, system_branch)
-
- # Write and commit the tree and update the temporary build ref.
- tree = self.app.runcmd(
- ['git', 'write-tree'], cwd=repo_dir, env=env).strip()
- commit = self.app.runcmd(
- ['git', 'commit-tree', tree, '-p', parent_sha1],
- feed_stdin=message, cwd=repo_dir, env=env).strip()
- self.app.runcmd(
- ['git', 'update-ref', '-m', message,
- 'refs/heads/%s' % build_ref, commit],
- cwd=repo_dir, env=env)
-
- def push_build_refs(self, build_repos):
- for repo, info in build_repos.iteritems():
- self.app.status(msg='%(repo)s: Pushing build branch', repo=repo)
- self.app.runcmd(['git', 'push', 'origin', '%s:%s' %
- (info['build-ref'], info['build-ref'])],
- cwd=info['dirname'])
-
- def delete_remote_build_refs(self, build_repos):
- for repo, info in build_repos.iteritems():
- self.app.status(msg='%(repo)s: Deleting remote build branch',
- repo=repo)
- self.app.runcmd(['git', 'push', 'origin',
- ':%s' % info['build-ref']], cwd=info['dirname'])
-
- def status(self, args):
- '''Show information about the current system branch or workspace
+ def _branch_status(self, ws, sb):
+ '''Show information about the current branch
This shows the status of every local git repository of the
current system branch. This is similar to running `git status`
- in each repository separately, but the output is nicer.
-
- If run in a Morph workspace, but not in a system branch checkout,
- it lists all checked out system branches in the workspace.
+ in each repository separately.
'''
+ branch = sb.get_config('branch.name')
+ root = sb.get_config('branch.root')
- if len(args) != 0:
- raise cliapp.AppException('morph status takes no arguments')
-
- workspace = self.deduce_workspace()
- try:
- branch, branch_path = self.deduce_system_branch()
- except cliapp.AppException:
- branch = None
-
- if branch is None:
- self.app.output.write("System branches in current workspace:\n")
- branch_dirs = sorted(self.walk_special_directories(
- workspace, special_subdir='.morph-system-branch'))
- for dirname in branch_dirs:
- branch = self.get_branch_config(dirname, 'branch.name')
- self.app.output.write(" %s\n" % branch)
- return
-
- root_repo = self.get_branch_config(branch_path, 'branch.root')
- root_repo_path = self.find_repository(branch_path, root_repo)
-
- self.app.output.write("On branch %s, root %s\n" % (branch, root_repo))
+ self.app.output.write("On branch %s, root %s\n" % (branch, root))
has_uncommitted_changes = False
- for d in self.iterate_branch_repos(branch_path, root_repo_path):
+ for gd in sorted(sb.list_git_directories(), key=lambda x: x.dirname):
try:
- repo = self.get_repo_config(d, 'morph.repository')
+ repo = gd.get_config('morph.repository')
except cliapp.AppException:
self.app.output.write(
- ' %s: not part of system branch\n' % d)
- continue
- head = self.get_head(d)
+ ' %s: not part of system branch\n' % gd.dirname)
+ # TODO: make this less vulnerable to a branch using
+ # refs/heads/foo instead of foo
+ head = gd.HEAD
if head != branch:
self.app.output.write(
- ' %s: unexpected ref checked out "%s"\n' % (repo, head))
- if len(self.get_uncommitted_changes(d)) > 0:
+ ' %s: unexpected ref checked out %r\n' % (repo, head))
+ if any(gd.get_index().get_uncommitted_changes()):
has_uncommitted_changes = True
self.app.output.write(' %s: uncommitted changes\n' % repo)
if not has_uncommitted_changes:
self.app.output.write("\nNo repos have outstanding changes.\n")
- def foreach(self, args):
- '''Run a command in each repository checked out in a system branch.
+ def branch_from_image(self, args):
+ '''Produce a branch of an existing system image.
- Use -- before specifying the command to separate its arguments from
- Morph's own arguments.
+ Given the metadata specified by --metadata-dir, create a new
+ branch then petrify it to the state of the commits at the time
+ the system was built.
- Command line arguments:
+ If --metadata-dir is not specified, it defaults to your currently
+ running system.
- * `--` indicates the end of option processing for Morph.
- * `COMMAND` is a command to run.
- * `ARGS` is a list of arguments or options to be passed onto
- `COMMAND`.
+ '''
+ if len(args) != 1:
+ raise cliapp.AppException(
+ "branch-from-image needs exactly 1 argument "
+ "of the new system branch's name")
+ system_branch = args[0]
+ metadata_path = self.app.settings['metadata-dir']
+ alias_resolver = morphlib.repoaliasresolver.RepoAliasResolver(
+ self.app.settings['repo-alias'])
- This runs the given `COMMAND` in each git repository belonging
- to the current system branch that exists locally in the current
- workspace. This can be a handy way to do the same thing in all
- the local git repositories.
+ self._require_git_user_config()
- For example:
+ ws = morphlib.workspace.open('.')
- morph foreach -- git push
+ system, metadata = self._load_system_metadata(metadata_path)
+ resolved_refs = dict(self._resolve_refs_from_metadata(alias_resolver,
+ metadata))
+ self.app.status(msg='Resolved refs: %r' % resolved_refs)
+ base_ref = system['sha1']
+ # The previous version would fall back to deducing this from the repo
+ # url and the repo alias resolver, but this does not always work, and
+ # new systems always have repo-alias in the metadata
+ root_url = system['repo-alias']
- The above command would push any committed changes in each
- repository to the git server.
+ lrc, rrc = morphlib.util.new_repo_caches(self.app)
+ cached_repo = lrc.get_updated_repo(root_url)
- '''
- # For simplicity, this simply iterates repositories in the directory
- # rather than walking through the morphologies as 'morph merge' does.
+ with self._initializing_system_branch(
+ ws, root_url, system_branch, cached_repo, base_ref) as (sb, gd):
- if len(args) == 0:
- raise cliapp.AppException('morph foreach expects a command to run')
+ # TODO: It's nasty to clone to a sha1 then create a branch
+ # of that sha1 then check it out, a nicer API may be the
+ # initial clone not checking out a branch at all, then
+ # the user creates and checks out their own branches
+ gd.branch(system_branch, base_ref)
+ gd.checkout(system_branch)
- workspace = self.deduce_workspace()
- branch, branch_path = self.deduce_system_branch()
+ loader = morphlib.morphloader.MorphologyLoader()
+ morphs = self._load_all_sysbranch_morphologies(sb, loader)
- root_repo = self.get_branch_config(branch_path, 'branch.root')
- root_repo_path = self.find_repository(branch_path, root_repo)
+ morphs.repoint_refs(sb.root_repository_url,
+ sb.system_branch_name)
- for d in self.iterate_branch_repos(branch_path, root_repo_path):
- try:
- repo = self.get_repo_config(d, 'morph.repository')
- except cliapp.AppException:
- continue
+ morphs.petrify_chunks(resolved_refs)
- if d != root_repo_path:
- self.app.output.write('\n')
- self.app.output.write('%s\n' % repo)
+ self._save_dirty_morphologies(loader, sb, morphs.morphologies)
- status, output, error = self.app.runcmd_unchecked(args, cwd=d)
- self.app.output.write(output)
- if status != 0:
- self.app.output.write(error)
- raise cliapp.AppException(
- 'Command failed at repo %s: %s' % (repo, ' '.join(args)))
+ @staticmethod
+ def _load_system_metadata(path):
+ '''Load all metadata in `path` corresponding to a single System.
+ '''
+
+ smd = morphlib.systemmetadatadir.SystemMetadataDir(path)
+ metadata = smd.values()
+ systems = [md for md in metadata
+ if 'kind' in md and md['kind'] == 'system']
+
+ if not systems:
+ raise cliapp.AppException(
+ 'Metadata directory does not contain any systems.')
+ if len(systems) > 1:
+ raise cliapp.AppException(
+ 'Metadata directory contains multiple systems.')
+ system_metadatum = systems[0]
+
+ metadata_cache_id_lookup = dict((md['cache-key'], md)
+ for md in metadata)
+
+ return system_metadatum, metadata_cache_id_lookup
+
+ @staticmethod
+ def _resolve_refs_from_metadata(alias_resolver, metadata):
+ '''Pre-resolve a set of refs from existing metadata.
+
+ Given the metadata, generate a mapping of all the (repo, ref)
+ pairs defined in the metadata and the commit id they resolved to.
+ '''
+ for md in metadata.itervalues():
+ repourls = set((md['repo-alias'], md['repo']))
+ repourls.update(alias_resolver.aliases_from_url(md['repo']))
+ for repourl in repourls:
+ yield ((repourl, md['original_ref']), md['sha1'])
diff --git a/morphlib/plugins/build_plugin.py b/morphlib/plugins/build_plugin.py
index fb7efa5b..1a4fb573 100644
--- a/morphlib/plugins/build_plugin.py
+++ b/morphlib/plugins/build_plugin.py
@@ -76,7 +76,7 @@ class BuildPlugin(cliapp.Plugin):
Example:
- morph distbuild devel-system-x86_64-generic
+ morph distbuild devel-system-x86_64-generic.morph
'''
@@ -103,8 +103,8 @@ class BuildPlugin(cliapp.Plugin):
Example:
- morph build-morphology baserock:baserock/morphs \
- master devel-system-x86_64-generic
+ morph build-morphology baserock:baserock/definitions \
+ master devel-system-x86_64-generic.morph
'''
@@ -141,7 +141,7 @@ class BuildPlugin(cliapp.Plugin):
Example:
- morph build devel-system-x86_64-generic
+ morph build devel-system-x86_64-generic.morph
'''
@@ -156,7 +156,7 @@ class BuildPlugin(cliapp.Plugin):
self.app.settings['cachedir'],
self.app.settings['cachedir-min-space'])
- system_name = morphlib.util.strip_morph_extension(args[0])
+ system_filename = morphlib.util.sanitise_morphology_path(args[0])
ws = morphlib.workspace.open('.')
sb = morphlib.sysbranchdir.open_from_within('.')
@@ -181,7 +181,8 @@ class BuildPlugin(cliapp.Plugin):
self.app.status(msg='Starting build %(uuid)s', uuid=build_uuid)
self.app.status(msg='Collecting morphologies involved in '
'building %(system)s from %(branch)s',
- system=system_name, branch=sb.system_branch_name)
+ system=system_filename,
+ branch=sb.system_branch_name)
bb = morphlib.buildbranch.BuildBranch(sb, build_ref_prefix,
push_temporary=push)
@@ -210,4 +211,4 @@ class BuildPlugin(cliapp.Plugin):
build_command.build([bb.root_repo_url,
bb.root_ref,
- system_name])
+ system_filename])
diff --git a/morphlib/plugins/cross-bootstrap_plugin.py b/morphlib/plugins/cross-bootstrap_plugin.py
index bfd0d047..cd8e355e 100644
--- a/morphlib/plugins/cross-bootstrap_plugin.py
+++ b/morphlib/plugins/cross-bootstrap_plugin.py
@@ -260,7 +260,7 @@ class CrossBootstrapPlugin(cliapp.Plugin):
self.app.settings, arch)
build_command = morphlib.buildcommand.BuildCommand(self.app, build_env)
- morph_name = system_name + '.morph'
+ morph_name = morphlib.util.sanitise_morphology_path(system_name)
builds_artifacts = [system_name + '-bootstrap-rootfs']
srcpool = build_command.create_source_pool(root_repo, ref, morph_name)
diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py
index 6fc0998c..9384c422 100644
--- a/morphlib/plugins/deploy_plugin.py
+++ b/morphlib/plugins/deploy_plugin.py
@@ -99,7 +99,7 @@ class DeployPlugin(cliapp.Plugin):
name: cluster-foo
kind: cluster
systems:
- - morph: devel-system-x86_64-generic
+ - morph: devel-system-x86_64-generic.morph
deploy:
cluster-foo-x86_64-1:
type: kvm
@@ -278,7 +278,7 @@ class DeployPlugin(cliapp.Plugin):
'/', 0)
self.app.settings['no-git-update'] = True
- cluster_name = morphlib.util.strip_morph_extension(args[0])
+ cluster_filename = morphlib.util.sanitise_morphology_path(args[0])
ws = morphlib.workspace.open('.')
sb = morphlib.sysbranchdir.open_from_within('.')
@@ -292,11 +292,9 @@ class DeployPlugin(cliapp.Plugin):
name = morphlib.git.get_user_name(self.app.runcmd)
email = morphlib.git.get_user_email(self.app.runcmd)
build_ref_prefix = self.app.settings['build-ref-prefix']
-
root_repo_dir = morphlib.gitdir.GitDirectory(
sb.get_git_directory_name(sb.root_repository_url))
- mf = morphlib.morphologyfinder.MorphologyFinder(root_repo_dir)
- cluster_text, cluster_filename = mf.read_morphology(cluster_name)
+ cluster_text = root_repo_dir.read_file(cluster_filename)
cluster_morphology = loader.load_from_string(cluster_text,
filename=cluster_filename)
@@ -388,9 +386,8 @@ class DeployPlugin(cliapp.Plugin):
self.app.status_prefix = system_status_prefix
try:
# Find the artifact to build
- morph = system['morph']
- srcpool = build_command.create_source_pool(build_repo, ref,
- morph + '.morph')
+ morph = morphlib.util.sanitise_morphology_path(system['morph'])
+ srcpool = build_command.create_source_pool(build_repo, ref, morph)
artifact = build_command.resolve_artifacts(srcpool)
diff --git a/morphlib/plugins/list_artifacts_plugin.py b/morphlib/plugins/list_artifacts_plugin.py
index 5e64f708..ad6bc772 100644
--- a/morphlib/plugins/list_artifacts_plugin.py
+++ b/morphlib/plugins/list_artifacts_plugin.py
@@ -55,21 +55,22 @@ class ListArtifactsPlugin(cliapp.Plugin):
'(see help)')
repo, ref = args[0], args[1]
- system_names = map(morphlib.util.strip_morph_extension, args[2:])
+ system_filenames = map(morphlib.util.sanitise_morphology_path,
+ args[2:])
self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app)
self.resolver = morphlib.artifactresolver.ArtifactResolver()
artifact_files = set()
- for system_name in system_names:
+ for system_filename in system_filenames:
system_artifact_files = self.list_artifacts_for_system(
- repo, ref, system_name)
+ repo, ref, system_filename)
artifact_files.update(system_artifact_files)
for artifact_file in sorted(artifact_files):
print artifact_file
- def list_artifacts_for_system(self, repo, ref, system_name):
+ def list_artifacts_for_system(self, repo, ref, system_filename):
'''List all artifact files in the build graph of a single system.'''
# Sadly, we must use a fresh source pool and a fresh list of artifacts
@@ -82,24 +83,24 @@ class ListArtifactsPlugin(cliapp.Plugin):
# different architectures right now.
self.app.status(
- msg='Creating source pool for %s' % system_name, chatty=True)
+ msg='Creating source pool for %s' % system_filename, chatty=True)
source_pool = self.app.create_source_pool(
- self.lrc, self.rrc, (repo, ref, system_name + '.morph'))
+ self.lrc, self.rrc, (repo, ref, system_filename))
self.app.status(
- msg='Resolving artifacts for %s' % system_name, chatty=True)
+ msg='Resolving artifacts for %s' % system_filename, chatty=True)
artifacts = self.resolver.resolve_artifacts(source_pool)
- def find_artifact_by_name(artifacts_list, name):
+ def find_artifact_by_name(artifacts_list, filename):
for a in artifacts_list:
- if a.source.filename == name + '.morph':
+ if a.source.filename == name:
return a
raise ValueError
- system_artifact = find_artifact_by_name(artifacts, system_name)
+ system_artifact = find_artifact_by_name(artifacts, system_filename)
self.app.status(
- msg='Computing cache keys for %s' % system_name, chatty=True)
+ msg='Computing cache keys for %s' % system_filename, chatty=True)
build_env = morphlib.buildenvironment.BuildEnvironment(
self.app.settings, system_artifact.source.morphology['arch'])
ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env)
diff --git a/morphlib/source.py b/morphlib/source.py
index 75a2e4de..2dbabad1 100644
--- a/morphlib/source.py
+++ b/morphlib/source.py
@@ -55,4 +55,4 @@ class Source(object):
def __str__(self): # pragma: no cover
return '%s|%s|%s' % (self.repo_name,
self.original_ref,
- self.filename[:-len('.morph')])
+ self.filename)
diff --git a/morphlib/sysbranchdir.py b/morphlib/sysbranchdir.py
index 9d96e974..b8953c2f 100644
--- a/morphlib/sysbranchdir.py
+++ b/morphlib/sysbranchdir.py
@@ -176,8 +176,9 @@ class SystemBranchDirectory(object):
gd_name = self.get_git_directory_name(self.root_repository_url)
gd = morphlib.gitdir.GitDirectory(gd_name)
mf = morphlib.morphologyfinder.MorphologyFinder(gd)
- for morph in mf.list_morphologies():
- text, filename = mf.read_morphology(morph)
+ for filename in (f for f in mf.list_morphologies()
+ if not gd.is_symlink(f)):
+ text = mf.read_morphology(filename)
m = loader.load_from_string(text, filename=filename)
m.repo_url = self.root_repository_url
m.ref = self.system_branch_name
diff --git a/morphlib/util.py b/morphlib/util.py
index 024de495..0c551296 100644
--- a/morphlib/util.py
+++ b/morphlib/util.py
@@ -61,13 +61,22 @@ def indent(string, spaces=4):
return '\n'.join(lines)
-def strip_morph_extension(morph_name):
- if morph_name.startswith('.'):
- raise morphlib.Error(
- 'Invalid morphology name: %s' % morph_name)
- elif morph_name.endswith('.morph'):
- return morph_name[:-len('.morph')]
- return morph_name
+def sanitise_morphology_path(morph_name):
+ '''Turn morph_name into a file path to a morphology.
+
+ We support both a file path being provided, and just the morphology
+ name for backwards compatibility.
+
+ '''
+ # If it has a / it must be a path, so return it unmolested
+ if '/' in morph_name:
+ return morph_name
+ # Must be an old format, which is always name + .morph
+ elif not morph_name.endswith('.morph'):
+ return morph_name + '.morph'
+ # morphology already ends with .morph
+ else:
+ return morph_name
def make_concurrency(cores=None):
diff --git a/morphlib/util_tests.py b/morphlib/util_tests.py
index 5a8ae797..715892b6 100644
--- a/morphlib/util_tests.py
+++ b/morphlib/util_tests.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2011-2013 Codethink Limited
+# Copyright (C) 2011-2014 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -38,26 +38,19 @@ class IndentTests(unittest.TestCase):
' foo\n bar')
-class StripMorphExtensionTests(unittest.TestCase):
+class SanitiseMorphologyPathTests(unittest.TestCase):
- def test_raises_error_when_string_starts_with_period(self):
- with self.assertRaises(morphlib.Error):
- morphlib.util.strip_morph_extension('.morph')
-
- def test_strips_morph_extension_from_string(self):
- self.assertEqual(morphlib.util.strip_morph_extension('a.morph'), 'a')
-
- def test_returns_morph_when_not_given_as_extension(self):
- self.assertEqual(morphlib.util.strip_morph_extension('morph'), 'morph')
-
- def test_strips_extension_only_once_from_string(self):
- self.assertEqual(morphlib.util.strip_morph_extension('a.morph.morph'),
+ def test_appends_morph_to_string(self):
+ self.assertEqual(morphlib.util.sanitise_morphology_path('a'),
'a.morph')
- def test_returns_input_without_modification_if_no_extension(self):
- self.assertEqual(
- morphlib.util.strip_morph_extension('completely not a path'),
- 'completely not a path')
+ def test_returns_morph_when_given_a_filename(self):
+ self.assertEqual(morphlib.util.sanitise_morphology_path('a.morph'),
+ 'a.morph')
+
+ def test_returns_morph_when_given_a_path(self):
+ self.assertEqual('stratum/a.morph',
+ morphlib.util.sanitise_morphology_path('stratum/a.morph'))
class MakeConcurrencyTests(unittest.TestCase):