diff options
Diffstat (limited to 'morphlib')
34 files changed, 661 insertions, 1559 deletions
diff --git a/morphlib/__init__.py b/morphlib/__init__.py index 0c928fd3..f98c11aa 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -68,10 +68,9 @@ import gitindex import localartifactcache import localrepocache import mountableimage -import morph2 import morphologyfactory import morphologyfinder -import morph3 +import morphology import morphloader import morphset import remoteartifactcache diff --git a/morphlib/app.py b/morphlib/app.py index a543443e..88eb58a4 100644 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -518,16 +518,15 @@ class Morph(cliapp.Application): self.output.write(text) def _help_topic(self, topic): - build_ref_prefix = self.settings['build-ref-prefix'] if topic in self.subcommands: usage = self._format_usage_for(topic) description = self._format_subcommand_help(topic) text = '%s\n\n%s' % (usage, description) self.output.write(text) - elif topic in extensions.list_extensions(build_ref_prefix): + elif topic in extensions.list_extensions(): name, kind = os.path.splitext(topic) try: - with extensions.get_extension_filename(build_ref_prefix, + with extensions.get_extension_filename( name, kind + '.help', executable=False) as fname: with open(fname, 'r') as f: diff --git a/morphlib/artifact_tests.py b/morphlib/artifact_tests.py index d4b15cba..62b1bfb9 100644 --- a/morphlib/artifact_tests.py +++ b/morphlib/artifact_tests.py @@ -23,24 +23,22 @@ import morphlib class ArtifactTests(unittest.TestCase): def setUp(self): - morph = morphlib.morph2.Morphology( - ''' - { - "name": "chunk", - "kind": "chunk", - "chunks": { - "chunk-runtime": [ - "usr/bin", - "usr/sbin", - "usr/lib", - "usr/libexec" - ], - "chunk-devel": [ - "usr/include" - ] - } - } - ''') + loader = morphlib.morphloader.MorphologyLoader() + morph = loader.load_from_string( + ''' + name: chunk + kind: chunk + products: + - artifact: chunk-runtime + include: + - usr/bin + - usr/sbin + - usr/lib + - usr/libexec + - artifact: chunk-devel + include: + - usr/include + ''') self.source = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'chunk.morph') self.artifact_name = 'chunk-runtime' diff --git a/morphlib/artifactresolver_tests.py b/morphlib/artifactresolver_tests.py index 6f62b4d1..96f7ced8 100644 --- a/morphlib/artifactresolver_tests.py +++ b/morphlib/artifactresolver_tests.py @@ -15,84 +15,68 @@ import itertools -import json import unittest +import yaml import morphlib -class FakeChunkMorphology(morphlib.morph2.Morphology): - - def __init__(self, name, artifact_names=[]): - assert(isinstance(artifact_names, list)) - - if artifact_names: - # fake a list of artifacts - artifacts = [] - for artifact_name in artifact_names: - artifacts.append({'artifact': artifact_name, - 'include': artifact_name}) - text = json.dumps({ - "name": name, - "kind": "chunk", - "products": artifacts - }) - self.builds_artifacts = artifact_names - else: - text = (''' - { - "name": "%s", - "kind": "chunk" - } - ''' % name) - self.builds_artifacts = [name] - morphlib.morph2.Morphology.__init__(self, text) - - -class FakeStratumMorphology(morphlib.morph2.Morphology): - - def __init__(self, name, chunks=[], build_depends=[]): - assert(isinstance(chunks, list)) - assert(isinstance(build_depends, list)) - - chunks_list = [] - for source_name, morph, repo, ref in chunks: - chunks_list.append({ - 'name': source_name, - 'morph': morph, - 'repo': repo, - 'ref': ref, - 'build-depends': [], - }) - build_depends_list = [] - for morph, repo, ref in build_depends: - build_depends_list.append({ - 'morph': morph, - 'repo': repo, - 'ref': ref - }) - if chunks_list: - text = (''' - { - "name": "%s", - "kind": "stratum", - "build-depends": %s, - "chunks": %s - } - ''' % (name, - json.dumps(build_depends_list), - json.dumps(chunks_list))) - else: - text = (''' - { - "name": "%s", - "kind": "stratum", - "build-depends": %s - } - ''' % (name, - json.dumps(build_depends_list))) - self.builds_artifacts = [name] - morphlib.morph2.Morphology.__init__(self, text) +def get_chunk_morphology(name, artifact_names=[]): + assert(isinstance(artifact_names, list)) + + if artifact_names: + # fake a list of artifacts + artifacts = [] + for artifact_name in artifact_names: + artifacts.append({'artifact': artifact_name, + 'include': [artifact_name]}) + text = yaml.dump({"name": name, + "kind": "chunk", + "products": artifacts}, default_flow_style=False) + builds_artifacts = artifact_names + else: + text = yaml.dump({'name': name, + 'kind': 'chunk'}, default_flow_style=False) + builds_artifacts = [name] + + loader = morphlib.morphloader.MorphologyLoader() + morph = loader.load_from_string(text) + morph.builds_artifacts = builds_artifacts + return morph + +def get_stratum_morphology(name, chunks=[], build_depends=[]): + assert(isinstance(chunks, list)) + assert(isinstance(build_depends, list)) + + chunks_list = [] + for source_name, morph, repo, ref in chunks: + chunks_list.append({ + 'name': source_name, + 'morph': morph, + 'repo': repo, + 'ref': ref, + 'build-depends': [], + }) + build_depends_list = [] + for morph in build_depends: + build_depends_list.append({ + 'morph': morph, + }) + if chunks_list: + text = yaml.dump({"name": name, + "kind": "stratum", + "build-depends": build_depends_list, + "chunks": chunks_list,}, default_flow_style=False) + else: + text = yaml.dump({"name": name, + "kind": "stratum", + "build-depends": build_depends_list}, + default_flow_style=False) + + loader = morphlib.morphloader.MorphologyLoader() + morph = loader.load_from_string(text) + morph.builds_artifacts = [name] + return morph class ArtifactResolverTests(unittest.TestCase): @@ -108,7 +92,7 @@ class ArtifactResolverTests(unittest.TestCase): def test_resolve_single_chunk_with_no_subartifacts(self): pool = morphlib.sourcepool.SourcePool() - morph = FakeChunkMorphology('chunk') + morph = get_chunk_morphology('chunk') source = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'chunk.morph') pool.add(source) @@ -127,7 +111,7 @@ class ArtifactResolverTests(unittest.TestCase): def test_resolve_single_chunk_with_one_new_artifact(self): pool = morphlib.sourcepool.SourcePool() - morph = FakeChunkMorphology('chunk', ['chunk-foobar']) + morph = get_chunk_morphology('chunk', ['chunk-foobar']) source = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'chunk.morph') pool.add(source) @@ -145,7 +129,7 @@ class ArtifactResolverTests(unittest.TestCase): def test_resolve_single_chunk_with_two_new_artifacts(self): pool = morphlib.sourcepool.SourcePool() - morph = FakeChunkMorphology('chunk', ['chunk-baz', 'chunk-qux']) + morph = get_chunk_morphology('chunk', ['chunk-baz', 'chunk-qux']) source = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'chunk.morph') pool.add(source) @@ -165,12 +149,12 @@ class ArtifactResolverTests(unittest.TestCase): def test_resolve_stratum_and_chunk(self): pool = morphlib.sourcepool.SourcePool() - morph = FakeChunkMorphology('chunk') + morph = get_chunk_morphology('chunk') chunk = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'chunk.morph') pool.add(chunk) - morph = FakeStratumMorphology( + morph = get_stratum_morphology( 'stratum', chunks=[('chunk', 'chunk', 'repo', 'ref')]) stratum = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'stratum.morph') @@ -199,12 +183,12 @@ class ArtifactResolverTests(unittest.TestCase): def test_resolve_stratum_and_chunk_with_two_new_artifacts(self): pool = morphlib.sourcepool.SourcePool() - morph = FakeChunkMorphology('chunk', ['chunk-foo', 'chunk-bar']) + morph = get_chunk_morphology('chunk', ['chunk-foo', 'chunk-bar']) chunk = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'chunk.morph') pool.add(chunk) - morph = FakeStratumMorphology( + morph = get_stratum_morphology( 'stratum', chunks=[ ('chunk', 'chunk', 'repo', 'ref'), @@ -236,51 +220,42 @@ class ArtifactResolverTests(unittest.TestCase): def test_resolving_artifacts_for_a_system_with_two_dependent_strata(self): pool = morphlib.sourcepool.SourcePool() - morph = FakeChunkMorphology('chunk1') + morph = get_chunk_morphology('chunk1') chunk1 = morphlib.source.Source( 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk1.morph') pool.add(chunk1) - morph = FakeStratumMorphology( + morph = get_stratum_morphology( 'stratum1', chunks=[('chunk1', 'chunk1', 'repo', 'original/ref')]) stratum1 = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'stratum1.morph') pool.add(stratum1) - morph = morphlib.morph2.Morphology( + loader = morphlib.morphloader.MorphologyLoader() + morph = loader.load_from_string( ''' - { - "name": "system", - "kind": "system", - "strata": [ - { - "repo": "repo", - "ref": "ref", - "morph": "stratum1" - }, - { - "repo": "repo", - "ref": "ref", - "morph": "stratum2" - } - ] - } + name: system + kind: system + arch: testarch + strata: + - morph: stratum1 + - morph: stratum2 ''') morph.builds_artifacts = ['system-rootfs'] system = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'system.morph') pool.add(system) - morph = FakeChunkMorphology('chunk2') + morph = get_chunk_morphology('chunk2') chunk2 = morphlib.source.Source( 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk2.morph') pool.add(chunk2) - morph = FakeStratumMorphology( + morph = get_stratum_morphology( 'stratum2', chunks=[('chunk2', 'chunk2', 'repo', 'original/ref')], - build_depends=[('stratum1', 'repo', 'ref')]) + build_depends=['stratum1']) stratum2 = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'stratum2.morph') pool.add(stratum2) @@ -337,52 +312,44 @@ class ArtifactResolverTests(unittest.TestCase): def test_resolving_stratum_with_explicit_chunk_dependencies(self): pool = morphlib.sourcepool.SourcePool() - morph = morphlib.morph2.Morphology( + loader = morphlib.morphloader.MorphologyLoader() + morph = loader.load_from_string( ''' - { - "name": "stratum", - "kind": "stratum", - "chunks": [ - { - "name": "chunk1", - "repo": "repo", - "ref": "original/ref", - "build-depends": [] - }, - { - "name": "chunk2", - "repo": "repo", - "ref": "original/ref", - "build-depends": [] - }, - { - "name": "chunk3", - "repo": "repo", - "ref": "original/ref", - "build-depends": [ - "chunk1", - "chunk2" - ] - } - ] - } + name: stratum + kind: stratum + build-depends: [] + chunks: + - name: chunk1 + repo: repo + ref: original/ref + build-depends: [] + - name: chunk2 + repo: repo + ref: original/ref + build-depends: [] + - name: chunk3 + repo: repo + ref: original/ref + build-depends: + - chunk1 + - chunk2 ''') morph.builds_artifacts = ['stratum'] stratum = morphlib.source.Source( 'repo', 'original/ref', 'sha1', 'tree', morph, 'stratum.morph') pool.add(stratum) - morph = FakeChunkMorphology('chunk1') + morph = get_chunk_morphology('chunk1') chunk1 = morphlib.source.Source( 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk1.morph') pool.add(chunk1) - morph = FakeChunkMorphology('chunk2') + morph = get_chunk_morphology('chunk2') chunk2 = morphlib.source.Source( 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk2.morph') pool.add(chunk2) - morph = FakeChunkMorphology('chunk3') + morph = get_chunk_morphology('chunk3') chunk3 = morphlib.source.Source( 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk3.morph') pool.add(chunk3) @@ -418,20 +385,33 @@ class ArtifactResolverTests(unittest.TestCase): for c3a in chunk_artifacts[2])) def test_detection_of_mutual_dependency_between_two_strata(self): + loader = morphlib.morphloader.MorphologyLoader() pool = morphlib.sourcepool.SourcePool() - morph = FakeStratumMorphology( + chunk = get_chunk_morphology('chunk1') + chunk1 = morphlib.source.Source( + 'repo', 'original/ref', 'sha1', 'tree', chunk, 'chunk1.morph') + pool.add(chunk1) + + morph = get_stratum_morphology( 'stratum1', - chunks=[], - build_depends=[('stratum2', 'repo', 'original/ref')]) + chunks=[(loader.save_to_string(chunk), 'chunk1.morph', + 'repo', 'original/ref')], + build_depends=['stratum2']) stratum1 = morphlib.source.Source( 'repo', 'original/ref', 'sha1', 'tree', morph, 'stratum1.morph') pool.add(stratum1) - morph = FakeStratumMorphology( + chunk = get_chunk_morphology('chunk2') + chunk2 = morphlib.source.Source( + 'repo', 'original/ref', 'sha1', 'tree', chunk, 'chunk2.morph') + pool.add(chunk2) + + morph = get_stratum_morphology( 'stratum2', - chunks=[], - build_depends=[('stratum1', 'repo', 'original/ref')]) + chunks=[(loader.save_to_string(chunk), 'chunk2.morph', + 'repo', 'original/ref')], + build_depends=['stratum1']) stratum2 = morphlib.source.Source( 'repo', 'original/ref', 'sha1', 'tree', morph, 'stratum2.morph') pool.add(stratum2) @@ -442,39 +422,34 @@ class ArtifactResolverTests(unittest.TestCase): def test_detection_of_chunk_dependencies_in_invalid_order(self): pool = morphlib.sourcepool.SourcePool() - morph = morphlib.morph2.Morphology( + loader = morphlib.morphloader.MorphologyLoader() + morph = loader.load_from_string( ''' - { - "name": "stratum", - "kind": "stratum", - "chunks": [ - { - "name": "chunk1", - "repo": "repo", - "ref": "original/ref", - "build-depends": [ - "chunk2" - ] - }, - { - "name": "chunk2", - "repo": "repo", - "ref": "original/ref" - } - ] - } + name: stratum + kind: stratum + build-depends: [] + chunks: + - name: chunk1 + repo: repo + ref: original/ref + build-depends: + - chunk2 + - name: chunk2 + repo: repo + ref: original/ref + build-depends: [] ''') morph.builds_artifacts = ['stratum'] stratum = morphlib.source.Source( 'repo', 'original/ref', 'sha1', 'tree', morph, 'stratum.morph') pool.add(stratum) - morph = FakeChunkMorphology('chunk1') + morph = get_chunk_morphology('chunk1') chunk1 = morphlib.source.Source( 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk1.morph') pool.add(chunk1) - morph = FakeChunkMorphology('chunk2') + morph = get_chunk_morphology('chunk2') chunk2 = morphlib.source.Source( 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk2.morph') pool.add(chunk2) @@ -485,27 +460,24 @@ class ArtifactResolverTests(unittest.TestCase): def test_detection_of_invalid_build_depends_format(self): pool = morphlib.sourcepool.SourcePool() - morph = morphlib.morph2.Morphology( + loader = morphlib.morphloader.MorphologyLoader() + morph = loader.load_from_string( ''' - { - "name": "stratum", - "kind": "stratum", - "chunks": [ - { - "name": "chunk", - "repo": "repo", - "ref": "original/ref", - "build-depends": "whatever" - } - ] - } + name: stratum + kind: stratum + build-depends: [] + chunks: + - name: chunk + repo: repo + ref: original/ref + build-depends: whatever ''') morph.builds_artifacts = ['stratum'] stratum = morphlib.source.Source( 'repo', 'original/ref', 'sha1', 'tree', morph, 'stratum.morph') pool.add(stratum) - morph = FakeChunkMorphology('chunk') + morph = get_chunk_morphology('chunk') chunk = morphlib.source.Source( 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk.morph') pool.add(chunk) diff --git a/morphlib/buildbranch.py b/morphlib/buildbranch.py index d415e7e1..885f5cf8 100644 --- a/morphlib/buildbranch.py +++ b/morphlib/buildbranch.py @@ -15,6 +15,7 @@ import collections +import contextlib import os import urlparse @@ -48,10 +49,9 @@ class BuildBranch(object): # would be better to not use local repositories and temporary refs, # so building from a workspace appears to be identical to using # `morph build-morphology` - def __init__(self, sb, build_ref_prefix, push_temporary): + def __init__(self, sb, build_ref_prefix): self._sb = sb - self._push_temporary = push_temporary self._cleanup = collections.deque() self._to_push = {} @@ -84,15 +84,18 @@ class BuildBranch(object): def _register_cleanup(self, func, *args, **kwargs): self._cleanup.append((func, args, kwargs)) - def add_uncommitted_changes(self): + def add_uncommitted_changes(self, add_cb=lambda **kwargs: None): '''Add any uncommitted changes to temporary build GitIndexes''' + changes_made = False for gd, (build_ref, index) in self._to_push.iteritems(): changed = [to_path for code, to_path, from_path in index.get_uncommitted_changes()] if not changed: continue - yield gd, build_ref + add_cb(gd=gd, build_ref=gd, changed=changed) + changes_made = True index.add_files_from_working_tree(changed) + return changes_made @staticmethod def _hash_morphologies(gd, morphologies, loader): @@ -102,7 +105,8 @@ class BuildBranch(object): sha1 = gd.store_blob(loader.save_to_string(morphology)) yield 0100644, sha1, morphology.filename - def inject_build_refs(self, loader): + def inject_build_refs(self, loader, use_local_repos, + inject_cb=lambda **kwargs: None): '''Update system and stratum morphologies to point to our branch. For all edited repositories, this alter the temporary GitIndex @@ -133,7 +137,7 @@ class BuildBranch(object): spec['repo'] = None spec['ref'] = None return True - if not self._push_temporary: + if use_local_repos: spec['repo'] = urlparse.urljoin('file://', gd.dirname) spec['ref'] = build_ref return True @@ -141,12 +145,15 @@ class BuildBranch(object): morphs.traverse_specs(process, filter) if any(m.dirty for m in morphs.morphologies): - yield self._root + inject_cb(gd=self._root) + # TODO: Prevent it hashing unchanged morphologies, while still + # hashing uncommitted ones. self._root_index.add_files_from_index_info( self._hash_morphologies(self._root, morphs.morphologies, loader)) - def update_build_refs(self, name, email, uuid): + def update_build_refs(self, name, email, uuid, + commit_cb=lambda **kwargs: None): '''Commit changes in temporary GitIndexes to temporary branches. `name` and `email` are required to construct the commit author info. @@ -176,12 +183,18 @@ class BuildBranch(object): with morphlib.branchmanager.LocalRefManager() as lrm: for gd, (build_ref, index) in self._to_push.iteritems(): - yield gd, build_ref tree = index.write_tree() try: parent = gd.resolve_ref_to_commit(build_ref) except morphlib.gitdir.InvalidRefError: parent = gd.resolve_ref_to_commit(gd.HEAD) + else: + # Skip updating ref if we already have a temporary + # build branch and have this tree on the branch + if tree == gd.resolve_ref_to_tree(build_ref): + continue + + commit_cb(gd=gd, build_ref=build_ref) commit = gd.commit_tree(tree, parent=parent, committer_name=committer_name, @@ -201,46 +214,55 @@ class BuildBranch(object): # a problem. lrm.update(gd, build_ref, commit, old_commit) - def push_build_branches(self): - '''Push all temporary build branches to the remote repositories. + def get_unpushed_branches(self): + '''Work out which, if any, local branches need to be pushed to build - This is a no-op if the BuildBranch was constructed with - `push_temporary` as False, so that the code flow for the user of - the BuildBranch can be the same when it can be pushed as when - it can't. + NOTE: This assumes that the refs in the morphologies and the + refs in the local checkouts match. ''' - # TODO: When BuildBranches become more context aware, if there - # are no uncommitted changes and the local versions are pushed - # we can skip pushing even if push_temporary is set. - # No uncommitted changes isn't sufficient reason to push the - # current HEAD - if self._push_temporary: - with morphlib.branchmanager.RemoteRefManager(False) as rrm: - for gd, (build_ref, index) in self._to_push.iteritems(): - remote = gd.get_remote('origin') - yield gd, build_ref, remote - refspec = morphlib.gitdir.RefSpec(build_ref) - rrm.push(remote, refspec) - self._register_cleanup(rrm.close) + for gd, (build_ref, index) in self._to_push.iteritems(): + remote = gd.get_remote('origin') + head_ref = gd.disambiguate_ref(gd.HEAD) + head_sha1 = gd.resolve_ref_to_commit(head_ref) + pushed_refs = sorted( + (remote_ref + for remote_sha1, remote_ref in remote.ls() + # substring match of refs, since ref may be a tag, + # in which case it would end with ^{} + if remote_sha1 == head_sha1 and head_ref in remote_ref), + key=len) + if not pushed_refs: + yield gd + + def push_build_branches(self, push_cb=lambda **kwargs: None): + '''Push all temporary build branches to the remote repositories. + ''' + with morphlib.branchmanager.RemoteRefManager(False) as rrm: + for gd, (build_ref, index) in self._to_push.iteritems(): + remote = gd.get_remote('origin') + refspec = morphlib.gitdir.RefSpec(build_ref) + push_cb(gd=gd, build_ref=build_ref, + remote=remote, refspec=refspec) + rrm.push(remote, refspec) + self._register_cleanup(rrm.close) @property def root_repo_url(self): '''URI of the repository that systems may be found in.''' - # TODO: When BuildBranches become more context aware, we only - # have to use the file:// URI when there's uncommitted changes - # and we can't push; or HEAD is not pushed and we can't push. - # All other times we can use the pushed branch - return (self._sb.get_config('branch.root') if self._push_temporary - else urlparse.urljoin('file://', self._root.dirname)) + return self._sb.get_config('branch.root') @property def root_ref(self): + return self._sb.get_config('branch.name') + + @property + def root_local_repo_url(self): + return urlparse.urljoin('file://', self._root.dirname) + + @property + def root_build_ref(self): '''Name of the ref of the repository that systems may be found in.''' - # TODO: When BuildBranches become more context aware, this can be - # HEAD when there's no uncommitted changes and we're not pushing; - # or we are pushing and there's no uncommitted changes and HEAD - # has been pushed. build_ref, index = self._to_push[self._root] return build_ref @@ -258,3 +280,47 @@ class BuildBranch(object): exceptions.append(e) if exceptions: raise BuildBranchCleanupError(self, exceptions) + + +@contextlib.contextmanager +def pushed_build_branch(bb, loader, changes_need_pushing, name, email, + build_uuid, status): + with contextlib.closing(bb) as bb: + def report_add(gd, build_ref, changed): + status(msg='Adding uncommitted changes '\ + 'in %(dirname)s to %(ref)s', + dirname=gd.dirname, ref=build_ref, chatty=True) + changes_made = bb.add_uncommitted_changes(add_cb=report_add) + unpushed = any(bb.get_unpushed_branches()) + + if not changes_made and not unpushed: + yield bb.root_repo_url, bb.root_ref + return + + def report_inject(gd): + status(msg='Injecting temporary build refs '\ + 'into morphologies in %(dirname)s', + dirname=gd.dirname, chatty=True) + bb.inject_build_refs(loader=loader, + use_local_repos=not changes_need_pushing, + inject_cb=report_inject) + + def report_commit(gd, build_ref): + status(msg='Committing changes in %(dirname)s '\ + 'to %(ref)s', + dirname=gd.dirname, ref=build_ref, + chatty=True) + bb.update_build_refs(name, email, build_uuid, + commit_cb=report_commit) + + if changes_need_pushing: + def report_push(gd, build_ref, remote, refspec): + status(msg='Pushing %(ref)s in %(dirname)s '\ + 'to %(remote)s', + ref=build_ref, dirname=gd.dirname, + remote=remote.get_push_url(), chatty=True) + bb.push_build_branches(push_cb=report_push) + + yield bb.root_repo_url, bb.root_build_ref + else: + yield bb.root_local_repo_url, bb.root_build_ref diff --git a/morphlib/builder2.py b/morphlib/builder2.py index d739dc13..c1a49221 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -405,7 +405,7 @@ class ChunkBuilder(BuilderBase): for step, in_parallel in steps: with self.build_watch(step): key = '%s-commands' % step - cmds = m.get_commands(key) + cmds = m[key] if cmds: with open(logfilepath, 'a') as log: self.app.status(msg='Running %(key)s', key=key) diff --git a/morphlib/buildsystem.py b/morphlib/buildsystem.py index 90cc15c2..fb99e70e 100644 --- a/morphlib/buildsystem.py +++ b/morphlib/buildsystem.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,6 +16,8 @@ import os +import morphlib + class BuildSystem(object): @@ -49,19 +51,14 @@ class BuildSystem(object): key = '_'.join(key.split('-')) return getattr(self, key) - def get_morphology_text(self, name): + def get_morphology(self, name): '''Return the text of an autodetected chunk morphology.''' - return ''' - { - "name": "%(name)s", - "kind": "chunk", - "build-system": "%(bs)s" - } - ''' % { + return morphlib.morphology.Morphology({ 'name': name, - 'bs': self.name, - } + 'kind': 'chunk', + 'build-system': self.name, + }) def used_by_project(self, file_list): '''Does a project use this build system? diff --git a/morphlib/buildsystem_tests.py b/morphlib/buildsystem_tests.py index 3171366b..56ba64d7 100644 --- a/morphlib/buildsystem_tests.py +++ b/morphlib/buildsystem_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -49,10 +49,10 @@ class BuildSystemTests(unittest.TestCase): def test_has_install_commands(self): self.assertEqual(self.bs['install-commands'], []) - def test_returns_morphology_text(self): + def test_returns_morphology(self): self.bs.name = 'fake' - text = self.bs.get_morphology_text('foobar') - self.assertTrue(type(text) in (str, unicode)) + morph = self.bs.get_morphology('foobar') + self.assertTrue(morph.__class__.__name__ == 'Morphology') class ManualBuildSystemTests(unittest.TestCase): diff --git a/morphlib/cachedrepo.py b/morphlib/cachedrepo.py index 996b42f7..88ceed48 100644 --- a/morphlib/cachedrepo.py +++ b/morphlib/cachedrepo.py @@ -195,15 +195,6 @@ class CachedRepo(object): self._checkout_ref(ref, target_dir) - def load_morphology(self, ref, name): - '''Loads a morphology from a given ref''' - - if not morphlib.git.is_valid_sha1(ref): - ref = self._rev_parse(ref) - text = self.cat(ref, '%s.morph' % name) - morphology = morphlib.morph2.Morphology(text) - return morphology - def ls_tree(self, ref): '''Return file names found in root tree. Does not recurse to subtrees. diff --git a/morphlib/cachedrepo_tests.py b/morphlib/cachedrepo_tests.py index 74c16591..d3ae331a 100644 --- a/morphlib/cachedrepo_tests.py +++ b/morphlib/cachedrepo_tests.py @@ -219,10 +219,6 @@ class CachedRepoTests(unittest.TestCase): morph_filename = os.path.join(unpack_dir, 'foo.morph') self.assertTrue(os.path.exists(morph_filename)) - def test_load_morphology_from_existing_ref(self): - morph = self.repo.load_morphology('master', 'foo') - self.assertTrue(morph['name'] == 'foo') - def test_ls_tree_in_existing_ref(self): data = self.repo.ls_tree('e28a23812eadf2fce6583b8819b9c5dbd36b9fb9') self.assertEqual(data, ['foo.morph']) diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py index b124b789..588fc8d3 100644 --- a/morphlib/cachekeycomputer.py +++ b/morphlib/cachekeycomputer.py @@ -114,7 +114,7 @@ class CacheKeyComputer(object): for prefix in ('pre-', '', 'post-'): for cmdtype in ('configure', 'build', 'test', 'install'): cmd_field = prefix + cmdtype + '-commands' - keys[cmd_field] = morphology.get_commands(cmd_field) + keys[cmd_field] = morphology[cmd_field] keys['devices'] = morphology.get('devices') keys['max-jobs'] = morphology.get('max-jobs') keys['system-integration'] = morphology.get('system-integration', @@ -122,10 +122,7 @@ class CacheKeyComputer(object): # products is omitted as they are part of the split-rules elif kind in ('system', 'stratum'): morphology = artifact.source.morphology - # Exclude fields starting with _orig_. This filtering can be - # removed once the morph2 code is gone. - morph_dict = dict((k, morphology[k]) for k in morphology.keys() - if not k.startswith('_orig_')) + morph_dict = dict((k, morphology[k]) for k in morphology.keys()) # Disregard all fields of a morphology that aren't important ignored_fields = ( diff --git a/morphlib/cachekeycomputer_tests.py b/morphlib/cachekeycomputer_tests.py index 9e18b19d..8558db6d 100644 --- a/morphlib/cachekeycomputer_tests.py +++ b/morphlib/cachekeycomputer_tests.py @@ -32,73 +32,60 @@ class DummyBuildEnvironment: class CacheKeyComputerTests(unittest.TestCase): def setUp(self): + loader = morphlib.morphloader.MorphologyLoader() self.source_pool = morphlib.sourcepool.SourcePool() for name, text in { - 'chunk.morph': '''{ - "name": "chunk", - "kind": "chunk", - "description": "A test chunk" - }''', - 'chunk2.morph': '''{ - "name": "chunk2", - "kind": "chunk", - "description": "A test chunk" - }''', - 'chunk3.morph': '''{ - "name": "chunk3", - "kind": "chunk", - "description": "A test chunk" - }''', - 'stratum.morph': '''{ - "name": "stratum", - "kind": "stratum", - "chunks": [ - { - "name": "chunk", - "repo": "repo", - "ref": "original/ref", - "build-depends": [] - } - ] - }''', - 'stratum2.morph': '''{ - "name": "stratum2", - "kind": "stratum", - "chunks": [ - { - "name": "chunk2", - "repo": "repo", - "ref": "original/ref", - "build-depends": [] - }, - { - "name": "chunk3", - "repo": "repo", - "ref": "original/ref", - "build-depends": [] - } - ] - }''', - 'system.morph': '''{ - "name": "system", - "kind": "system", - "strata": [ - { - "morph": "stratum", - "repo": "repo", - "ref": "original/ref" - }, - { - "morph": "stratum2", - "repo": "repo", - "ref": "original/ref" - } - ] - }''', + 'chunk.morph': ''' + name: chunk + kind: chunk + description: A test chunk + ''', + 'chunk2.morph': ''' + name: chunk2 + kind: chunk + description: A test chunk + ''', + 'chunk3.morph': ''' + name: chunk3 + kind: chunk + description: A test chunk + ''', + 'stratum.morph': ''' + name: stratum + kind: stratum + build-depends: [] + chunks: + - name: chunk + repo: repo + ref: original/ref + build-depends: [] + ''', + 'stratum2.morph': ''' + name: stratum2 + kind: stratum + build-depends: [] + chunks: + - name: chunk2 + repo: repo + ref: original/ref + build-depends: [] + - name: chunk3 + repo: repo + ref: original/ref + build-depends: [] + ''', + 'system.morph': ''' + name: system + kind: system + arch: testarch + strata: + - morph: stratum + - morph: stratum2 + ''', }.iteritems(): source = morphlib.source.Source( 'repo', 'original/ref', 'sha', 'tree', - morphlib.morph2.Morphology(text), name) + loader.load_from_string(text), name) self.source_pool.add(source) # FIXME: This should use MorphologyFactory m = source.morphology @@ -202,7 +189,12 @@ class CacheKeyComputerTests(unittest.TestCase): self.assertEqual(old_sha, new_sha) def test_same_morphology_added_to_source_pool_only_appears_once(self): - m = morphlib.morph2.Morphology('{"name": "chunk", "kind": "chunk"}') + loader = morphlib.morphloader.MorphologyLoader() + m = loader.load_from_string( + ''' + name: chunk + kind: chunk + ''') src = morphlib.source.Source('repo', 'original/ref', 'sha', 'tree', m, 'chunk.morph') sp = morphlib.sourcepool.SourcePool() diff --git a/morphlib/extensions.py b/morphlib/extensions.py index be551fdd..55478418 100644 --- a/morphlib/extensions.py +++ b/morphlib/extensions.py @@ -30,26 +30,20 @@ class ExtensionNotFoundError(ExtensionError): class ExtensionNotExecutableError(ExtensionError): pass -def _get_root_repo(build_ref_prefix): +def _get_root_repo(): system_branch = morphlib.sysbranchdir.open_from_within('.') root_repo_dir = morphlib.gitdir.GitDirectory( system_branch.get_git_directory_name( system_branch.root_repository_url)) - build_branch = morphlib.buildbranch.BuildBranch(system_branch, - build_ref_prefix, - push_temporary=False) - ref = build_branch.root_ref - - return (build_branch.root_ref, root_repo_dir) + return root_repo_dir def _get_morph_extension_directory(): code_dir = os.path.dirname(morphlib.__file__) return os.path.join(code_dir, 'exts') -def _list_repo_extension_filenames(build_ref_prefix, - kind): #pragma: no cover - (ref, repo_dir) = _get_root_repo(build_ref_prefix) - files = repo_dir.list_files(ref) +def _list_repo_extension_filenames(kind): #pragma: no cover + repo_dir = _get_root_repo() + files = repo_dir.list_files() return (f for f in files if os.path.splitext(f)[1] == kind) def _list_morph_extension_filenames(kind): @@ -59,9 +53,9 @@ def _list_morph_extension_filenames(kind): def _get_extension_name(filename): return os.path.basename(filename) -def _get_repo_extension_contents(build_ref_prefix, name, kind): - (ref, repo_dir) = _get_root_repo(build_ref_prefix) - return repo_dir.get_file_from_ref(ref, name + kind) +def _get_repo_extension_contents(name, kind): + repo_dir = _get_root_repo() + return repo_dir.read_file(name + kind) def _get_morph_extension_filename(name, kind): return os.path.join(_get_morph_extension_directory(), name + kind) @@ -71,11 +65,11 @@ def _is_executable(filename): mask = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH return (stat.S_IMODE(st.st_mode) & mask) != 0 -def _list_extensions(build_ref_prefix, kind): +def _list_extensions(kind): repo_extension_filenames = [] try: repo_extension_filenames = \ - _list_repo_extension_filenames(build_ref_prefix, kind) + _list_repo_extension_filenames(kind) except (sysbranchdir.NotInSystemBranch): # Squash this and just return no system branch extensions pass @@ -90,7 +84,7 @@ def _list_extensions(build_ref_prefix, kind): extension_names.update(set(morph_extension_names)) return list(extension_names) -def list_extensions(build_ref_prefix, kind=None): +def list_extensions(kind=None): """ List all available extensions by 'kind'. @@ -102,10 +96,10 @@ def list_extensions(build_ref_prefix, kind=None): be associated with a '.write' extension of the same name. """ if kind: - return _list_extensions(build_ref_prefix, kind) + return _list_extensions(kind) else: - configure_extensions = _list_extensions(build_ref_prefix, '.configure') - write_extensions = _list_extensions(build_ref_prefix, '.write') + configure_extensions = _list_extensions('.configure') + write_extensions = _list_extensions('.write') return configure_extensions + write_extensions @@ -121,8 +115,7 @@ class get_extension_filename(): If the extension is in the build repository then a temporary file will be created, which will be deleted on exting the with block. """ - def __init__(self, build_ref_prefix, name, kind, executable=True): - self.build_ref_prefix = build_ref_prefix + def __init__(self, name, kind, executable=True): self.name = name self.kind = kind self.executable = executable @@ -131,10 +124,9 @@ class get_extension_filename(): def __enter__(self): ext_filename = None try: - ext_contents = _get_repo_extension_contents(self.build_ref_prefix, - self.name, + ext_contents = _get_repo_extension_contents(self.name, self.kind) - except cliapp.AppException, sysbranchdir.NotInSystemBranch: + except (IOError, cliapp.AppException, sysbranchdir.NotInSystemBranch): # Not found: look for it in the Morph code. ext_filename = _get_morph_extension_filename(self.name, self.kind) if not os.path.exists(ext_filename): diff --git a/morphlib/gitdir.py b/morphlib/gitdir.py index 5b0693cb..fea26c2e 100644 --- a/morphlib/gitdir.py +++ b/morphlib/gitdir.py @@ -282,6 +282,16 @@ class Remote(object): return self._get_remote_url(self.name, 'push') @staticmethod + def _parse_ls_remote_output(output): # pragma: no cover + for line in output.splitlines(): + sha1, refname = line.split(None, 1) + yield sha1, refname + + def ls(self): # pragma: no cover + out = self.gd._runcmd(['git', 'ls-remote', self.get_fetch_url()]) + return self._parse_ls_remote_output(out) + + @staticmethod def _parse_push_output(output): for line in output.splitlines(): m = PUSH_FORMAT.match(line) @@ -484,6 +494,19 @@ class GitDirectory(object): except cliapp.AppException as e: raise InvalidRefError(self, ref) + def disambiguate_ref(self, ref): # pragma: no cover + try: + out = self._runcmd(['git', 'rev-parse', '--symbolic-full-name', + ref]) + return out.strip() + except cliapp.AppException: # ref not found + if ref.startswith('refs/heads/'): + return ref + elif ref.startswith('heads/'): + return 'refs/' + ref + else: + return 'refs/heads/' + ref + def resolve_ref_to_commit(self, ref): return self._rev_parse('%s^{commit}' % ref) diff --git a/morphlib/gitindex.py b/morphlib/gitindex.py index 978ea0e2..6be4aacb 100644 --- a/morphlib/gitindex.py +++ b/morphlib/gitindex.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -97,7 +97,8 @@ class GitIndex(object): def get_uncommitted_changes(self): for code, to_path, from_path in self._get_status(): - if code not in (STATUS_UNTRACKED, STATUS_IGNORED): + if (code not in (STATUS_UNTRACKED, STATUS_IGNORED) + or code == (STATUS_UNTRACKED) and to_path.endswith('.morph')): yield code, to_path, from_path def set_to_tree(self, treeish): diff --git a/morphlib/localartifactcache_tests.py b/morphlib/localartifactcache_tests.py index f400a645..6283c833 100644 --- a/morphlib/localartifactcache_tests.py +++ b/morphlib/localartifactcache_tests.py @@ -27,23 +27,21 @@ class LocalArtifactCacheTests(unittest.TestCase): def setUp(self): self.tempfs = fs.tempfs.TempFS() - morph = morphlib.morph2.Morphology( + loader = morphlib.morphloader.MorphologyLoader() + morph = loader.load_from_string( ''' - { - "name": "chunk", - "kind": "chunk", - "artifacts": { - "chunk-runtime": [ - "usr/bin", - "usr/sbin", - "usr/lib", - "usr/libexec" - ], - "chunk-devel": [ - "usr/include" - ] - } - } + name: chunk + kind: chunk + products: + - artifact: chunk-runtime + include: + - usr/bin + - usr/sbin + - usr/lib + - usr/libexec + - artifact: chunk-devel + include: + - usr/include ''') self.source = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'chunk.morph') diff --git a/morphlib/morph2.py b/morphlib/morph2.py deleted file mode 100644 index b49c0f73..00000000 --- a/morphlib/morph2.py +++ /dev/null @@ -1,313 +0,0 @@ -# Copyright (C) 2012-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import re - -import morphlib -from morphlib.util import OrderedDict, json - -class Morphology(object): - - '''An in-memory representation of a morphology. - - This is a parsed version of the morphology, with rules for default - values applied. No other processing. - - ''' - - static_defaults = { - 'chunk': [ - ('description', ''), - ('pre-configure-commands', None), - ('configure-commands', None), - ('post-configure-commands', None), - ('pre-build-commands', None), - ('build-commands', None), - ('post-build-commands', None), - ('pre-test-commands', None), - ('test-commands', None), - ('post-test-commands', None), - ('pre-install-commands', None), - ('install-commands', None), - ('post-install-commands', None), - ('devices', None), - ('products', []), - ('max-jobs', None), - ('build-system', 'manual') - ], - 'stratum': [ - ('chunks', []), - ('description', ''), - ('build-depends', None), - ], - 'system': [ - ('strata', []), - ('description', ''), - ('arch', None), - ('configuration-extensions', []), - ], - 'cluster': [ - ('description', ''), - ], - } - - @staticmethod - def _load_json(text): - return json.loads(text, object_pairs_hook=OrderedDict, - encoding='unicode-escape') - - @staticmethod - def _dump_json(obj, f): - text = json.dumps(obj, indent=4, encoding='unicode-escape') - text = re.sub(" \n", "\n", text) - f.write(text) - f.write('\n') - - def __init__(self, text): - self._dict, self._dumper = self._load_morphology_dict(text) - self._set_defaults() - self._validate_children() - - def __getitem__(self, key): - return self._dict[key] - - def __contains__(self, key): - return key in self._dict - - # Not covered by tests, since it's trivial, morph2 is going away - # and only exists so the new morphology validation code can use it. - def get(self, key, default=None): # pragma: no cover - try: - return self[key] - except KeyError: - return default - - def get_commands(self, which): - '''Return the commands to run from a morphology or the build system''' - if self[which] is None: - attr = '_'.join(which.split('-')) - bs = morphlib.buildsystem.lookup_build_system(self['build-system']) - return getattr(bs, attr) - else: - return self[which] - - def keys(self): - return self._dict.keys() - - def _load_morphology_dict(self, text): - '''Load morphology, identifying whether it is JSON or YAML''' - - try: - data = self._load_json(text) - dumper = self._dump_json - except ValueError as e: # pragma: no cover - data = morphlib.yamlparse.load(text) - dumper = morphlib.yamlparse.dump - - if data is None: - raise morphlib.YAMLError("Morphology is empty") - if type(data) not in [dict, OrderedDict]: - raise morphlib.YAMLError("Morphology did not parse as a dict") - - return data, dumper - - def _validate_children(self): - if self['kind'] == 'system': - names = set() - for info in self['strata']: - name = info.get('alias', info['morph']) - if name in names: - raise ValueError('Duplicate stratum "%s"' % name) - names.add(name) - elif self['kind'] == 'stratum': - names = set() - for info in self['chunks']: - name = info.get('alias', info['name']) - if name in names: - raise ValueError('Duplicate chunk "%s"' % name) - names.add(name) - elif self['kind'] == 'cluster': - if not 'systems' in self: - raise KeyError('"systems" not found') - if not self['systems']: - raise ValueError('"systems" is empty') - for system in self['systems']: - if 'morph' not in system: - raise KeyError('"morph" not found') - if 'deploy-defaults' in system: - if not isinstance(system['deploy-defaults'], dict): - raise ValueError('deploy defaults for morph "%s" ' - 'are not a mapping: %r' - % (system['morph'], - system['deploy-defaults'])) - if 'deploy' in system: - for system_id, deploy_params in system['deploy'].items(): - if not isinstance(deploy_params, dict): - raise ValueError('deployment parameters for ' - 'system "%s" are not a mapping:' - ' %r' - % (system_id, deploy_params)) - - def _set_default_value(self, target_dict, key, value): - '''Change a value in the in-memory representation of the morphology - - Record the default value separately, so that when writing out the - morphology we can determine whether the change from the on-disk value - was done at load time, or later on (we want to only write back out - the later, deliberate changes). - - ''' - target_dict[key] = value - target_dict['_orig_' + key] = value - - def _set_defaults(self): - if 'max-jobs' in self: - self._set_default_value(self._dict, 'max-jobs', - int(self['max-jobs'])) - - for name, value in self.static_defaults[self['kind']]: - if name not in self._dict: - self._set_default_value(self._dict, name, value) - - if self['kind'] == 'stratum': - self._set_stratum_defaults() - elif self['kind'] == 'cluster': - self._set_cluster_defaults() - - def _set_stratum_defaults(self): - for source in self['chunks']: - if 'repo' not in source: - self._set_default_value(source, 'repo', source['name']) - if 'build-depends' not in source: - self._set_default_value(source, 'build-depends', None) - if 'build-mode' not in source: - self._set_default_value(source, 'build-mode', 'staging') - if 'prefix' not in source: - self._set_default_value(source, 'prefix', '/usr') - - def _set_cluster_defaults(self): - if 'systems' in self and self['systems']: - for system in self['systems']: - if 'deploy-defaults' not in system: - self._set_default_value(system, - 'deploy-defaults', - dict()) - if 'deploy' not in system: - self._set_default_value(system, - 'deploy', - dict()) - - def lookup_child_by_name(self, name): - '''Find child reference by its name. - - This lookup honors aliases. - - ''' - - if self['kind'] == 'system': - for info in self['strata']: - source_name = info.get('alias', info['morph']) - if source_name == name: - return info - elif self['kind'] == 'stratum': - for info in self['chunks']: - source_name = info.get('alias', info['name']) - if source_name == name: - return info - raise KeyError('"%s" not found' % name) - - def _apply_changes(self, live_dict, original_dict): - '''Returns a new dict updated with changes from the in-memory object - - This allows us to write out a morphology including only the changes - that were done after the morphology was loaded -- not the changes done - to set default values during construction. - - ''' - output_dict = {} - - for key in live_dict.keys(): - if key.startswith('_orig_'): - continue - - value = self._apply_changes_for_key(key, live_dict, original_dict) - # VILE HACK to preserve nulls in repo/ref fields - if value is not None or key in ('repo', 'ref'): - output_dict[key] = value - return output_dict - - def _apply_changes_for_key(self, key, live_dict, original_dict): - '''Return value to write out for one key, recursing if necessary''' - - live_value = live_dict.get(key, None) - orig_value = original_dict.get(key, None) - - if type(live_value) in [dict, OrderedDict] and orig_value is not None: - # Recursively apply changes for dict - result = self._apply_changes(live_value, orig_value) - elif type(live_value) is list and orig_value is not None: - # Recursively apply changes for list (existing, then new items). - result = [] - for i in range(0, min(len(orig_value), len(live_value))): - if type(live_value[i]) in [dict, OrderedDict]: - item = self._apply_changes(live_value[i], orig_value[i]) - else: - item = live_value[i] - result.append(item) - for i in range(len(orig_value), len(live_value)): - if type(live_value[i]) in [dict, OrderedDict]: - item = self._apply_changes(live_value[i], {}) - else: - item = live_value[i] - result.append(item) - else: - # Simple values. Use original value unless it has been changed from - # the default in memmory. - if live_dict[key] == live_dict.get('_orig_' + key, None): - result = original_dict.get(key, None) - else: - result = live_dict[key] - return result - - def update_text(self, text, output_fd, convert_to=None): - '''Write out in-memory changes to loaded morphology text - - Similar in function to update_file(). - - ''' - original_dict, dumper = self._load_morphology_dict(text) - - if convert_to == 'json': # pragma: no cover - dumper = self._dump_json - elif convert_to == 'yaml': # pragma: no cover - dumper = morphlib.yamlparse.dump - - output_dict = self._apply_changes(self._dict, original_dict) - dumper(output_dict, output_fd) - - def update_file(self, filename, output_fd=None, **kws): # pragma: no cover - '''Write out in-memory changes to on-disk morphology file - - This function reads the original morphology text from 'filename', so - that it can avoid writing out properties that are set in memory - to their default value but weren't specified by the user at all. - - ''' - with open(filename, 'r') as f: - text = f.read() - - with output_fd or morphlib.savefile.SaveFile(filename, 'w') as f: - self.update_text(text, f, **kws) diff --git a/morphlib/morph2_tests.py b/morphlib/morph2_tests.py deleted file mode 100644 index c9957ad5..00000000 --- a/morphlib/morph2_tests.py +++ /dev/null @@ -1,391 +0,0 @@ -# Copyright (C) 2012-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import copy -import json -import StringIO -import unittest - -import yaml - -import morphlib -from morphlib.morph2 import Morphology - - -class MorphologyTests(unittest.TestCase): - - def test_parses_simple_json_chunk(self): - m = Morphology(''' - { - "name": "foo", - "kind": "chunk", - "build-system": "manual" - } - ''') - - self.assertEqual(m['name'], 'foo') - self.assertEqual(m['kind'], 'chunk') - self.assertEqual(m['build-system'], 'manual') - self.assertEqual(m['pre-configure-commands'], None) - self.assertEqual(m['configure-commands'], None) - self.assertEqual(m['post-configure-commands'], None) - self.assertEqual(m['pre-build-commands'], None) - self.assertEqual(m['build-commands'], None) - self.assertEqual(m['post-build-commands'], None) - self.assertEqual(m['pre-test-commands'], None) - self.assertEqual(m['test-commands'], None) - self.assertEqual(m['post-test-commands'], None) - self.assertEqual(m['pre-install-commands'], None) - self.assertEqual(m['install-commands'], None) - self.assertEqual(m['post-install-commands'], None) - self.assertEqual(m['max-jobs'], None) - self.assertEqual(m['products'], []) - - if morphlib.got_yaml: - def test_parses_simple_yaml_chunk(self): - m = Morphology(''' - name: foo - kind: chunk - build-system: manual - ''') - - self.assertEqual(m['name'], 'foo') - self.assertEqual(m['kind'], 'chunk') - self.assertEqual(m['build-system'], 'manual') - self.assertEqual(m['pre-configure-commands'], None) - self.assertEqual(m['configure-commands'], None) - self.assertEqual(m['post-configure-commands'], None) - self.assertEqual(m['pre-build-commands'], None) - self.assertEqual(m['build-commands'], None) - self.assertEqual(m['post-build-commands'], None) - self.assertEqual(m['pre-test-commands'], None) - self.assertEqual(m['test-commands'], None) - self.assertEqual(m['post-test-commands'], None) - self.assertEqual(m['pre-install-commands'], None) - self.assertEqual(m['install-commands'], None) - self.assertEqual(m['post-install-commands'], None) - self.assertEqual(m['max-jobs'], None) - self.assertEqual(m['products'], []) - - def test_sets_stratum_chunks_repo_and_morph_from_name(self): - m = Morphology(''' - { - "name": "foo", - "kind": "stratum", - "chunks": [ - { - "name": "le-chunk", - "ref": "ref" - } - ] - } - ''') - - self.assertEqual(m['chunks'][0]['repo'], 'le-chunk') - self.assertEqual(m['chunks'][0]['build-depends'], None) - - def test_returns_dict_keys(self): - m = Morphology(''' - { - "name": "foo", - "kind": "system", - } - ''') - - self.assertTrue('name' in m.keys()) - self.assertTrue('kind' in m.keys()) - - def test_system_indexes_strata(self): - m = Morphology(''' - { - "kind": "system", - "strata": [ - { - "morph": "stratum1", - "repo": "repo", - "ref": "ref" - }, - { - "alias": "aliased-stratum", - "morph": "stratum2", - "repo": "repo", - "ref": "ref" - } - ] - } - ''') - self.assertEqual(m.lookup_child_by_name('stratum1'), - {'morph': 'stratum1', 'repo': 'repo', 'ref': 'ref' }) - self.assertEqual(m.lookup_child_by_name('aliased-stratum'), - {'alias': 'aliased-stratum', 'morph': 'stratum2', - 'repo': 'repo', 'ref': 'ref'}) - - def test_stratum_indexes_chunks(self): - m = Morphology(''' - { - "kind": "stratum", - "chunks": [ - { - "name": "chunk", - "repo": "repo", - "ref": "ref" - } - ] - } - ''') - - child = m.lookup_child_by_name('chunk') - self.assertEqual(child['name'], 'chunk') - self.assertEqual(child['repo'], 'repo') - self.assertEqual(child['ref'], 'ref') - - def test_raises_error_when_child_lookup_fails(self): - m = Morphology(''' - { - "kind": "stratum", - "chunks": [ - { - "name": "chunk", - "repo": "repo", - "ref": "ref" - } - ] - } - ''') - - self.assertRaises(KeyError, m.lookup_child_by_name, 'foo') - - ## Validation tests - - def test_not_empty(self): - self.assertRaises(morphlib.YAMLError, Morphology, '') - - def test_is_dict(self): - self.assertRaises(morphlib.YAMLError, Morphology, 'foo') - - def test_makes_max_jobs_be_an_integer(self): - m = Morphology(''' - { - "name": "foo", - "kind": "chunk", - "max-jobs": "42" - } - ''') - self.assertEqual(m['max-jobs'], 42) - - def test_stratum_names_must_be_unique_within_a_system(self): - text = ''' - { - "kind": "system", - "strata": [ - { - "morph": "stratum", - "repo": "test1", - "ref": "ref" - }, - { - "morph": "stratum", - "repo": "test2", - "ref": "ref" - } - ] - } - ''' - self.assertRaises(ValueError, - Morphology, - text) - - def test_chunk_names_must_be_unique_within_a_stratum(self): - text = ''' - { - "kind": "stratum", - "chunks": [ - { - "name": "chunk", - "repo": "test1", - "ref": "ref" - }, - { - "name": "chunk", - "repo": "test2", - "ref": "ref" - } - ] - } - ''' - self.assertRaises(ValueError, - Morphology, - text) - - ## Writing tests - - stratum_text = '''{ - "kind": "stratum", - "chunks": [ - { - "name": "foo", - "repo": "morphs", - "ref": "ref", - "build-depends": [] - }, - { - "name": "bar", - "repo": "morphs", - "ref": "ref", - "build-depends": [ - "foo" - ] - } - ] -}''' - - def test_writing_handles_added_chunks(self): - text_lines = self.stratum_text.splitlines() - text_lines = text_lines[0:16] + text_lines[8:17] + text_lines[17:] - text_lines[18] = ' "name": "baz",' - - # Add a new chunk to the list - morphology = Morphology(self.stratum_text) - morphology['chunks'].append(copy.copy(morphology['chunks'][1])) - morphology['chunks'][2]['name'] = 'baz' - - output = StringIO.StringIO() - morphology.update_text(self.stratum_text, output) - d = yaml.load(output.getvalue()) - self.assertEqual(d['chunks'][2]['name'], 'baz') - - def test_writing_handles_deleted_chunks(self): - text_lines = self.stratum_text.splitlines() - text_lines = text_lines[0:3] + text_lines[9:] - - # Delete a chunk - morphology = Morphology(self.stratum_text) - del morphology['chunks'][0] - - output = StringIO.StringIO() - morphology.update_text(self.stratum_text, output) - d = yaml.load(output.getvalue()) - self.assertEqual(len(d['chunks']), 1) - - system_text = '''{ - "kind": "system", - "arch": "x86_64", -}''' - - def test_nested_dict(self): - # Real morphologies don't trigger this code path, so we test manually - original_dict = { - 'dict': { '1': 'fee', '2': 'fie', '3': 'foe', '4': 'foo' } - } - live_dict = copy.deepcopy(original_dict) - live_dict['_orig_dict'] = live_dict['dict'] - - dummy = Morphology(self.stratum_text) - output_dict = dummy._apply_changes(live_dict, original_dict) - self.assertEqual(original_dict, output_dict) - - def test_uses_morphology_commands_when_given(self): - m = Morphology(''' - { - 'name': 'foo', - 'kind': 'chunk', - 'build-system': 'dummy', - 'build-commands': ['build-it'] - } - ''') - cmds = m.get_commands('build-commands') - self.assertEqual(cmds, ['build-it']) - - def test_uses_build_system_commands_when_morphology_doesnt(self): - m = Morphology(''' - { - 'name': 'foo', - 'kind': 'chunk', - 'build-system': 'dummy', - } - ''') - cmds = m.get_commands('build-commands') - self.assertEqual(cmds, ['echo dummy build']) - - def test_uses_morphology_commands_when_morphology_has_empty_list(self): - m = Morphology(''' - { - 'name': 'foo', - 'kind': 'chunk', - 'build-system': 'dummy', - 'build-commands': [] - } - ''') - cmds = m.get_commands('build-commands') - self.assertEqual(cmds, []) - - ## Cluster morphologies tests - - def test_parses_simple_cluster_morph(self): - m = Morphology(''' - name: foo - kind: cluster - systems: - - morph: bar - ''') - self.assertEqual(m['name'], 'foo') - self.assertEqual(m['kind'], 'cluster') - self.assertEqual(m['systems'][0]['morph'], 'bar') - - def test_fails_without_systems(self): - text = ''' - name: foo - kind: cluster - ''' - self.assertRaises(KeyError, Morphology, text) - - def test_fails_with_empty_systems(self): - text = ''' - name: foo - kind: cluster - systems: - ''' - self.assertRaises(ValueError, Morphology, text) - - def test_fails_without_morph(self): - text = ''' - name: foo - kind: cluster - systems: - - deploy: - ''' - self.assertRaises(KeyError, Morphology, text) - - def test_fails_with_invalid_deploy_defaults(self): - text = ''' - name: foo - kind: cluster - systems: - - morph: bar - deploy-defaults: ooops_i_am_not_a_mapping - ''' - self.assertRaises(ValueError, Morphology, text) - - def test_fails_with_invalid_deployment_params(self): - text = ''' - name: foo - kind: cluster - systems: - - morph: bar - deploy: - qux: ooops_i_am_not_a_mapping - ''' - self.assertRaises(ValueError, Morphology, text) diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py index 21e10827..05dcb62c 100644 --- a/morphlib/morphloader.py +++ b/morphlib/morphloader.py @@ -354,6 +354,8 @@ class MorphologyLoader(object): 'products': [], 'max-jobs': None, 'build-system': 'manual', + 'build-mode': 'staging', + 'prefix': '/usr', }, 'stratum': { 'chunks': [], @@ -392,7 +394,7 @@ class MorphologyLoader(object): if not isinstance(obj, dict): raise NotADictionaryError(morph_filename) - return morphlib.morph3.Morphology(obj) + return morphlib.morphology.Morphology(obj) def load_from_string(self, string, filename='string'): '''Load a morphology from a string. @@ -404,6 +406,7 @@ class MorphologyLoader(object): m = self.parse_morphology_text(string, filename) m.filename = filename self.validate(m) + self.set_commands(m) self.set_defaults(m) return m @@ -438,9 +441,6 @@ class MorphologyLoader(object): self._require_field('kind', morph) # The rest of the validation is dependent on the kind. - - # FIXME: move validation of clusters from morph2 to - # here, and use morphload to load the morphology kind = morph['kind'] if kind not in ('system', 'stratum', 'chunk', 'cluster'): raise UnknownKindError(morph['kind'], morph.filename) @@ -731,19 +731,37 @@ class MorphologyLoader(object): for spec in morph['chunks']: if 'repo' not in spec: spec['repo'] = spec['name'] - if 'morph' not in spec: - spec['morph'] = spec['name'] + if 'build-mode' not in spec: + spec['build-mode'] = \ + self._static_defaults['chunk']['build-mode'] + if 'prefix' not in spec: + spec['prefix'] = \ + self._static_defaults['chunk']['prefix'] self._set_stratum_specs_defaults(morph, 'build-depends') def _unset_stratum_defaults(self, morph): for spec in morph['chunks']: if 'repo' in spec and spec['repo'] == spec['name']: del spec['repo'] - if 'morph' in spec and spec['morph'] == spec['name']: - del spec['morph'] + if 'build-mode' in spec and spec['build-mode'] == \ + self._static_defaults['chunk']['build-mode']: + del spec['build-mode'] + if 'prefix' in spec and spec['prefix'] == \ + self._static_defaults['chunk']['prefix']: + del spec['prefix'] self._unset_stratum_specs_defaults(morph, 'strata') def _set_chunk_defaults(self, morph): if morph['max-jobs'] is not None: morph['max-jobs'] = int(morph['max-jobs']) + def set_commands(self, morph): + if morph['kind'] == 'chunk': + for key in self._static_defaults['chunk']: + if 'commands' not in key: continue + if key not in morph: + attr = '_'.join(key.split('-')) + default = self._static_defaults['chunk']['build-system'] + bs = morphlib.buildsystem.lookup_build_system( + morph.get('build-system', default)) + morph[key] = getattr(bs, attr) diff --git a/morphlib/morphloader_tests.py b/morphlib/morphloader_tests.py index f4d2f9b6..d47ec750 100644 --- a/morphlib/morphloader_tests.py +++ b/morphlib/morphloader_tests.py @@ -59,21 +59,21 @@ build-system: dummy self.loader.parse_morphology_text, '- item1\n- item2\n', 'test') def test_fails_to_validate_dict_without_kind(self): - m = morphlib.morph3.Morphology({ + m = morphlib.morphology.Morphology({ 'invalid': 'field', }) self.assertRaises( morphlib.morphloader.MissingFieldError, self.loader.validate, m) def test_fails_to_validate_chunk_with_no_fields(self): - m = morphlib.morph3.Morphology({ + m = morphlib.morphology.Morphology({ 'kind': 'chunk', }) self.assertRaises( morphlib.morphloader.MissingFieldError, self.loader.validate, m) def test_fails_to_validate_chunk_with_invalid_field(self): - m = morphlib.morph3.Morphology({ + m = morphlib.morphology.Morphology({ 'kind': 'chunk', 'name': 'foo', 'invalid': 'field', @@ -82,7 +82,7 @@ build-system: dummy morphlib.morphloader.InvalidFieldError, self.loader.validate, m) def test_validate_requires_products_list(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( kind='chunk', name='foo', products={ @@ -98,7 +98,7 @@ build-system: dummy self.assertEqual(e.morphology_name, 'foo') def test_validate_requires_products_list_of_mappings(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( kind='chunk', name='foo', products=[ @@ -113,7 +113,7 @@ build-system: dummy self.assertEqual(e.morphology_name, 'foo') def test_validate_requires_products_list_required_fields(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( kind='chunk', name='foo', products=[ @@ -136,7 +136,7 @@ build-system: dummy self.assertEqual(exs[3].field, 'products[0].factiart') def test_validate_requires_products_list_include_is_list(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( kind='chunk', name='foo', products=[ @@ -154,7 +154,7 @@ build-system: dummy self.assertEqual(ex.morphology_name, 'foo') def test_validate_requires_products_list_include_is_list_of_strings(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( kind='chunk', name='foo', products=[ @@ -175,14 +175,14 @@ build-system: dummy def test_fails_to_validate_stratum_with_no_fields(self): - m = morphlib.morph3.Morphology({ + m = morphlib.morphology.Morphology({ 'kind': 'stratum', }) self.assertRaises( morphlib.morphloader.MissingFieldError, self.loader.validate, m) def test_fails_to_validate_stratum_with_invalid_field(self): - m = morphlib.morph3.Morphology({ + m = morphlib.morphology.Morphology({ 'kind': 'stratum', 'name': 'foo', 'invalid': 'field', @@ -191,7 +191,7 @@ build-system: dummy morphlib.morphloader.InvalidFieldError, self.loader.validate, m) def test_validate_requires_chunk_refs_in_stratum_to_be_strings(self): - m = morphlib.morph3.Morphology({ + m = morphlib.morphology.Morphology({ 'kind': 'stratum', 'name': 'foo', 'build-depends': [], @@ -209,7 +209,7 @@ build-system: dummy self.loader.validate(m) def test_fails_to_validate_stratum_with_empty_refs_for_a_chunk(self): - m = morphlib.morph3.Morphology({ + m = morphlib.morphology.Morphology({ 'kind': 'stratum', 'name': 'foo', 'build-depends': [], @@ -227,7 +227,7 @@ build-system: dummy self.loader.validate(m) def test_fails_to_validate_system_with_obsolete_system_kind_field(self): - m = morphlib.morph3.Morphology({ + m = morphlib.morphology.Morphology({ 'kind': 'system', 'name': 'foo', 'arch': 'x86_64', @@ -240,7 +240,7 @@ build-system: dummy morphlib.morphloader.ObsoleteFieldsError, self.loader.validate, m) def test_fails_to_validate_system_with_obsolete_disk_size_field(self): - m = morphlib.morph3.Morphology({ + m = morphlib.morphology.Morphology({ 'kind': 'system', 'name': 'foo', 'arch': 'x86_64', @@ -253,14 +253,14 @@ build-system: dummy morphlib.morphloader.ObsoleteFieldsError, self.loader.validate, m) def test_fails_to_validate_system_with_no_fields(self): - m = morphlib.morph3.Morphology({ + m = morphlib.morphology.Morphology({ 'kind': 'system', }) self.assertRaises( morphlib.morphloader.MissingFieldError, self.loader.validate, m) def test_fails_to_validate_system_with_invalid_field(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( kind="system", name="foo", arch="blah", @@ -272,14 +272,14 @@ build-system: dummy morphlib.morphloader.InvalidFieldError, self.loader.validate, m) def test_fails_to_validate_morphology_with_unknown_kind(self): - m = morphlib.morph3.Morphology({ + m = morphlib.morphology.Morphology({ 'kind': 'invalid', }) self.assertRaises( morphlib.morphloader.UnknownKindError, self.loader.validate, m) def test_validate_requires_unique_stratum_names_within_a_system(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( { "kind": "system", "name": "foo", @@ -301,7 +301,7 @@ build-system: dummy self.loader.validate, m) def test_validate_requires_unique_chunk_names_within_a_stratum(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( { "kind": "stratum", "name": "foo", @@ -322,7 +322,7 @@ build-system: dummy self.loader.validate, m) def test_validate_requires_a_valid_architecture(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( kind="system", name="foo", arch="blah", @@ -334,7 +334,7 @@ build-system: dummy self.loader.validate, m) def test_validate_normalises_architecture_armv7_to_armv7l(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( kind="system", name="foo", arch="armv7", @@ -345,7 +345,7 @@ build-system: dummy self.assertEqual(m['arch'], 'armv7l') def test_validate_requires_build_deps_for_chunks_in_strata(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( { "kind": "stratum", "name": "foo", @@ -365,7 +365,7 @@ build-system: dummy self.loader.validate, m) def test_validate_requires_build_deps_or_bootstrap_mode_for_strata(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( { "name": "stratum-no-bdeps-no-bootstrap", "kind": "stratum", @@ -395,7 +395,7 @@ build-system: dummy self.loader.validate(m) def test_validate_requires_chunks_in_strata(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( { "name": "stratum", "kind": "stratum", @@ -415,7 +415,7 @@ build-system: dummy self.loader.validate, m) def test_validate_requires_strata_in_system(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( name='system', kind='system', arch='testarch') @@ -425,7 +425,7 @@ build-system: dummy def test_validate_requires_list_of_strata_in_system(self): for v in (None, {}): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( name='system', kind='system', arch='testarch', @@ -437,7 +437,7 @@ build-system: dummy self.assertEqual(cm.exception.strata_type, type(v)) def test_validate_requires_non_empty_strata_in_system(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( name='system', kind='system', arch='testarch', @@ -447,7 +447,7 @@ build-system: dummy self.loader.validate, m) def test_validate_requires_stratum_specs_in_system(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( name='system', kind='system', arch='testarch', @@ -460,7 +460,7 @@ build-system: dummy def test_validate_requires_unique_deployment_names_in_cluster(self): subsystem = [{'morph': 'baz', 'deploy': {'foobar': None}}] - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( name='cluster', kind='cluster', systems=[{'morph': 'foo', @@ -513,7 +513,7 @@ build-system: dummy self.assertEqual(morph['build-system'], 'dummy') def test_saves_to_string(self): - morph = morphlib.morph3.Morphology({ + morph = morphlib.morphology.Morphology({ 'name': 'foo', 'kind': 'chunk', 'build-system': 'dummy', @@ -529,7 +529,7 @@ build-system: dummy ''') def test_saves_to_file(self): - morph = morphlib.morph3.Morphology({ + morph = morphlib.morphology.Morphology({ 'name': 'foo', 'kind': 'chunk', 'build-system': 'dummy', @@ -548,7 +548,7 @@ build-system: dummy ''') def test_validate_does_not_set_defaults(self): - m = morphlib.morph3.Morphology({ + m = morphlib.morphology.Morphology({ 'kind': 'chunk', 'name': 'foo', }) @@ -556,7 +556,7 @@ build-system: dummy self.assertEqual(sorted(m.keys()), sorted(['kind', 'name'])) def test_sets_defaults_for_chunks(self): - m = morphlib.morph3.Morphology({ + m = morphlib.morphology.Morphology({ 'kind': 'chunk', 'name': 'foo', }) @@ -569,6 +569,7 @@ build-system: dummy 'name': 'foo', 'description': '', 'build-system': 'manual', + 'build-mode': 'staging', 'configure-commands': [], 'pre-configure-commands': [], @@ -589,10 +590,11 @@ build-system: dummy 'products': [], 'devices': [], 'max-jobs': None, + 'prefix': '/usr', }) def test_unsets_defaults_for_chunks(self): - m = morphlib.morph3.Morphology({ + m = morphlib.morphology.Morphology({ 'kind': 'chunk', 'name': 'foo', 'build-system': 'manual', @@ -606,7 +608,7 @@ build-system: dummy }) def test_sets_defaults_for_strata(self): - m = morphlib.morph3.Morphology({ + m = morphlib.morphology.Morphology({ 'kind': 'stratum', 'name': 'foo', 'chunks': [ @@ -637,6 +639,7 @@ build-system: dummy "morph": "bar", 'build-mode': 'bootstrap', 'build-depends': [], + 'prefix': '/usr', }, ], 'products': [], @@ -650,21 +653,22 @@ build-system: dummy { 'name': 'bar', "ref": "bar", - 'build-mode': 'bootstrap', + 'build-mode': 'staging', 'build-depends': [], + 'prefix': '/usr', }, ], } test_dict_with_build_depends = dict(test_dict) test_dict_with_build_depends["build-depends"] = [] - m = morphlib.morph3.Morphology(test_dict_with_build_depends) + m = morphlib.morphology.Morphology(test_dict_with_build_depends) self.loader.unset_defaults(m) self.assertEqual( dict(m), test_dict) def test_sets_defaults_for_system(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( kind='system', name='foo', arch='testarch', @@ -692,7 +696,7 @@ build-system: dummy dict(m)) def test_unsets_defaults_for_system(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( { 'description': '', 'kind': 'system', @@ -720,7 +724,7 @@ build-system: dummy }) def test_sets_defaults_for_cluster(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( name='foo', kind='cluster', systems=[ @@ -737,7 +741,7 @@ build-system: dummy 'deploy': {}}]) def test_unsets_defaults_for_cluster(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( name='foo', kind='cluster', description='', @@ -754,8 +758,8 @@ build-system: dummy [{'morph': 'foo'}, {'morph': 'bar'}]) - def test_sets_stratum_chunks_repo_and_morph_from_name(self): - m = morphlib.morph3.Morphology( + def test_sets_stratum_chunks_repo_from_name(self): + m = morphlib.morphology.Morphology( { "name": "foo", "kind": "stratum", @@ -771,10 +775,9 @@ build-system: dummy self.loader.set_defaults(m) self.loader.validate(m) self.assertEqual(m['chunks'][0]['repo'], 'le-chunk') - self.assertEqual(m['chunks'][0]['morph'], 'le-chunk') - def test_collapses_stratum_chunks_repo_and_morph_from_name(self): - m = morphlib.morph3.Morphology( + def test_collapses_stratum_chunks_repo_from_name(self): + m = morphlib.morphology.Morphology( { "name": "foo", "kind": "stratum", @@ -791,10 +794,9 @@ build-system: dummy self.loader.unset_defaults(m) self.assertTrue('repo' not in m['chunks'][0]) - self.assertTrue('morph' not in m['chunks'][0]) def test_convertes_max_jobs_to_an_integer(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( { "name": "foo", "kind": "chunk", @@ -827,7 +829,7 @@ build-system: dummy def test_warns_when_systems_refer_to_strata_with_repo_or_ref(self): for obsolete_field in ('repo', 'ref'): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( name="foo", kind="system", arch="testarch", @@ -850,7 +852,7 @@ build-system: dummy def test_warns_when_strata_refer_to_build_depends_with_repo_or_ref(self): for obsolete_field in ('repo', 'ref'): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( { 'name': 'foo', 'kind': 'stratum', @@ -883,7 +885,7 @@ build-system: dummy def test_unordered_asciibetically_after_ordered(self): # We only get morphologies with arbitrary keys in clusters - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( name='foo', kind='cluster', systems=[ @@ -924,21 +926,21 @@ build-system: dummy self.assertEqual(s, self.loader.save_to_string(m)) def test_smoketest_multi_line_unicode(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( name=u'foo', description=u'1 2 3\n4 5 6\n7 8 9\n', ) s = self.loader.save_to_string(m) def test_smoketest_multi_line_unicode_encoded(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( name=u'foo \u263A'.encode('utf-8'), description=u'1 \u263A\n2 \u263A\n3 \u263A\n'.encode('utf-8'), ) s = self.loader.save_to_string(m) def test_smoketest_binary_garbage(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( description='\x92', ) s = self.loader.save_to_string(m) diff --git a/morphlib/morph3.py b/morphlib/morphology.py index 477cac1a..314c315a 100644 --- a/morphlib/morph3.py +++ b/morphlib/morphology.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/morphlib/morph3_tests.py b/morphlib/morphology_tests.py index e150bf33..385f62ee 100644 --- a/morphlib/morph3_tests.py +++ b/morphlib/morphology_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -24,7 +24,7 @@ import morphlib class MorphologyTests(unittest.TestCase): def setUp(self): - self.morph = morphlib.morph3.Morphology() + self.morph = morphlib.morphology.Morphology() def test_has_repo_url_attribute(self): self.assertEqual(self.morph.repo_url, None) diff --git a/morphlib/morphologyfactory.py b/morphlib/morphologyfactory.py index cd1972be..1a8e374e 100644 --- a/morphlib/morphologyfactory.py +++ b/morphlib/morphologyfactory.py @@ -37,31 +37,6 @@ class NotcachedError(MorphologyFactoryError): "remote cache specified" % repo_name) -class StratumError(MorphologyFactoryError): - pass - - -class NoChunkBuildDependsError(StratumError): - def __init__(self, stratum, chunk): - StratumError.__init__( - self, 'No build dependencies in stratum %s for chunk %s ' - '(build-depends is a mandatory field)' % (stratum, chunk)) - - -class EmptyStratumError(StratumError): - - def __init__(self, stratum): - cliapp.AppException.__init__(self, - "Stratum %s is empty (has no dependencies)" % stratum) - - -class NoStratumBuildDependsError(StratumError): - def __init__(self, stratum): - StratumError.__init__( - self, 'Stratum %s has no build-dependencies listed ' - 'and has no bootstrap chunks.' % stratum) - - class MorphologyFactory(object): '''A way of creating morphologies which will provide a default''' @@ -75,16 +50,17 @@ class MorphologyFactory(object): if self._app is not None: self._app.status(*args, **kwargs) - def _get_morphology_text(self, reponame, sha1, filename): + def _load_morphology(self, reponame, sha1, filename): morph_name = os.path.splitext(os.path.basename(filename))[0] + loader = morphlib.morphloader.MorphologyLoader() if self._lrc.has_repo(reponame): self.status(msg="Looking for %s in local repo cache" % filename, chatty=True) try: repo = self._lrc.get_repo(reponame) - text = repo.cat(sha1, filename) + morph = loader.load_from_string(repo.cat(sha1, filename)) except IOError: - text = None + morph = None file_list = repo.ls_tree(sha1) elif self._rrc is not None: self.status(msg="Retrieving %(reponame)s %(sha1)s %(filename)s" @@ -93,35 +69,28 @@ class MorphologyFactory(object): chatty=True) try: text = self._rrc.cat_file(reponame, sha1, filename) + morph = loader.load_from_string(text) except morphlib.remoterepocache.CatFileError: - text = None + morph = None file_list = self._rrc.ls_tree(reponame, sha1) else: raise NotcachedError(reponame) - if text is None: + if morph is None: self.status(msg="File %s doesn't exist: attempting to infer " "chunk morph from repo's build system" % filename, chatty=True) bs = morphlib.buildsystem.detect_build_system(file_list) if bs is None: raise MorphologyNotFoundError(filename) - text = bs.get_morphology_text(morph_name) - return morph_name, text + morph = bs.get_morphology(morph_name) + loader.validate(morph) + loader.set_commands(morph) + loader.set_defaults(morph) + return morph def get_morphology(self, reponame, sha1, filename): - morph_name, text = self._get_morphology_text(reponame, sha1, filename) - - try: - morphology = morphlib.morph2.Morphology(text) - except morphlib.YAMLError as e: # pragma: no cover - raise morphlib.Error("Error parsing %s: %s" % - (filename, str(e))) - - if morph_name != morphology['name']: - raise morphlib.Error( - "Name %s does not match basename of morphology file %s" % - (morphology['name'], filename)) + morphology = self._load_morphology(reponame, sha1, filename) method_name = '_check_and_tweak_%s' % morphology['kind'] if hasattr(self, method_name): @@ -133,54 +102,17 @@ class MorphologyFactory(object): def _check_and_tweak_system(self, morphology, reponame, sha1, filename): '''Check and tweak a system morphology.''' - if morphology['arch'] is None: # pragma: no cover - raise morphlib.Error('No arch specified in system %s ' - '(arch is a mandatory field)' % - filename) - - if morphology['arch'] == 'armv7': - morphology._dict['arch'] = 'armv7l' - - if morphology['arch'] not in morphlib.valid_archs: - raise morphlib.Error('Unknown arch %s. This version of Morph ' - 'supports the following architectures: %s' % - (morphology['arch'], - ', '.join(morphlib.valid_archs))) - name = morphology['name'] morphology.builds_artifacts = [name + '-rootfs'] morphology.needs_artifact_metadata_cached = False - morphlib.morphloader.MorphologyLoader._validate_stratum_specs_fields( - morphology, 'strata') - morphlib.morphloader.MorphologyLoader._set_stratum_specs_defaults( - morphology, 'strata') - def _check_and_tweak_stratum(self, morphology, reponame, sha1, filename): '''Check and tweak a stratum morphology.''' - if len(morphology['chunks']) == 0: - raise EmptyStratumError(morphology['name']) - - for source in morphology['chunks']: - if source.get('build-depends', None) is None: - name = source.get('name', source.get('repo', 'unknown')) - raise NoChunkBuildDependsError(filename, name) - - if (len(morphology['build-depends'] or []) == 0 and - not any(c.get('build-mode') in ('bootstrap', 'test') - for c in morphology['chunks'])): - raise NoStratumBuildDependsError(filename) - morphology.builds_artifacts = [morphology['name']] morphology.needs_artifact_metadata_cached = True - morphlib.morphloader.MorphologyLoader._validate_stratum_specs_fields( - morphology, 'build-depends') - morphlib.morphloader.MorphologyLoader._set_stratum_specs_defaults( - morphology, 'build-depends') - def _check_and_tweak_chunk(self, morphology, reponame, sha1, filename): '''Check and tweak a chunk morphology.''' @@ -191,5 +123,3 @@ class MorphologyFactory(object): morphology.builds_artifacts = [morphology['name']] morphology.needs_artifact_metadata_cached = False - - morphlib.morphloader.MorphologyLoader._validate_chunk(morphology) diff --git a/morphlib/morphologyfactory_tests.py b/morphlib/morphologyfactory_tests.py index 47bf3153..0b3253da 100644 --- a/morphlib/morphologyfactory_tests.py +++ b/morphlib/morphologyfactory_tests.py @@ -17,7 +17,6 @@ import unittest import morphlib -from morphlib.morph2 import Morphology from morphlib.morphologyfactory import (MorphologyFactory, MorphologyNotFoundError, NotcachedError) @@ -31,7 +30,7 @@ class FakeRemoteRepoCache(object): return '''{ "name": "%s", "kind": "chunk", - "build-system": "bar" + "build-system": "dummy" }''' % filename[:-len('.morph')] return 'text' @@ -41,94 +40,76 @@ class FakeRemoteRepoCache(object): class FakeLocalRepo(object): morphologies = { - 'chunk.morph': '''{ - "name": "chunk", - "kind": "chunk", - "build-system": "bar" - }''', - 'chunk-split.morph': '''{ - "name": "chunk-split", - "kind": "chunk", - "build-system": "bar", - "products": [ - { - "artifact": "chunk-split-runtime", - "include": [] - }, - { - "artifact": "chunk-split-devel", - "include": [] - } - ] - }''', - 'stratum.morph': '''{ - "name": "stratum", - "kind": "stratum", - "chunks": [ - { - "name": "chunk", - "repo": "test:repo", - "ref": "sha1", - "build-mode": "bootstrap", - "build-depends": [] - } - ] - }''', - 'stratum-no-chunk-bdeps.morph': '''{ - "name": "stratum-no-chunk-bdeps", - "kind": "stratum", - "chunks": [ - { - "name": "chunk", - "repo": "test:repo", - "ref": "sha1", - "build-mode": "bootstrap" - } - ] - }''', - 'stratum-no-bdeps-no-bootstrap.morph': '''{ - "name": "stratum-no-bdeps-no-bootstrap", - "kind": "stratum", - "chunks": [ - { - "name": "chunk", - "repo": "test:repo", - "ref": "sha1", - "build-depends": [] - } - ] - }''', - 'stratum-bdeps-no-bootstrap.morph': '''{ - "name": "stratum-bdeps-no-bootstrap", - "kind": "stratum", - "build-depends": [ - { - "morph": "stratum" - } - ], - "chunks": [ - { - "name": "chunk", - "repo": "test:repo", - "ref": "sha1", - "build-depends": [] - } - ] - }''', - 'stratum-empty.morph': '''{ - "name": "stratum-empty", - "kind": "stratum" - }''', - 'system.morph': '''{ - "name": "system", - "kind": "system", - "arch": "%(arch)s" - }''', - 'parse-error.morph': '''{ "name"''', - 'name-mismatch.morph': '''{ - "name": "fred", - "kind": "stratum" - }''', + 'chunk.morph': ''' + name: chunk + kind: chunk + build-system: dummy + ''', + 'chunk-split.morph': ''' + name: chunk-split + kind: chunk + build-system: dummy + products: + - artifact: chunk-split-runtime + include: [] + - artifact: chunk-split-devel + include: [] + ''', + 'stratum.morph': ''' + name: stratum + kind: stratum + chunks: + - name: chunk + repo: test:repo + ref: sha1 + build-mode: bootstrap + build-depends: [] + ''', + 'stratum-no-chunk-bdeps.morph': ''' + name: stratum-no-chunk-bdeps + kind: stratum + chunks: + - name: chunk + repo: test:repo + ref: sha1 + build-mode: bootstrap + ''', + 'stratum-no-bdeps-no-bootstrap.morph': ''' + name: stratum-no-bdeps-no-bootstrap + kind: stratum + chunks: + - name: chunk + repo: test:repo + ref: sha1 + build-depends: [] + ''', + 'stratum-bdeps-no-bootstrap.morph': ''' + name: stratum-bdeps-no-bootstrap + kind: stratum + build-depends: + - morph: stratum + chunks: + - name: chunk + repo: test:repo + ref: sha1 + build-depends: [] + ''', + 'stratum-empty.morph': ''' + name: stratum-empty + kind: stratum + ''', + 'system.morph': ''' + name: system + kind: system + arch: %(arch)s + strata: + - morph: stratum + ''', + 'parse-error.morph': ''' name''', + 'name-mismatch.morph': ''' + name: fred + kind: stratum + ''', } def __init__(self): @@ -144,7 +125,7 @@ class FakeLocalRepo(object): return '''{ "name": "%s", "kind": "chunk", - "build-system": "bar" + "build-system": "dummy" }''' % filename[:-len('.morph')] return 'text' @@ -313,13 +294,13 @@ class MorphologyFactoryTests(unittest.TestCase): 'reponame', 'sha1', 'parse-error.morph') def test_fails_on_no_chunk_bdeps(self): - self.assertRaises(morphlib.morphologyfactory.NoChunkBuildDependsError, + self.assertRaises(morphlib.morphloader.NoBuildDependenciesError, self.mf.get_morphology, 'reponame', 'sha1', 'stratum-no-chunk-bdeps.morph') def test_fails_on_no_bdeps_or_bootstrap(self): self.assertRaises( - morphlib.morphologyfactory.NoStratumBuildDependsError, + morphlib.morphloader.NoStratumBuildDependenciesError, self.mf.get_morphology, 'reponame', 'sha1', 'stratum-no-bdeps-no-bootstrap.morph') @@ -330,6 +311,6 @@ class MorphologyFactoryTests(unittest.TestCase): def test_fails_on_empty_stratum(self): self.assertRaises( - morphlib.morphologyfactory.EmptyStratumError, + morphlib.morphloader.EmptyStratumError, self.mf.get_morphology, 'reponame', 'sha1', 'stratum-empty.morph') diff --git a/morphlib/morphset.py b/morphlib/morphset.py index 590ac51e..bf061f94 100644 --- a/morphlib/morphset.py +++ b/morphlib/morphset.py @@ -127,7 +127,8 @@ class MorphologySet(object): specs = m[kind] for spec in specs: if cb_filter(m, kind, spec): - fn = morphlib.util.sanitise_morphology_path(spec['morph']) + fn = morphlib.util.sanitise_morphology_path( + spec['morph'] if 'morph' in spec else spec['name']) orig_spec = (spec.get('repo'), spec.get('ref'), fn) dirtied = cb_process(m, kind, spec) if dirtied: @@ -148,7 +149,8 @@ class MorphologySet(object): if m.ref != spec.get('ref'): m.ref = spec.get('ref') m.dirty = True - file = morphlib.util.sanitise_morphology_path(spec['morph']) + file = morphlib.util.sanitise_morphology_path( + spec['morph'] if 'morph' in spec else spec['name']) assert (m.filename == file or m.repo_url == spec.get('repo')), \ 'Moving morphologies is not supported.' @@ -162,7 +164,7 @@ class MorphologySet(object): ''' def wanted_spec(m, kind, spec): - spec_name = spec.get('name', spec['morph']) + spec_name = spec['name'] if 'name' in spec else spec['morph'] return (spec.get('repo') == repo_url and spec.get('ref') == orig_ref and spec_name == morph_name) @@ -243,17 +245,3 @@ class MorphologySet(object): return True self.traverse_specs(process_chunk_spec, wanted_chunk_spec) - - def unpetrify_all(self): - '''If a spec is petrified, unpetrify it. - - ''' - - def wanted_spec(m, kind, spec): - return ('unpetrify-ref' in spec and - morphlib.git.is_valid_sha1(spec.get('ref'))) - def process_spec(m, kind, spec): - spec['ref'] = spec.pop('unpetrify-ref') - return True - - self.traverse_specs(process_spec, wanted_spec) diff --git a/morphlib/morphset_tests.py b/morphlib/morphset_tests.py index 8679c64a..81b5810f 100644 --- a/morphlib/morphset_tests.py +++ b/morphlib/morphset_tests.py @@ -26,7 +26,7 @@ class MorphologySetTests(unittest.TestCase): def setUp(self): self.morphs = morphlib.morphset.MorphologySet() - self.system = morphlib.morph3.Morphology({ + self.system = morphlib.morphology.Morphology({ 'kind': 'system', 'name': 'foo-system', 'strata': [ @@ -41,7 +41,7 @@ class MorphologySetTests(unittest.TestCase): self.system.ref = 'master' self.system.filename = 'foo-system.morph' - self.stratum = morphlib.morph3.Morphology({ + self.stratum = morphlib.morphology.Morphology({ 'kind': 'stratum', 'name': 'foo-stratum', 'chunks': [ @@ -111,7 +111,7 @@ class MorphologySetTests(unittest.TestCase): }) def test_changes_stratum_ref_in_build_depends(self): - other_stratum = morphlib.morph3.Morphology({ + other_stratum = morphlib.morphology.Morphology({ 'name': 'other-stratum', 'kind': 'stratum', 'chunks': [], @@ -200,18 +200,3 @@ class MorphologySetTests(unittest.TestCase): 'unpetrify-ref': 'master', } ]) - - def test_unpetrify_all(self): - self.morphs.add_morphology(self.system) - self.morphs.add_morphology(self.stratum) - self.morphs.petrify_chunks({('test:foo-chunk', 'master'): '0'*40}) - self.morphs.unpetrify_all() - self.assertEqual( - self.stratum['chunks'], - [ - { - 'repo': 'test:foo-chunk', - 'ref': 'master', - 'morph': 'foo-chunk', - } - ]) diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py index a66098b8..a258cd70 100644 --- a/morphlib/plugins/branch_and_merge_plugin.py +++ b/morphlib/plugins/branch_and_merge_plugin.py @@ -38,10 +38,6 @@ class BranchAndMergePlugin(cliapp.Plugin): self.app.add_subcommand( 'edit', self.edit, arg_synopsis='SYSTEM STRATUM [CHUNK]') self.app.add_subcommand( - 'petrify', self.petrify, arg_synopsis='') - self.app.add_subcommand( - 'unpetrify', self.unpetrify, arg_synopsis='') - self.app.add_subcommand( 'show-system-branch', self.show_system_branch, arg_synopsis='') self.app.add_subcommand( 'show-branch-root', self.show_branch_root, arg_synopsis='') @@ -313,9 +309,17 @@ class BranchAndMergePlugin(cliapp.Plugin): cached_repo = lrc.get_updated_repo(chunk_url) gd = sb.clone_cached_repo(cached_repo, chunk_ref) - if chunk_ref != sb.system_branch_name: - gd.branch(sb.system_branch_name, chunk_ref) - gd.checkout(sb.system_branch_name) + system_branch_ref = gd.disambiguate_ref(sb.system_branch_name) + sha1 = gd.resolve_ref_to_commit(chunk_ref) + + try: + old_sha1 = gd.resolve_ref_to_commit(system_branch_ref) + except morphlib.gitdir.InvalidRefError as e: + pass + else: + gd.delete_ref(system_branch_ref, old_sha1) + gd.branch(sb.system_branch_name, sha1) + gd.checkout(sb.system_branch_name) gd.update_submodules(self.app) gd.update_remotes() if gd.has_fat(): @@ -377,16 +381,17 @@ class BranchAndMergePlugin(cliapp.Plugin): This would, for example, write out something like: - /src/ws/master/baserock:baserock/morphs + /src/ws/master/baserock/baserock/definitions - when the master branch of the `baserock:baserock/morphs` + when the master branch of the `baserock/baserock/definitions` repository is checked out. ''' ws = morphlib.workspace.open('.') sb = morphlib.sysbranchdir.open_from_within('.') - self.app.output.write('%s\n' % sb.get_config('branch.root')) + repo_url = sb.get_config('branch.root') + self.app.output.write('%s\n' % sb.get_git_directory_name(repo_url)) def _remove_branch_dir_safe(self, workspace_root, system_branch_root): # This function avoids throwing any exceptions, so it is safe to call @@ -486,100 +491,6 @@ class BranchAndMergePlugin(cliapp.Plugin): morphs.add_morphology(morph) return morphs - def petrify(self, args): - '''Convert all chunk refs in a system branch to be fixed SHA1s. - - This modifies all git commit references in system and stratum - morphologies, in the current system branch, to be fixed SHA - commit identifiers, rather than symbolic branch or tag names. - This is useful for making sure none of the components in a system - branch change accidentally. - - Consider the following scenario: - - * The `master` system branch refers to `gcc` using the - `baserock/morph` ref. This is appropriate, since the main line - of development should use the latest curated code. - - * You create a system branch to prepare for a release, called - `TROVE_ID/release/2.0`. The reference to `gcc` is still - `baserock/morph`. - - * You test everything, and make a release. You deploy the release - images onto devices, which get shipped to your customers. - - * A new version GCC is committed to the `baserock/morph` branch. - - * Your release branch suddenly uses a new compiler, which may - or may not work for your particular system at that release. - - To avoid this, you need to _petrify_ all git references - so that they do not change accidentally. If you've tested - your release with the GCC release that is stored in commit - `94c50665324a7aeb32f3096393ec54b2e63bfb28`, then you should - continue to use that version of GCC, regardless of what might - happen in the master system branch. If, and only if, you decide - that a new compiler would be good for your release should you - include it in your release branch. This way, only the things - that you change intentionally change in your release branch. - - ''' - - if args: - raise cliapp.AppException('morph petrify takes no arguments') - - ws = morphlib.workspace.open('.') - sb = morphlib.sysbranchdir.open_from_within('.') - loader = morphlib.morphloader.MorphologyLoader() - lrc, rrc = morphlib.util.new_repo_caches(self.app) - update_repos = not self.app.settings['no-git-update'] - - morphs = self._load_all_sysbranch_morphologies(sb, loader) - - #TODO: Stop using app.resolve_ref - def resolve_refs(morphs): - for repo, ref in morphs.list_refs(): - # You can't resolve null refs, so don't attempt to. - if repo is None or ref is None: - continue - # TODO: Handle refs that are only in workspace in general - if (repo == sb.root_repository_url - and ref == sb.system_branch_name): - continue - commit_sha1, tree_sha1 = self.app.resolve_ref( - lrc, rrc, repo, ref, update=update_repos) - yield ((repo, ref), commit_sha1) - - morphs.repoint_refs(sb.root_repository_url, - sb.system_branch_name) - - morphs.petrify_chunks(dict(resolve_refs(morphs))) - - # Write morphologies back out again. - self._save_dirty_morphologies(loader, sb, morphs.morphologies) - - def unpetrify(self, args): - '''Reverse the process of petrification. - - This undoes the changes `morph petrify` did. - - ''' - - if args: - raise cliapp.AppException('morph petrify takes no arguments') - - ws = morphlib.workspace.open('.') - sb = morphlib.sysbranchdir.open_from_within('.') - loader = morphlib.morphloader.MorphologyLoader() - - morphs = self._load_all_sysbranch_morphologies(sb, loader) - - # Restore the ref for each stratum and chunk - morphs.unpetrify_all() - - # Write morphologies back out again. - self._save_dirty_morphologies(loader, sb, morphs.morphologies) - def status(self, args): '''Show information about the current system branch or workspace diff --git a/morphlib/plugins/build_plugin.py b/morphlib/plugins/build_plugin.py index 1a4fb573..64630c2b 100644 --- a/morphlib/plugins/build_plugin.py +++ b/morphlib/plugins/build_plugin.py @@ -184,31 +184,10 @@ class BuildPlugin(cliapp.Plugin): system=system_filename, branch=sb.system_branch_name) - bb = morphlib.buildbranch.BuildBranch(sb, build_ref_prefix, - push_temporary=push) - with contextlib.closing(bb) as bb: - - for gd, build_ref in bb.add_uncommitted_changes(): - self.app.status(msg='Adding uncommitted changes '\ - 'in %(dirname)s to %(ref)s', - dirname=gd.dirname, ref=build_ref, chatty=True) - - for gd in bb.inject_build_refs(loader): - self.app.status(msg='Injecting temporary build refs '\ - 'into morphologies in %(dirname)s', - dirname=gd.dirname, chatty=True) - - for gd, build_ref in bb.update_build_refs(name, email, build_uuid): - self.app.status(msg='Committing changes in %(dirname)s '\ - 'to %(ref)s', - dirname=gd.dirname, ref=build_ref, chatty=True) - - for gd, build_ref, remote in bb.push_build_branches(): - self.app.status(msg='Pushing %(ref)s in %(dirname)s '\ - 'to %(remote)s', - ref=build_ref, dirname=gd.dirname, - remote=remote.get_push_url(), chatty=True) - - build_command.build([bb.root_repo_url, - bb.root_ref, - system_filename]) + bb = morphlib.buildbranch.BuildBranch(sb, build_ref_prefix) + pbb = morphlib.buildbranch.pushed_build_branch( + bb, loader=loader, changes_need_pushing=push, + name=name, email=email, build_uuid=build_uuid, + status=self.app.status) + with pbb as (repo, ref): + build_command.build([repo, ref, system_filename]) diff --git a/morphlib/plugins/cross-bootstrap_plugin.py b/morphlib/plugins/cross-bootstrap_plugin.py index cd8e355e..0c3e3a4a 100644 --- a/morphlib/plugins/cross-bootstrap_plugin.py +++ b/morphlib/plugins/cross-bootstrap_plugin.py @@ -182,7 +182,7 @@ class BootstrapSystemBuilder(morphlib.builder2.BuilderBase): for step, in_parallel in steps: key = '%s-commands' % step - cmds = m.get_commands(key) + cmds = m[key] for cmd in cmds: f.write('(') if in_parallel: diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index 38c17bc2..61b8145e 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -319,40 +319,21 @@ class DeployPlugin(cliapp.Plugin): self.validate_deployment_options( env_vars, all_deployments, all_subsystems) - bb = morphlib.buildbranch.BuildBranch(sb, build_ref_prefix, - push_temporary=False) - with contextlib.closing(bb) as bb: - - for gd, build_ref in bb.add_uncommitted_changes(): - self.app.status(msg='Adding uncommitted changes '\ - 'in %(dirname)s to %(ref)s', - dirname=gd.dirname, ref=build_ref, chatty=True) - - for gd in bb.inject_build_refs(loader): - self.app.status(msg='Injecting temporary build refs '\ - 'into morphologies in %(dirname)s', - dirname=gd.dirname, chatty=True) - - for gd, build_ref in bb.update_build_refs(name, email, build_uuid): - self.app.status(msg='Committing changes in %(dirname)s '\ - 'to %(ref)s', - dirname=gd.dirname, ref=build_ref, chatty=True) - - for gd, build_ref, remote in bb.push_build_branches(): - self.app.status(msg='Pushing %(ref)s in %(dirname)s '\ - 'to %(remote)s', - ref=build_ref, dirname=gd.dirname, - remote=remote.get_push_url(), chatty=True) - + bb = morphlib.buildbranch.BuildBranch(sb, build_ref_prefix) + pbb = morphlib.buildbranch.pushed_build_branch( + bb, loader=loader, changes_need_pushing=False, + name=name, email=email, build_uuid=build_uuid, + status=self.app.status) + with pbb as (repo, ref): # Create a tempdir for this deployment to work in deploy_tempdir = tempfile.mkdtemp( dir=os.path.join(self.app.settings['tempdir'], 'deployments')) try: for system in cluster_morphology['systems']: self.deploy_system(build_command, deploy_tempdir, - root_repo_dir, bb.root_repo_url, - bb.root_ref, system, env_vars, - deployments, parent_location='') + root_repo_dir, repo, ref, system, + env_vars, deployments, + parent_location='') finally: shutil.rmtree(deploy_tempdir) @@ -547,9 +528,8 @@ class DeployPlugin(cliapp.Plugin): system morphology (repo, ref), or with the Morph code. ''' - build_ref_prefix = self.app.settings['build-ref-prefix'] with morphlib.extensions.get_extension_filename( - build_ref_prefix, name, kind) as ext_filename: + name, kind) as ext_filename: self.app.status(msg='Running extension %(name)s%(kind)s', name=name, kind=kind) self.app.runcmd( diff --git a/morphlib/plugins/show_dependencies_plugin.py b/morphlib/plugins/show_dependencies_plugin.py index c59cf507..3a1cb7ad 100644 --- a/morphlib/plugins/show_dependencies_plugin.py +++ b/morphlib/plugins/show_dependencies_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -65,7 +65,7 @@ class ShowDependenciesPlugin(cliapp.Plugin): # traverse the morphs to list all the sources for repo, ref, filename in self.app.itertriplets(args): - morph = filename[:-len('.morph')] + morph = morphlib.util.sanitise_morphology_path(filename) self.app.output.write('dependency graph for %s|%s|%s:\n' % (repo, ref, morph)) diff --git a/morphlib/remoteartifactcache_tests.py b/morphlib/remoteartifactcache_tests.py index d11bf264..ca959ebf 100644 --- a/morphlib/remoteartifactcache_tests.py +++ b/morphlib/remoteartifactcache_tests.py @@ -24,26 +24,24 @@ import morphlib class RemoteArtifactCacheTests(unittest.TestCase): def setUp(self): - morph = morphlib.morph2.Morphology( + loader = morphlib.morphloader.MorphologyLoader() + morph = loader.load_from_string( ''' - { - "name": "chunk", - "kind": "chunk", - "artifacts": { - "chunk-runtime": [ - "usr/bin", - "usr/sbin", - "usr/lib", - "usr/libexec" - ], - "chunk-devel": [ - "usr/include" - ], - "chunk-doc": [ - "usr/share/doc" - ] - } - } + name: chunk + kind: chunk + products: + - artifact: chunk-runtime + include: + - usr/bin + - usr/sbin + - usr/lib + - usr/libexec + - artifact: chunk-devel + include: + - usr/include + - artifact: chunk-doc + include: + - usr/share/doc ''') self.source = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'chunk.morph') diff --git a/morphlib/source_tests.py b/morphlib/source_tests.py index 6643f0fc..f5ce5d4d 100644 --- a/morphlib/source_tests.py +++ b/morphlib/source_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -22,10 +22,8 @@ import morphlib class SourceTests(unittest.TestCase): morphology_text = ''' - { - "name": "foo", - "kind": "chunk" - } + name: foo + kind: chunk ''' def setUp(self): @@ -33,7 +31,8 @@ class SourceTests(unittest.TestCase): self.original_ref = 'original/ref' self.sha1 = 'CAFEF00D' self.tree = 'F000000D' - self.morphology = morphlib.morph2.Morphology(self.morphology_text) + loader = morphlib.morphloader.MorphologyLoader() + self.morphology = loader.load_from_string(self.morphology_text) self.filename = 'foo.morph' self.source = morphlib.source.Source( self.repo_name, self.original_ref, self.sha1, self.tree, diff --git a/morphlib/sysbranchdir.py b/morphlib/sysbranchdir.py index b8953c2f..19fba695 100644 --- a/morphlib/sysbranchdir.py +++ b/morphlib/sysbranchdir.py @@ -68,19 +68,13 @@ class SystemBranchDirectory(object): value = cliapp.runcmd(['git', 'config', '-f', self._config_path, key]) return value.strip() - def get_git_directory_name(self, repo_url): - '''Return directory pathname for a given git repository. - - If the URL is a real one (not aliased), the schema and leading // - are removed from it, as is a .git suffix. - - Any colons in the URL path or network location are replaced - with slashes, so that directory paths do not contain colons. - This avoids problems with PYTHONPATH, PATH, and other things - that use colon as a separator. - - ''' + def _find_git_directory(self, repo_url): + for gd in self.list_git_directories(): + if gd.get_config('morph.repository') == repo_url: + return gd.dirname + return None + def _fabricate_git_directory_name(self, repo_url): # Parse the URL. If the path component is absolute, we assume # it's a real URL; otherwise, an aliased URL. parts = urlparse.urlparse(repo_url) @@ -107,6 +101,26 @@ class SystemBranchDirectory(object): return os.path.join(self.root_directory, relative) + def get_git_directory_name(self, repo_url): + '''Return directory pathname for a given git repository. + + If the repository has already been cloned, then it returns the + path to that, if not it will fabricate a path based on the url. + + If the URL is a real one (not aliased), the schema and leading // + are removed from it, as is a .git suffix. + + Any colons in the URL path or network location are replaced + with slashes, so that directory paths do not contain colons. + This avoids problems with PYTHONPATH, PATH, and other things + that use colon as a separator. + + ''' + found_repo = self._find_git_directory(repo_url) + if not found_repo: + return self._fabricate_git_directory_name(repo_url) + return found_repo + def get_filename(self, repo_url, relative): '''Return full pathname to a file in a checked out repository. |