diff options
80 files changed, 1522 insertions, 3191 deletions
diff --git a/doc/branching-merging-systems.mdwn b/doc/branching-merging-systems.mdwn index c2e24d77..3bc19aab 100644 --- a/doc/branching-merging-systems.mdwn +++ b/doc/branching-merging-systems.mdwn @@ -129,7 +129,7 @@ Also, clone the `morphs` git repository inside the system branch directory. cd ~/baserock/liw/foo/morphs - morph petrify base-system.morph devel-system.morph + edit base-system.morph devel-system.morph git commit -a Modify the specified morphologies (or the stratum morphologies they @@ -225,36 +225,6 @@ it to exist instead. * Run `git checkout BRANCH` in the `morphs` repository. -Implementation: `morph petrify` --------------- - -Usage: - - morph petrify [MORPH]... - morph petrify --petrify-from-system FILE - -This needs to be run in the `morphs` git repository in a system branch. - -In the first form: - -* read each of the given morphologies; if the morphology is a system one, - follow references to stratum morphologies and process those instead -* in each stratum morphology, replace a reference to a chunk with the - absolute SHA-1: if the original reference was, say, `baserock/morph`, - get the SHA-1 of the current tip commit in that branch and replace - the reference in the morphology - -In the second form: - -* extract the system and stratum morphologies used in the system image file; - these are in a petrified form already -* copy the morphologies to the current working directory, overwriting the - files from git - -In either case, the results need to be committed (with normal git commands) -by the user. - - Implementation: `morph edit` -------------- diff --git a/morphlib/__init__.py b/morphlib/__init__.py index 0c928fd3..f98c11aa 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -68,10 +68,9 @@ import gitindex import localartifactcache import localrepocache import mountableimage -import morph2 import morphologyfactory import morphologyfinder -import morph3 +import morphology import morphloader import morphset import remoteartifactcache diff --git a/morphlib/app.py b/morphlib/app.py index a543443e..88eb58a4 100644 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -518,16 +518,15 @@ class Morph(cliapp.Application): self.output.write(text) def _help_topic(self, topic): - build_ref_prefix = self.settings['build-ref-prefix'] if topic in self.subcommands: usage = self._format_usage_for(topic) description = self._format_subcommand_help(topic) text = '%s\n\n%s' % (usage, description) self.output.write(text) - elif topic in extensions.list_extensions(build_ref_prefix): + elif topic in extensions.list_extensions(): name, kind = os.path.splitext(topic) try: - with extensions.get_extension_filename(build_ref_prefix, + with extensions.get_extension_filename( name, kind + '.help', executable=False) as fname: with open(fname, 'r') as f: diff --git a/morphlib/artifact_tests.py b/morphlib/artifact_tests.py index d4b15cba..62b1bfb9 100644 --- a/morphlib/artifact_tests.py +++ b/morphlib/artifact_tests.py @@ -23,24 +23,22 @@ import morphlib class ArtifactTests(unittest.TestCase): def setUp(self): - morph = morphlib.morph2.Morphology( - ''' - { - "name": "chunk", - "kind": "chunk", - "chunks": { - "chunk-runtime": [ - "usr/bin", - "usr/sbin", - "usr/lib", - "usr/libexec" - ], - "chunk-devel": [ - "usr/include" - ] - } - } - ''') + loader = morphlib.morphloader.MorphologyLoader() + morph = loader.load_from_string( + ''' + name: chunk + kind: chunk + products: + - artifact: chunk-runtime + include: + - usr/bin + - usr/sbin + - usr/lib + - usr/libexec + - artifact: chunk-devel + include: + - usr/include + ''') self.source = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'chunk.morph') self.artifact_name = 'chunk-runtime' diff --git a/morphlib/artifactresolver_tests.py b/morphlib/artifactresolver_tests.py index 6f62b4d1..96f7ced8 100644 --- a/morphlib/artifactresolver_tests.py +++ b/morphlib/artifactresolver_tests.py @@ -15,84 +15,68 @@ import itertools -import json import unittest +import yaml import morphlib -class FakeChunkMorphology(morphlib.morph2.Morphology): - - def __init__(self, name, artifact_names=[]): - assert(isinstance(artifact_names, list)) - - if artifact_names: - # fake a list of artifacts - artifacts = [] - for artifact_name in artifact_names: - artifacts.append({'artifact': artifact_name, - 'include': artifact_name}) - text = json.dumps({ - "name": name, - "kind": "chunk", - "products": artifacts - }) - self.builds_artifacts = artifact_names - else: - text = (''' - { - "name": "%s", - "kind": "chunk" - } - ''' % name) - self.builds_artifacts = [name] - morphlib.morph2.Morphology.__init__(self, text) - - -class FakeStratumMorphology(morphlib.morph2.Morphology): - - def __init__(self, name, chunks=[], build_depends=[]): - assert(isinstance(chunks, list)) - assert(isinstance(build_depends, list)) - - chunks_list = [] - for source_name, morph, repo, ref in chunks: - chunks_list.append({ - 'name': source_name, - 'morph': morph, - 'repo': repo, - 'ref': ref, - 'build-depends': [], - }) - build_depends_list = [] - for morph, repo, ref in build_depends: - build_depends_list.append({ - 'morph': morph, - 'repo': repo, - 'ref': ref - }) - if chunks_list: - text = (''' - { - "name": "%s", - "kind": "stratum", - "build-depends": %s, - "chunks": %s - } - ''' % (name, - json.dumps(build_depends_list), - json.dumps(chunks_list))) - else: - text = (''' - { - "name": "%s", - "kind": "stratum", - "build-depends": %s - } - ''' % (name, - json.dumps(build_depends_list))) - self.builds_artifacts = [name] - morphlib.morph2.Morphology.__init__(self, text) +def get_chunk_morphology(name, artifact_names=[]): + assert(isinstance(artifact_names, list)) + + if artifact_names: + # fake a list of artifacts + artifacts = [] + for artifact_name in artifact_names: + artifacts.append({'artifact': artifact_name, + 'include': [artifact_name]}) + text = yaml.dump({"name": name, + "kind": "chunk", + "products": artifacts}, default_flow_style=False) + builds_artifacts = artifact_names + else: + text = yaml.dump({'name': name, + 'kind': 'chunk'}, default_flow_style=False) + builds_artifacts = [name] + + loader = morphlib.morphloader.MorphologyLoader() + morph = loader.load_from_string(text) + morph.builds_artifacts = builds_artifacts + return morph + +def get_stratum_morphology(name, chunks=[], build_depends=[]): + assert(isinstance(chunks, list)) + assert(isinstance(build_depends, list)) + + chunks_list = [] + for source_name, morph, repo, ref in chunks: + chunks_list.append({ + 'name': source_name, + 'morph': morph, + 'repo': repo, + 'ref': ref, + 'build-depends': [], + }) + build_depends_list = [] + for morph in build_depends: + build_depends_list.append({ + 'morph': morph, + }) + if chunks_list: + text = yaml.dump({"name": name, + "kind": "stratum", + "build-depends": build_depends_list, + "chunks": chunks_list,}, default_flow_style=False) + else: + text = yaml.dump({"name": name, + "kind": "stratum", + "build-depends": build_depends_list}, + default_flow_style=False) + + loader = morphlib.morphloader.MorphologyLoader() + morph = loader.load_from_string(text) + morph.builds_artifacts = [name] + return morph class ArtifactResolverTests(unittest.TestCase): @@ -108,7 +92,7 @@ class ArtifactResolverTests(unittest.TestCase): def test_resolve_single_chunk_with_no_subartifacts(self): pool = morphlib.sourcepool.SourcePool() - morph = FakeChunkMorphology('chunk') + morph = get_chunk_morphology('chunk') source = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'chunk.morph') pool.add(source) @@ -127,7 +111,7 @@ class ArtifactResolverTests(unittest.TestCase): def test_resolve_single_chunk_with_one_new_artifact(self): pool = morphlib.sourcepool.SourcePool() - morph = FakeChunkMorphology('chunk', ['chunk-foobar']) + morph = get_chunk_morphology('chunk', ['chunk-foobar']) source = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'chunk.morph') pool.add(source) @@ -145,7 +129,7 @@ class ArtifactResolverTests(unittest.TestCase): def test_resolve_single_chunk_with_two_new_artifacts(self): pool = morphlib.sourcepool.SourcePool() - morph = FakeChunkMorphology('chunk', ['chunk-baz', 'chunk-qux']) + morph = get_chunk_morphology('chunk', ['chunk-baz', 'chunk-qux']) source = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'chunk.morph') pool.add(source) @@ -165,12 +149,12 @@ class ArtifactResolverTests(unittest.TestCase): def test_resolve_stratum_and_chunk(self): pool = morphlib.sourcepool.SourcePool() - morph = FakeChunkMorphology('chunk') + morph = get_chunk_morphology('chunk') chunk = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'chunk.morph') pool.add(chunk) - morph = FakeStratumMorphology( + morph = get_stratum_morphology( 'stratum', chunks=[('chunk', 'chunk', 'repo', 'ref')]) stratum = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'stratum.morph') @@ -199,12 +183,12 @@ class ArtifactResolverTests(unittest.TestCase): def test_resolve_stratum_and_chunk_with_two_new_artifacts(self): pool = morphlib.sourcepool.SourcePool() - morph = FakeChunkMorphology('chunk', ['chunk-foo', 'chunk-bar']) + morph = get_chunk_morphology('chunk', ['chunk-foo', 'chunk-bar']) chunk = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'chunk.morph') pool.add(chunk) - morph = FakeStratumMorphology( + morph = get_stratum_morphology( 'stratum', chunks=[ ('chunk', 'chunk', 'repo', 'ref'), @@ -236,51 +220,42 @@ class ArtifactResolverTests(unittest.TestCase): def test_resolving_artifacts_for_a_system_with_two_dependent_strata(self): pool = morphlib.sourcepool.SourcePool() - morph = FakeChunkMorphology('chunk1') + morph = get_chunk_morphology('chunk1') chunk1 = morphlib.source.Source( 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk1.morph') pool.add(chunk1) - morph = FakeStratumMorphology( + morph = get_stratum_morphology( 'stratum1', chunks=[('chunk1', 'chunk1', 'repo', 'original/ref')]) stratum1 = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'stratum1.morph') pool.add(stratum1) - morph = morphlib.morph2.Morphology( + loader = morphlib.morphloader.MorphologyLoader() + morph = loader.load_from_string( ''' - { - "name": "system", - "kind": "system", - "strata": [ - { - "repo": "repo", - "ref": "ref", - "morph": "stratum1" - }, - { - "repo": "repo", - "ref": "ref", - "morph": "stratum2" - } - ] - } + name: system + kind: system + arch: testarch + strata: + - morph: stratum1 + - morph: stratum2 ''') morph.builds_artifacts = ['system-rootfs'] system = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'system.morph') pool.add(system) - morph = FakeChunkMorphology('chunk2') + morph = get_chunk_morphology('chunk2') chunk2 = morphlib.source.Source( 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk2.morph') pool.add(chunk2) - morph = FakeStratumMorphology( + morph = get_stratum_morphology( 'stratum2', chunks=[('chunk2', 'chunk2', 'repo', 'original/ref')], - build_depends=[('stratum1', 'repo', 'ref')]) + build_depends=['stratum1']) stratum2 = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'stratum2.morph') pool.add(stratum2) @@ -337,52 +312,44 @@ class ArtifactResolverTests(unittest.TestCase): def test_resolving_stratum_with_explicit_chunk_dependencies(self): pool = morphlib.sourcepool.SourcePool() - morph = morphlib.morph2.Morphology( + loader = morphlib.morphloader.MorphologyLoader() + morph = loader.load_from_string( ''' - { - "name": "stratum", - "kind": "stratum", - "chunks": [ - { - "name": "chunk1", - "repo": "repo", - "ref": "original/ref", - "build-depends": [] - }, - { - "name": "chunk2", - "repo": "repo", - "ref": "original/ref", - "build-depends": [] - }, - { - "name": "chunk3", - "repo": "repo", - "ref": "original/ref", - "build-depends": [ - "chunk1", - "chunk2" - ] - } - ] - } + name: stratum + kind: stratum + build-depends: [] + chunks: + - name: chunk1 + repo: repo + ref: original/ref + build-depends: [] + - name: chunk2 + repo: repo + ref: original/ref + build-depends: [] + - name: chunk3 + repo: repo + ref: original/ref + build-depends: + - chunk1 + - chunk2 ''') morph.builds_artifacts = ['stratum'] stratum = morphlib.source.Source( 'repo', 'original/ref', 'sha1', 'tree', morph, 'stratum.morph') pool.add(stratum) - morph = FakeChunkMorphology('chunk1') + morph = get_chunk_morphology('chunk1') chunk1 = morphlib.source.Source( 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk1.morph') pool.add(chunk1) - morph = FakeChunkMorphology('chunk2') + morph = get_chunk_morphology('chunk2') chunk2 = morphlib.source.Source( 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk2.morph') pool.add(chunk2) - morph = FakeChunkMorphology('chunk3') + morph = get_chunk_morphology('chunk3') chunk3 = morphlib.source.Source( 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk3.morph') pool.add(chunk3) @@ -418,20 +385,33 @@ class ArtifactResolverTests(unittest.TestCase): for c3a in chunk_artifacts[2])) def test_detection_of_mutual_dependency_between_two_strata(self): + loader = morphlib.morphloader.MorphologyLoader() pool = morphlib.sourcepool.SourcePool() - morph = FakeStratumMorphology( + chunk = get_chunk_morphology('chunk1') + chunk1 = morphlib.source.Source( + 'repo', 'original/ref', 'sha1', 'tree', chunk, 'chunk1.morph') + pool.add(chunk1) + + morph = get_stratum_morphology( 'stratum1', - chunks=[], - build_depends=[('stratum2', 'repo', 'original/ref')]) + chunks=[(loader.save_to_string(chunk), 'chunk1.morph', + 'repo', 'original/ref')], + build_depends=['stratum2']) stratum1 = morphlib.source.Source( 'repo', 'original/ref', 'sha1', 'tree', morph, 'stratum1.morph') pool.add(stratum1) - morph = FakeStratumMorphology( + chunk = get_chunk_morphology('chunk2') + chunk2 = morphlib.source.Source( + 'repo', 'original/ref', 'sha1', 'tree', chunk, 'chunk2.morph') + pool.add(chunk2) + + morph = get_stratum_morphology( 'stratum2', - chunks=[], - build_depends=[('stratum1', 'repo', 'original/ref')]) + chunks=[(loader.save_to_string(chunk), 'chunk2.morph', + 'repo', 'original/ref')], + build_depends=['stratum1']) stratum2 = morphlib.source.Source( 'repo', 'original/ref', 'sha1', 'tree', morph, 'stratum2.morph') pool.add(stratum2) @@ -442,39 +422,34 @@ class ArtifactResolverTests(unittest.TestCase): def test_detection_of_chunk_dependencies_in_invalid_order(self): pool = morphlib.sourcepool.SourcePool() - morph = morphlib.morph2.Morphology( + loader = morphlib.morphloader.MorphologyLoader() + morph = loader.load_from_string( ''' - { - "name": "stratum", - "kind": "stratum", - "chunks": [ - { - "name": "chunk1", - "repo": "repo", - "ref": "original/ref", - "build-depends": [ - "chunk2" - ] - }, - { - "name": "chunk2", - "repo": "repo", - "ref": "original/ref" - } - ] - } + name: stratum + kind: stratum + build-depends: [] + chunks: + - name: chunk1 + repo: repo + ref: original/ref + build-depends: + - chunk2 + - name: chunk2 + repo: repo + ref: original/ref + build-depends: [] ''') morph.builds_artifacts = ['stratum'] stratum = morphlib.source.Source( 'repo', 'original/ref', 'sha1', 'tree', morph, 'stratum.morph') pool.add(stratum) - morph = FakeChunkMorphology('chunk1') + morph = get_chunk_morphology('chunk1') chunk1 = morphlib.source.Source( 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk1.morph') pool.add(chunk1) - morph = FakeChunkMorphology('chunk2') + morph = get_chunk_morphology('chunk2') chunk2 = morphlib.source.Source( 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk2.morph') pool.add(chunk2) @@ -485,27 +460,24 @@ class ArtifactResolverTests(unittest.TestCase): def test_detection_of_invalid_build_depends_format(self): pool = morphlib.sourcepool.SourcePool() - morph = morphlib.morph2.Morphology( + loader = morphlib.morphloader.MorphologyLoader() + morph = loader.load_from_string( ''' - { - "name": "stratum", - "kind": "stratum", - "chunks": [ - { - "name": "chunk", - "repo": "repo", - "ref": "original/ref", - "build-depends": "whatever" - } - ] - } + name: stratum + kind: stratum + build-depends: [] + chunks: + - name: chunk + repo: repo + ref: original/ref + build-depends: whatever ''') morph.builds_artifacts = ['stratum'] stratum = morphlib.source.Source( 'repo', 'original/ref', 'sha1', 'tree', morph, 'stratum.morph') pool.add(stratum) - morph = FakeChunkMorphology('chunk') + morph = get_chunk_morphology('chunk') chunk = morphlib.source.Source( 'repo', 'original/ref', 'sha1', 'tree', morph, 'chunk.morph') pool.add(chunk) diff --git a/morphlib/buildbranch.py b/morphlib/buildbranch.py index d415e7e1..885f5cf8 100644 --- a/morphlib/buildbranch.py +++ b/morphlib/buildbranch.py @@ -15,6 +15,7 @@ import collections +import contextlib import os import urlparse @@ -48,10 +49,9 @@ class BuildBranch(object): # would be better to not use local repositories and temporary refs, # so building from a workspace appears to be identical to using # `morph build-morphology` - def __init__(self, sb, build_ref_prefix, push_temporary): + def __init__(self, sb, build_ref_prefix): self._sb = sb - self._push_temporary = push_temporary self._cleanup = collections.deque() self._to_push = {} @@ -84,15 +84,18 @@ class BuildBranch(object): def _register_cleanup(self, func, *args, **kwargs): self._cleanup.append((func, args, kwargs)) - def add_uncommitted_changes(self): + def add_uncommitted_changes(self, add_cb=lambda **kwargs: None): '''Add any uncommitted changes to temporary build GitIndexes''' + changes_made = False for gd, (build_ref, index) in self._to_push.iteritems(): changed = [to_path for code, to_path, from_path in index.get_uncommitted_changes()] if not changed: continue - yield gd, build_ref + add_cb(gd=gd, build_ref=gd, changed=changed) + changes_made = True index.add_files_from_working_tree(changed) + return changes_made @staticmethod def _hash_morphologies(gd, morphologies, loader): @@ -102,7 +105,8 @@ class BuildBranch(object): sha1 = gd.store_blob(loader.save_to_string(morphology)) yield 0100644, sha1, morphology.filename - def inject_build_refs(self, loader): + def inject_build_refs(self, loader, use_local_repos, + inject_cb=lambda **kwargs: None): '''Update system and stratum morphologies to point to our branch. For all edited repositories, this alter the temporary GitIndex @@ -133,7 +137,7 @@ class BuildBranch(object): spec['repo'] = None spec['ref'] = None return True - if not self._push_temporary: + if use_local_repos: spec['repo'] = urlparse.urljoin('file://', gd.dirname) spec['ref'] = build_ref return True @@ -141,12 +145,15 @@ class BuildBranch(object): morphs.traverse_specs(process, filter) if any(m.dirty for m in morphs.morphologies): - yield self._root + inject_cb(gd=self._root) + # TODO: Prevent it hashing unchanged morphologies, while still + # hashing uncommitted ones. self._root_index.add_files_from_index_info( self._hash_morphologies(self._root, morphs.morphologies, loader)) - def update_build_refs(self, name, email, uuid): + def update_build_refs(self, name, email, uuid, + commit_cb=lambda **kwargs: None): '''Commit changes in temporary GitIndexes to temporary branches. `name` and `email` are required to construct the commit author info. @@ -176,12 +183,18 @@ class BuildBranch(object): with morphlib.branchmanager.LocalRefManager() as lrm: for gd, (build_ref, index) in self._to_push.iteritems(): - yield gd, build_ref tree = index.write_tree() try: parent = gd.resolve_ref_to_commit(build_ref) except morphlib.gitdir.InvalidRefError: parent = gd.resolve_ref_to_commit(gd.HEAD) + else: + # Skip updating ref if we already have a temporary + # build branch and have this tree on the branch + if tree == gd.resolve_ref_to_tree(build_ref): + continue + + commit_cb(gd=gd, build_ref=build_ref) commit = gd.commit_tree(tree, parent=parent, committer_name=committer_name, @@ -201,46 +214,55 @@ class BuildBranch(object): # a problem. lrm.update(gd, build_ref, commit, old_commit) - def push_build_branches(self): - '''Push all temporary build branches to the remote repositories. + def get_unpushed_branches(self): + '''Work out which, if any, local branches need to be pushed to build - This is a no-op if the BuildBranch was constructed with - `push_temporary` as False, so that the code flow for the user of - the BuildBranch can be the same when it can be pushed as when - it can't. + NOTE: This assumes that the refs in the morphologies and the + refs in the local checkouts match. ''' - # TODO: When BuildBranches become more context aware, if there - # are no uncommitted changes and the local versions are pushed - # we can skip pushing even if push_temporary is set. - # No uncommitted changes isn't sufficient reason to push the - # current HEAD - if self._push_temporary: - with morphlib.branchmanager.RemoteRefManager(False) as rrm: - for gd, (build_ref, index) in self._to_push.iteritems(): - remote = gd.get_remote('origin') - yield gd, build_ref, remote - refspec = morphlib.gitdir.RefSpec(build_ref) - rrm.push(remote, refspec) - self._register_cleanup(rrm.close) + for gd, (build_ref, index) in self._to_push.iteritems(): + remote = gd.get_remote('origin') + head_ref = gd.disambiguate_ref(gd.HEAD) + head_sha1 = gd.resolve_ref_to_commit(head_ref) + pushed_refs = sorted( + (remote_ref + for remote_sha1, remote_ref in remote.ls() + # substring match of refs, since ref may be a tag, + # in which case it would end with ^{} + if remote_sha1 == head_sha1 and head_ref in remote_ref), + key=len) + if not pushed_refs: + yield gd + + def push_build_branches(self, push_cb=lambda **kwargs: None): + '''Push all temporary build branches to the remote repositories. + ''' + with morphlib.branchmanager.RemoteRefManager(False) as rrm: + for gd, (build_ref, index) in self._to_push.iteritems(): + remote = gd.get_remote('origin') + refspec = morphlib.gitdir.RefSpec(build_ref) + push_cb(gd=gd, build_ref=build_ref, + remote=remote, refspec=refspec) + rrm.push(remote, refspec) + self._register_cleanup(rrm.close) @property def root_repo_url(self): '''URI of the repository that systems may be found in.''' - # TODO: When BuildBranches become more context aware, we only - # have to use the file:// URI when there's uncommitted changes - # and we can't push; or HEAD is not pushed and we can't push. - # All other times we can use the pushed branch - return (self._sb.get_config('branch.root') if self._push_temporary - else urlparse.urljoin('file://', self._root.dirname)) + return self._sb.get_config('branch.root') @property def root_ref(self): + return self._sb.get_config('branch.name') + + @property + def root_local_repo_url(self): + return urlparse.urljoin('file://', self._root.dirname) + + @property + def root_build_ref(self): '''Name of the ref of the repository that systems may be found in.''' - # TODO: When BuildBranches become more context aware, this can be - # HEAD when there's no uncommitted changes and we're not pushing; - # or we are pushing and there's no uncommitted changes and HEAD - # has been pushed. build_ref, index = self._to_push[self._root] return build_ref @@ -258,3 +280,47 @@ class BuildBranch(object): exceptions.append(e) if exceptions: raise BuildBranchCleanupError(self, exceptions) + + +@contextlib.contextmanager +def pushed_build_branch(bb, loader, changes_need_pushing, name, email, + build_uuid, status): + with contextlib.closing(bb) as bb: + def report_add(gd, build_ref, changed): + status(msg='Adding uncommitted changes '\ + 'in %(dirname)s to %(ref)s', + dirname=gd.dirname, ref=build_ref, chatty=True) + changes_made = bb.add_uncommitted_changes(add_cb=report_add) + unpushed = any(bb.get_unpushed_branches()) + + if not changes_made and not unpushed: + yield bb.root_repo_url, bb.root_ref + return + + def report_inject(gd): + status(msg='Injecting temporary build refs '\ + 'into morphologies in %(dirname)s', + dirname=gd.dirname, chatty=True) + bb.inject_build_refs(loader=loader, + use_local_repos=not changes_need_pushing, + inject_cb=report_inject) + + def report_commit(gd, build_ref): + status(msg='Committing changes in %(dirname)s '\ + 'to %(ref)s', + dirname=gd.dirname, ref=build_ref, + chatty=True) + bb.update_build_refs(name, email, build_uuid, + commit_cb=report_commit) + + if changes_need_pushing: + def report_push(gd, build_ref, remote, refspec): + status(msg='Pushing %(ref)s in %(dirname)s '\ + 'to %(remote)s', + ref=build_ref, dirname=gd.dirname, + remote=remote.get_push_url(), chatty=True) + bb.push_build_branches(push_cb=report_push) + + yield bb.root_repo_url, bb.root_build_ref + else: + yield bb.root_local_repo_url, bb.root_build_ref diff --git a/morphlib/builder2.py b/morphlib/builder2.py index d739dc13..c1a49221 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -405,7 +405,7 @@ class ChunkBuilder(BuilderBase): for step, in_parallel in steps: with self.build_watch(step): key = '%s-commands' % step - cmds = m.get_commands(key) + cmds = m[key] if cmds: with open(logfilepath, 'a') as log: self.app.status(msg='Running %(key)s', key=key) diff --git a/morphlib/buildsystem.py b/morphlib/buildsystem.py index 90cc15c2..fb99e70e 100644 --- a/morphlib/buildsystem.py +++ b/morphlib/buildsystem.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,6 +16,8 @@ import os +import morphlib + class BuildSystem(object): @@ -49,19 +51,14 @@ class BuildSystem(object): key = '_'.join(key.split('-')) return getattr(self, key) - def get_morphology_text(self, name): + def get_morphology(self, name): '''Return the text of an autodetected chunk morphology.''' - return ''' - { - "name": "%(name)s", - "kind": "chunk", - "build-system": "%(bs)s" - } - ''' % { + return morphlib.morphology.Morphology({ 'name': name, - 'bs': self.name, - } + 'kind': 'chunk', + 'build-system': self.name, + }) def used_by_project(self, file_list): '''Does a project use this build system? diff --git a/morphlib/buildsystem_tests.py b/morphlib/buildsystem_tests.py index 3171366b..56ba64d7 100644 --- a/morphlib/buildsystem_tests.py +++ b/morphlib/buildsystem_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -49,10 +49,10 @@ class BuildSystemTests(unittest.TestCase): def test_has_install_commands(self): self.assertEqual(self.bs['install-commands'], []) - def test_returns_morphology_text(self): + def test_returns_morphology(self): self.bs.name = 'fake' - text = self.bs.get_morphology_text('foobar') - self.assertTrue(type(text) in (str, unicode)) + morph = self.bs.get_morphology('foobar') + self.assertTrue(morph.__class__.__name__ == 'Morphology') class ManualBuildSystemTests(unittest.TestCase): diff --git a/morphlib/cachedrepo.py b/morphlib/cachedrepo.py index 996b42f7..88ceed48 100644 --- a/morphlib/cachedrepo.py +++ b/morphlib/cachedrepo.py @@ -195,15 +195,6 @@ class CachedRepo(object): self._checkout_ref(ref, target_dir) - def load_morphology(self, ref, name): - '''Loads a morphology from a given ref''' - - if not morphlib.git.is_valid_sha1(ref): - ref = self._rev_parse(ref) - text = self.cat(ref, '%s.morph' % name) - morphology = morphlib.morph2.Morphology(text) - return morphology - def ls_tree(self, ref): '''Return file names found in root tree. Does not recurse to subtrees. diff --git a/morphlib/cachedrepo_tests.py b/morphlib/cachedrepo_tests.py index 74c16591..d3ae331a 100644 --- a/morphlib/cachedrepo_tests.py +++ b/morphlib/cachedrepo_tests.py @@ -219,10 +219,6 @@ class CachedRepoTests(unittest.TestCase): morph_filename = os.path.join(unpack_dir, 'foo.morph') self.assertTrue(os.path.exists(morph_filename)) - def test_load_morphology_from_existing_ref(self): - morph = self.repo.load_morphology('master', 'foo') - self.assertTrue(morph['name'] == 'foo') - def test_ls_tree_in_existing_ref(self): data = self.repo.ls_tree('e28a23812eadf2fce6583b8819b9c5dbd36b9fb9') self.assertEqual(data, ['foo.morph']) diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py index b124b789..588fc8d3 100644 --- a/morphlib/cachekeycomputer.py +++ b/morphlib/cachekeycomputer.py @@ -114,7 +114,7 @@ class CacheKeyComputer(object): for prefix in ('pre-', '', 'post-'): for cmdtype in ('configure', 'build', 'test', 'install'): cmd_field = prefix + cmdtype + '-commands' - keys[cmd_field] = morphology.get_commands(cmd_field) + keys[cmd_field] = morphology[cmd_field] keys['devices'] = morphology.get('devices') keys['max-jobs'] = morphology.get('max-jobs') keys['system-integration'] = morphology.get('system-integration', @@ -122,10 +122,7 @@ class CacheKeyComputer(object): # products is omitted as they are part of the split-rules elif kind in ('system', 'stratum'): morphology = artifact.source.morphology - # Exclude fields starting with _orig_. This filtering can be - # removed once the morph2 code is gone. - morph_dict = dict((k, morphology[k]) for k in morphology.keys() - if not k.startswith('_orig_')) + morph_dict = dict((k, morphology[k]) for k in morphology.keys()) # Disregard all fields of a morphology that aren't important ignored_fields = ( diff --git a/morphlib/cachekeycomputer_tests.py b/morphlib/cachekeycomputer_tests.py index 9e18b19d..8558db6d 100644 --- a/morphlib/cachekeycomputer_tests.py +++ b/morphlib/cachekeycomputer_tests.py @@ -32,73 +32,60 @@ class DummyBuildEnvironment: class CacheKeyComputerTests(unittest.TestCase): def setUp(self): + loader = morphlib.morphloader.MorphologyLoader() self.source_pool = morphlib.sourcepool.SourcePool() for name, text in { - 'chunk.morph': '''{ - "name": "chunk", - "kind": "chunk", - "description": "A test chunk" - }''', - 'chunk2.morph': '''{ - "name": "chunk2", - "kind": "chunk", - "description": "A test chunk" - }''', - 'chunk3.morph': '''{ - "name": "chunk3", - "kind": "chunk", - "description": "A test chunk" - }''', - 'stratum.morph': '''{ - "name": "stratum", - "kind": "stratum", - "chunks": [ - { - "name": "chunk", - "repo": "repo", - "ref": "original/ref", - "build-depends": [] - } - ] - }''', - 'stratum2.morph': '''{ - "name": "stratum2", - "kind": "stratum", - "chunks": [ - { - "name": "chunk2", - "repo": "repo", - "ref": "original/ref", - "build-depends": [] - }, - { - "name": "chunk3", - "repo": "repo", - "ref": "original/ref", - "build-depends": [] - } - ] - }''', - 'system.morph': '''{ - "name": "system", - "kind": "system", - "strata": [ - { - "morph": "stratum", - "repo": "repo", - "ref": "original/ref" - }, - { - "morph": "stratum2", - "repo": "repo", - "ref": "original/ref" - } - ] - }''', + 'chunk.morph': ''' + name: chunk + kind: chunk + description: A test chunk + ''', + 'chunk2.morph': ''' + name: chunk2 + kind: chunk + description: A test chunk + ''', + 'chunk3.morph': ''' + name: chunk3 + kind: chunk + description: A test chunk + ''', + 'stratum.morph': ''' + name: stratum + kind: stratum + build-depends: [] + chunks: + - name: chunk + repo: repo + ref: original/ref + build-depends: [] + ''', + 'stratum2.morph': ''' + name: stratum2 + kind: stratum + build-depends: [] + chunks: + - name: chunk2 + repo: repo + ref: original/ref + build-depends: [] + - name: chunk3 + repo: repo + ref: original/ref + build-depends: [] + ''', + 'system.morph': ''' + name: system + kind: system + arch: testarch + strata: + - morph: stratum + - morph: stratum2 + ''', }.iteritems(): source = morphlib.source.Source( 'repo', 'original/ref', 'sha', 'tree', - morphlib.morph2.Morphology(text), name) + loader.load_from_string(text), name) self.source_pool.add(source) # FIXME: This should use MorphologyFactory m = source.morphology @@ -202,7 +189,12 @@ class CacheKeyComputerTests(unittest.TestCase): self.assertEqual(old_sha, new_sha) def test_same_morphology_added_to_source_pool_only_appears_once(self): - m = morphlib.morph2.Morphology('{"name": "chunk", "kind": "chunk"}') + loader = morphlib.morphloader.MorphologyLoader() + m = loader.load_from_string( + ''' + name: chunk + kind: chunk + ''') src = morphlib.source.Source('repo', 'original/ref', 'sha', 'tree', m, 'chunk.morph') sp = morphlib.sourcepool.SourcePool() diff --git a/morphlib/extensions.py b/morphlib/extensions.py index be551fdd..55478418 100644 --- a/morphlib/extensions.py +++ b/morphlib/extensions.py @@ -30,26 +30,20 @@ class ExtensionNotFoundError(ExtensionError): class ExtensionNotExecutableError(ExtensionError): pass -def _get_root_repo(build_ref_prefix): +def _get_root_repo(): system_branch = morphlib.sysbranchdir.open_from_within('.') root_repo_dir = morphlib.gitdir.GitDirectory( system_branch.get_git_directory_name( system_branch.root_repository_url)) - build_branch = morphlib.buildbranch.BuildBranch(system_branch, - build_ref_prefix, - push_temporary=False) - ref = build_branch.root_ref - - return (build_branch.root_ref, root_repo_dir) + return root_repo_dir def _get_morph_extension_directory(): code_dir = os.path.dirname(morphlib.__file__) return os.path.join(code_dir, 'exts') -def _list_repo_extension_filenames(build_ref_prefix, - kind): #pragma: no cover - (ref, repo_dir) = _get_root_repo(build_ref_prefix) - files = repo_dir.list_files(ref) +def _list_repo_extension_filenames(kind): #pragma: no cover + repo_dir = _get_root_repo() + files = repo_dir.list_files() return (f for f in files if os.path.splitext(f)[1] == kind) def _list_morph_extension_filenames(kind): @@ -59,9 +53,9 @@ def _list_morph_extension_filenames(kind): def _get_extension_name(filename): return os.path.basename(filename) -def _get_repo_extension_contents(build_ref_prefix, name, kind): - (ref, repo_dir) = _get_root_repo(build_ref_prefix) - return repo_dir.get_file_from_ref(ref, name + kind) +def _get_repo_extension_contents(name, kind): + repo_dir = _get_root_repo() + return repo_dir.read_file(name + kind) def _get_morph_extension_filename(name, kind): return os.path.join(_get_morph_extension_directory(), name + kind) @@ -71,11 +65,11 @@ def _is_executable(filename): mask = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH return (stat.S_IMODE(st.st_mode) & mask) != 0 -def _list_extensions(build_ref_prefix, kind): +def _list_extensions(kind): repo_extension_filenames = [] try: repo_extension_filenames = \ - _list_repo_extension_filenames(build_ref_prefix, kind) + _list_repo_extension_filenames(kind) except (sysbranchdir.NotInSystemBranch): # Squash this and just return no system branch extensions pass @@ -90,7 +84,7 @@ def _list_extensions(build_ref_prefix, kind): extension_names.update(set(morph_extension_names)) return list(extension_names) -def list_extensions(build_ref_prefix, kind=None): +def list_extensions(kind=None): """ List all available extensions by 'kind'. @@ -102,10 +96,10 @@ def list_extensions(build_ref_prefix, kind=None): be associated with a '.write' extension of the same name. """ if kind: - return _list_extensions(build_ref_prefix, kind) + return _list_extensions(kind) else: - configure_extensions = _list_extensions(build_ref_prefix, '.configure') - write_extensions = _list_extensions(build_ref_prefix, '.write') + configure_extensions = _list_extensions('.configure') + write_extensions = _list_extensions('.write') return configure_extensions + write_extensions @@ -121,8 +115,7 @@ class get_extension_filename(): If the extension is in the build repository then a temporary file will be created, which will be deleted on exting the with block. """ - def __init__(self, build_ref_prefix, name, kind, executable=True): - self.build_ref_prefix = build_ref_prefix + def __init__(self, name, kind, executable=True): self.name = name self.kind = kind self.executable = executable @@ -131,10 +124,9 @@ class get_extension_filename(): def __enter__(self): ext_filename = None try: - ext_contents = _get_repo_extension_contents(self.build_ref_prefix, - self.name, + ext_contents = _get_repo_extension_contents(self.name, self.kind) - except cliapp.AppException, sysbranchdir.NotInSystemBranch: + except (IOError, cliapp.AppException, sysbranchdir.NotInSystemBranch): # Not found: look for it in the Morph code. ext_filename = _get_morph_extension_filename(self.name, self.kind) if not os.path.exists(ext_filename): diff --git a/morphlib/gitdir.py b/morphlib/gitdir.py index 5b0693cb..fea26c2e 100644 --- a/morphlib/gitdir.py +++ b/morphlib/gitdir.py @@ -282,6 +282,16 @@ class Remote(object): return self._get_remote_url(self.name, 'push') @staticmethod + def _parse_ls_remote_output(output): # pragma: no cover + for line in output.splitlines(): + sha1, refname = line.split(None, 1) + yield sha1, refname + + def ls(self): # pragma: no cover + out = self.gd._runcmd(['git', 'ls-remote', self.get_fetch_url()]) + return self._parse_ls_remote_output(out) + + @staticmethod def _parse_push_output(output): for line in output.splitlines(): m = PUSH_FORMAT.match(line) @@ -484,6 +494,19 @@ class GitDirectory(object): except cliapp.AppException as e: raise InvalidRefError(self, ref) + def disambiguate_ref(self, ref): # pragma: no cover + try: + out = self._runcmd(['git', 'rev-parse', '--symbolic-full-name', + ref]) + return out.strip() + except cliapp.AppException: # ref not found + if ref.startswith('refs/heads/'): + return ref + elif ref.startswith('heads/'): + return 'refs/' + ref + else: + return 'refs/heads/' + ref + def resolve_ref_to_commit(self, ref): return self._rev_parse('%s^{commit}' % ref) diff --git a/morphlib/gitindex.py b/morphlib/gitindex.py index 978ea0e2..6be4aacb 100644 --- a/morphlib/gitindex.py +++ b/morphlib/gitindex.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -97,7 +97,8 @@ class GitIndex(object): def get_uncommitted_changes(self): for code, to_path, from_path in self._get_status(): - if code not in (STATUS_UNTRACKED, STATUS_IGNORED): + if (code not in (STATUS_UNTRACKED, STATUS_IGNORED) + or code == (STATUS_UNTRACKED) and to_path.endswith('.morph')): yield code, to_path, from_path def set_to_tree(self, treeish): diff --git a/morphlib/localartifactcache_tests.py b/morphlib/localartifactcache_tests.py index f400a645..6283c833 100644 --- a/morphlib/localartifactcache_tests.py +++ b/morphlib/localartifactcache_tests.py @@ -27,23 +27,21 @@ class LocalArtifactCacheTests(unittest.TestCase): def setUp(self): self.tempfs = fs.tempfs.TempFS() - morph = morphlib.morph2.Morphology( + loader = morphlib.morphloader.MorphologyLoader() + morph = loader.load_from_string( ''' - { - "name": "chunk", - "kind": "chunk", - "artifacts": { - "chunk-runtime": [ - "usr/bin", - "usr/sbin", - "usr/lib", - "usr/libexec" - ], - "chunk-devel": [ - "usr/include" - ] - } - } + name: chunk + kind: chunk + products: + - artifact: chunk-runtime + include: + - usr/bin + - usr/sbin + - usr/lib + - usr/libexec + - artifact: chunk-devel + include: + - usr/include ''') self.source = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'chunk.morph') diff --git a/morphlib/morph2.py b/morphlib/morph2.py deleted file mode 100644 index b49c0f73..00000000 --- a/morphlib/morph2.py +++ /dev/null @@ -1,313 +0,0 @@ -# Copyright (C) 2012-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import re - -import morphlib -from morphlib.util import OrderedDict, json - -class Morphology(object): - - '''An in-memory representation of a morphology. - - This is a parsed version of the morphology, with rules for default - values applied. No other processing. - - ''' - - static_defaults = { - 'chunk': [ - ('description', ''), - ('pre-configure-commands', None), - ('configure-commands', None), - ('post-configure-commands', None), - ('pre-build-commands', None), - ('build-commands', None), - ('post-build-commands', None), - ('pre-test-commands', None), - ('test-commands', None), - ('post-test-commands', None), - ('pre-install-commands', None), - ('install-commands', None), - ('post-install-commands', None), - ('devices', None), - ('products', []), - ('max-jobs', None), - ('build-system', 'manual') - ], - 'stratum': [ - ('chunks', []), - ('description', ''), - ('build-depends', None), - ], - 'system': [ - ('strata', []), - ('description', ''), - ('arch', None), - ('configuration-extensions', []), - ], - 'cluster': [ - ('description', ''), - ], - } - - @staticmethod - def _load_json(text): - return json.loads(text, object_pairs_hook=OrderedDict, - encoding='unicode-escape') - - @staticmethod - def _dump_json(obj, f): - text = json.dumps(obj, indent=4, encoding='unicode-escape') - text = re.sub(" \n", "\n", text) - f.write(text) - f.write('\n') - - def __init__(self, text): - self._dict, self._dumper = self._load_morphology_dict(text) - self._set_defaults() - self._validate_children() - - def __getitem__(self, key): - return self._dict[key] - - def __contains__(self, key): - return key in self._dict - - # Not covered by tests, since it's trivial, morph2 is going away - # and only exists so the new morphology validation code can use it. - def get(self, key, default=None): # pragma: no cover - try: - return self[key] - except KeyError: - return default - - def get_commands(self, which): - '''Return the commands to run from a morphology or the build system''' - if self[which] is None: - attr = '_'.join(which.split('-')) - bs = morphlib.buildsystem.lookup_build_system(self['build-system']) - return getattr(bs, attr) - else: - return self[which] - - def keys(self): - return self._dict.keys() - - def _load_morphology_dict(self, text): - '''Load morphology, identifying whether it is JSON or YAML''' - - try: - data = self._load_json(text) - dumper = self._dump_json - except ValueError as e: # pragma: no cover - data = morphlib.yamlparse.load(text) - dumper = morphlib.yamlparse.dump - - if data is None: - raise morphlib.YAMLError("Morphology is empty") - if type(data) not in [dict, OrderedDict]: - raise morphlib.YAMLError("Morphology did not parse as a dict") - - return data, dumper - - def _validate_children(self): - if self['kind'] == 'system': - names = set() - for info in self['strata']: - name = info.get('alias', info['morph']) - if name in names: - raise ValueError('Duplicate stratum "%s"' % name) - names.add(name) - elif self['kind'] == 'stratum': - names = set() - for info in self['chunks']: - name = info.get('alias', info['name']) - if name in names: - raise ValueError('Duplicate chunk "%s"' % name) - names.add(name) - elif self['kind'] == 'cluster': - if not 'systems' in self: - raise KeyError('"systems" not found') - if not self['systems']: - raise ValueError('"systems" is empty') - for system in self['systems']: - if 'morph' not in system: - raise KeyError('"morph" not found') - if 'deploy-defaults' in system: - if not isinstance(system['deploy-defaults'], dict): - raise ValueError('deploy defaults for morph "%s" ' - 'are not a mapping: %r' - % (system['morph'], - system['deploy-defaults'])) - if 'deploy' in system: - for system_id, deploy_params in system['deploy'].items(): - if not isinstance(deploy_params, dict): - raise ValueError('deployment parameters for ' - 'system "%s" are not a mapping:' - ' %r' - % (system_id, deploy_params)) - - def _set_default_value(self, target_dict, key, value): - '''Change a value in the in-memory representation of the morphology - - Record the default value separately, so that when writing out the - morphology we can determine whether the change from the on-disk value - was done at load time, or later on (we want to only write back out - the later, deliberate changes). - - ''' - target_dict[key] = value - target_dict['_orig_' + key] = value - - def _set_defaults(self): - if 'max-jobs' in self: - self._set_default_value(self._dict, 'max-jobs', - int(self['max-jobs'])) - - for name, value in self.static_defaults[self['kind']]: - if name not in self._dict: - self._set_default_value(self._dict, name, value) - - if self['kind'] == 'stratum': - self._set_stratum_defaults() - elif self['kind'] == 'cluster': - self._set_cluster_defaults() - - def _set_stratum_defaults(self): - for source in self['chunks']: - if 'repo' not in source: - self._set_default_value(source, 'repo', source['name']) - if 'build-depends' not in source: - self._set_default_value(source, 'build-depends', None) - if 'build-mode' not in source: - self._set_default_value(source, 'build-mode', 'staging') - if 'prefix' not in source: - self._set_default_value(source, 'prefix', '/usr') - - def _set_cluster_defaults(self): - if 'systems' in self and self['systems']: - for system in self['systems']: - if 'deploy-defaults' not in system: - self._set_default_value(system, - 'deploy-defaults', - dict()) - if 'deploy' not in system: - self._set_default_value(system, - 'deploy', - dict()) - - def lookup_child_by_name(self, name): - '''Find child reference by its name. - - This lookup honors aliases. - - ''' - - if self['kind'] == 'system': - for info in self['strata']: - source_name = info.get('alias', info['morph']) - if source_name == name: - return info - elif self['kind'] == 'stratum': - for info in self['chunks']: - source_name = info.get('alias', info['name']) - if source_name == name: - return info - raise KeyError('"%s" not found' % name) - - def _apply_changes(self, live_dict, original_dict): - '''Returns a new dict updated with changes from the in-memory object - - This allows us to write out a morphology including only the changes - that were done after the morphology was loaded -- not the changes done - to set default values during construction. - - ''' - output_dict = {} - - for key in live_dict.keys(): - if key.startswith('_orig_'): - continue - - value = self._apply_changes_for_key(key, live_dict, original_dict) - # VILE HACK to preserve nulls in repo/ref fields - if value is not None or key in ('repo', 'ref'): - output_dict[key] = value - return output_dict - - def _apply_changes_for_key(self, key, live_dict, original_dict): - '''Return value to write out for one key, recursing if necessary''' - - live_value = live_dict.get(key, None) - orig_value = original_dict.get(key, None) - - if type(live_value) in [dict, OrderedDict] and orig_value is not None: - # Recursively apply changes for dict - result = self._apply_changes(live_value, orig_value) - elif type(live_value) is list and orig_value is not None: - # Recursively apply changes for list (existing, then new items). - result = [] - for i in range(0, min(len(orig_value), len(live_value))): - if type(live_value[i]) in [dict, OrderedDict]: - item = self._apply_changes(live_value[i], orig_value[i]) - else: - item = live_value[i] - result.append(item) - for i in range(len(orig_value), len(live_value)): - if type(live_value[i]) in [dict, OrderedDict]: - item = self._apply_changes(live_value[i], {}) - else: - item = live_value[i] - result.append(item) - else: - # Simple values. Use original value unless it has been changed from - # the default in memmory. - if live_dict[key] == live_dict.get('_orig_' + key, None): - result = original_dict.get(key, None) - else: - result = live_dict[key] - return result - - def update_text(self, text, output_fd, convert_to=None): - '''Write out in-memory changes to loaded morphology text - - Similar in function to update_file(). - - ''' - original_dict, dumper = self._load_morphology_dict(text) - - if convert_to == 'json': # pragma: no cover - dumper = self._dump_json - elif convert_to == 'yaml': # pragma: no cover - dumper = morphlib.yamlparse.dump - - output_dict = self._apply_changes(self._dict, original_dict) - dumper(output_dict, output_fd) - - def update_file(self, filename, output_fd=None, **kws): # pragma: no cover - '''Write out in-memory changes to on-disk morphology file - - This function reads the original morphology text from 'filename', so - that it can avoid writing out properties that are set in memory - to their default value but weren't specified by the user at all. - - ''' - with open(filename, 'r') as f: - text = f.read() - - with output_fd or morphlib.savefile.SaveFile(filename, 'w') as f: - self.update_text(text, f, **kws) diff --git a/morphlib/morph2_tests.py b/morphlib/morph2_tests.py deleted file mode 100644 index c9957ad5..00000000 --- a/morphlib/morph2_tests.py +++ /dev/null @@ -1,391 +0,0 @@ -# Copyright (C) 2012-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import copy -import json -import StringIO -import unittest - -import yaml - -import morphlib -from morphlib.morph2 import Morphology - - -class MorphologyTests(unittest.TestCase): - - def test_parses_simple_json_chunk(self): - m = Morphology(''' - { - "name": "foo", - "kind": "chunk", - "build-system": "manual" - } - ''') - - self.assertEqual(m['name'], 'foo') - self.assertEqual(m['kind'], 'chunk') - self.assertEqual(m['build-system'], 'manual') - self.assertEqual(m['pre-configure-commands'], None) - self.assertEqual(m['configure-commands'], None) - self.assertEqual(m['post-configure-commands'], None) - self.assertEqual(m['pre-build-commands'], None) - self.assertEqual(m['build-commands'], None) - self.assertEqual(m['post-build-commands'], None) - self.assertEqual(m['pre-test-commands'], None) - self.assertEqual(m['test-commands'], None) - self.assertEqual(m['post-test-commands'], None) - self.assertEqual(m['pre-install-commands'], None) - self.assertEqual(m['install-commands'], None) - self.assertEqual(m['post-install-commands'], None) - self.assertEqual(m['max-jobs'], None) - self.assertEqual(m['products'], []) - - if morphlib.got_yaml: - def test_parses_simple_yaml_chunk(self): - m = Morphology(''' - name: foo - kind: chunk - build-system: manual - ''') - - self.assertEqual(m['name'], 'foo') - self.assertEqual(m['kind'], 'chunk') - self.assertEqual(m['build-system'], 'manual') - self.assertEqual(m['pre-configure-commands'], None) - self.assertEqual(m['configure-commands'], None) - self.assertEqual(m['post-configure-commands'], None) - self.assertEqual(m['pre-build-commands'], None) - self.assertEqual(m['build-commands'], None) - self.assertEqual(m['post-build-commands'], None) - self.assertEqual(m['pre-test-commands'], None) - self.assertEqual(m['test-commands'], None) - self.assertEqual(m['post-test-commands'], None) - self.assertEqual(m['pre-install-commands'], None) - self.assertEqual(m['install-commands'], None) - self.assertEqual(m['post-install-commands'], None) - self.assertEqual(m['max-jobs'], None) - self.assertEqual(m['products'], []) - - def test_sets_stratum_chunks_repo_and_morph_from_name(self): - m = Morphology(''' - { - "name": "foo", - "kind": "stratum", - "chunks": [ - { - "name": "le-chunk", - "ref": "ref" - } - ] - } - ''') - - self.assertEqual(m['chunks'][0]['repo'], 'le-chunk') - self.assertEqual(m['chunks'][0]['build-depends'], None) - - def test_returns_dict_keys(self): - m = Morphology(''' - { - "name": "foo", - "kind": "system", - } - ''') - - self.assertTrue('name' in m.keys()) - self.assertTrue('kind' in m.keys()) - - def test_system_indexes_strata(self): - m = Morphology(''' - { - "kind": "system", - "strata": [ - { - "morph": "stratum1", - "repo": "repo", - "ref": "ref" - }, - { - "alias": "aliased-stratum", - "morph": "stratum2", - "repo": "repo", - "ref": "ref" - } - ] - } - ''') - self.assertEqual(m.lookup_child_by_name('stratum1'), - {'morph': 'stratum1', 'repo': 'repo', 'ref': 'ref' }) - self.assertEqual(m.lookup_child_by_name('aliased-stratum'), - {'alias': 'aliased-stratum', 'morph': 'stratum2', - 'repo': 'repo', 'ref': 'ref'}) - - def test_stratum_indexes_chunks(self): - m = Morphology(''' - { - "kind": "stratum", - "chunks": [ - { - "name": "chunk", - "repo": "repo", - "ref": "ref" - } - ] - } - ''') - - child = m.lookup_child_by_name('chunk') - self.assertEqual(child['name'], 'chunk') - self.assertEqual(child['repo'], 'repo') - self.assertEqual(child['ref'], 'ref') - - def test_raises_error_when_child_lookup_fails(self): - m = Morphology(''' - { - "kind": "stratum", - "chunks": [ - { - "name": "chunk", - "repo": "repo", - "ref": "ref" - } - ] - } - ''') - - self.assertRaises(KeyError, m.lookup_child_by_name, 'foo') - - ## Validation tests - - def test_not_empty(self): - self.assertRaises(morphlib.YAMLError, Morphology, '') - - def test_is_dict(self): - self.assertRaises(morphlib.YAMLError, Morphology, 'foo') - - def test_makes_max_jobs_be_an_integer(self): - m = Morphology(''' - { - "name": "foo", - "kind": "chunk", - "max-jobs": "42" - } - ''') - self.assertEqual(m['max-jobs'], 42) - - def test_stratum_names_must_be_unique_within_a_system(self): - text = ''' - { - "kind": "system", - "strata": [ - { - "morph": "stratum", - "repo": "test1", - "ref": "ref" - }, - { - "morph": "stratum", - "repo": "test2", - "ref": "ref" - } - ] - } - ''' - self.assertRaises(ValueError, - Morphology, - text) - - def test_chunk_names_must_be_unique_within_a_stratum(self): - text = ''' - { - "kind": "stratum", - "chunks": [ - { - "name": "chunk", - "repo": "test1", - "ref": "ref" - }, - { - "name": "chunk", - "repo": "test2", - "ref": "ref" - } - ] - } - ''' - self.assertRaises(ValueError, - Morphology, - text) - - ## Writing tests - - stratum_text = '''{ - "kind": "stratum", - "chunks": [ - { - "name": "foo", - "repo": "morphs", - "ref": "ref", - "build-depends": [] - }, - { - "name": "bar", - "repo": "morphs", - "ref": "ref", - "build-depends": [ - "foo" - ] - } - ] -}''' - - def test_writing_handles_added_chunks(self): - text_lines = self.stratum_text.splitlines() - text_lines = text_lines[0:16] + text_lines[8:17] + text_lines[17:] - text_lines[18] = ' "name": "baz",' - - # Add a new chunk to the list - morphology = Morphology(self.stratum_text) - morphology['chunks'].append(copy.copy(morphology['chunks'][1])) - morphology['chunks'][2]['name'] = 'baz' - - output = StringIO.StringIO() - morphology.update_text(self.stratum_text, output) - d = yaml.load(output.getvalue()) - self.assertEqual(d['chunks'][2]['name'], 'baz') - - def test_writing_handles_deleted_chunks(self): - text_lines = self.stratum_text.splitlines() - text_lines = text_lines[0:3] + text_lines[9:] - - # Delete a chunk - morphology = Morphology(self.stratum_text) - del morphology['chunks'][0] - - output = StringIO.StringIO() - morphology.update_text(self.stratum_text, output) - d = yaml.load(output.getvalue()) - self.assertEqual(len(d['chunks']), 1) - - system_text = '''{ - "kind": "system", - "arch": "x86_64", -}''' - - def test_nested_dict(self): - # Real morphologies don't trigger this code path, so we test manually - original_dict = { - 'dict': { '1': 'fee', '2': 'fie', '3': 'foe', '4': 'foo' } - } - live_dict = copy.deepcopy(original_dict) - live_dict['_orig_dict'] = live_dict['dict'] - - dummy = Morphology(self.stratum_text) - output_dict = dummy._apply_changes(live_dict, original_dict) - self.assertEqual(original_dict, output_dict) - - def test_uses_morphology_commands_when_given(self): - m = Morphology(''' - { - 'name': 'foo', - 'kind': 'chunk', - 'build-system': 'dummy', - 'build-commands': ['build-it'] - } - ''') - cmds = m.get_commands('build-commands') - self.assertEqual(cmds, ['build-it']) - - def test_uses_build_system_commands_when_morphology_doesnt(self): - m = Morphology(''' - { - 'name': 'foo', - 'kind': 'chunk', - 'build-system': 'dummy', - } - ''') - cmds = m.get_commands('build-commands') - self.assertEqual(cmds, ['echo dummy build']) - - def test_uses_morphology_commands_when_morphology_has_empty_list(self): - m = Morphology(''' - { - 'name': 'foo', - 'kind': 'chunk', - 'build-system': 'dummy', - 'build-commands': [] - } - ''') - cmds = m.get_commands('build-commands') - self.assertEqual(cmds, []) - - ## Cluster morphologies tests - - def test_parses_simple_cluster_morph(self): - m = Morphology(''' - name: foo - kind: cluster - systems: - - morph: bar - ''') - self.assertEqual(m['name'], 'foo') - self.assertEqual(m['kind'], 'cluster') - self.assertEqual(m['systems'][0]['morph'], 'bar') - - def test_fails_without_systems(self): - text = ''' - name: foo - kind: cluster - ''' - self.assertRaises(KeyError, Morphology, text) - - def test_fails_with_empty_systems(self): - text = ''' - name: foo - kind: cluster - systems: - ''' - self.assertRaises(ValueError, Morphology, text) - - def test_fails_without_morph(self): - text = ''' - name: foo - kind: cluster - systems: - - deploy: - ''' - self.assertRaises(KeyError, Morphology, text) - - def test_fails_with_invalid_deploy_defaults(self): - text = ''' - name: foo - kind: cluster - systems: - - morph: bar - deploy-defaults: ooops_i_am_not_a_mapping - ''' - self.assertRaises(ValueError, Morphology, text) - - def test_fails_with_invalid_deployment_params(self): - text = ''' - name: foo - kind: cluster - systems: - - morph: bar - deploy: - qux: ooops_i_am_not_a_mapping - ''' - self.assertRaises(ValueError, Morphology, text) diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py index 21e10827..05dcb62c 100644 --- a/morphlib/morphloader.py +++ b/morphlib/morphloader.py @@ -354,6 +354,8 @@ class MorphologyLoader(object): 'products': [], 'max-jobs': None, 'build-system': 'manual', + 'build-mode': 'staging', + 'prefix': '/usr', }, 'stratum': { 'chunks': [], @@ -392,7 +394,7 @@ class MorphologyLoader(object): if not isinstance(obj, dict): raise NotADictionaryError(morph_filename) - return morphlib.morph3.Morphology(obj) + return morphlib.morphology.Morphology(obj) def load_from_string(self, string, filename='string'): '''Load a morphology from a string. @@ -404,6 +406,7 @@ class MorphologyLoader(object): m = self.parse_morphology_text(string, filename) m.filename = filename self.validate(m) + self.set_commands(m) self.set_defaults(m) return m @@ -438,9 +441,6 @@ class MorphologyLoader(object): self._require_field('kind', morph) # The rest of the validation is dependent on the kind. - - # FIXME: move validation of clusters from morph2 to - # here, and use morphload to load the morphology kind = morph['kind'] if kind not in ('system', 'stratum', 'chunk', 'cluster'): raise UnknownKindError(morph['kind'], morph.filename) @@ -731,19 +731,37 @@ class MorphologyLoader(object): for spec in morph['chunks']: if 'repo' not in spec: spec['repo'] = spec['name'] - if 'morph' not in spec: - spec['morph'] = spec['name'] + if 'build-mode' not in spec: + spec['build-mode'] = \ + self._static_defaults['chunk']['build-mode'] + if 'prefix' not in spec: + spec['prefix'] = \ + self._static_defaults['chunk']['prefix'] self._set_stratum_specs_defaults(morph, 'build-depends') def _unset_stratum_defaults(self, morph): for spec in morph['chunks']: if 'repo' in spec and spec['repo'] == spec['name']: del spec['repo'] - if 'morph' in spec and spec['morph'] == spec['name']: - del spec['morph'] + if 'build-mode' in spec and spec['build-mode'] == \ + self._static_defaults['chunk']['build-mode']: + del spec['build-mode'] + if 'prefix' in spec and spec['prefix'] == \ + self._static_defaults['chunk']['prefix']: + del spec['prefix'] self._unset_stratum_specs_defaults(morph, 'strata') def _set_chunk_defaults(self, morph): if morph['max-jobs'] is not None: morph['max-jobs'] = int(morph['max-jobs']) + def set_commands(self, morph): + if morph['kind'] == 'chunk': + for key in self._static_defaults['chunk']: + if 'commands' not in key: continue + if key not in morph: + attr = '_'.join(key.split('-')) + default = self._static_defaults['chunk']['build-system'] + bs = morphlib.buildsystem.lookup_build_system( + morph.get('build-system', default)) + morph[key] = getattr(bs, attr) diff --git a/morphlib/morphloader_tests.py b/morphlib/morphloader_tests.py index f4d2f9b6..d47ec750 100644 --- a/morphlib/morphloader_tests.py +++ b/morphlib/morphloader_tests.py @@ -59,21 +59,21 @@ build-system: dummy self.loader.parse_morphology_text, '- item1\n- item2\n', 'test') def test_fails_to_validate_dict_without_kind(self): - m = morphlib.morph3.Morphology({ + m = morphlib.morphology.Morphology({ 'invalid': 'field', }) self.assertRaises( morphlib.morphloader.MissingFieldError, self.loader.validate, m) def test_fails_to_validate_chunk_with_no_fields(self): - m = morphlib.morph3.Morphology({ + m = morphlib.morphology.Morphology({ 'kind': 'chunk', }) self.assertRaises( morphlib.morphloader.MissingFieldError, self.loader.validate, m) def test_fails_to_validate_chunk_with_invalid_field(self): - m = morphlib.morph3.Morphology({ + m = morphlib.morphology.Morphology({ 'kind': 'chunk', 'name': 'foo', 'invalid': 'field', @@ -82,7 +82,7 @@ build-system: dummy morphlib.morphloader.InvalidFieldError, self.loader.validate, m) def test_validate_requires_products_list(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( kind='chunk', name='foo', products={ @@ -98,7 +98,7 @@ build-system: dummy self.assertEqual(e.morphology_name, 'foo') def test_validate_requires_products_list_of_mappings(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( kind='chunk', name='foo', products=[ @@ -113,7 +113,7 @@ build-system: dummy self.assertEqual(e.morphology_name, 'foo') def test_validate_requires_products_list_required_fields(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( kind='chunk', name='foo', products=[ @@ -136,7 +136,7 @@ build-system: dummy self.assertEqual(exs[3].field, 'products[0].factiart') def test_validate_requires_products_list_include_is_list(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( kind='chunk', name='foo', products=[ @@ -154,7 +154,7 @@ build-system: dummy self.assertEqual(ex.morphology_name, 'foo') def test_validate_requires_products_list_include_is_list_of_strings(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( kind='chunk', name='foo', products=[ @@ -175,14 +175,14 @@ build-system: dummy def test_fails_to_validate_stratum_with_no_fields(self): - m = morphlib.morph3.Morphology({ + m = morphlib.morphology.Morphology({ 'kind': 'stratum', }) self.assertRaises( morphlib.morphloader.MissingFieldError, self.loader.validate, m) def test_fails_to_validate_stratum_with_invalid_field(self): - m = morphlib.morph3.Morphology({ + m = morphlib.morphology.Morphology({ 'kind': 'stratum', 'name': 'foo', 'invalid': 'field', @@ -191,7 +191,7 @@ build-system: dummy morphlib.morphloader.InvalidFieldError, self.loader.validate, m) def test_validate_requires_chunk_refs_in_stratum_to_be_strings(self): - m = morphlib.morph3.Morphology({ + m = morphlib.morphology.Morphology({ 'kind': 'stratum', 'name': 'foo', 'build-depends': [], @@ -209,7 +209,7 @@ build-system: dummy self.loader.validate(m) def test_fails_to_validate_stratum_with_empty_refs_for_a_chunk(self): - m = morphlib.morph3.Morphology({ + m = morphlib.morphology.Morphology({ 'kind': 'stratum', 'name': 'foo', 'build-depends': [], @@ -227,7 +227,7 @@ build-system: dummy self.loader.validate(m) def test_fails_to_validate_system_with_obsolete_system_kind_field(self): - m = morphlib.morph3.Morphology({ + m = morphlib.morphology.Morphology({ 'kind': 'system', 'name': 'foo', 'arch': 'x86_64', @@ -240,7 +240,7 @@ build-system: dummy morphlib.morphloader.ObsoleteFieldsError, self.loader.validate, m) def test_fails_to_validate_system_with_obsolete_disk_size_field(self): - m = morphlib.morph3.Morphology({ + m = morphlib.morphology.Morphology({ 'kind': 'system', 'name': 'foo', 'arch': 'x86_64', @@ -253,14 +253,14 @@ build-system: dummy morphlib.morphloader.ObsoleteFieldsError, self.loader.validate, m) def test_fails_to_validate_system_with_no_fields(self): - m = morphlib.morph3.Morphology({ + m = morphlib.morphology.Morphology({ 'kind': 'system', }) self.assertRaises( morphlib.morphloader.MissingFieldError, self.loader.validate, m) def test_fails_to_validate_system_with_invalid_field(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( kind="system", name="foo", arch="blah", @@ -272,14 +272,14 @@ build-system: dummy morphlib.morphloader.InvalidFieldError, self.loader.validate, m) def test_fails_to_validate_morphology_with_unknown_kind(self): - m = morphlib.morph3.Morphology({ + m = morphlib.morphology.Morphology({ 'kind': 'invalid', }) self.assertRaises( morphlib.morphloader.UnknownKindError, self.loader.validate, m) def test_validate_requires_unique_stratum_names_within_a_system(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( { "kind": "system", "name": "foo", @@ -301,7 +301,7 @@ build-system: dummy self.loader.validate, m) def test_validate_requires_unique_chunk_names_within_a_stratum(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( { "kind": "stratum", "name": "foo", @@ -322,7 +322,7 @@ build-system: dummy self.loader.validate, m) def test_validate_requires_a_valid_architecture(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( kind="system", name="foo", arch="blah", @@ -334,7 +334,7 @@ build-system: dummy self.loader.validate, m) def test_validate_normalises_architecture_armv7_to_armv7l(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( kind="system", name="foo", arch="armv7", @@ -345,7 +345,7 @@ build-system: dummy self.assertEqual(m['arch'], 'armv7l') def test_validate_requires_build_deps_for_chunks_in_strata(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( { "kind": "stratum", "name": "foo", @@ -365,7 +365,7 @@ build-system: dummy self.loader.validate, m) def test_validate_requires_build_deps_or_bootstrap_mode_for_strata(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( { "name": "stratum-no-bdeps-no-bootstrap", "kind": "stratum", @@ -395,7 +395,7 @@ build-system: dummy self.loader.validate(m) def test_validate_requires_chunks_in_strata(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( { "name": "stratum", "kind": "stratum", @@ -415,7 +415,7 @@ build-system: dummy self.loader.validate, m) def test_validate_requires_strata_in_system(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( name='system', kind='system', arch='testarch') @@ -425,7 +425,7 @@ build-system: dummy def test_validate_requires_list_of_strata_in_system(self): for v in (None, {}): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( name='system', kind='system', arch='testarch', @@ -437,7 +437,7 @@ build-system: dummy self.assertEqual(cm.exception.strata_type, type(v)) def test_validate_requires_non_empty_strata_in_system(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( name='system', kind='system', arch='testarch', @@ -447,7 +447,7 @@ build-system: dummy self.loader.validate, m) def test_validate_requires_stratum_specs_in_system(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( name='system', kind='system', arch='testarch', @@ -460,7 +460,7 @@ build-system: dummy def test_validate_requires_unique_deployment_names_in_cluster(self): subsystem = [{'morph': 'baz', 'deploy': {'foobar': None}}] - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( name='cluster', kind='cluster', systems=[{'morph': 'foo', @@ -513,7 +513,7 @@ build-system: dummy self.assertEqual(morph['build-system'], 'dummy') def test_saves_to_string(self): - morph = morphlib.morph3.Morphology({ + morph = morphlib.morphology.Morphology({ 'name': 'foo', 'kind': 'chunk', 'build-system': 'dummy', @@ -529,7 +529,7 @@ build-system: dummy ''') def test_saves_to_file(self): - morph = morphlib.morph3.Morphology({ + morph = morphlib.morphology.Morphology({ 'name': 'foo', 'kind': 'chunk', 'build-system': 'dummy', @@ -548,7 +548,7 @@ build-system: dummy ''') def test_validate_does_not_set_defaults(self): - m = morphlib.morph3.Morphology({ + m = morphlib.morphology.Morphology({ 'kind': 'chunk', 'name': 'foo', }) @@ -556,7 +556,7 @@ build-system: dummy self.assertEqual(sorted(m.keys()), sorted(['kind', 'name'])) def test_sets_defaults_for_chunks(self): - m = morphlib.morph3.Morphology({ + m = morphlib.morphology.Morphology({ 'kind': 'chunk', 'name': 'foo', }) @@ -569,6 +569,7 @@ build-system: dummy 'name': 'foo', 'description': '', 'build-system': 'manual', + 'build-mode': 'staging', 'configure-commands': [], 'pre-configure-commands': [], @@ -589,10 +590,11 @@ build-system: dummy 'products': [], 'devices': [], 'max-jobs': None, + 'prefix': '/usr', }) def test_unsets_defaults_for_chunks(self): - m = morphlib.morph3.Morphology({ + m = morphlib.morphology.Morphology({ 'kind': 'chunk', 'name': 'foo', 'build-system': 'manual', @@ -606,7 +608,7 @@ build-system: dummy }) def test_sets_defaults_for_strata(self): - m = morphlib.morph3.Morphology({ + m = morphlib.morphology.Morphology({ 'kind': 'stratum', 'name': 'foo', 'chunks': [ @@ -637,6 +639,7 @@ build-system: dummy "morph": "bar", 'build-mode': 'bootstrap', 'build-depends': [], + 'prefix': '/usr', }, ], 'products': [], @@ -650,21 +653,22 @@ build-system: dummy { 'name': 'bar', "ref": "bar", - 'build-mode': 'bootstrap', + 'build-mode': 'staging', 'build-depends': [], + 'prefix': '/usr', }, ], } test_dict_with_build_depends = dict(test_dict) test_dict_with_build_depends["build-depends"] = [] - m = morphlib.morph3.Morphology(test_dict_with_build_depends) + m = morphlib.morphology.Morphology(test_dict_with_build_depends) self.loader.unset_defaults(m) self.assertEqual( dict(m), test_dict) def test_sets_defaults_for_system(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( kind='system', name='foo', arch='testarch', @@ -692,7 +696,7 @@ build-system: dummy dict(m)) def test_unsets_defaults_for_system(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( { 'description': '', 'kind': 'system', @@ -720,7 +724,7 @@ build-system: dummy }) def test_sets_defaults_for_cluster(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( name='foo', kind='cluster', systems=[ @@ -737,7 +741,7 @@ build-system: dummy 'deploy': {}}]) def test_unsets_defaults_for_cluster(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( name='foo', kind='cluster', description='', @@ -754,8 +758,8 @@ build-system: dummy [{'morph': 'foo'}, {'morph': 'bar'}]) - def test_sets_stratum_chunks_repo_and_morph_from_name(self): - m = morphlib.morph3.Morphology( + def test_sets_stratum_chunks_repo_from_name(self): + m = morphlib.morphology.Morphology( { "name": "foo", "kind": "stratum", @@ -771,10 +775,9 @@ build-system: dummy self.loader.set_defaults(m) self.loader.validate(m) self.assertEqual(m['chunks'][0]['repo'], 'le-chunk') - self.assertEqual(m['chunks'][0]['morph'], 'le-chunk') - def test_collapses_stratum_chunks_repo_and_morph_from_name(self): - m = morphlib.morph3.Morphology( + def test_collapses_stratum_chunks_repo_from_name(self): + m = morphlib.morphology.Morphology( { "name": "foo", "kind": "stratum", @@ -791,10 +794,9 @@ build-system: dummy self.loader.unset_defaults(m) self.assertTrue('repo' not in m['chunks'][0]) - self.assertTrue('morph' not in m['chunks'][0]) def test_convertes_max_jobs_to_an_integer(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( { "name": "foo", "kind": "chunk", @@ -827,7 +829,7 @@ build-system: dummy def test_warns_when_systems_refer_to_strata_with_repo_or_ref(self): for obsolete_field in ('repo', 'ref'): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( name="foo", kind="system", arch="testarch", @@ -850,7 +852,7 @@ build-system: dummy def test_warns_when_strata_refer_to_build_depends_with_repo_or_ref(self): for obsolete_field in ('repo', 'ref'): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( { 'name': 'foo', 'kind': 'stratum', @@ -883,7 +885,7 @@ build-system: dummy def test_unordered_asciibetically_after_ordered(self): # We only get morphologies with arbitrary keys in clusters - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( name='foo', kind='cluster', systems=[ @@ -924,21 +926,21 @@ build-system: dummy self.assertEqual(s, self.loader.save_to_string(m)) def test_smoketest_multi_line_unicode(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( name=u'foo', description=u'1 2 3\n4 5 6\n7 8 9\n', ) s = self.loader.save_to_string(m) def test_smoketest_multi_line_unicode_encoded(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( name=u'foo \u263A'.encode('utf-8'), description=u'1 \u263A\n2 \u263A\n3 \u263A\n'.encode('utf-8'), ) s = self.loader.save_to_string(m) def test_smoketest_binary_garbage(self): - m = morphlib.morph3.Morphology( + m = morphlib.morphology.Morphology( description='\x92', ) s = self.loader.save_to_string(m) diff --git a/morphlib/morph3.py b/morphlib/morphology.py index 477cac1a..314c315a 100644 --- a/morphlib/morph3.py +++ b/morphlib/morphology.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/morphlib/morph3_tests.py b/morphlib/morphology_tests.py index e150bf33..385f62ee 100644 --- a/morphlib/morph3_tests.py +++ b/morphlib/morphology_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -24,7 +24,7 @@ import morphlib class MorphologyTests(unittest.TestCase): def setUp(self): - self.morph = morphlib.morph3.Morphology() + self.morph = morphlib.morphology.Morphology() def test_has_repo_url_attribute(self): self.assertEqual(self.morph.repo_url, None) diff --git a/morphlib/morphologyfactory.py b/morphlib/morphologyfactory.py index cd1972be..1a8e374e 100644 --- a/morphlib/morphologyfactory.py +++ b/morphlib/morphologyfactory.py @@ -37,31 +37,6 @@ class NotcachedError(MorphologyFactoryError): "remote cache specified" % repo_name) -class StratumError(MorphologyFactoryError): - pass - - -class NoChunkBuildDependsError(StratumError): - def __init__(self, stratum, chunk): - StratumError.__init__( - self, 'No build dependencies in stratum %s for chunk %s ' - '(build-depends is a mandatory field)' % (stratum, chunk)) - - -class EmptyStratumError(StratumError): - - def __init__(self, stratum): - cliapp.AppException.__init__(self, - "Stratum %s is empty (has no dependencies)" % stratum) - - -class NoStratumBuildDependsError(StratumError): - def __init__(self, stratum): - StratumError.__init__( - self, 'Stratum %s has no build-dependencies listed ' - 'and has no bootstrap chunks.' % stratum) - - class MorphologyFactory(object): '''A way of creating morphologies which will provide a default''' @@ -75,16 +50,17 @@ class MorphologyFactory(object): if self._app is not None: self._app.status(*args, **kwargs) - def _get_morphology_text(self, reponame, sha1, filename): + def _load_morphology(self, reponame, sha1, filename): morph_name = os.path.splitext(os.path.basename(filename))[0] + loader = morphlib.morphloader.MorphologyLoader() if self._lrc.has_repo(reponame): self.status(msg="Looking for %s in local repo cache" % filename, chatty=True) try: repo = self._lrc.get_repo(reponame) - text = repo.cat(sha1, filename) + morph = loader.load_from_string(repo.cat(sha1, filename)) except IOError: - text = None + morph = None file_list = repo.ls_tree(sha1) elif self._rrc is not None: self.status(msg="Retrieving %(reponame)s %(sha1)s %(filename)s" @@ -93,35 +69,28 @@ class MorphologyFactory(object): chatty=True) try: text = self._rrc.cat_file(reponame, sha1, filename) + morph = loader.load_from_string(text) except morphlib.remoterepocache.CatFileError: - text = None + morph = None file_list = self._rrc.ls_tree(reponame, sha1) else: raise NotcachedError(reponame) - if text is None: + if morph is None: self.status(msg="File %s doesn't exist: attempting to infer " "chunk morph from repo's build system" % filename, chatty=True) bs = morphlib.buildsystem.detect_build_system(file_list) if bs is None: raise MorphologyNotFoundError(filename) - text = bs.get_morphology_text(morph_name) - return morph_name, text + morph = bs.get_morphology(morph_name) + loader.validate(morph) + loader.set_commands(morph) + loader.set_defaults(morph) + return morph def get_morphology(self, reponame, sha1, filename): - morph_name, text = self._get_morphology_text(reponame, sha1, filename) - - try: - morphology = morphlib.morph2.Morphology(text) - except morphlib.YAMLError as e: # pragma: no cover - raise morphlib.Error("Error parsing %s: %s" % - (filename, str(e))) - - if morph_name != morphology['name']: - raise morphlib.Error( - "Name %s does not match basename of morphology file %s" % - (morphology['name'], filename)) + morphology = self._load_morphology(reponame, sha1, filename) method_name = '_check_and_tweak_%s' % morphology['kind'] if hasattr(self, method_name): @@ -133,54 +102,17 @@ class MorphologyFactory(object): def _check_and_tweak_system(self, morphology, reponame, sha1, filename): '''Check and tweak a system morphology.''' - if morphology['arch'] is None: # pragma: no cover - raise morphlib.Error('No arch specified in system %s ' - '(arch is a mandatory field)' % - filename) - - if morphology['arch'] == 'armv7': - morphology._dict['arch'] = 'armv7l' - - if morphology['arch'] not in morphlib.valid_archs: - raise morphlib.Error('Unknown arch %s. This version of Morph ' - 'supports the following architectures: %s' % - (morphology['arch'], - ', '.join(morphlib.valid_archs))) - name = morphology['name'] morphology.builds_artifacts = [name + '-rootfs'] morphology.needs_artifact_metadata_cached = False - morphlib.morphloader.MorphologyLoader._validate_stratum_specs_fields( - morphology, 'strata') - morphlib.morphloader.MorphologyLoader._set_stratum_specs_defaults( - morphology, 'strata') - def _check_and_tweak_stratum(self, morphology, reponame, sha1, filename): '''Check and tweak a stratum morphology.''' - if len(morphology['chunks']) == 0: - raise EmptyStratumError(morphology['name']) - - for source in morphology['chunks']: - if source.get('build-depends', None) is None: - name = source.get('name', source.get('repo', 'unknown')) - raise NoChunkBuildDependsError(filename, name) - - if (len(morphology['build-depends'] or []) == 0 and - not any(c.get('build-mode') in ('bootstrap', 'test') - for c in morphology['chunks'])): - raise NoStratumBuildDependsError(filename) - morphology.builds_artifacts = [morphology['name']] morphology.needs_artifact_metadata_cached = True - morphlib.morphloader.MorphologyLoader._validate_stratum_specs_fields( - morphology, 'build-depends') - morphlib.morphloader.MorphologyLoader._set_stratum_specs_defaults( - morphology, 'build-depends') - def _check_and_tweak_chunk(self, morphology, reponame, sha1, filename): '''Check and tweak a chunk morphology.''' @@ -191,5 +123,3 @@ class MorphologyFactory(object): morphology.builds_artifacts = [morphology['name']] morphology.needs_artifact_metadata_cached = False - - morphlib.morphloader.MorphologyLoader._validate_chunk(morphology) diff --git a/morphlib/morphologyfactory_tests.py b/morphlib/morphologyfactory_tests.py index 47bf3153..0b3253da 100644 --- a/morphlib/morphologyfactory_tests.py +++ b/morphlib/morphologyfactory_tests.py @@ -17,7 +17,6 @@ import unittest import morphlib -from morphlib.morph2 import Morphology from morphlib.morphologyfactory import (MorphologyFactory, MorphologyNotFoundError, NotcachedError) @@ -31,7 +30,7 @@ class FakeRemoteRepoCache(object): return '''{ "name": "%s", "kind": "chunk", - "build-system": "bar" + "build-system": "dummy" }''' % filename[:-len('.morph')] return 'text' @@ -41,94 +40,76 @@ class FakeRemoteRepoCache(object): class FakeLocalRepo(object): morphologies = { - 'chunk.morph': '''{ - "name": "chunk", - "kind": "chunk", - "build-system": "bar" - }''', - 'chunk-split.morph': '''{ - "name": "chunk-split", - "kind": "chunk", - "build-system": "bar", - "products": [ - { - "artifact": "chunk-split-runtime", - "include": [] - }, - { - "artifact": "chunk-split-devel", - "include": [] - } - ] - }''', - 'stratum.morph': '''{ - "name": "stratum", - "kind": "stratum", - "chunks": [ - { - "name": "chunk", - "repo": "test:repo", - "ref": "sha1", - "build-mode": "bootstrap", - "build-depends": [] - } - ] - }''', - 'stratum-no-chunk-bdeps.morph': '''{ - "name": "stratum-no-chunk-bdeps", - "kind": "stratum", - "chunks": [ - { - "name": "chunk", - "repo": "test:repo", - "ref": "sha1", - "build-mode": "bootstrap" - } - ] - }''', - 'stratum-no-bdeps-no-bootstrap.morph': '''{ - "name": "stratum-no-bdeps-no-bootstrap", - "kind": "stratum", - "chunks": [ - { - "name": "chunk", - "repo": "test:repo", - "ref": "sha1", - "build-depends": [] - } - ] - }''', - 'stratum-bdeps-no-bootstrap.morph': '''{ - "name": "stratum-bdeps-no-bootstrap", - "kind": "stratum", - "build-depends": [ - { - "morph": "stratum" - } - ], - "chunks": [ - { - "name": "chunk", - "repo": "test:repo", - "ref": "sha1", - "build-depends": [] - } - ] - }''', - 'stratum-empty.morph': '''{ - "name": "stratum-empty", - "kind": "stratum" - }''', - 'system.morph': '''{ - "name": "system", - "kind": "system", - "arch": "%(arch)s" - }''', - 'parse-error.morph': '''{ "name"''', - 'name-mismatch.morph': '''{ - "name": "fred", - "kind": "stratum" - }''', + 'chunk.morph': ''' + name: chunk + kind: chunk + build-system: dummy + ''', + 'chunk-split.morph': ''' + name: chunk-split + kind: chunk + build-system: dummy + products: + - artifact: chunk-split-runtime + include: [] + - artifact: chunk-split-devel + include: [] + ''', + 'stratum.morph': ''' + name: stratum + kind: stratum + chunks: + - name: chunk + repo: test:repo + ref: sha1 + build-mode: bootstrap + build-depends: [] + ''', + 'stratum-no-chunk-bdeps.morph': ''' + name: stratum-no-chunk-bdeps + kind: stratum + chunks: + - name: chunk + repo: test:repo + ref: sha1 + build-mode: bootstrap + ''', + 'stratum-no-bdeps-no-bootstrap.morph': ''' + name: stratum-no-bdeps-no-bootstrap + kind: stratum + chunks: + - name: chunk + repo: test:repo + ref: sha1 + build-depends: [] + ''', + 'stratum-bdeps-no-bootstrap.morph': ''' + name: stratum-bdeps-no-bootstrap + kind: stratum + build-depends: + - morph: stratum + chunks: + - name: chunk + repo: test:repo + ref: sha1 + build-depends: [] + ''', + 'stratum-empty.morph': ''' + name: stratum-empty + kind: stratum + ''', + 'system.morph': ''' + name: system + kind: system + arch: %(arch)s + strata: + - morph: stratum + ''', + 'parse-error.morph': ''' name''', + 'name-mismatch.morph': ''' + name: fred + kind: stratum + ''', } def __init__(self): @@ -144,7 +125,7 @@ class FakeLocalRepo(object): return '''{ "name": "%s", "kind": "chunk", - "build-system": "bar" + "build-system": "dummy" }''' % filename[:-len('.morph')] return 'text' @@ -313,13 +294,13 @@ class MorphologyFactoryTests(unittest.TestCase): 'reponame', 'sha1', 'parse-error.morph') def test_fails_on_no_chunk_bdeps(self): - self.assertRaises(morphlib.morphologyfactory.NoChunkBuildDependsError, + self.assertRaises(morphlib.morphloader.NoBuildDependenciesError, self.mf.get_morphology, 'reponame', 'sha1', 'stratum-no-chunk-bdeps.morph') def test_fails_on_no_bdeps_or_bootstrap(self): self.assertRaises( - morphlib.morphologyfactory.NoStratumBuildDependsError, + morphlib.morphloader.NoStratumBuildDependenciesError, self.mf.get_morphology, 'reponame', 'sha1', 'stratum-no-bdeps-no-bootstrap.morph') @@ -330,6 +311,6 @@ class MorphologyFactoryTests(unittest.TestCase): def test_fails_on_empty_stratum(self): self.assertRaises( - morphlib.morphologyfactory.EmptyStratumError, + morphlib.morphloader.EmptyStratumError, self.mf.get_morphology, 'reponame', 'sha1', 'stratum-empty.morph') diff --git a/morphlib/morphset.py b/morphlib/morphset.py index 590ac51e..bf061f94 100644 --- a/morphlib/morphset.py +++ b/morphlib/morphset.py @@ -127,7 +127,8 @@ class MorphologySet(object): specs = m[kind] for spec in specs: if cb_filter(m, kind, spec): - fn = morphlib.util.sanitise_morphology_path(spec['morph']) + fn = morphlib.util.sanitise_morphology_path( + spec['morph'] if 'morph' in spec else spec['name']) orig_spec = (spec.get('repo'), spec.get('ref'), fn) dirtied = cb_process(m, kind, spec) if dirtied: @@ -148,7 +149,8 @@ class MorphologySet(object): if m.ref != spec.get('ref'): m.ref = spec.get('ref') m.dirty = True - file = morphlib.util.sanitise_morphology_path(spec['morph']) + file = morphlib.util.sanitise_morphology_path( + spec['morph'] if 'morph' in spec else spec['name']) assert (m.filename == file or m.repo_url == spec.get('repo')), \ 'Moving morphologies is not supported.' @@ -162,7 +164,7 @@ class MorphologySet(object): ''' def wanted_spec(m, kind, spec): - spec_name = spec.get('name', spec['morph']) + spec_name = spec['name'] if 'name' in spec else spec['morph'] return (spec.get('repo') == repo_url and spec.get('ref') == orig_ref and spec_name == morph_name) @@ -243,17 +245,3 @@ class MorphologySet(object): return True self.traverse_specs(process_chunk_spec, wanted_chunk_spec) - - def unpetrify_all(self): - '''If a spec is petrified, unpetrify it. - - ''' - - def wanted_spec(m, kind, spec): - return ('unpetrify-ref' in spec and - morphlib.git.is_valid_sha1(spec.get('ref'))) - def process_spec(m, kind, spec): - spec['ref'] = spec.pop('unpetrify-ref') - return True - - self.traverse_specs(process_spec, wanted_spec) diff --git a/morphlib/morphset_tests.py b/morphlib/morphset_tests.py index 8679c64a..81b5810f 100644 --- a/morphlib/morphset_tests.py +++ b/morphlib/morphset_tests.py @@ -26,7 +26,7 @@ class MorphologySetTests(unittest.TestCase): def setUp(self): self.morphs = morphlib.morphset.MorphologySet() - self.system = morphlib.morph3.Morphology({ + self.system = morphlib.morphology.Morphology({ 'kind': 'system', 'name': 'foo-system', 'strata': [ @@ -41,7 +41,7 @@ class MorphologySetTests(unittest.TestCase): self.system.ref = 'master' self.system.filename = 'foo-system.morph' - self.stratum = morphlib.morph3.Morphology({ + self.stratum = morphlib.morphology.Morphology({ 'kind': 'stratum', 'name': 'foo-stratum', 'chunks': [ @@ -111,7 +111,7 @@ class MorphologySetTests(unittest.TestCase): }) def test_changes_stratum_ref_in_build_depends(self): - other_stratum = morphlib.morph3.Morphology({ + other_stratum = morphlib.morphology.Morphology({ 'name': 'other-stratum', 'kind': 'stratum', 'chunks': [], @@ -200,18 +200,3 @@ class MorphologySetTests(unittest.TestCase): 'unpetrify-ref': 'master', } ]) - - def test_unpetrify_all(self): - self.morphs.add_morphology(self.system) - self.morphs.add_morphology(self.stratum) - self.morphs.petrify_chunks({('test:foo-chunk', 'master'): '0'*40}) - self.morphs.unpetrify_all() - self.assertEqual( - self.stratum['chunks'], - [ - { - 'repo': 'test:foo-chunk', - 'ref': 'master', - 'morph': 'foo-chunk', - } - ]) diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py index a66098b8..a258cd70 100644 --- a/morphlib/plugins/branch_and_merge_plugin.py +++ b/morphlib/plugins/branch_and_merge_plugin.py @@ -38,10 +38,6 @@ class BranchAndMergePlugin(cliapp.Plugin): self.app.add_subcommand( 'edit', self.edit, arg_synopsis='SYSTEM STRATUM [CHUNK]') self.app.add_subcommand( - 'petrify', self.petrify, arg_synopsis='') - self.app.add_subcommand( - 'unpetrify', self.unpetrify, arg_synopsis='') - self.app.add_subcommand( 'show-system-branch', self.show_system_branch, arg_synopsis='') self.app.add_subcommand( 'show-branch-root', self.show_branch_root, arg_synopsis='') @@ -313,9 +309,17 @@ class BranchAndMergePlugin(cliapp.Plugin): cached_repo = lrc.get_updated_repo(chunk_url) gd = sb.clone_cached_repo(cached_repo, chunk_ref) - if chunk_ref != sb.system_branch_name: - gd.branch(sb.system_branch_name, chunk_ref) - gd.checkout(sb.system_branch_name) + system_branch_ref = gd.disambiguate_ref(sb.system_branch_name) + sha1 = gd.resolve_ref_to_commit(chunk_ref) + + try: + old_sha1 = gd.resolve_ref_to_commit(system_branch_ref) + except morphlib.gitdir.InvalidRefError as e: + pass + else: + gd.delete_ref(system_branch_ref, old_sha1) + gd.branch(sb.system_branch_name, sha1) + gd.checkout(sb.system_branch_name) gd.update_submodules(self.app) gd.update_remotes() if gd.has_fat(): @@ -377,16 +381,17 @@ class BranchAndMergePlugin(cliapp.Plugin): This would, for example, write out something like: - /src/ws/master/baserock:baserock/morphs + /src/ws/master/baserock/baserock/definitions - when the master branch of the `baserock:baserock/morphs` + when the master branch of the `baserock/baserock/definitions` repository is checked out. ''' ws = morphlib.workspace.open('.') sb = morphlib.sysbranchdir.open_from_within('.') - self.app.output.write('%s\n' % sb.get_config('branch.root')) + repo_url = sb.get_config('branch.root') + self.app.output.write('%s\n' % sb.get_git_directory_name(repo_url)) def _remove_branch_dir_safe(self, workspace_root, system_branch_root): # This function avoids throwing any exceptions, so it is safe to call @@ -486,100 +491,6 @@ class BranchAndMergePlugin(cliapp.Plugin): morphs.add_morphology(morph) return morphs - def petrify(self, args): - '''Convert all chunk refs in a system branch to be fixed SHA1s. - - This modifies all git commit references in system and stratum - morphologies, in the current system branch, to be fixed SHA - commit identifiers, rather than symbolic branch or tag names. - This is useful for making sure none of the components in a system - branch change accidentally. - - Consider the following scenario: - - * The `master` system branch refers to `gcc` using the - `baserock/morph` ref. This is appropriate, since the main line - of development should use the latest curated code. - - * You create a system branch to prepare for a release, called - `TROVE_ID/release/2.0`. The reference to `gcc` is still - `baserock/morph`. - - * You test everything, and make a release. You deploy the release - images onto devices, which get shipped to your customers. - - * A new version GCC is committed to the `baserock/morph` branch. - - * Your release branch suddenly uses a new compiler, which may - or may not work for your particular system at that release. - - To avoid this, you need to _petrify_ all git references - so that they do not change accidentally. If you've tested - your release with the GCC release that is stored in commit - `94c50665324a7aeb32f3096393ec54b2e63bfb28`, then you should - continue to use that version of GCC, regardless of what might - happen in the master system branch. If, and only if, you decide - that a new compiler would be good for your release should you - include it in your release branch. This way, only the things - that you change intentionally change in your release branch. - - ''' - - if args: - raise cliapp.AppException('morph petrify takes no arguments') - - ws = morphlib.workspace.open('.') - sb = morphlib.sysbranchdir.open_from_within('.') - loader = morphlib.morphloader.MorphologyLoader() - lrc, rrc = morphlib.util.new_repo_caches(self.app) - update_repos = not self.app.settings['no-git-update'] - - morphs = self._load_all_sysbranch_morphologies(sb, loader) - - #TODO: Stop using app.resolve_ref - def resolve_refs(morphs): - for repo, ref in morphs.list_refs(): - # You can't resolve null refs, so don't attempt to. - if repo is None or ref is None: - continue - # TODO: Handle refs that are only in workspace in general - if (repo == sb.root_repository_url - and ref == sb.system_branch_name): - continue - commit_sha1, tree_sha1 = self.app.resolve_ref( - lrc, rrc, repo, ref, update=update_repos) - yield ((repo, ref), commit_sha1) - - morphs.repoint_refs(sb.root_repository_url, - sb.system_branch_name) - - morphs.petrify_chunks(dict(resolve_refs(morphs))) - - # Write morphologies back out again. - self._save_dirty_morphologies(loader, sb, morphs.morphologies) - - def unpetrify(self, args): - '''Reverse the process of petrification. - - This undoes the changes `morph petrify` did. - - ''' - - if args: - raise cliapp.AppException('morph petrify takes no arguments') - - ws = morphlib.workspace.open('.') - sb = morphlib.sysbranchdir.open_from_within('.') - loader = morphlib.morphloader.MorphologyLoader() - - morphs = self._load_all_sysbranch_morphologies(sb, loader) - - # Restore the ref for each stratum and chunk - morphs.unpetrify_all() - - # Write morphologies back out again. - self._save_dirty_morphologies(loader, sb, morphs.morphologies) - def status(self, args): '''Show information about the current system branch or workspace diff --git a/morphlib/plugins/build_plugin.py b/morphlib/plugins/build_plugin.py index 1a4fb573..64630c2b 100644 --- a/morphlib/plugins/build_plugin.py +++ b/morphlib/plugins/build_plugin.py @@ -184,31 +184,10 @@ class BuildPlugin(cliapp.Plugin): system=system_filename, branch=sb.system_branch_name) - bb = morphlib.buildbranch.BuildBranch(sb, build_ref_prefix, - push_temporary=push) - with contextlib.closing(bb) as bb: - - for gd, build_ref in bb.add_uncommitted_changes(): - self.app.status(msg='Adding uncommitted changes '\ - 'in %(dirname)s to %(ref)s', - dirname=gd.dirname, ref=build_ref, chatty=True) - - for gd in bb.inject_build_refs(loader): - self.app.status(msg='Injecting temporary build refs '\ - 'into morphologies in %(dirname)s', - dirname=gd.dirname, chatty=True) - - for gd, build_ref in bb.update_build_refs(name, email, build_uuid): - self.app.status(msg='Committing changes in %(dirname)s '\ - 'to %(ref)s', - dirname=gd.dirname, ref=build_ref, chatty=True) - - for gd, build_ref, remote in bb.push_build_branches(): - self.app.status(msg='Pushing %(ref)s in %(dirname)s '\ - 'to %(remote)s', - ref=build_ref, dirname=gd.dirname, - remote=remote.get_push_url(), chatty=True) - - build_command.build([bb.root_repo_url, - bb.root_ref, - system_filename]) + bb = morphlib.buildbranch.BuildBranch(sb, build_ref_prefix) + pbb = morphlib.buildbranch.pushed_build_branch( + bb, loader=loader, changes_need_pushing=push, + name=name, email=email, build_uuid=build_uuid, + status=self.app.status) + with pbb as (repo, ref): + build_command.build([repo, ref, system_filename]) diff --git a/morphlib/plugins/cross-bootstrap_plugin.py b/morphlib/plugins/cross-bootstrap_plugin.py index cd8e355e..0c3e3a4a 100644 --- a/morphlib/plugins/cross-bootstrap_plugin.py +++ b/morphlib/plugins/cross-bootstrap_plugin.py @@ -182,7 +182,7 @@ class BootstrapSystemBuilder(morphlib.builder2.BuilderBase): for step, in_parallel in steps: key = '%s-commands' % step - cmds = m.get_commands(key) + cmds = m[key] for cmd in cmds: f.write('(') if in_parallel: diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index 38c17bc2..61b8145e 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -319,40 +319,21 @@ class DeployPlugin(cliapp.Plugin): self.validate_deployment_options( env_vars, all_deployments, all_subsystems) - bb = morphlib.buildbranch.BuildBranch(sb, build_ref_prefix, - push_temporary=False) - with contextlib.closing(bb) as bb: - - for gd, build_ref in bb.add_uncommitted_changes(): - self.app.status(msg='Adding uncommitted changes '\ - 'in %(dirname)s to %(ref)s', - dirname=gd.dirname, ref=build_ref, chatty=True) - - for gd in bb.inject_build_refs(loader): - self.app.status(msg='Injecting temporary build refs '\ - 'into morphologies in %(dirname)s', - dirname=gd.dirname, chatty=True) - - for gd, build_ref in bb.update_build_refs(name, email, build_uuid): - self.app.status(msg='Committing changes in %(dirname)s '\ - 'to %(ref)s', - dirname=gd.dirname, ref=build_ref, chatty=True) - - for gd, build_ref, remote in bb.push_build_branches(): - self.app.status(msg='Pushing %(ref)s in %(dirname)s '\ - 'to %(remote)s', - ref=build_ref, dirname=gd.dirname, - remote=remote.get_push_url(), chatty=True) - + bb = morphlib.buildbranch.BuildBranch(sb, build_ref_prefix) + pbb = morphlib.buildbranch.pushed_build_branch( + bb, loader=loader, changes_need_pushing=False, + name=name, email=email, build_uuid=build_uuid, + status=self.app.status) + with pbb as (repo, ref): # Create a tempdir for this deployment to work in deploy_tempdir = tempfile.mkdtemp( dir=os.path.join(self.app.settings['tempdir'], 'deployments')) try: for system in cluster_morphology['systems']: self.deploy_system(build_command, deploy_tempdir, - root_repo_dir, bb.root_repo_url, - bb.root_ref, system, env_vars, - deployments, parent_location='') + root_repo_dir, repo, ref, system, + env_vars, deployments, + parent_location='') finally: shutil.rmtree(deploy_tempdir) @@ -547,9 +528,8 @@ class DeployPlugin(cliapp.Plugin): system morphology (repo, ref), or with the Morph code. ''' - build_ref_prefix = self.app.settings['build-ref-prefix'] with morphlib.extensions.get_extension_filename( - build_ref_prefix, name, kind) as ext_filename: + name, kind) as ext_filename: self.app.status(msg='Running extension %(name)s%(kind)s', name=name, kind=kind) self.app.runcmd( diff --git a/morphlib/plugins/show_dependencies_plugin.py b/morphlib/plugins/show_dependencies_plugin.py index c59cf507..3a1cb7ad 100644 --- a/morphlib/plugins/show_dependencies_plugin.py +++ b/morphlib/plugins/show_dependencies_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -65,7 +65,7 @@ class ShowDependenciesPlugin(cliapp.Plugin): # traverse the morphs to list all the sources for repo, ref, filename in self.app.itertriplets(args): - morph = filename[:-len('.morph')] + morph = morphlib.util.sanitise_morphology_path(filename) self.app.output.write('dependency graph for %s|%s|%s:\n' % (repo, ref, morph)) diff --git a/morphlib/remoteartifactcache_tests.py b/morphlib/remoteartifactcache_tests.py index d11bf264..ca959ebf 100644 --- a/morphlib/remoteartifactcache_tests.py +++ b/morphlib/remoteartifactcache_tests.py @@ -24,26 +24,24 @@ import morphlib class RemoteArtifactCacheTests(unittest.TestCase): def setUp(self): - morph = morphlib.morph2.Morphology( + loader = morphlib.morphloader.MorphologyLoader() + morph = loader.load_from_string( ''' - { - "name": "chunk", - "kind": "chunk", - "artifacts": { - "chunk-runtime": [ - "usr/bin", - "usr/sbin", - "usr/lib", - "usr/libexec" - ], - "chunk-devel": [ - "usr/include" - ], - "chunk-doc": [ - "usr/share/doc" - ] - } - } + name: chunk + kind: chunk + products: + - artifact: chunk-runtime + include: + - usr/bin + - usr/sbin + - usr/lib + - usr/libexec + - artifact: chunk-devel + include: + - usr/include + - artifact: chunk-doc + include: + - usr/share/doc ''') self.source = morphlib.source.Source( 'repo', 'ref', 'sha1', 'tree', morph, 'chunk.morph') diff --git a/morphlib/source_tests.py b/morphlib/source_tests.py index 6643f0fc..f5ce5d4d 100644 --- a/morphlib/source_tests.py +++ b/morphlib/source_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -22,10 +22,8 @@ import morphlib class SourceTests(unittest.TestCase): morphology_text = ''' - { - "name": "foo", - "kind": "chunk" - } + name: foo + kind: chunk ''' def setUp(self): @@ -33,7 +31,8 @@ class SourceTests(unittest.TestCase): self.original_ref = 'original/ref' self.sha1 = 'CAFEF00D' self.tree = 'F000000D' - self.morphology = morphlib.morph2.Morphology(self.morphology_text) + loader = morphlib.morphloader.MorphologyLoader() + self.morphology = loader.load_from_string(self.morphology_text) self.filename = 'foo.morph' self.source = morphlib.source.Source( self.repo_name, self.original_ref, self.sha1, self.tree, diff --git a/morphlib/sysbranchdir.py b/morphlib/sysbranchdir.py index b8953c2f..19fba695 100644 --- a/morphlib/sysbranchdir.py +++ b/morphlib/sysbranchdir.py @@ -68,19 +68,13 @@ class SystemBranchDirectory(object): value = cliapp.runcmd(['git', 'config', '-f', self._config_path, key]) return value.strip() - def get_git_directory_name(self, repo_url): - '''Return directory pathname for a given git repository. - - If the URL is a real one (not aliased), the schema and leading // - are removed from it, as is a .git suffix. - - Any colons in the URL path or network location are replaced - with slashes, so that directory paths do not contain colons. - This avoids problems with PYTHONPATH, PATH, and other things - that use colon as a separator. - - ''' + def _find_git_directory(self, repo_url): + for gd in self.list_git_directories(): + if gd.get_config('morph.repository') == repo_url: + return gd.dirname + return None + def _fabricate_git_directory_name(self, repo_url): # Parse the URL. If the path component is absolute, we assume # it's a real URL; otherwise, an aliased URL. parts = urlparse.urlparse(repo_url) @@ -107,6 +101,26 @@ class SystemBranchDirectory(object): return os.path.join(self.root_directory, relative) + def get_git_directory_name(self, repo_url): + '''Return directory pathname for a given git repository. + + If the repository has already been cloned, then it returns the + path to that, if not it will fabricate a path based on the url. + + If the URL is a real one (not aliased), the schema and leading // + are removed from it, as is a .git suffix. + + Any colons in the URL path or network location are replaced + with slashes, so that directory paths do not contain colons. + This avoids problems with PYTHONPATH, PATH, and other things + that use colon as a separator. + + ''' + found_repo = self._find_git_directory(repo_url) + if not found_repo: + return self._fabricate_git_directory_name(repo_url) + return found_repo + def get_filename(self, repo_url, relative): '''Return full pathname to a file in a checked out repository. diff --git a/scripts/edit-morph b/scripts/edit-morph index d0b793a4..90679b23 100755 --- a/scripts/edit-morph +++ b/scripts/edit-morph @@ -32,13 +32,8 @@ class EditMorph(cliapp.Application): 'automatically') def load_morphology(self, file_name, expected_kind = None): - with open(file_name) as f: - text = f.read() - try: - morphology = morphlib.morph2.Morphology(text) - except ValueError as e: - raise morphlib.Error("Error parsing %s: %s" % - (file_name, str(e))) + loader = morphlib.morphloader.MorphologyLoader() + morphology = loader.load_from_file(file_name) if expected_kind is not None and morphology['kind'] != expected_kind: raise morphlib.Error("Expected: a %s morphology" % expected_kind) @@ -67,10 +62,10 @@ class EditMorph(cliapp.Application): elif chunk_name in info['build-depends']: info['build-depends'].remove(chunk_name) build_depends_count += 1 - morphology._dict['chunks'] = new_chunks + morphology['chunks'] = new_chunks - with morphlib.savefile.SaveFile(file_name, 'w') as f: - morphology.update_text(text, f) + loader = morphlib.morphloader.MorphologyLoader() + loader.save_to_file(file_name, morphology) self.output.write("Removed: %i chunk(s) and %i build depend(s).\n" % (component_count, build_depends_count)) @@ -88,49 +83,10 @@ class EditMorph(cliapp.Application): for chunk in morphology['chunks']: chunk['build-depends'].sort() - morphology._dict['chunks'] = self.sort_chunks(morphology['chunks']) + morphology['chunks'] = self.sort_chunks(morphology['chunks']) - with morphlib.savefile.SaveFile(file_name, 'w') as f: - morphology.update_text(text, f) - - def cmd_to_json(self, args): - """Convert one or more FILES to JSON. - - Assumes a .yaml extension, which will be removed in the output file. - """ - - if len(args) == 0: - raise cliapp.AppException("to-json expects one or more filenames") - - for file_name in args: - try: - morphology, text = self.load_morphology(file_name) - - if not file_name.endswith('.yaml'): - raise morphlib.Error('file name does not end with .yaml') - out_file_name = file_name[:-len('.yaml')] - - with morphlib.savefile.SaveFile(out_file_name, 'w') as f: - morphology.update_text(text, f, convert_to='json') - except Exception as e: - self.output.write('%s: %s\n' % (file_name, e)) - - def cmd_to_yaml(self, args): - """Convert one or more FILES to YAML. - - Adds a .yaml extension for each input file.""" - - if len(args) == 0: - raise cliapp.AppException("to-yaml expects one or more filenames") - - for file_name in args: - try: - morphology, text = self.load_morphology(file_name) - - with morphlib.savefile.SaveFile(file_name + '.yaml', 'w') as f: - morphology.update_text(text, f, convert_to='yaml') - except Exception as e: - self.output.write('%s: %s\n' % (file_name, e)) + loader = morphlib.morphloader.MorphologyLoader() + loader.save_to_file(file_name, morphology) def sort_chunks(self, chunks_list): """Sort stratum chunks diff --git a/tests.as-root/archless-system-fails.script b/tests.as-root/archless-system-fails.script index 2fdeb018..e34e9ad6 100755 --- a/tests.as-root/archless-system-fails.script +++ b/tests.as-root/archless-system-fails.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2011-2013 Codethink Limited +# Copyright (C) 2011-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -23,17 +23,10 @@ set -eu cd "$DATADIR/morphs" git checkout --quiet -b archless master cat <<EOF >archless-system.morph -{ - "name": "archless-system", - "kind": "system", - "strata": [ - { - "morph": "hello-stratum", - "repo": "tests:morphs", - "ref": "archless" - } - ] -} +name: archless-system +kind: system +strata: + - morph: hello-stratum EOF git add archless-system.morph git commit --quiet -m "add archless system" diff --git a/tests.as-root/archless-system-fails.stderr b/tests.as-root/archless-system-fails.stderr index 585eb635..71e7dbb0 100644 --- a/tests.as-root/archless-system-fails.stderr +++ b/tests.as-root/archless-system-fails.stderr @@ -1 +1 @@ -ERROR: No arch specified in system archless-system.morph (arch is a mandatory field) +ERROR: Missing field arch from morphology string diff --git a/tests.as-root/metadata-includes-morph-version.setup b/tests.as-root/metadata-includes-morph-version.setup index d7fc96e3..e4557302 100755 --- a/tests.as-root/metadata-includes-morph-version.setup +++ b/tests.as-root/metadata-includes-morph-version.setup @@ -24,19 +24,12 @@ morphsrepo="$DATADIR/morphs" cd "$morphsrepo" git checkout -b tarball cat <<EOF > hello-tarball.morph -{ - "name": "hello-tarball", - "kind": "system", - "arch": "$("$SRCDIR/scripts/test-morph" print-architecture)", - "strata": [ - { - "morph": "hello-stratum", - }, - { - "morph": "linux-stratum", - } - ] -} +name: hello-tarball +kind: system +arch: $("$SRCDIR/scripts/test-morph" print-architecture) +strata: + - morph: hello-stratum + - morph: linux-stratum EOF git add hello-tarball.morph diff --git a/tests.as-root/metadata-includes-repo-alias.setup b/tests.as-root/metadata-includes-repo-alias.setup index d7fc96e3..e4557302 100755 --- a/tests.as-root/metadata-includes-repo-alias.setup +++ b/tests.as-root/metadata-includes-repo-alias.setup @@ -24,19 +24,12 @@ morphsrepo="$DATADIR/morphs" cd "$morphsrepo" git checkout -b tarball cat <<EOF > hello-tarball.morph -{ - "name": "hello-tarball", - "kind": "system", - "arch": "$("$SRCDIR/scripts/test-morph" print-architecture)", - "strata": [ - { - "morph": "hello-stratum", - }, - { - "morph": "linux-stratum", - } - ] -} +name: hello-tarball +kind: system +arch: $("$SRCDIR/scripts/test-morph" print-architecture) +strata: + - morph: hello-stratum + - morph: linux-stratum EOF git add hello-tarball.morph diff --git a/tests.as-root/rootfs-tarball-builds-rootfs-and-kernel.script b/tests.as-root/rootfs-tarball-builds-rootfs-and-kernel.script index e0829968..ca1b1302 100755 --- a/tests.as-root/rootfs-tarball-builds-rootfs-and-kernel.script +++ b/tests.as-root/rootfs-tarball-builds-rootfs-and-kernel.script @@ -27,14 +27,11 @@ arch=$("$SRCDIR/scripts/test-morph" print-architecture) cd "$DATADIR/kernel-repo" cat <<EOF >linux.morph -{ - "name": "linux", - "kind": "chunk", - "install-commands": [ - "mkdir -p \"\$DESTDIR/boot\"", - "touch \"\$DESTDIR\"/boot/zImage" - ] -} +name: linux +kind: chunk +install-commands: + - mkdir -p "\$DESTDIR/boot" + - touch "\$DESTDIR/boot/zImage" EOF git add linux.morph git commit --quiet -m 'Make the kernel create a dummy zImage' diff --git a/tests.as-root/system-overlap.script b/tests.as-root/system-overlap.script index 1ce8379d..9be6df13 100755 --- a/tests.as-root/system-overlap.script +++ b/tests.as-root/system-overlap.script @@ -28,63 +28,44 @@ morphsrepo="$DATADIR/morphs" cd "$morphsrepo" git checkout --quiet -b overlap master cat <<EOF >overlap-system.morph -{ - "name": "overlap-system", - "kind": "system", - "arch": "$("$SRCDIR/scripts/test-morph" print-architecture)", - "strata": [ - { - "morph": "foo-baz-stratum", - }, - { - "morph": "foo-barqux-stratum", - } - ] -} +name: overlap-system +kind: system +arch: $("$SRCDIR/scripts/test-morph" print-architecture) +strata: + - morph: foo-baz-stratum + - morph: foo-barqux-stratum EOF cat <<EOF >foo-baz-stratum.morph -{ - "name": "foo-baz-stratum", - "kind": "stratum", - "chunks": [ - { - "name": "overlap-foo-baz", - "repo": "test:chunk-repo", - "ref": "overlap", - "build-mode": "test", - "build-depends": [] - }, - { - "name": "linux", - "repo": "test:kernel-repo", - "ref": "master", - "build-mode": "test", - "build-depends": ["overlap-foo-baz"] - } - ] -} +name: foo-baz-stratum +kind: stratum +chunks: + - name: overlap-foo-baz + repo: test:chunk-repo + ref: overlap + build-mode: test + build-depends: [] + - name: linux + repo: test:kernel-repo + ref: master + build-mode: test + build-depends: + - overlap-foo-baz EOF cat <<EOF >foo-barqux-stratum.morph -{ - "name": "foo-barqux-stratum", - "kind": "stratum", - "chunks": [ - { - "name": "overlap-foobar", - "repo": "test:chunk-repo", - "ref": "overlap", - "build-mode": "test", - "build-depends": [] - }, - { - "name": "overlap-fooqux", - "repo": "test:chunk-repo", - "ref": "overlap", - "build-mode": "test", - "build-depends": ["overlap-foobar"] - } - ] -} +name: foo-barqux-stratum +kind: stratum +chunks: + - name: overlap-foobar + repo: test:chunk-repo + ref: overlap + build-mode: test + build-depends: [] + - name: overlap-fooqux + repo: test:chunk-repo + ref: overlap + build-mode: test + build-depends: + - overlap-foobar EOF git add overlap-system.morph foo-baz-stratum.morph foo-barqux-stratum.morph git commit --quiet -m "add overlapping system" @@ -92,39 +73,30 @@ git commit --quiet -m "add overlapping system" cd "$chunkrepo" git checkout --quiet -b overlap master cat <<EOF >overlap-foo-baz.morph -{ - "name": "overlap-foo-baz", - "kind": "chunk", - "install-commands": [ - "mkdir -p \$DESTDIR/bin", - "for f in foo bar baz; do echo echo \$f >\$DESTDIR/bin/\$f; done" - ] -} +name: overlap-foo-baz +kind: chunk +install-commands: + - mkdir -p "\$DESTDIR"/bin + - for f in foo bar baz; do echo echo \$f >"\$DESTDIR"/bin/\$f; done EOF cat <<EOF >overlap-foobar.morph -{ - "name": "overlap-foobar", - "kind": "chunk", - "install-commands": [ - "mkdir -p \$DESTDIR/usr/bin \$DESTDIR/bin", - "echo echo foobar >\$DESTDIR/usr/bin/foobar", - "ln -s /usr/bin/foobar \$DESTDIR/bin/foo", - "ln -s /usr/bin/foobar \$DESTDIR/bin/bar" - ] -} +name: overlap-foobar +kind: chunk +install-commands: + - mkdir -p "\$DESTDIR"/usr/bin "\$DESTDIR"/bin + - echo echo foobar >"\$DESTDIR"/usr/bin/foobar + - ln -s /usr/bin/foobar "\$DESTDIR"/bin/foo + - ln -s /usr/bin/foobar "\$DESTDIR"/bin/bar EOF cat <<EOF >overlap-fooqux.morph -{ - "name": "overlap-fooqux", - "kind": "chunk", - "install-commands": [ - "mkdir -p \$DESTDIR/usr/bin \$DESTDIR/bin", - "for f in qux fooqux; do echo echo \$f >\$DESTDIR/usr/bin/\$f; done", - "ln -s /usr/bin/fooqux \$DESTDIR/bin/foo" - ] -} +name: overlap-fooqux +kind: chunk +install-commands: + - mkdir -p "\$DESTDIR"/usr/bin "\$DESTDIR"/bin + - for f in qux fooqux; do echo echo \$f >"\$DESTDIR"/usr/bin/\$f; done + - ln -s /usr/bin/fooqux "\$DESTDIR"/bin/foo EOF git add overlap-*.morph diff --git a/tests.as-root/tarball-image-is-sensible.setup b/tests.as-root/tarball-image-is-sensible.setup index c47a5336..a687b691 100755 --- a/tests.as-root/tarball-image-is-sensible.setup +++ b/tests.as-root/tarball-image-is-sensible.setup @@ -24,17 +24,14 @@ chunkrepo="$DATADIR/chunk-repo" cd "$chunkrepo" git checkout -b tarball-links cat >links.morph <<'EOF' -{ - "name": "links", - "kind": "chunk", - "build-system": "manual", - "install-commands": [ - "mkdir -p \"$DESTDIR/bin\"", - "touch \"$DESTDIR/bin/true\"", - "cd \"$DESTDIR/bin\" && ln true true-hardlink", - "cd \"$DESTDIR/bin\" && ln -s true true-symlink" - ] -} +name: links +kind: chunk +build-system: manual +install-commands: + - mkdir -p "$DESTDIR/bin" + - touch "$DESTDIR/bin/true" + - cd "$DESTDIR/bin" && ln true true-hardlink + - cd "$DESTDIR/bin" && ln -s true true-symlink EOF git add links.morph git commit --quiet -m 'Add link adding chunk' @@ -43,22 +40,13 @@ morphsrepo="$DATADIR/morphs" cd "$morphsrepo" git checkout -b tarball-links cat <<EOF > hello-tarball.morph -{ - "name": "hello-tarball", - "kind": "system", - "arch": "$("$SRCDIR/scripts/test-morph" print-architecture)", - "strata": [ - { - "morph": "link-stratum", - }, - { - "morph": "hello-stratum", - }, - { - "morph": "linux-stratum", - } - ] -} +name: hello-tarball +kind: system +arch: $("$SRCDIR/scripts/test-morph" print-architecture) +strata: + - morph: link-stratum + - morph: hello-stratum + - morph: linux-stratum EOF git add hello-tarball.morph @@ -69,19 +57,14 @@ sed -i linux-stratum.morph \ git add linux-stratum.morph cat <<EOF > link-stratum.morph -{ - "name": "link-stratum", - "kind": "stratum", - "chunks": [ - { - "name": "links", - "repo": "test:chunk-repo", - "ref": "tarball-links", - "build-mode": "test", - "build-depends": [] - } - ] -} +name: link-stratum +kind: stratum +chunks: + - name: links + repo: test:chunk-repo + ref: tarball-links + build-mode: test + build-depends: [] EOF git add link-stratum.morph git commit --quiet -m "add morphs" diff --git a/tests.as-root/unimportant-morphology-contents-do-not-change-cache-keys.script b/tests.as-root/unimportant-morphology-contents-do-not-change-cache-keys.script deleted file mode 100755 index a540cdee..00000000 --- a/tests.as-root/unimportant-morphology-contents-do-not-change-cache-keys.script +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2012-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -## Some contents of morphologies (description, build-depends, chunks) -## should not change the cache keys of those morphs because they are -## either already considered (build-depends and chunks) or do not affect -## building (description). This test checks that changes to these parts -## of a morphology do not force rebuilds. - -# FIXME: This seems to break because the new "morph edit" makes correct -# changes to build-dependencies, which breaks the old "morph build". -# Disable test now, re-enable it after "morph build" is fixed. --liw -exit 0 - -set -eu - -source "$SRCDIR/tests.as-root/setup-build" - -# Build once. -cd "$DATADIR/workspace" -"$SRCDIR/scripts/test-morph" build linux-system -ARTIFACT_COUNT="$(ls "$DATADIR/cache/artifacts" | wc -l)" - -# Petrify the refs, so the morphologies will be different -(set -e && cd branch1/test/morphs && git push --quiet origin HEAD) -(set -e && cd branch1/test/kernel-repo && git push --quiet origin HEAD) -"$SRCDIR/scripts/test-morph" petrify - -# Build with the petrified morphologies. -"$SRCDIR/scripts/test-morph" build linux-system -[ "$ARTIFACT_COUNT" -eq $(ls "$DATADIR/cache/artifacts" | wc -l) ] diff --git a/tests.branching.disabled/workflow-petrify.script b/tests.branching.disabled/workflow-petrify.script deleted file mode 100755 index 3c561d5b..00000000 --- a/tests.branching.disabled/workflow-petrify.script +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2012-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## Do a complete workflow test, with strata outside the main morphologies -## repository. - -set -eu - -. "$SRCDIR/scripts/setup-3rd-party-strata" - -cd "$DATADIR/workspace" -"$SRCDIR/scripts/test-morph" branch test:morphs test/petrify - -cd test/petrify -"$SRCDIR/scripts/test-morph" petrify - -echo "test/petrify after petrifying:" -cat test:morphs/hello-system.morph -cat test:external-strata/stratum2.morph -cat test:external-strata/stratum3.morph - -"$SRCDIR/scripts/test-morph" edit hello - -echo -echo "test/petrify after editing a chunk:" -cat test:morphs/hello-system.morph -cat test:external-strata/stratum2.morph -cat test:external-strata/stratum3.morph - -# me/readme-fixes in test:stratum2-hello should be 'master', but -# that's the only ref upstream so we can infer that it is if we got here - -cd test:morphs -git commit --quiet --all -m "Petrify branch test/petrify" -git push --quiet origin test/petrify -cd ../test:external-strata -git commit --quiet --all -m "Petrify branch test/petrify" -git push --quiet origin test/petrify - -# unpetrify is easy enough. Let's try unpetrifying a different branch that -# was forked off the original while it was petrified, instead. -cd "$DATADIR/workspace" -"$SRCDIR/scripts/test-morph" branch test:morphs test/unpetrify test/petrify - -cd test/unpetrify -"$SRCDIR/scripts/test-morph" unpetrify - -echo -echo "test/unpetrify after unpetrifying:" -cat test:morphs/hello-system.morph -cat test:external-strata/stratum2.morph -cat test:external-strata/stratum3.morph diff --git a/tests.branching.disabled/workflow-petrify.stdout b/tests.branching.disabled/workflow-petrify.stdout deleted file mode 100644 index a0ce82f4..00000000 --- a/tests.branching.disabled/workflow-petrify.stdout +++ /dev/null @@ -1,152 +0,0 @@ -test/petrify after petrifying: -{ - "name": "hello-system", - "kind": "system", - "arch": "x86_64", - "strata": [ - { - "morph": "hello-stratum", - "repo": "test:morphs", - "ref": "test/petrify" - }, - { - "morph": "stratum2", - "repo": "test:external-strata", - "ref": "test/petrify" - }, - { - "morph": "stratum3", - "repo": "test:external-strata", - "ref": "test/petrify" - } - ] -} -{ - "name": "stratum2", - "kind": "stratum", - "chunks": [ - { - "name": "hello", - "repo": "test:stratum2-hello", - "ref": "f4730636e429149bb923fa16be3aa9802d484b23", - "build-mode": "test", - "build-depends": [], - "unpetrify-ref": "master" - } - ] -} -{ - "name": "stratum3", - "kind": "stratum", - "chunks": [ - { - "name": "hello", - "repo": "test:stratum3-hello", - "ref": "f4730636e429149bb923fa16be3aa9802d484b23", - "build-mode": "test", - "build-depends": [], - "unpetrify-ref": "master" - } - ] -} - -test/petrify after editing a chunk: -{ - "name": "hello-system", - "kind": "system", - "arch": "x86_64", - "strata": [ - { - "morph": "hello-stratum", - "repo": "test:morphs", - "ref": "test/petrify" - }, - { - "morph": "stratum2", - "repo": "test:external-strata", - "ref": "test/petrify" - }, - { - "morph": "stratum3", - "repo": "test:external-strata", - "ref": "test/petrify" - } - ] -} -{ - "name": "stratum2", - "kind": "stratum", - "chunks": [ - { - "name": "hello", - "repo": "test:stratum2-hello", - "ref": "test/petrify", - "build-mode": "test", - "build-depends": [] - } - ] -} -{ - "name": "stratum3", - "kind": "stratum", - "chunks": [ - { - "name": "hello", - "repo": "test:stratum3-hello", - "ref": "f4730636e429149bb923fa16be3aa9802d484b23", - "build-mode": "test", - "build-depends": [], - "unpetrify-ref": "master" - } - ] -} - -test/unpetrify after unpetrifying: -{ - "name": "hello-system", - "kind": "system", - "arch": "x86_64", - "strata": [ - { - "morph": "hello-stratum", - "repo": "test:morphs", - "ref": "test/unpetrify" - }, - { - "morph": "stratum2", - "repo": "test:external-strata", - "ref": "test/unpetrify" - }, - { - "morph": "stratum3", - "repo": "test:external-strata", - "ref": "test/unpetrify" - } - ] -} -{ - "name": "stratum2", - "kind": "stratum", - "chunks": [ - { - "name": "hello", - "repo": "test:stratum2-hello", - "ref": "test/petrify", - "build-mode": "test", - "build-depends": [] - } - ] -} -{ - "name": "stratum3", - "kind": "stratum", - "chunks": [ - { - "name": "hello", - "repo": "test:stratum3-hello", - "ref": "master", - "build-mode": "test", - "build-depends": [] - } - ] -} diff --git a/tests.branching/ambiguous-refs.script b/tests.branching/ambiguous-refs.script deleted file mode 100755 index aeec61a1..00000000 --- a/tests.branching/ambiguous-refs.script +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2012-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## Guard against a bug that occurs if 'git show-ref' is used to resolve refs -## instead of 'git rev-parse --verify': show-ref returns a list of partial -## matches sorted alphabetically, so any code using it may resolve refs -## incorrectly. - -set -eu - -. "$SRCDIR/scripts/fix-committer-info" - -cd "$DATADIR/morphs" -git mv hello-stratum.morph goodbye-stratum.morph -sed -e '/morph: hello-stratum/s/hello-stratum/goodbye-stratum/' \ - -i hello-system.morph - -git commit --quiet -am "Rename hello-system" - -cd "$DATADIR/workspace" -"$SRCDIR/scripts/test-morph" init -"$SRCDIR/scripts/test-morph" branch test:morphs release - -# Create an extra ref to confuse any users of git show-ref -cd release/test/morphs -git checkout --quiet -b alpha/master HEAD~1 -git checkout --quiet release - -# The petrify will fail if we resolved 'master' as 'alpha/master' by mistake. -cd "$DATADIR/workspace/release/test/morphs" -"$SRCDIR/scripts/test-morph" petrify diff --git a/tests.branching/build-after-petrify.script b/tests.branching/build-after-petrify.script deleted file mode 100755 index d3b75f07..00000000 --- a/tests.branching/build-after-petrify.script +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## Verify systems can be built after "morph petrify" - -set -eu - -. "$SRCDIR/tests.branching/setup-second-chunk" - -cd "$DATADIR/workspace" -"$SRCDIR/scripts/test-morph" init -"$SRCDIR/scripts/test-morph" branch test:morphs test/build-petrify master - -"$SRCDIR/scripts/test-morph" petrify - -"$SRCDIR/scripts/test-morph" build hello-system diff --git a/tests.branching/petrify-no-double-petrify.script b/tests.branching/petrify-no-double-petrify.script deleted file mode 100755 index 3c9185dc..00000000 --- a/tests.branching/petrify-no-double-petrify.script +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2012-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## It should be impossible to lose the original refs, or we risk making -## unpetrify impossible. - -set -eu - -cd "$DATADIR/workspace" -"$SRCDIR/scripts/test-morph" init -"$SRCDIR/scripts/test-morph" branch test:morphs test/petrify - -cd test/petrify/test/morphs -git push --quiet origin HEAD -"$SRCDIR/scripts/test-morph" petrify -"$SRCDIR/scripts/test-morph" petrify -"$SRCDIR/scripts/test-morph" petrify - -cat hello-stratum.morph diff --git a/tests.branching/petrify-no-double-petrify.stdout b/tests.branching/petrify-no-double-petrify.stdout deleted file mode 100644 index 50da61ba..00000000 --- a/tests.branching/petrify-no-double-petrify.stdout +++ /dev/null @@ -1,9 +0,0 @@ -name: hello-stratum -kind: stratum -chunks: -- name: hello - repo: test:hello - ref: 293fa0b08f0382c63181c36b6efa602876aa8c87 - unpetrify-ref: master - build-depends: [] - build-mode: test diff --git a/tests.branching/petrify.script b/tests.branching/petrify.script deleted file mode 100755 index f8e7c1e9..00000000 --- a/tests.branching/petrify.script +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2012-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## Verify "morph petrify", and ensure it doesn't petrify chunks that have -## already been edited. - -set -eu - -. "$SRCDIR/tests.branching/setup-second-chunk" - -cd "$DATADIR/workspace" -"$SRCDIR/scripts/test-morph" init -"$SRCDIR/scripts/test-morph" branch test:morphs test/petrify master - -cd test/petrify/test/morphs -git push --quiet origin HEAD -"$SRCDIR/scripts/test-morph" edit goodbye -(cd ../goodbye && git push --quiet origin HEAD) - -"$SRCDIR/scripts/test-morph" petrify -echo "Petrified:" -cat hello-stratum.morph - -"$SRCDIR/scripts/test-morph" unpetrify -echo -echo "Unpetrified:" -cat hello-stratum.morph diff --git a/tests.branching/petrify.stdout b/tests.branching/petrify.stdout deleted file mode 100644 index f9f35342..00000000 --- a/tests.branching/petrify.stdout +++ /dev/null @@ -1,31 +0,0 @@ -Petrified: -name: hello-stratum -kind: stratum -chunks: -- name: hello - repo: test:hello - ref: 293fa0b08f0382c63181c36b6efa602876aa8c87 - unpetrify-ref: master - build-depends: [] - build-mode: test -- name: goodbye - repo: test:goodbye - ref: 717c4a523fb5a94ca2f0a61e665fbc2da6a1f6ac - unpetrify-ref: test/petrify - build-depends: [] - build-mode: test - -Unpetrified: -name: hello-stratum -kind: stratum -chunks: -- name: hello - repo: test:hello - ref: master - build-depends: [] - build-mode: test -- name: goodbye - repo: test:goodbye - ref: test/petrify - build-depends: [] - build-mode: test diff --git a/tests.branching/show-branch-root-in-branched-branch.script b/tests.branching/show-branch-root-in-branched-branch.script deleted file mode 100755 index 4598d6a9..00000000 --- a/tests.branching/show-branch-root-in-branched-branch.script +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2012 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## Check that 'morph show-branch-root' works when being run in a -## local system branch created using 'morph branch ...'. - -set -eu - -# Create a workspace and branch. -cd "$DATADIR/workspace" -"$SRCDIR/scripts/test-morph" init -"$SRCDIR/scripts/test-morph" branch test:morphs testbranch - -# Try to print the branch root repository. -"$SRCDIR/scripts/test-morph" show-branch-root diff --git a/tests.branching/show-branch-root-in-branched-branch.stdout b/tests.branching/show-branch-root-in-branched-branch.stdout deleted file mode 100644 index b2804d56..00000000 --- a/tests.branching/show-branch-root-in-branched-branch.stdout +++ /dev/null @@ -1 +0,0 @@ -test:morphs diff --git a/tests.branching/show-branch-root-in-checked-out-branch.script b/tests.branching/show-branch-root-in-checked-out-branch.script deleted file mode 100755 index 97f892b9..00000000 --- a/tests.branching/show-branch-root-in-checked-out-branch.script +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2012 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## Check that 'morph show-branch-root' works when being run in a -## local system branch created using 'morph checkout ...'. - -set -eu - -# Create a workspace and branch. -cd "$DATADIR/workspace" -"$SRCDIR/scripts/test-morph" init -"$SRCDIR/scripts/test-morph" checkout test:morphs master - -# Try to print the branch root repository. -"$SRCDIR/scripts/test-morph" show-branch-root diff --git a/tests.branching/show-branch-root-in-checked-out-branch.stdout b/tests.branching/show-branch-root-in-checked-out-branch.stdout deleted file mode 100644 index b2804d56..00000000 --- a/tests.branching/show-branch-root-in-checked-out-branch.stdout +++ /dev/null @@ -1 +0,0 @@ -test:morphs diff --git a/tests.branching/show-branch-root-with-repo-url.script b/tests.branching/show-branch-root-with-repo-url.script deleted file mode 100755 index 5480c3ef..00000000 --- a/tests.branching/show-branch-root-with-repo-url.script +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2012 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## Check that 'morph show-branch-root' works when being run in a -## local system branch created from a full repository URL. - -set -eu - -# Create a workspace and branch. -cd "$DATADIR/workspace" -"$SRCDIR/scripts/test-morph" init -"$SRCDIR/scripts/test-morph" branch "file://${DATADIR}/morphs.git" testbranch - -# Try to print the branch root repository. -"$SRCDIR/scripts/test-morph" show-branch-root diff --git a/tests.branching/show-branch-root-with-repo-url.stdout b/tests.branching/show-branch-root-with-repo-url.stdout deleted file mode 100644 index 2d47eb40..00000000 --- a/tests.branching/show-branch-root-with-repo-url.stdout +++ /dev/null @@ -1 +0,0 @@ -file://TMP/morphs.git diff --git a/tests.build/build-chunk-failures-dump-log.script b/tests.build/build-chunk-failures-dump-log.script index e5c7c38d..645fd59a 100755 --- a/tests.build/build-chunk-failures-dump-log.script +++ b/tests.build/build-chunk-failures-dump-log.script @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (C) 2011-2013 Codethink Limited +# Copyright (C) 2011-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -25,15 +25,12 @@ chunkrepo="$DATADIR/chunk-repo" cd "$chunkrepo" git checkout --quiet farrokh cat <<EOF >hello.morph -{ - "name": "hello", - "kind": "chunk", - "build-system": "dummy", - "build-commands": [ - "echo The next command will fail", - "false" - ] -} +name: hello +kind: chunk +build-system: dummy +build-commands: + - echo The next command will fail + - "false" EOF git add hello.morph git commit --quiet -m "Make morphology fail to build." diff --git a/tests.build/build-stratum-with-submodules.script b/tests.build/build-stratum-with-submodules.script index c996e769..015db3f2 100755 --- a/tests.build/build-stratum-with-submodules.script +++ b/tests.build/build-stratum-with-submodules.script @@ -26,14 +26,11 @@ set -eu parent="$DATADIR/parent-repo" mkdir "$parent" cat <<EOF > "$parent/parent.morph" -{ - "name": "parent", - "kind": "chunk", - "build-system": "manual", - "build-commands": [ - "test -f le-sub/README" - ] -} +name: parent +kind: chunk +build-system: manual +build-commands: + - test -f le-sub/README EOF "$SRCDIR/scripts/run-git-in" "$parent" init --quiet @@ -47,19 +44,14 @@ EOF morphs="$DATADIR/morphs-repo" cat <<EOF > "$morphs/hello-stratum.morph" -{ - "name": "hello-stratum", - "kind": "stratum", - "chunks": [ - { - "name": "parent", - "repo": "test:parent-repo", - "ref": "master", - "build-depends": [], - "build-mode": "test" - } - ] -} +name: hello-stratum +kind: stratum +chunks: + - name: parent + repo: test:parent-repo + ref: master + build-depends: [] + build-mode: test EOF "$SRCDIR/scripts/run-git-in" "$morphs" add hello-stratum.morph "$SRCDIR/scripts/run-git-in" "$morphs" commit --quiet -m 'foo' diff --git a/tests.build/build-system-autotools.script b/tests.build/build-system-autotools.script index ba5cd32f..2ea53174 100755 --- a/tests.build/build-system-autotools.script +++ b/tests.build/build-system-autotools.script @@ -37,12 +37,10 @@ EOF git add Makefile cat <<EOF > hello.morph -{ - "name": "hello", - "kind": "chunk", - "build-system": "autotools", - "configure-commands": [] -} +name: hello +kind: chunk +build-system: autotools +configure-commands: [] EOF git add hello.morph git commit --quiet -m "Convert hello to an autotools project" diff --git a/tests.build/build-system-cmake.script b/tests.build/build-system-cmake.script index ab5186d7..570a9af7 100755 --- a/tests.build/build-system-cmake.script +++ b/tests.build/build-system-cmake.script @@ -38,12 +38,11 @@ EOF git add CMakeLists.txt cat <<EOF > hello.morph -{ - "name": "hello", - "kind": "chunk", - "build-system": "cmake", - "install-commands": ["make DESTDIR=\"\$DESTDIR\" install"] -} +name: hello +kind: chunk +build-system: cmake +install-commands: + - make DESTDIR="\$DESTDIR" install EOF git add hello.morph git commit --quiet -m "Convert hello to a cmake project" diff --git a/tests.build/build-system-cpan.script b/tests.build/build-system-cpan.script index f66d4027..735dac84 100755 --- a/tests.build/build-system-cpan.script +++ b/tests.build/build-system-cpan.script @@ -44,11 +44,9 @@ EOF git add Makefile.PL cat <<EOF >hello.morph -{ - "name": "hello", - "kind": "chunk", - "build-system": "cpan" -} +name: hello +kind: chunk +build-system: cpan EOF git add hello.morph @@ -57,20 +55,15 @@ git commit --quiet -m 'convert hello into a perl cpan project' # Set 'prefix' of hello to something custom cd "$DATADIR/morphs-repo" cat <<EOF > hello-stratum.morph -{ - "name": "hello-stratum", - "kind": "stratum", - "chunks": [ - { - "name": "hello", - "repo": "test:chunk-repo", - "ref": "farrokh", - "build-depends": [], - "build-mode": "test", - "prefix": "/" - } - ] -} +name: hello-stratum +kind: stratum +chunks: + - name: hello + repo: test:chunk-repo + ref: farrokh + build-depends: [] + build-mode: test + prefix: / EOF git add hello-stratum.morph git commit -q -m "Set custom install prefix for hello" diff --git a/tests.build/build-system-python-distutils.script b/tests.build/build-system-python-distutils.script index e1dccb4b..9a751491 100755 --- a/tests.build/build-system-python-distutils.script +++ b/tests.build/build-system-python-distutils.script @@ -41,11 +41,9 @@ EOF git add setup.py cat <<EOF >hello.morph -{ - "name": "hello", - "kind": "chunk", - "build-system": "python-distutils" -} +name: hello +kind: chunk +build-system: python-distutils EOF git add hello.morph @@ -55,20 +53,15 @@ git commit --quiet -m 'convert hello into a python project' # Set 'prefix' of hello to something custom cd "$DATADIR/morphs-repo" cat <<EOF > hello-stratum.morph -{ - "name": "hello-stratum", - "kind": "stratum", - "chunks": [ - { - "name": "hello", - "repo": "test:chunk-repo", - "ref": "farrokh", - "build-depends": [], - "build-mode": "test", - "prefix": "" - } - ] -} +name: hello-stratum +kind: stratum +chunks: + - name: hello + repo: test:chunk-repo + ref: farrokh + build-depends: [] + build-mode: test + prefix: "" EOF git add hello-stratum.morph git commit -q -m "Set custom install prefix for hello" diff --git a/tests.build/build-system-qmake.script b/tests.build/build-system-qmake.script index d9e21fba..b3861936 100755 --- a/tests.build/build-system-qmake.script +++ b/tests.build/build-system-qmake.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2011-2013 Codethink Limited +# Copyright (C) 2011-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -46,12 +46,11 @@ EOF git add hello.pro cat <<EOF > hello.morph -{ - "name": "hello", - "kind": "chunk", - "build-system": "qmake", - "install-commands": ["make INSTALL_ROOT=\"\$DESTDIR\" install"] -} +name: hello +kind: chunk +build-system: qmake +install-commands: + - make INSTALL_ROOT="\$DESTDIR" install EOF git add hello.morph git commit --quiet -m "Convert hello to an qmake project" diff --git a/tests.build/empty-stratum.script b/tests.build/empty-stratum.script index 6856b2bd..19c36558 100755 --- a/tests.build/empty-stratum.script +++ b/tests.build/empty-stratum.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -24,10 +24,8 @@ git checkout --quiet -b empty-stratum # Create empty stratum to test S4585 cat <<EOF > hello-stratum.morph -{ - "name": "hello-stratum", - "kind": "stratum" -} +name: hello-stratum +kind: stratum EOF sed -i 's/master/empty-stratum/' hello-system.morph git add hello-stratum.morph hello-system.morph diff --git a/tests.build/empty-stratum.stderr b/tests.build/empty-stratum.stderr index d8c9bf28..6a4ecb05 100644 --- a/tests.build/empty-stratum.stderr +++ b/tests.build/empty-stratum.stderr @@ -1 +1 @@ -ERROR: Stratum hello-stratum is empty (has no dependencies) +ERROR: Stratum hello-stratum has no chunks in string diff --git a/tests.build/prefix.script b/tests.build/prefix.script index ca9648c9..209c1a54 100755 --- a/tests.build/prefix.script +++ b/tests.build/prefix.script @@ -24,24 +24,18 @@ set -eu cd "$DATADIR/chunk-repo" git checkout -q master cat <<\EOF > xyzzy.morph -{ - "name": "xyzzy", - "kind": "chunk", - "configure-commands": [ - "echo First chunk: prefix $PREFIX" - ] -} +name: xyzzy +kind: chunk +configure-commands: + - "echo First chunk: prefix $PREFIX" EOF cat <<\EOF > plugh.morph -{ - "name": "plugh", - "kind": "chunk", - "configure-commands": [ - "echo Second chunk: prefix $PREFIX", - "echo Path: $(echo $PATH | grep -o '/plover')" - ] -} +name: plugh +kind: chunk +configure-commands: + - "echo Second chunk: prefix $PREFIX" + - "echo Path: $(echo $PATH | grep -o '/plover')" EOF git add xyzzy.morph @@ -51,29 +45,21 @@ git commit -q -m "Add chunks" # Change stratum to include those two chunks, and use a custom install prefix cd "$DATADIR/morphs-repo" cat <<EOF > hello-stratum.morph -{ - "name": "hello-stratum", - "kind": "stratum", - "chunks": [ - { - "name": "xyzzy", - "repo": "test:chunk-repo", - "ref": "master", - "build-depends": [], - "build-mode": "test", - "prefix": "/plover" - }, - { - "name": "plugh", - "repo": "test:chunk-repo", - "ref": "master", - "build-mode": "test", - "build-depends": [ - "xyzzy" - ] - } - ] -} +name: hello-stratum +kind: stratum +chunks: + - name: xyzzy + repo: test:chunk-repo + ref: master + build-depends: [] + build-mode: test + prefix: /plover + - name: plugh + repo: test:chunk-repo + ref: master + build-mode: test + build-depends: + - xyzzy EOF git add hello-stratum.morph git commit -q -m "Update stratum" diff --git a/tests.build/setup b/tests.build/setup index 559825b1..833f132d 100755 --- a/tests.build/setup +++ b/tests.build/setup @@ -57,19 +57,15 @@ EOF git add hello.c cat <<EOF > hello.morph -{ - "name": "hello", - "kind": "chunk", - "build-system": "dummy", - "build-commands": [ - "gcc -o hello hello.c" - ], - "install-commands": [ - "install -d \\"\$DESTDIR\\"/etc", - "install -d \\"\$DESTDIR\\"/bin", - "install hello \\"\$DESTDIR\\"/bin/hello" - ] -} +name: hello +kind: chunk +build-system: dummy +build-commands: + - gcc -o hello hello.c +install-commands: + - install -d "\$DESTDIR"/etc + - install -d "\$DESTDIR"/bin + - install hello "\$DESTDIR"/bin/hello EOF git add hello.morph @@ -87,33 +83,23 @@ cd "$morphsrepo" git init --quiet cat <<EOF > hello-stratum.morph -{ - "name": "hello-stratum", - "kind": "stratum", - "chunks": [ - { - "name": "hello", - "repo": "test:chunk-repo", - "ref": "farrokh", - "build-mode": "test", - "build-depends": [] - } - ] -} +name: hello-stratum +kind: stratum +chunks: + - name: hello + repo: test:chunk-repo + ref: farrokh + build-mode: test + build-depends: [] EOF git add hello-stratum.morph cat <<EOF > hello-system.morph -{ - "name": "hello-system", - "kind": "system", - "arch": "$("$SRCDIR/scripts/test-morph" print-architecture)", - "strata": [ - { - "morph": "hello-stratum", - } - ] -} +name: hello-system +kind: system +arch: $("$SRCDIR/scripts/test-morph" print-architecture) +strata: + - morph: hello-stratum EOF git add hello-system.morph diff --git a/tests.build/setup-build-essential b/tests.build/setup-build-essential index 5674020d..9ffb7774 100755 --- a/tests.build/setup-build-essential +++ b/tests.build/setup-build-essential @@ -28,28 +28,21 @@ EOF chmod +x morph-test-cc cat <<EOF > "stage1-cc.morph" -{ - "name": "stage1-cc", - "kind": "chunk", - "install-commands": [ - "install -d \"\$DESTDIR\$PREFIX/bin\"", - "install -m 755 morph-test-cc \"\$DESTDIR\$PREFIX/bin/morph-test-cc\"" - ] -} +name: stage1-cc +kind: chunk +install-commands: + - install -d "\$DESTDIR\$PREFIX/bin" + - install -m 755 morph-test-cc "\$DESTDIR\$PREFIX/bin/morph-test-cc" EOF cat <<EOF > "cc.morph" -{ - "name": "cc", - "kind": "chunk", - "configure-commands": [ - "[ -e ../tools/bin/morph-test-cc ]" - ], - "install-commands": [ - "install -d \"\$DESTDIR\$PREFIX/bin\"", - "install -m 755 morph-test-cc \"\$DESTDIR\$PREFIX/bin/morph-test-cc\"" - ] -} +name: cc +kind: chunk +configure-commands: + - [ -e ../tools/bin/morph-test-cc ] +install-commands: + - install -d "\$DESTDIR\$PREFIX/bin" + - install -m 755 morph-test-cc "\$DESTDIR\$PREFIX/bin/morph-test-cc" EOF git init -q @@ -61,21 +54,16 @@ git commit -q -m "Create compiler chunk" cd "$DATADIR/chunk-repo" git checkout -q farrokh cat <<EOF > "hello.morph" -{ - "name": "hello", - "kind": "chunk", - "configure-commands": [ - "[ ! -e ../tools/bin/morph-test-cc ]", - "[ -e ../usr/bin/morph-test-cc ]" - ], - "build-commands": [ - "../usr/bin/morph-test-cc > hello" - ], - "install-commands": [ - "install -d \"\$DESTDIR\$PREFIX/bin\"", - "install hello \"\$DESTDIR\$PREFIX/bin/hello\"" - ] -} +name: hello +kind: chunk +configure-commands: + - [ ! -e ../tools/bin/morph-test-cc ] + - [ -e ../usr/bin/morph-test-cc ] +build-commands: + - ../usr/bin/morph-test-cc > hello +install-commands: + - install -d "\$DESTDIR\$PREFIX/bin" + - install hello "\$DESTDIR\$PREFIX/bin/hello" EOF git add hello.morph git commit -q -m "Make 'hello' require our mock compiler" @@ -85,50 +73,34 @@ git commit -q -m "Make 'hello' require our mock compiler" # artifact, and neither should make it into the system. cd "$DATADIR/morphs-repo" cat <<EOF > "build-essential.morph" -{ - "name": "build-essential", - "kind": "stratum", - "chunks": [ - { - "name": "stage1-cc", - "repo": "test:cc-repo", - "ref": "master", - "build-depends": [], - "build-mode": "bootstrap", - "prefix": "/tools" - }, - { - "name": "cc", - "repo": "test:cc-repo", - "ref": "master", - "build-depends": [ - "stage1-cc" - ], - "build-mode": "test" - } - ] -} +name: build-essential +kind: stratum +chunks: + - name: stage1-cc + repo: test:cc-repo + ref: master + build-depends: [] + build-mode: bootstrap + prefix: /tools + - name: cc + repo: test:cc-repo + ref: master + build-depends: + - stage1-cc + build-mode: test EOF cat <<EOF > "hello-stratum.morph" -{ - "name": "hello-stratum", - "kind": "stratum", - "build-depends": [ - { - "morph": "build-essential", - } - ], - "chunks": [ - { - "name": "hello", - "repo": "test:chunk-repo", - "ref": "farrokh", - "build-depends": [], - "build-mode": "test" - } - ] -} +name: hello-stratum +kind: stratum +build-depends: + - morph: build-essential +chunks: + - name: hello + repo: test:chunk-repo + ref: farrokh + build-depends: [] + build-mode: test EOF git add build-essential.morph hello-stratum.morph hello-system.morph diff --git a/tests.build/stratum-overlap-warns.setup b/tests.build/stratum-overlap-warns.setup index 5ebd217c..b969822d 100755 --- a/tests.build/stratum-overlap-warns.setup +++ b/tests.build/stratum-overlap-warns.setup @@ -3,7 +3,7 @@ # If a stratum has multiple chunks that have the same files in them, # then this should be notified # -# Copyright (C) 2011-2013 Codethink Limited +# Copyright (C) 2011-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -26,40 +26,32 @@ morphsrepo="$DATADIR/morphs-repo" cd "$morphsrepo" git checkout --quiet -b overlap master cat <<EOF >hello-stratum.morph -{ - "name": "hello-stratum", - "kind": "stratum", - "chunks": [ - { - "name": "dirs", - "repo": "test:chunk-repo", - "ref": "overlap", - "build-depends": [], - "build-mode": "test" - }, - { - "name": "overlap-foobar", - "repo": "test:chunk-repo", - "ref": "overlap", - "build-depends": ["dirs"], - "build-mode": "test" - }, - { - "name": "overlap-fooqux", - "repo": "test:chunk-repo", - "ref": "overlap", - "build-depends": ["overlap-foobar"], - "build-mode": "test" - }, - { - "name": "overlap-foo-baz", - "repo": "test:chunk-repo", - "ref": "overlap", - "build-depends": ["overlap-fooqux"], - "build-mode": "test" - } - ] -} +name: hello-stratum +kind: stratum +chunks: + - name: dirs + repo: test:chunk-repo + ref: overlap + build-depends: [] + build-mode: test + - name: overlap-foobar + repo: test:chunk-repo + ref: overlap + build-depends: + - dirs + build-mode: test + - name: overlap-fooqux + repo: test:chunk-repo + ref: overlap + build-depends: + - overlap-foobar + build-mode: test + - name: overlap-foo-baz + repo: test:chunk-repo + ref: overlap + build-depends: + - overlap-fooqux + build-mode: test EOF sed -i 's/master/overlap/' hello-system.morph git add hello-stratum.morph hello-system.morph @@ -69,53 +61,41 @@ cd "$chunkrepo" git checkout --quiet -b overlap master cat <<EOF >dirs.morph -{ - "name": "dirs", - "kind": "chunk", - "install-commands": [ - "mkdir -p \$DESTDIR/bin", - "ln -s .. \$DESTDIR/usr" - ] -} +name: dirs +kind: chunk +install-commands: + - mkdir -p "\$DESTDIR/bin" + - ln -s .. "\$DESTDIR/usr" EOF git add dirs.morph cat <<EOF >overlap-foo-baz.morph -{ - "name": "overlap-foo-baz", - "kind": "chunk", - "install-commands": [ - "mkdir -p \$DESTDIR/bin", - "for f in foo bar baz; do echo echo \$f >\$DESTDIR/bin/\$f; done" - ] -} +name: overlap-foo-baz +kind: chunk +install-commands: + - mkdir -p "\$DESTDIR/bin" + - for f in foo bar baz; do echo echo \$f >"\$DESTDIR/bin/\$f"; done EOF git add overlap-foo-baz.morph cat <<EOF >overlap-foobar.morph -{ - "name": "overlap-foobar", - "kind": "chunk", - "install-commands": [ - "mkdir -p \$DESTDIR/usr/bin \$DESTDIR/bin", - "echo echo foobar >\$DESTDIR/usr/bin/foobar", - "ln -s /usr/bin/foobar \$DESTDIR/bin/foo", - "ln -s /usr/bin/foobar \$DESTDIR/bin/bar" - ] -} +name: overlap-foobar +kind: chunk +install-commands: + - mkdir -p "\$DESTDIR/usr/bin" "\$DESTDIR/bin" + - echo echo foobar >"\$DESTDIR/usr/bin/foobar" + - ln -s /usr/bin/foobar "\$DESTDIR/bin/foo" + - ln -s /usr/bin/foobar "\$DESTDIR/bin/bar" EOF git add overlap-foobar.morph cat <<EOF >overlap-fooqux.morph -{ - "name": "overlap-fooqux", - "kind": "chunk", - "install-commands": [ - "mkdir -p \$DESTDIR/usr/bin \$DESTDIR/bin", - "for f in qux fooqux; do echo echo \$f >\$DESTDIR/usr/bin/\$f; done", - "ln -s /usr/bin/fooqux \$DESTDIR/bin/foo" - ] -} +name: overlap-fooqux +kind: chunk +install-commands: + - mkdir -p "\$DESTDIR/usr/bin" "\$DESTDIR/bin" + - for f in qux fooqux; do echo echo \$f >"\$DESTDIR/usr/bin/\$f"; done + - ln -s /usr/bin/fooqux "\$DESTDIR/bin/foo" EOF git add overlap-fooqux.morph diff --git a/tests.deploy/setup b/tests.deploy/setup index ece8819a..033598bc 100755 --- a/tests.deploy/setup +++ b/tests.deploy/setup @@ -62,19 +62,15 @@ EOF git add hello.c cat <<EOF > hello.morph -{ - "name": "hello", - "kind": "chunk", - "build-system": "dummy", - "build-commands": [ - "gcc -o hello hello.c" - ], - "install-commands": [ - "install -d \\"\$DESTDIR\\"/etc", - "install -d \\"\$DESTDIR\\"/bin", - "install hello \\"\$DESTDIR\\"/bin/hello" - ] -} +name: hello +kind: chunk +build-system: dummy +build-commands: + - gcc -o hello hello.c +install-commands: + - install -d "\$DESTDIR/etc" + - install -d "\$DESTDIR/bin" + - install hello "\$DESTDIR/bin/hello" EOF git add hello.morph @@ -92,76 +88,49 @@ cd "$morphsrepo" git init --quiet cat <<EOF > hello-stratum.morph -{ - "name": "hello-stratum", - "kind": "stratum", - "chunks": [ - { - "name": "hello", - "repo": "test:chunk-repo", - "ref": "farrokh", - "build-depends": [], - "build-mode": "test" - } - ] -} +name: hello-stratum +kind: stratum +chunks: + - name: hello + repo: test:chunk-repo + ref: farrokh + build-depends: [] + build-mode: test EOF git add hello-stratum.morph cat <<EOF > hello-system.morph -{ - "name": "hello-system", - "kind": "system", - "arch": "$("$SRCDIR/scripts/test-morph" print-architecture)", - "strata": [ - { - "morph": "hello-stratum", - } - ] -} +name: hello-system +kind: system +arch: $("$SRCDIR/scripts/test-morph" print-architecture) +strata: + - morph: hello-stratum EOF git add hello-system.morph cat <<EOF > linux-stratum.morph -{ - "name": "linux-stratum", - "kind": "stratum", - "build-depends": [ - { - "morph": "hello-stratum", - } - ], - "chunks": [ - { - "name": "linux", - "repo": "test:kernel-repo", - "ref": "master", - "build-depends": [], - "build-mode": "test" - } - ] -} +name: linux-stratum +kind: stratum +build-depends: + - morph: hello-stratum +chunks: + - name: linux + repo: test:kernel-repo + ref: master + build-depends: [] + build-mode: test EOF git add linux-stratum.morph cat <<EOF > linux-system.morph -{ - "name": "linux-system", - "kind": "system", - "arch": "$("$SRCDIR/scripts/test-morph" print-architecture)", - "strata": [ - { - "morph": "hello-stratum", - }, - { - "morph": "linux-stratum", - } - ], - "configuration-extensions": [ - set-hostname - ] - -} +name: linux-system +kind: system +arch: $("$SRCDIR/scripts/test-morph" print-architecture) +strata: + - morph: hello-stratum + - morph: linux-stratum +configuration-extensions: + - set-hostname EOF git add linux-system.morph @@ -222,16 +191,13 @@ git commit --quiet -m "add morphs" # Make a dummy kernel chunk. mkdir "$DATADIR/kernel-repo" cat <<EOF > "$DATADIR/kernel-repo/linux.morph" -{ - "name": "linux", - "kind": "chunk", - "install-commands": [ - "mkdir -p \"\$DESTDIR/boot\"", - "touch \"\$DESTDIR\"/extlinux.conf", - "touch \"\$DESTDIR\"/boot/vmlinuz", - "touch \"\$DESTDIR\"/boot/System.map" - ] -} +name: linux +kind: chunk +install-commands: + - mkdir -p "\$DESTDIR/boot" + - touch "\$DESTDIR/extlinux.conf" + - touch "\$DESTDIR/boot/vmlinuz" + - touch "\$DESTDIR/boot/System.map" EOF "$SRCDIR/scripts/run-git-in" "$DATADIR/kernel-repo" init --quiet "$SRCDIR/scripts/run-git-in" "$DATADIR/kernel-repo" add . diff --git a/tests/setup b/tests/setup index 07643ddc..02ddb3af 100755 --- a/tests/setup +++ b/tests/setup @@ -10,7 +10,7 @@ # The stratum repository contains a single branch, "master", with a # stratum and a system morphology that include the chunk above. # -# Copyright (C) 2011-2013 Codethink Limited +# Copyright (C) 2011-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -57,19 +57,15 @@ EOF git add hello.c cat <<EOF > hello.morph -{ - "name": "hello", - "kind": "chunk", - "build-system": "dummy", - "build-commands": [ - "gcc -o hello hello.c" - ], - "install-commands": [ - "install -d \\"\$DESTDIR\\"/etc", - "install -d \\"\$DESTDIR\\"/bin", - "install hello \\"\$DESTDIR\\"/bin/hello" - ] -} +name: hello +kind: chunk +build-system: dummy +build-commands: + - gcc -o hello hello.c +install-commands: + - install -d "\$DESTDIR/etc" + - install -d "\$DESTDIR/bin" + - install hello "\$DESTDIR/bin/hello" EOF git add hello.morph @@ -87,34 +83,22 @@ cd "$morphsrepo" git init --quiet cat <<EOF > hello-stratum.morph -{ - "name": "hello-stratum", - "kind": "stratum", - "chunks": [ - { - "name": "hello", - "repo": "test:chunk-repo", - "ref": "farrokh", - "build-mode": "test", - "build-depends": [] - } - ] -} +name: hello-stratum +kind: stratum +chunks: + - name: hello + repo: test:chunk-repo + ref: farrokh + build-mode: test + build-depends: [] EOF git add hello-stratum.morph cat <<EOF > hello-system.morph -{ - "name": "hello-system", - "kind": "system", - "strata": [ - { - "morph": "hello-stratum", - "repo": "test:morphs-repo", - "ref": "master" - } - ] -} +name: hello-system +kind: system +strata: + - morph: hello-stratum EOF git add hello-system.morph diff --git a/tests/show-dependencies.setup b/tests/show-dependencies.setup index 510656e6..c99e90a9 100755 --- a/tests/show-dependencies.setup +++ b/tests/show-dependencies.setup @@ -42,18 +42,14 @@ gtkcomponents=(freetype fontconfig cairo pango glib gdk-pixbuf gtk for component in "${gtkcomponents[@]}" do cat <<EOF > $component.morph -{ - "name": "$component", - "kind": "chunk", - "build-commands": [ - "gcc -o hello hello.c" - ], - "install-commands": [ - "install -d \\"\$DESTDIR\\"/etc", - "install -d \\"\$DESTDIR\\"/bin", - "install hello \\"\$DESTDIR\\"/bin/$component" - ] -} +name: $component +kind: chunk +build-commands: + - gcc -o hello hello.c +install-commands: + - install -d "\$DESTDIR"/etc + - install -d "\$DESTDIR"/bin + - install hello "\$DESTDIR"/bin/$component EOF git add $component.morph done @@ -61,95 +57,64 @@ git commit --quiet -m "add .c source file and GTK chunk morphologies" # Define a stratum for the GTK stack cat <<EOF > gtk-stack.morph -{ - "name": "gtk-stack", - "kind": "stratum", - "build-depends": [ - ], - "chunks": [ - { - "name": "freetype", - "repo": "test:test-repo", - "ref": "master", - "build-mode": "test", - "build-depends": [ - ] - }, - { - "name": "fontconfig", - "repo": "test:test-repo", - "ref": "master", - "build-mode": "test", - "build-depends": [ - ] - }, - { - "name": "cairo", - "repo": "test:test-repo", - "ref": "master", - "build-mode": "test", - "build-depends": [ - ] - }, - { - "name": "pango", - "repo": "test:test-repo", - "ref": "master", - "build-mode": "test", - "build-depends": [ - "freetype", - "fontconfig" - ] - }, - { - "name": "glib", - "repo": "test:test-repo", - "ref": "master", - "build-mode": "test", - "build-depends": [ - ] - }, - { - "name": "gdk-pixbuf", - "repo": "test:test-repo", - "ref": "master", - "build-mode": "test", - "build-depends": [ - "glib" - ] - }, - { - "name": "gtk", - "repo": "test:test-repo", - "ref": "master", - "build-mode": "test", - "build-depends": [ - "cairo", - "gdk-pixbuf", - "glib", - "pango" - ] - }, - { - "name": "dbus", - "repo": "test:test-repo", - "ref": "master", - "build-mode": "test", - "build-depends": [ - ] - }, - { - "name": "dbus-glib", - "repo": "test:test-repo", - "ref": "master", - "build-mode": "test", - "build-depends": [ - "dbus", - "glib" - ] - } - ] -} +name: gtk-stack +kind: stratum +build-depends: [] +chunks: + - name: freetype + repo: test:test-repo + ref: master + build-mode: test + build-depends: [] + - name: fontconfig + repo: test:test-repo + ref: master + build-mode: test + build-depends: [] + - name: cairo + repo: test:test-repo + ref: master + build-mode: test + build-depends: [] + - name: pango + repo: test:test-repo + ref: master + build-mode: test + build-depends: + - freetype + - fontconfig + - name: glib + repo: test:test-repo + ref: master + build-mode: test + build-depends: [] + - name: gdk-pixbuf + repo: test:test-repo + ref: master + build-mode: test + build-depends: + - glib + - name: gtk + repo: test:test-repo + ref: master + build-mode: test + build-depends: + - cairo + - gdk-pixbuf + - glib + - pango + - name: dbus + repo: test:test-repo + ref: master + build-mode: test + build-depends: [] + - name: dbus-glib + repo: test:test-repo + ref: master + build-mode: test + build-depends: + - dbus + - glib EOF git add gtk-stack.morph git commit --quiet -m "add gtk-stack.morph stratum" @@ -172,18 +137,14 @@ xfcecomponents=(xfce4-dev-tools libxfce4util libxfce4ui exo xfconf garcon for component in "${xfcecomponents[@]}" do cat <<EOF > $component.morph -{ - "name": "$component", - "kind": "chunk", - "build-commands": [ - "gcc -o hello hello.c" - ], - "install-commands": [ - "install -d \\"\$DESTDIR\\"/etc", - "install -d \\"\$DESTDIR\\"/bin", - "install hello \\"\$DESTDIR\\"/bin/$component" - ] -} +name: $component +kind: chunk +build-commands: + - gcc -o hello hello.c +install-commands: + - install -d "\$DESTDIR"/etc + - install -d "\$DESTDIR"/bin + - install hello "\$DESTDIR"/bin/$component EOF git add $component.morph done @@ -191,170 +152,117 @@ git commit --quiet -m "add .c source file and GTK chunk morphologies" # Define a stratum for the Xfce core cat <<EOF > xfce-core.morph -{ - "name": "xfce-core", - "kind": "stratum", - "build-depends": [ - { - "morph": "gtk-stack", - } - ], - "chunks": [ - { - "name": "libxfce4util", - "repo": "test:test-repo", - "ref": "master", - "build-mode": "test", - "build-depends": [ - ] - }, - { - "name": "xfconf", - "repo": "test:test-repo", - "ref": "master", - "build-mode": "test", - "build-depends": [ - "libxfce4util" - ] - }, - { - "name": "libxfce4ui", - "repo": "test:test-repo", - "ref": "master", - "build-mode": "test", - "build-depends": [ - "xfconf" - ] - }, - { - "name": "exo", - "repo": "test:test-repo", - "ref": "master", - "build-mode": "test", - "build-depends": [ - "libxfce4util" - ] - }, - { - "name": "garcon", - "repo": "test:test-repo", - "ref": "master", - "build-mode": "test", - "build-depends": [ - "libxfce4util" - ] - }, - { - "name": "thunar", - "repo": "test:test-repo", - "ref": "master", - "build-mode": "test", - "build-depends": [ - "libxfce4ui", - "exo" - ] - }, - { - "name": "tumbler", - "repo": "test:test-repo", - "ref": "master", - "build-mode": "test", - "build-depends": [ - ] - }, - { - "name": "xfce4-panel", - "repo": "test:test-repo", - "ref": "master", - "build-mode": "test", - "build-depends": [ - "libxfce4ui", - "exo", - "garcon" - ] - }, - { - "name": "xfce4-settings", - "repo": "test:test-repo", - "ref": "master", - "build-mode": "test", - "build-depends": [ - "libxfce4ui", - "exo", - "xfconf" - ] - }, - { - "name": "xfce4-session", - "repo": "test:test-repo", - "ref": "master", - "build-mode": "test", - "build-depends": [ - "libxfce4ui", - "exo", - "xfconf" - ] - }, - { - "name": "xfwm4", - "repo": "test:test-repo", - "ref": "master", - "build-mode": "test", - "build-depends": [ - "libxfce4ui", - "xfconf" - ] - }, - { - "name": "xfdesktop", - "repo": "test:test-repo", - "ref": "master", - "build-mode": "test", - "build-depends": [ - "libxfce4ui", - "xfconf" - ] - }, - { - "name": "xfce4-appfinder", - "repo": "test:test-repo", - "ref": "master", - "build-mode": "test", - "build-depends": [ - "libxfce4ui", - "garcon", - "xfconf" - ] - }, - { - "name": "gtk-xfce-engine", - "repo": "test:test-repo", - "ref": "master", - "build-mode": "test", - "build-depends": [ - "libxfce4ui", - "garcon", - "xfconf" - ] - } - ] -} +name: xfce-core +kind: stratum +build-depends: + - morph: gtk-stack +chunks: + - name: libxfce4util + repo: test:test-repo + ref: master + build-mode: test + build-depends: [] + - name: xfconf + repo: test:test-repo + ref: master + build-mode: test + build-depends: + - libxfce4util + - name: libxfce4ui + repo: test:test-repo + ref: master + build-mode: test + build-depends: + - xfconf + - name: exo + repo: test:test-repo + ref: master + build-mode: test + build-depends: + - libxfce4util + - name: garcon + repo: test:test-repo + ref: master + build-mode: test + build-depends: + - libxfce4util + - name: thunar + repo: test:test-repo + ref: master + build-mode: test + build-depends: + - libxfce4ui + - exo + - name: tumbler + repo: test:test-repo + ref: master + build-mode: test + build-depends: [] + - name: xfce4-panel + repo: test:test-repo + ref: master + build-mode: test + build-depends: + - libxfce4ui + - exo + - garcon + - name: xfce4-settings + repo: test:test-repo + ref: master + build-mode: test + build-depends: + - libxfce4ui + - exo + - xfconf + - name: xfce4-session + repo: test:test-repo + ref: master + build-mode: test + build-depends: + - libxfce4ui + - exo + - xfconf + - name: xfwm4 + repo: test:test-repo + ref: master + build-mode: test + build-depends: + - libxfce4ui + - xfconf + - name: xfdesktop + repo: test:test-repo + ref: master + build-mode: test + build-depends: + - libxfce4ui + - xfconf + - name: xfce4-appfinder + repo: test:test-repo + ref: master + build-mode: test + build-depends: + - libxfce4ui + - garcon + - xfconf + - name: gtk-xfce-engine + repo: test:test-repo + ref: master + build-mode: test + build-depends: + - libxfce4ui + - garcon + - xfconf EOF git add xfce-core.morph git commit --quiet -m "add xfce-core.morph stratum" cat <<EOF > xfce-system.morph -{ - "name": "xfce-system", - "kind": "system", - "arch": "$("$SRCDIR/scripts/test-morph" print-architecture)", - "strata": [ - { - "build-mode": "test", - "morph": "xfce-core" - } - ] -} +name: xfce-system +kind: system +arch: $("$SRCDIR/scripts/test-morph" print-architecture) +strata: + - build-mode: test + morph: xfce-core EOF git add xfce-system.morph git commit --quiet -m "add xfce-system" diff --git a/tests/show-dependencies.stdout b/tests/show-dependencies.stdout index 5d6a6e58..91d289a9 100644 --- a/tests/show-dependencies.stdout +++ b/tests/show-dependencies.stdout @@ -1,4 +1,4 @@ -dependency graph for test-repo|master|xfce-system: +dependency graph for test-repo|master|xfce-system.morph: test-repo|master|xfce-system.morph|xfce-system-rootfs -> test-repo|master|xfce-core.morph|xfce-core-devel -> test-repo|master|xfce-core.morph|xfce-core-runtime diff --git a/yarns/branches-workspaces.yarn b/yarns/branches-workspaces.yarn index 96fbbd12..7c85d6e0 100644 --- a/yarns/branches-workspaces.yarn +++ b/yarns/branches-workspaces.yarn @@ -39,7 +39,7 @@ existing workspace, initialising it should fail. WHEN the user attempts to initialise a workspace THEN morph failed -Checking out or branching system branches +Checking out system branches ----------------------------------------- Once we have a workspace, we can check out a system branch. @@ -50,6 +50,12 @@ Once we have a workspace, we can check out a system branch. WHEN the user checks out the system branch called master THEN the system branch master is checked out +Edit is probably not the best name for is, but we can use `morph edit` +to investigate chunks in existing branches. + + WHEN the user edits the chunk test-chunk in branch master + THEN the edited chunk test:test-chunk has git branch master + Checking out a system branch should fail, if the branch doesn't exist. SCENARIO checking out a system branch that doesn't exist @@ -58,6 +64,9 @@ Checking out a system branch should fail, if the branch doesn't exist. WHEN the user attempts to check out the system branch called foo THEN morph failed +Branching system branches +----------------------------------------- + We can, instead, create a new system branch, off master. SCENARIO branch off master @@ -142,19 +151,19 @@ current directory, things should fail. AND the user attempts to report the system branch from the directory . THEN morph failed -`morph show-branch-root` reports the URL (possibly aliases) of the -system branch root repository. It can be run inside a checkout, or -somewhere outside a checkout, where exactly one checkout exists below. +`morph show-branch-root` reports the path of the system branch root +repository. It can be run inside a checkout, or somewhere outside a +checkout, where exactly one checkout exists below. SCENARIO morph reports system branch root repository GIVEN a workspace AND a git server WHEN the user checks out the system branch called master AND the user reports the system branch root repository from the directory master - THEN the system branch root repository is reported as test:morphs + THEN the system branch root repository is reported as workspace/master/test/morphs WHEN the user reports the system branch root repository from the directory . - THEN the system branch root repository is reported as test:morphs + THEN the system branch root repository is reported as workspace/master/test/morphs However, it fails if run outside a checkout and there's no system branches checked out. @@ -168,8 +177,8 @@ branches checked out. Editing components ------------------ -`morph edit` can edit refs for a stratum only, or it can do that for -a chunk, and check out the chunk's repository. +`morph edit` can edit refs for a chunk, and check out the chunk's +repository. First of all, we verify that that when we create a system branch, all the refs are unchanged. @@ -178,7 +187,6 @@ all the refs are unchanged. GIVEN a workspace AND a git server WHEN the user creates a system branch called foo - THEN in branch foo, stratum strata/test-stratum.morph refs test-chunk in master Edit the chunk. We make use of special knowledge here: `test:test-chunk` is a chunk repository created in the mocked git server, for testing @@ -194,6 +202,175 @@ fields when referring to strata, when it didn't before. AND in branch foo, system systems/test-system.morph refers to test-stratum without repo AND in branch foo, system systems/test-system.morph refers to test-stratum without ref +Temporary Build Branch behaviour +-------------------------------- + +Morph always builds from committed changes, but it's not always convenient +to commit and push changes, so `morph build` can create temporary build +branches when necessary. + + SCENARIO morph makes temporary build branches for uncommitted changes when necessary + GIVEN a workspace + AND a git server + WHEN the user checks out the system branch called master + +The user hasn't made any changes yet, so attempts to build require no +temporary build branches. + + GIVEN the workspace contains no temporary build branches + AND we can build with local branches + WHEN the user builds systems/test-system.morph of the master branch + THEN the morphs repository in the workspace for master has no temporary build branches + +Similarly, if we need to build from pushed branches, such as when we're +distbuilding, we don't need temporary build branches yet, since we have +no local changes. + + GIVEN the workspace contains no temporary build branches + AND the git server contains no temporary build branches + AND we must build from pushed branches + WHEN the user builds systems/test-system.morph of the master branch + THEN the morphs repository in the workspace for master has no temporary build branches + AND no temporary build branches were pushed to the morphs repository + +If we actually want to be able to push our changes for review, we need to +use a different branch from master, since we require code to be reviewed +then merged, rather than pushing directly to master. + + WHEN the user creates a system branch called baserock/test + +When we start making changes we do need temporary build branches, since +the chunk specifiers in the strata now need to refer to the local changes +to the repository. + + WHEN the user edits the chunk test-chunk in branch baserock/test + +If we don't need to build from pushed branches then we have temporary +build branches only in the local clones of the repositories. + + GIVEN the workspace contains no temporary build branches + AND the git server contains no temporary build branches + AND we can build with local branches + WHEN the user builds systems/test-system.morph of the baserock/test branch + THEN the morphs repository in the workspace for baserock/test has temporary build branches + AND no temporary build branches were pushed to the morphs repository + +If we do need to build from pushed changes, then the temporary build +branch needs to be pushed. + + GIVEN the workspace contains no temporary build branches + AND the git server contains no temporary build branches + AND we must build from pushed branches + WHEN the user builds systems/test-system.morph of the baserock/test branch + THEN the morphs repository in the workspace for baserock/test has temporary build branches + AND temporary build branches were pushed to the morphs repository + +NOTE: We're not checking whether the test-chunk repo has changes since +it's currently an implementation detail that it does, but it would +be possible to build without a temporary build branch for the chunk +repository. + +Now that we have the chunk repository available, we can make our changes. + + WHEN the user makes changes to test-chunk in branch baserock/test + +When we have uncommitted changes to chunk repositories, we need +temporary build branches locally for local builds. + + GIVEN the workspace contains no temporary build branches + AND the git server contains no temporary build branches + AND we can build with local branches + WHEN the user builds systems/test-system.morph of the baserock/test branch + THEN the morphs repository in the workspace for baserock/test has temporary build branches + AND the test-chunk repository in the workspace for baserock/test has temporary build branches + AND no temporary build branches were pushed to the morphs repository + AND no temporary build branches were pushed to the test-chunk repository + +As before, we also need temporary build branches to have been pushed + + GIVEN the workspace contains no temporary build branches + AND the git server contains no temporary build branches + AND we must build from pushed branches + WHEN the user builds systems/test-system.morph of the baserock/test branch + THEN the morphs repository in the workspace for baserock/test has temporary build branches + AND the test-chunk repository in the workspace for baserock/test has temporary build branches + AND temporary build branches were pushed to the morphs repository + AND temporary build branches were pushed to the test-chunk repository + +Now that we've made our changes, we can commit them. + + WHEN the user commits changes to morphs in branch baserock/test + AND the user commits changes to test-chunk in branch baserock/test + +For local builds we should be able to use these committed changes, +provided the ref in the morphology matches the committed ref in the +chunk repository. + +However, since we do not currently do this integrity check, as it requires +extra tracking between edited morphologies and the local repositories, +it's easier to just require a temporary build branch. + + GIVEN the workspace contains no temporary build branches + AND the git server contains no temporary build branches + AND we can build with local branches + WHEN the user builds systems/test-system.morph of the baserock/test branch + THEN the morphs repository in the workspace for baserock/test has temporary build branches + AND the test-chunk repository in the workspace for baserock/test has temporary build branches + AND no temporary build branches were pushed to the morphs repository + AND no temporary build branches were pushed to the test-chunk repository + +For distributed building, it being committed locally is not sufficient, +as remote workers need to be able to access the changes, and dist-build +workers tunneling into the developer's machine and using those +repositories would be madness, so we require temporary build branches +to be pushed. + + GIVEN the workspace contains no temporary build branches + AND the git server contains no temporary build branches + AND we must build from pushed branches + WHEN the user builds systems/test-system.morph of the baserock/test branch + THEN the morphs repository in the workspace for baserock/test has temporary build branches + AND the test-chunk repository in the workspace for baserock/test has temporary build branches + AND temporary build branches were pushed to the morphs repository + AND temporary build branches were pushed to the test-chunk repository + +We can now push our committed changes. + + WHEN the user pushes the system branch called baserock/test to the git server + +We now don't need temporary build branches for local builds. + + GIVEN the workspace contains no temporary build branches + AND the git server contains no temporary build branches + AND we can build with local branches + WHEN the user builds systems/test-system.morph of the baserock/test branch + THEN the morphs repository in the workspace for baserock/test has no temporary build branches + AND the test-chunk repository in the workspace for baserock/test has no temporary build branches + AND no temporary build branches were pushed to the morphs repository + AND no temporary build branches were pushed to the test-chunk repository + +Nor do we need temporary build branches for distributed builds. + + GIVEN the workspace contains no temporary build branches + AND the git server contains no temporary build branches + AND we must build from pushed branches + WHEN the user builds systems/test-system.morph of the baserock/test branch + THEN the morphs repository in the workspace for baserock/test has no temporary build branches + AND the test-chunk repository in the workspace for baserock/test has no temporary build branches + AND no temporary build branches were pushed to the morphs repository + AND no temporary build branches were pushed to the test-chunk repository + + IMPLEMENTS WHEN the user makes changes to test-chunk in branch (\S+) + chunkdir="$(slashify_colons "test:test-chunk")" + cd "$DATADIR/workspace/$MATCH_1/$chunkdir" + sed -i -e 's/Hello/Goodbye/g' test-bin + + IMPLEMENTS WHEN the user commits changes to (\S+) in branch (\S+) + chunkdir="$(slashify_colons "test:$MATCH_1")" + cd "$DATADIR/workspace/$MATCH_2/$chunkdir" + git commit -a -m 'Commit local changes' + + Status of system branch checkout -------------------------------- @@ -237,41 +414,6 @@ branch checkout. THEN morph ran command in test/morphs in foo AND morph ran command in test/test-chunk in foo -Explicit petrification ----------------------- - -We petrify branches explicitly (though this may later change so that -`morph branch` does it automatically). To test this, we create a branch, -petrify it, and verify that every ref looks like a SHA1. We then -unpetrify and verify that we have all the same refs as before. - - SCENARIO morph petrifies and unpetrifies - GIVEN a workspace - AND a git server - WHEN the user creates a system branch called foo - AND the user pushes the system branch called foo to the git server - AND remembering all refs in foo - AND petrifying foo - THEN foo is petrified - -Petrifying a morphology should not cause it to start having repo or ref -fields when referring to strata, when it didn't before. - - AND in branch foo, system systems/test-system.morph refers to test-stratum without repo - AND in branch foo, system systems/test-system.morph refers to test-stratum without ref - -Unpetrify must put the morphologies back in the same logical state they -were in before. - - WHEN unpetrifying foo - THEN foo refs are as remembered - -Unpetrifying a morphology should not cause it to start having repo or -ref fields when referring to strata, when it didn't before. - - AND in branch foo, system systems/test-system.morph refers to test-stratum without repo - AND in branch foo, system systems/test-system.morph refers to test-stratum without ref - Generating a manifest works SCENARIO morph generates a manifest diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn index 5b5b1724..d2e72ccf 100644 --- a/yarns/implementations.yarn +++ b/yarns/implementations.yarn @@ -67,7 +67,28 @@ another to hold a chunk. # Create a directory for all the git repositories. mkdir "$DATADIR/gits" - # Create a repo for the system and stratum morphologies. + # Create the chunk repository. + + mkdir "$DATADIR/gits/test-chunk" + + run_in "$DATADIR/gits/test-chunk" git init . + cat > "$DATADIR/gits/test-chunk/test-bin" <<'EOF' + #!/bin/sh + echo Hello World + EOF + cat > "$DATADIR/gits/test-chunk/test.h" <<'EOF' + int foo(void); + EOF + cat > "$DATADIR/gits/test-chunk/test.pc" <<'EOF' + prefix=/usr + includedir=${prefix}/include + Name: test + Cflags: -I{includedir} + EOF + run_in "$DATADIR/gits/test-chunk" git add . + run_in "$DATADIR/gits/test-chunk" git commit --allow-empty -m Initial. + + # Create a repo for the morphologies. mkdir "$DATADIR/gits/morphs" @@ -88,15 +109,12 @@ another to hold a chunk. - name: test-chunk repo: test:test-chunk morph: test-chunk.morph - ref: master + unpetrify-ref: master + ref: $(run_in "$DATADIR/gits/test-chunk" git rev-parse master) build-mode: test build-depends: [] EOF - # Create the chunk repository. - - mkdir "$DATADIR/gits/test-chunk" - # To verify that chunk splitting works, we have a chunk that installs # dummy files in all the places that different kinds of files are # usually installed. e.g. executables in `/bin` and `/usr/bin` @@ -240,23 +258,6 @@ another to hold a chunk. run_in "$DATADIR/gits/morphs" git commit -m Initial. run_in "$DATADIR/gits/morphs" git tag -a "test-tag" -m "Tagging test-tag" - run_in "$DATADIR/gits/test-chunk" git init . - cat > "$DATADIR/gits/test-chunk/test-bin" <<'EOF' - #!/bin/sh - echo Hello World - EOF - cat > "$DATADIR/gits/test-chunk/test.h" <<'EOF' - int foo(void); - EOF - cat > "$DATADIR/gits/test-chunk/test.pc" <<'EOF' - prefix=/usr - includedir=${prefix}/include - Name: test - Cflags: -I{includedir} - EOF - run_in "$DATADIR/gits/test-chunk" git add . - run_in "$DATADIR/gits/test-chunk" git commit --allow-empty -m Initial. - # Create the Morph configuration file so we can access the repos # using test:foo URL aliases. @@ -293,7 +294,8 @@ have a morphology using the test architecture. - name: stage1-chunk morph: stage1-chunk.morph repo: test:test-chunk - ref: master + ref: $(run_in "$DATADIR/gits/test-chunk" git rev-parse master) + unpetrify-ref: master build-mode: bootstrap build-depends: [] EOF @@ -307,7 +309,8 @@ have a morphology using the test architecture. - name: test-chunk morph: test-chunk.morph repo: test:test-chunk - ref: master + unpetrify-ref: master + ref: $(run_in "$DATADIR/gits/test-chunk" git rev-parse master) build-mode: test build-depends: [] EOF @@ -380,8 +383,7 @@ Attempt to branch a system branch from a root that had no systems. Pushing all changes in a system branch checkout to the git server. IMPLEMENTS WHEN the user pushes the system branch called (\S+) to the git server - # FIXME: For now, this is just the morphs checkout. - run_in "$DATADIR/workspace/$MATCH_1/test/morphs" git push origin HEAD + run_in "$DATADIR/workspace/$MATCH_1/" morph foreach -- sh -c 'git push -u origin HEAD 2>&1' Report workspace path. @@ -417,7 +419,7 @@ Report system branch root repository. else attempt_morph show-branch-root > "$@"; fi IMPLEMENTS THEN the system branch root repository is reported as (.*) - echo "$MATCH_1" > "$DATADIR/branch-root.actual" + echo "$DATADIR/$MATCH_1" > "$DATADIR/branch-root.actual" diff -u "$DATADIR/branch-root.actual" "$DATADIR/branch-root.reported" Editing morphologies with `morph edit`. @@ -444,7 +446,7 @@ Editing morphologies with `morph edit`. ls -l "$DATADIR/workspace/$MATCH_2" chunkdir="$(slashify_colons "$MATCH_1")" cd "$DATADIR/workspace/$MATCH_2/$chunkdir" - git branch | awk '$1 == "*" { print $2 }' > "$DATADIR/git-branch.actual" + git rev-parse --abbrev-ref HEAD > "$DATADIR/git-branch.actual" echo "$MATCH_2" > "$DATADIR/git-branch.wanted" diff -u "$DATADIR/git-branch.wanted" "$DATADIR/git-branch.actual" @@ -519,54 +521,6 @@ Running shell command in each checked out repository: grep -Fx "$MATCH_1" "$DATADIR/morph.stdout" grep -Fx "$DATADIR/workspace/$MATCH_2/$MATCH_1" "$DATADIR/morph.stdout" -Petrification and unpetrification: - - IMPLEMENTS WHEN remembering all refs in (\S+) - cd "$DATADIR/workspace/$MATCH_1/test/morphs" - list_refs $(find . -type f) > "$DATADIR/refs.remembered" - - IMPLEMENTS THEN (\S+) refs are as remembered - cd "$DATADIR/workspace/$MATCH_1/test/morphs" - - # FIXME: petrify/unpetrify doesn't work quite right at this time: - # petrify can change a ref to a stratum to point at the system - # branch, but does it without adding an unpetrify-ref, and so - # unpetrify doesn't undo the change. We ignore this bug for the - # time being, in order to make the test suite pass. When the - # petrification code has been cleaned up to not be so hairy, - # we'll fix the test and the code. - # - # We would like to verify the result like this: - # - # list_refs $(find . -type f) > "$DATADIR/refs.now" - # diff -u "$DATADIR/refs.remembered" "$DATADIR/refs.now" - # - # However, due to the bug, we have to do it in a more complicated - # manner. - - list_refs $(find . -type f) | - while read filename ref - do - orig=$(awk -v "f=$filename" '$1 == f { print $2 }' \ - "$DATADIR/refs.remembered") - if [ "$orig" != "$ref" ] && [ "$ref" != "$MATCH_1" ] - then - die "Un-petrified ref: $filename $ref (should be $orig)" - fi - done - - IMPLEMENTS WHEN petrifying (\S+) - cd "$DATADIR/workspace/$MATCH_1/test/morphs" - run_morph petrify - - IMPLEMENTS WHEN unpetrifying (\S+) - cd "$DATADIR/workspace/$MATCH_1/test/morphs" - run_morph unpetrify - - IMPLEMENTS THEN (\S+) is petrified - cd "$DATADIR/workspace/$MATCH_1/test/morphs" - assert_morphologies_are_petrified "$MATCH_1" $(find . -type f) - Generating a manifest. IMPLEMENTS GIVEN a system artifact @@ -605,6 +559,72 @@ Generating a manifest. die "Output isn't what we expect" fi +Implementations for temporary build branch handling +--------------------------------------------------- + + IMPLEMENTS GIVEN the workspace contains no temporary build branches + build_ref_prefix=baserock/builds/ + cd "$DATADIR/workspace" + # Want to use -execdir here, but busybox find doesn't support it + find . -name .git -print | while read gitdir; do ( + cd "$(dirname "$gitdir")" + eval "$(git for-each-ref --shell \ + --format='git update-ref -d %(refname) %(objectname)' \ + "refs/heads/$build_ref_prefix")" + ); done + + IMPLEMENTS GIVEN the git server contains no temporary build branches + build_ref_prefix=refs/heads/baserock/builds/ + cd "$DATADIR/gits" + # Want to use -execdir here, but busybox find doesn't support it + find . -name .git -print | while read gitdir; do ( + cd "$(dirname "$gitdir")" + eval "$(git for-each-ref --shell \ + --format='git update-ref -d %(refname) %(objectname)' \ + "$build_ref_prefix")" + git config receive.denyCurrentBranch ignore + rm -f .git/morph-pushed-branches + mkdir -p .git/hooks + cat >.git/hooks/post-receive <<'EOF' + #!/bin/sh + touch "$GIT_DIR/hook-ever-run" + exec cat >>"$GIT_DIR/morph-pushed-branches" + EOF + chmod +x .git/hooks/post-receive + ); done + + IMPLEMENTS GIVEN we can build with local branches + sed -i -e '/push-build-branches/d' "$DATADIR/morph.conf" + + IMPLEMENTS GIVEN we must build from pushed branches + cat >>"$DATADIR/morph.conf" <<'EOF' + push-build-branches = True + EOF + + IMPLEMENTS THEN the (\S+) repository in the workspace for (\S+) has temporary build branches + build_ref_prefix=refs/heads/baserock/builds/ + cd "$DATADIR/workspace/$MATCH_2/$(slashify_colons "test:$MATCH_1")" + git for-each-ref | grep -F "$build_ref_prefix" + + IMPLEMENTS THEN the (\S+) repository in the workspace for (\S+) has no temporary build branches + build_ref_prefix=refs/heads/baserock/builds/ + cd "$DATADIR/workspace/$MATCH_2/$(slashify_colons "test:$MATCH_1")" + if git for-each-ref | grep -F "$build_ref_prefix"; then + die Did not expect repo to contain build branches + fi + + IMPLEMENTS THEN no temporary build branches were pushed to the (\S+) repository + build_ref_prefix=refs/heads/baserock/builds/ + cd "$DATADIR/gits/$MATCH_1/.git" + if test -e morph-pushed-branches && grep -F "$build_ref_prefix" morph-pushed-branches; then + die Did not expect any pushed build branches + fi + + IMPLEMENTS THEN temporary build branches were pushed to the (\S+) repository + build_ref_prefix=refs/heads/baserock/builds/ + cd "$DATADIR/gits/$MATCH_1/.git" + test -e morph-pushed-branches && grep -F "$build_ref_prefix" morph-pushed-branches + Implementation sections for building ==================================== @@ -801,6 +821,10 @@ Implementations for building systems cd "$DATADIR/workspace/$MATCH_3" run_morph build "$MATCH_1" + IMPLEMENTS WHEN the user builds (\S+) of the (\S+) (branch|tag) + cd "$DATADIR/workspace/$MATCH_2" + run_morph build "$MATCH_1" + Implementations for tarball inspection -------------------------------------- diff --git a/yarns/morph.shell-lib b/yarns/morph.shell-lib index 05c11bcc..9d67f2ab 100644 --- a/yarns/morph.shell-lib +++ b/yarns/morph.shell-lib @@ -37,11 +37,12 @@ run_morph() { { set +e - "$SRCDIR"/morph \ + "$SRCDIR"/morph --verbose \ --cachedir-min-space=0 --tempdir-min-space=0 \ --no-default-config --config "$DATADIR/morph.conf" "$@" \ - 2> "$DATADIR/result-$1" + 2> "$DATADIR/result-$1" > "$DATADIR/out-$1" local exit_code="$?" + cat "$DATADIR/out-$1" cat "$DATADIR/result-$1" >&2 return "$exit_code" } diff --git a/yarns/regression.yarn b/yarns/regression.yarn index e5b3d875..6f499d90 100644 --- a/yarns/regression.yarn +++ b/yarns/regression.yarn @@ -90,7 +90,8 @@ Implementations - name: bootstrap-chunk morph: bootstrap-chunk.morph repo: test:test-chunk - ref: master + unpetrify-ref: master + ref: $(run_in "$DATADIR/gits/test-chunk" git rev-parse master) build-mode: bootstrap build-depends: [] EOF diff --git a/yarns/splitting.yarn b/yarns/splitting.yarn index 1b67a881..d4b942d8 100644 --- a/yarns/splitting.yarn +++ b/yarns/splitting.yarn @@ -141,7 +141,8 @@ Implementations chunks: - name: test-chunk repo: test:test-chunk - ref: master + unpetrify-ref: master + ref: $(run_in "$DATADIR/gits/test-chunk" git rev-parse master) morph: test-chunk.morph build-mode: test build-depends: [] @@ -189,13 +190,15 @@ Implementations chunks: - name: test-chunk repo: test:test-chunk - ref: master morph: test-chunk.morph + unpetrify-ref: master + ref: $(run_in "$DATADIR/gits/test-chunk" git rev-parse master) build-mode: test build-depends: [] - name: unbuildable-chunk repo: test:test-chunk - ref: refs/heads/master + unpetrify-ref: refs/heads/master + ref: $(run_in "$DATADIR/gits/test-chunk" git rev-parse master) morph: unbuildable-chunk.morph build-mode: test build-depends: |