diff options
60 files changed, 732 insertions, 378 deletions
@@ -118,6 +118,9 @@ then fi case "$x" in + # Excluding yarn files since it's not possible to split up the + # IMPLEMENTS lines of them + *.yarn) ;; *) if awk 'length > 79' "$x" | grep . > /dev/null then diff --git a/morphlib/__init__.py b/morphlib/__init__.py index 7eb3f975..4954f812 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -38,7 +38,7 @@ __version__ = gitversion.version # List of architectures that Morph supports -valid_archs = ['armv7l', 'armv7b', 'x86_32', 'x86_64'] +valid_archs = ['armv7l', 'armv7b', 'testarch', 'x86_32', 'x86_64'] class Error(cliapp.AppException): diff --git a/morphlib/app.py b/morphlib/app.py index a0833d45..aecee42b 100644 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -20,10 +20,17 @@ import logging import os import sys import time +import urlparse import warnings import morphlib +class InvalidUrlError(cliapp.AppException): + + def __init__(self, parameter, url): + cliapp.AppException.__init__( + self, 'Value %s for argument %s is not a url' % + (url, parameter)) defaults = { 'trove-host': 'git.baserock.org', @@ -85,6 +92,20 @@ class Morph(cliapp.Application): metavar='URL', default=None, group=group_advanced) + self.settings.string( + ['artifact-cache-server'], + 'HTTP URL for the artifact cache server; ' + 'if not set, then the cache-server setting is used instead', + metavar='URL', + default=None, + group=group_advanced) + self.settings.string( + ['git-resolve-cache-server'], + 'HTTP URL for the git ref resolving cache server; ' + 'if not set, then the cache-server setting is used instead', + metavar='URL', + default=None, + group=group_advanced) self.settings.string(['tarball-server'], 'base URL to download tarballs. ' 'If not provided, defaults to ' @@ -200,6 +221,13 @@ class Morph(cliapp.Application): tmpdir = os.path.join(tmpdir_base, 'morph_tmp') self.settings['tempdir'] = tmpdir + if self.settings['tarball-server']: + url_split = urlparse.urlparse(self.settings['tarball-server']) + if not (url_split.netloc and + url_split.scheme in ('http', 'https', 'file')): + raise InvalidUrlError('tarball-server', + self.settings['tarball-server']) + if 'MORPH_DUMP_PROCESSED_CONFIG' in os.environ: self.settings.dump_config(sys.stdout) sys.exit(0) @@ -417,7 +445,7 @@ class Morph(cliapp.Application): # Log the environment. prev = getattr(self, 'prev_env', {}) - morphlib.util.log_dict_diff(kwargs['env'], prev) + morphlib.util.log_dict_diff(self, kwargs['env'], prev) self.prev_env = kwargs['env'] # run the command line diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index e76b7a14..7b7612a4 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -47,9 +47,10 @@ class BuildCommand(object): for repo_name, ref, filename in self.app.itertriplets(args): self.app.status(msg='Building %(repo_name)s %(ref)s %(filename)s', repo_name=repo_name, ref=ref, filename=filename) - self.app.status(msg='Figuring out the right build order') + self.app.status(msg='Deciding on task order') srcpool = self.create_source_pool(repo_name, ref, filename) root_artifact = self.resolve_artifacts(srcpool) + self._validate_architecture(root_artifact) self.build_in_order(root_artifact) self.app.status(msg='Build ends successfully') @@ -88,6 +89,17 @@ class BuildCommand(object): return srcpool + def _validate_architecture(self, root_artifact): + '''Perform the validation between root and target architectures.''' + + root_arch = root_artifact.source.morphology['arch'] + host_arch = morphlib.util.get_host_architecture() + if root_arch != host_arch: + raise morphlib.Error( + 'Are you trying to cross-build? ' + 'Host architecture is %s but target is %s' + % (host_arch, root_arch)) + def resolve_artifacts(self, srcpool): '''Resolve the artifacts that will be built for a set of sources''' @@ -395,8 +407,10 @@ class BuildCommand(object): if artifact.source.build_mode == 'bootstrap': if not self.in_same_stratum(artifact, target_artifact): continue - self.app.status(msg='Installing chunk %(chunk_name)s', - chunk_name=artifact.name) + self.app.status(msg='Installing chunk %(chunk_name)s ' + 'from cache %(cache)s', + chunk_name=artifact.name, + cache=artifact.cache_key[:7]) handle = self.lac.get(artifact) staging_area.install_artifact(handle) diff --git a/morphlib/builder2.py b/morphlib/builder2.py index e1eaaa86..bab89aa2 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -439,8 +439,10 @@ class ChunkBuilder(BuilderBase): contents = morphlib.bins.chunk_contents(destdir, patterns) self.write_metadata(destdir, artifact_name, contents) - logging.debug('assembling chunk %s' % artifact_name) - logging.debug('assembling into %s' % f.name) + self.app.status(msg='assembling chunk %s' % artifact_name, + chatty=True) + self.app.status(msg='assembling into %s' % f.name, + chatty=True) self.app.status(msg='Creating chunk artifact %(name)s', name=artifact.name) morphlib.bins.create_chunk(destdir, f, patterns) @@ -700,8 +702,10 @@ class Builder(object): # pragma: no cover self.remote_artifact_cache, artifact, self.repo_cache, self.max_jobs, self.setup_mounts) - logging.debug('Builder.build: artifact %s with %s' % - (artifact.name, repr(o))) + self.app.status(msg='Builder.build: artifact %s with %s' % + (artifact.name, repr(o)), + chatty=True) built_artifacts = o.build_and_cache() - logging.debug('Builder.build: done') + self.app.status(msg='Builder.build: done', + chatty=True) return built_artifacts diff --git a/morphlib/cachedrepo.py b/morphlib/cachedrepo.py index 0a085bb2..e85b0059 100644 --- a/morphlib/cachedrepo.py +++ b/morphlib/cachedrepo.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -126,7 +126,11 @@ class CachedRepo(object): except cliapp.AppException: raise InvalidReferenceError(self, ref) - tree = self._show_tree_hash(absref) + try: + tree = self._show_tree_hash(absref) + except cliapp.AppException: + raise InvalidReferenceError(self, ref) + return absref, tree def cat(self, ref, filename): @@ -239,11 +243,12 @@ class CachedRepo(object): return self.app.runcmd(*args, **kwargs) def _rev_parse(self, ref): # pragma: no cover - return self._runcmd(['git', 'rev-parse', '--verify', ref])[0:40] + return self._runcmd( + ['git', 'rev-parse', '--verify', '%s^{commit}' % ref])[0:40] def _show_tree_hash(self, absref): # pragma: no cover return self._runcmd( - ['git', 'log', '-1', '--format=format:%T', absref]).strip() + ['git', 'rev-parse', '--verify', '%s^{tree}' % absref]).strip() def _ls_tree(self, ref): # pragma: no cover result = self._runcmd(['git', 'ls-tree', '--name-only', ref]) diff --git a/morphlib/cachedrepo_tests.py b/morphlib/cachedrepo_tests.py index 18704f58..b1825eba 100644 --- a/morphlib/cachedrepo_tests.py +++ b/morphlib/cachedrepo_tests.py @@ -32,8 +32,11 @@ class CachedRepoTests(unittest.TestCase): "kind": "chunk" }''' + bad_sha1_known_to_rev_parse = 'cafecafecafecafecafecafecafecafecafecafe' + def rev_parse(self, ref): output = { + self.bad_sha1_known_to_rev_parse: self.bad_sha1_known_to_rev_parse, 'a4da32f5a81c8bc6d660404724cedc3bc0914a75': 'a4da32f5a81c8bc6d660404724cedc3bc0914a75', 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9': @@ -159,7 +162,7 @@ class CachedRepoTests(unittest.TestCase): def test_fail_resolving_an_invalid_sha1_ref(self): self.assertRaises(cachedrepo.InvalidReferenceError, self.repo.resolve_ref, - '079bbfd447c8534e464ce5d40b80114c2022ebf4') + self.bad_sha1_known_to_rev_parse) def test_cat_existing_file_in_existing_ref(self): data = self.repo.cat('e28a23812eadf2fce6583b8819b9c5dbd36b9fb9', diff --git a/morphlib/exts/fstab.configure b/morphlib/exts/fstab.configure new file mode 100755 index 00000000..a1287ea4 --- /dev/null +++ b/morphlib/exts/fstab.configure @@ -0,0 +1,40 @@ +#!/usr/bin/python +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# =*= License: GPL-2 =*= + + +import os +import sys + + +def asciibetical(strings): + + def key(s): + return [ord(c) for c in s] + + return sorted(strings, key=key) + + +fstab_filename = os.path.join(sys.argv[1], 'etc', 'fstab') + +fstab_vars = asciibetical(x for x in os.environ if x.startswith('FSTAB_')) +with open(fstab_filename, 'a') as f: + for var in fstab_vars: + f.write('%s\n' % os.environ[var]) + +os.chown(fstab_filename, 0, 0) +os.chmod(fstab_filename, 0644) diff --git a/morphlib/exts/kvm.write b/morphlib/exts/kvm.write index 9a6be135..4f877c22 100755 --- a/morphlib/exts/kvm.write +++ b/morphlib/exts/kvm.write @@ -68,6 +68,7 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): except BaseException: sys.stderr.write('Error deploying to libvirt') os.remove(raw_disk) + cliapp.ssh_runcmd(ssh_host, ['rm', '-f', vm_path]) raise else: os.remove(raw_disk) diff --git a/morphlib/exts/rawdisk.write b/morphlib/exts/rawdisk.write index a74d6905..8723ac0c 100755 --- a/morphlib/exts/rawdisk.write +++ b/morphlib/exts/rawdisk.write @@ -46,8 +46,14 @@ class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): if os.path.isfile(location): self.upgrade_local_system(location, temp_root) else: - self.create_local_system(temp_root, location) - self.status(msg='Disk image has been created at %s' % location) + try: + self.create_local_system(temp_root, location) + self.status(msg='Disk image has been created at %s' % location) + except Exception: + os.remove(location) + self.status(msg='Failure to create disk image at %s' % + location) + raise def upgrade_local_system(self, raw_disk, temp_root): mp = self.mount(raw_disk) diff --git a/morphlib/exts/virtualbox-ssh.write b/morphlib/exts/virtualbox-ssh.write index 2374db31..f18ef804 100755 --- a/morphlib/exts/virtualbox-ssh.write +++ b/morphlib/exts/virtualbox-ssh.write @@ -76,10 +76,10 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): except BaseException: sys.stderr.write('Error deploying to VirtualBox') os.remove(raw_disk) + cliapp.ssh_runcmd(ssh_host, ['rm', '-f', vdi_path]) raise else: os.remove(raw_disk) - self.status( msg='Virtual machine %(vm_name)s has been created', vm_name=vm_name) diff --git a/morphlib/morph2.py b/morphlib/morph2.py index 6975e699..0e0d9201 100644 --- a/morphlib/morph2.py +++ b/morphlib/morph2.py @@ -58,10 +58,11 @@ class Morphology(object): ('strata', []), ('description', ''), ('arch', None), - ('system-kind', None), ('configuration-extensions', []), ], - 'cluster': [] + 'cluster': [ + ('description', ''), + ], } @staticmethod @@ -169,10 +170,6 @@ class Morphology(object): self._set_default_value(self._dict, 'max-jobs', int(self['max-jobs'])) - if 'disk-size' in self: - self._set_default_value(self._dict, 'disk-size', - self._parse_size(self['disk-size'])) - for name, value in self.static_defaults[self['kind']]: if name not in self._dict: self._set_default_value(self._dict, name, value) @@ -207,17 +204,6 @@ class Morphology(object): 'deploy', dict()) - def _parse_size(self, size): - if isinstance(size, basestring): - size = size.lower() - if size.endswith('g'): - return int(size[:-1]) * 1024 ** 3 - elif size.endswith('m'): # pragma: no cover - return int(size[:-1]) * 1024 ** 2 - elif size.endswith('k'): # pragma: no cover - return int(size[:-1]) * 1024 - return int(size) # pragma: no cover - def lookup_child_by_name(self, name): '''Find child reference by its name. @@ -285,10 +271,7 @@ class Morphology(object): # Simple values. Use original value unless it has been changed from # the default in memmory. if live_dict[key] == live_dict.get('_orig_' + key, None): - if key in original_dict: - result = original_dict[key] - else: - result = None + result = original_dict.get(key, None) else: result = live_dict[key] return result diff --git a/morphlib/morph2_tests.py b/morphlib/morph2_tests.py index bf32d3c2..aaa1d1cc 100644 --- a/morphlib/morph2_tests.py +++ b/morphlib/morph2_tests.py @@ -96,29 +96,16 @@ class MorphologyTests(unittest.TestCase): self.assertEqual(m['chunks'][0]['morph'], 'le-chunk') self.assertEqual(m['chunks'][0]['build-depends'], None) - def test_parses_system_disk_size(self): - m = Morphology(''' - { - "name": "foo", - "kind": "system", - "disk-size": "1g" - } - ''') - - self.assertEqual(m['disk-size'], 1024 ** 3) - def test_returns_dict_keys(self): m = Morphology(''' { "name": "foo", "kind": "system", - "disk-size": "1g" } ''') self.assertTrue('name' in m.keys()) self.assertTrue('kind' in m.keys()) - self.assertTrue('disk-size' in m.keys()) def test_system_indexes_strata(self): m = Morphology(''' @@ -307,32 +294,9 @@ class MorphologyTests(unittest.TestCase): system_text = '''{ "kind": "system", - "disk-size": "1g", "arch": "x86_64", - "system-kind": "rootfs-tarball" }''' - def test_writing_preserves_disk_size(self): - text_lines = self.system_text.splitlines() - morphology = Morphology(self.system_text) - - output = StringIO.StringIO() - morphology.update_text(self.system_text, output) - output_lines = output.getvalue().splitlines() - self.assertEqual(text_lines, output_lines) - - def test_writing_updates_disk_size(self): - text_lines = self.system_text.splitlines() - text_lines[2] = ' "disk-size": 512,' - - morphology = Morphology(self.system_text) - morphology._dict['disk-size'] = 512 - - output = StringIO.StringIO() - morphology.update_text(self.system_text, output) - output_lines = output.getvalue().splitlines() - self.assertEqual(text_lines, output_lines) - def test_nested_dict(self): # Real morphologies don't trigger this code path, so we test manually original_dict = { diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py index c94078f9..702a330c 100644 --- a/morphlib/morphloader.py +++ b/morphlib/morphloader.py @@ -54,6 +54,12 @@ class InvalidFieldError(morphlib.Error): self.msg = ( 'Field %s not allowed in morphology %s' % (field, morphology)) +class ObsoleteFieldsError(morphlib.Error): + + def __init__(self, fields, morphology): + self.msg = ( + 'Morphology %s uses obsolete fields: %s' % + (morphology, ' '.join(fields))) class UnknownArchitectureError(morphlib.Error): @@ -62,14 +68,6 @@ class UnknownArchitectureError(morphlib.Error): 'Unknown architecture %s in morphology %s' % (arch, morphology)) -class InvalidSystemKindError(morphlib.Error): - - def __init__(self, system_kind, morphology): - self.msg = ( - 'system-kind %s not allowed (must be rootfs-tarball), in %s' % - (system_kind, morphology)) - - class NoBuildDependenciesError(morphlib.Error): def __init__(self, stratum_name, chunk_name, morphology): @@ -115,6 +113,13 @@ class MorphologyLoader(object): ], } + _obsolete_fields = { + 'system': [ + 'system-kind', + 'disk-size', + ], + } + _static_defaults = { 'chunk': { 'description': '', @@ -144,11 +149,11 @@ class MorphologyLoader(object): 'strata': [], 'description': '', 'arch': None, - 'system-kind': 'rootfs-tarball', 'configuration-extensions': [], - 'disk-size': '1G', }, - 'cluster': {}, + 'cluster': { + 'description': '', + }, } def parse_morphology_text(self, text, whence): @@ -229,8 +234,10 @@ class MorphologyLoader(object): raise UnknownKindError(morph['kind'], morph.filename) required = ['kind'] + self._required_fields[kind] + obsolete = self._obsolete_fields.get(kind, []) allowed = self._static_defaults[kind].keys() self._require_fields(required, morph) + self._deny_obsolete_fields(obsolete, morph) self._deny_unknown_fields(required + allowed, morph) if kind == 'system': @@ -260,12 +267,6 @@ class MorphologyLoader(object): if morph['arch'] not in morphlib.valid_archs: raise UnknownArchitectureError(morph['arch'], morph.filename) - # If system-kind is present, it must be rootfs-tarball. - if 'system-kind' in morph: - if morph['system-kind'] not in (None, 'rootfs-tarball'): - raise InvalidSystemKindError( - morph['system-kind'], morph.filename) - def _validate_stratum(self, morph): # Require at least one chunk. if len(morph.get('chunks', [])) == 0: @@ -308,6 +309,11 @@ class MorphologyLoader(object): for field in fields: self._require_field(field, morphology) + def _deny_obsolete_fields(self, fields, morphology): + obsolete_ones = [x for x in morphology if x in fields] + if obsolete_ones: + raise ObsoleteFieldsError(obsolete_ones, morphology.filename) + def _deny_unknown_fields(self, allowed, morphology): for field in morphology: if field not in allowed: diff --git a/morphlib/morphloader_tests.py b/morphlib/morphloader_tests.py index ac0fef53..f38d58e8 100644 --- a/morphlib/morphloader_tests.py +++ b/morphlib/morphloader_tests.py @@ -94,6 +94,26 @@ build-system: dummy self.assertRaises( morphlib.morphloader.InvalidFieldError, self.loader.validate, m) + def test_fails_to_validate_system_with_obsolete_system_kind_field(self): + m = morphlib.morph3.Morphology({ + 'kind': 'system', + 'name': 'foo', + 'arch': 'x86_64', + 'system-kind': 'foo', + }) + self.assertRaises( + morphlib.morphloader.ObsoleteFieldsError, self.loader.validate, m) + + def test_fails_to_validate_system_with_obsolete_disk_size_field(self): + m = morphlib.morph3.Morphology({ + 'kind': 'system', + 'name': 'foo', + 'arch': 'x86_64', + 'disk-size': 'over 9000', + }) + self.assertRaises( + morphlib.morphloader.ObsoleteFieldsError, self.loader.validate, m) + def test_fails_to_validate_system_with_no_fields(self): m = morphlib.morph3.Morphology({ 'kind': 'system', @@ -182,22 +202,6 @@ build-system: dummy self.loader.validate(m) self.assertEqual(m['arch'], 'armv7l') - def test_validate_requires_system_kind_to_be_tarball_if_present(self): - m = morphlib.morph3.Morphology( - { - "kind": "system", - "name": "foo", - "arch": "armv7l", - "strata": [], - "system-kind": "blah", - }) - - self.assertRaises( - morphlib.morphloader.InvalidSystemKindError, - self.loader.validate, m) - m['system-kind'] = 'rootfs-tarball' - self.loader.validate(m) - def test_validate_requires_build_deps_for_chunks_in_strata(self): m = morphlib.morph3.Morphology( { @@ -468,13 +472,11 @@ name: foo dict(m), { 'kind': 'system', - 'system-kind': 'rootfs-tarball', 'name': 'foo', 'description': '', 'arch': 'x86_64', 'strata': [], 'configuration-extensions': [], - 'disk-size': '1G', }) def test_unsets_defaults_for_system(self): diff --git a/morphlib/morphologyfactory.py b/morphlib/morphologyfactory.py index ae5a4332..5afafefb 100644 --- a/morphlib/morphologyfactory.py +++ b/morphlib/morphologyfactory.py @@ -140,19 +140,6 @@ class MorphologyFactory(object): (morphology['arch'], ', '.join(morphlib.valid_archs))) - kind = morphology['system-kind'] - if kind == 'rootfs-tarball': # pragma: no cover - self._app.status( - msg='WARNING: Obsolete field system-kind used in morphology ' - '(it is harmless, but should be removed)') - elif kind: - raise morphlib.Error( - 'System kind %s is not supported (anymore), ' - 'the whole system-kind field is deprecated. ' - 'Please remove system-kind from your system ' - 'morphologies and morph deploy to create ' - 'the desired output format.' % kind) - name = morphology['name'] morphology.builds_artifacts = [name + '-rootfs'] diff --git a/morphlib/morphologyfactory_tests.py b/morphlib/morphologyfactory_tests.py index 06489085..6e1e67d3 100644 --- a/morphlib/morphologyfactory_tests.py +++ b/morphlib/morphologyfactory_tests.py @@ -118,7 +118,6 @@ class FakeLocalRepo(object): 'system.morph': '''{ "name": "system", "kind": "system", - "system-kind": "%(system_kind)s", "arch": "%(arch)s" }''', 'parse-error.morph': '''{ "name"''', @@ -130,13 +129,11 @@ class FakeLocalRepo(object): def __init__(self): self.arch = 'x86_64' - self.system_kind = '' def cat(self, sha1, filename): if filename in self.morphologies: values = { 'arch': self.arch, - 'system_kind': self.system_kind, } return self.morphologies[filename] % values elif filename.endswith('.morph'): @@ -308,11 +305,6 @@ class MorphologyFactoryTests(unittest.TestCase): morph = self.mf.get_morphology('reponame', 'sha1', 'system.morph') self.assertEqual(morph['arch'], 'armv7l') - def test_fails_if_system_define_system_kind_that_is_not_tarball(self): - self.lr.system_kind = 'blahblah' - self.assertRaises(morphlib.Error, self.mf.get_morphology, - 'reponame', 'sha1', 'system.morph') - def test_fails_on_parse_error(self): self.assertRaises(morphlib.Error, self.mf.get_morphology, 'reponame', 'sha1', 'parse-error.morph') diff --git a/morphlib/plugins/artifact_inspection_plugin.py b/morphlib/plugins/artifact_inspection_plugin.py index 6c321abb..9181d488 100644 --- a/morphlib/plugins/artifact_inspection_plugin.py +++ b/morphlib/plugins/artifact_inspection_plugin.py @@ -78,7 +78,8 @@ class AutotoolsVersionGuesser(ProjectVersionGuesser): break # Then, try running autoconf against the configure script - version = self._check_autoconf_package_version(filename, data) + version = self._check_autoconf_package_version( + repo, ref, filename, data) if version: self.app.status( msg='%(repo)s: Version of %(ref)s detected ' @@ -107,7 +108,7 @@ class AutotoolsVersionGuesser(ProjectVersionGuesser): return version return None - def _check_autoconf_package_version(self, filename, data): + def _check_autoconf_package_version(self, repo, ref, filename, data): tempdir = morphlib.tempdir.Tempdir(self.app.settings['tempdir']) with open(tempdir.join(filename), 'w') as f: f.write(data) diff --git a/morphlib/plugins/branch_and_merge_new_plugin.py b/morphlib/plugins/branch_and_merge_new_plugin.py index 3d0a71a5..9c4cd53e 100644 --- a/morphlib/plugins/branch_and_merge_new_plugin.py +++ b/morphlib/plugins/branch_and_merge_new_plugin.py @@ -134,6 +134,9 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): gd.update_submodules(self.app) gd.update_remotes() + except morphlib.sysbranchdir.SystemBranchDirectoryAlreadyExists as e: + logging.error('Caught exception: %s' % str(e)) + raise except BaseException as e: # Oops. Clean up. logging.error('Caught exception: %s' % str(e)) @@ -434,22 +437,14 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): sb = morphlib.sysbranchdir.open_from_within('.') loader = morphlib.morphloader.MorphologyLoader() - # FIXME: The old "morph edit" code did its own morphology validation, - # which was much laxer than what MorphologyFactory does, or the - # new MorphologyLoader does. This new "morph edit" uses - # MorphologyLoader, and the stricter validation breaks the test - # suite. However, I want to keep the test suite as untouched as - # possible, until all the old code is gone (after which the test - # suite will be refactored). Thus, to work around the test suite - # breaking, we disable morphology validation for now. - loader.validate = lambda *args: None - # Load the system morphology, and all stratum morphologies, including # all the strata that are being build-depended on. logging.debug('Loading system morphology') system_morph = loader.load_from_file( sb.get_filename(sb.root_repository_url, system_name + '.morph')) + if system_morph['kind'] != 'system': + raise cliapp.AppException("%s is not a system" % system_name) system_morph.repo_url = sb.root_repository_url system_morph.ref = sb.system_branch_name system_morph.filename = system_name + '.morph' @@ -832,7 +827,7 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): system, metadata = self._load_system_metadata(metadata_path) resolved_refs = dict(self._resolve_refs_from_metadata(alias_resolver, metadata)) - logging.debug('Resolved refs: %r' % resolved_refs) + self.app.status(msg='Resolved refs: %r' % resolved_refs) base_ref = system['sha1'] # The previous version would fall back to deducing this from the repo # url and the repo alias resolver, but this does not always work, and diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py index 2f8560d0..a2d2174a 100644 --- a/morphlib/plugins/branch_and_merge_plugin.py +++ b/morphlib/plugins/branch_and_merge_plugin.py @@ -333,7 +333,6 @@ class BranchAndMergePlugin(cliapp.Plugin): required = { 'system': [ 'name', - 'system-kind', 'arch', 'strata', ], @@ -354,8 +353,6 @@ class BranchAndMergePlugin(cliapp.Plugin): 'system': [ 'kind', 'description', - 'disk-size', - '_disk-size', 'configuration-extensions', ], 'stratum': [ @@ -1641,7 +1638,7 @@ class BranchAndMergePlugin(cliapp.Plugin): } def add_morphology_info(info, category): - repo = info['repo'] + repo = info['repo'] or branch_root if repo in build_repos: repo_dir = build_repos[repo]['dirname'] else: @@ -1661,7 +1658,7 @@ class BranchAndMergePlugin(cliapp.Plugin): # building this system from the system branch. system_morphology = self.load_morphology(branch_root_dir, system_name) for info in system_morphology['strata']: - if info['ref'] == system_branch: + if info['ref'] == system_branch or info['ref'] is None: repo_dir = add_morphology_info(info, 'strata') if repo_dir: stratum_morphology = self.load_morphology( @@ -1721,8 +1718,9 @@ class BranchAndMergePlugin(cliapp.Plugin): # Obtain parent SHA1 for the temporary ref tree to be committed. # This will either be the current commit of the temporary ref or # HEAD in case the temporary ref does not exist yet. - system_branch_sha1 = self.resolve_ref(repo_dir, system_branch) - parent_sha1 = self.resolve_ref(repo_dir, build_ref) + system_branch_sha1 = self.resolve_ref( + repo_dir, '%s^{commit}' % system_branch) + parent_sha1 = self.resolve_ref(repo_dir, '%s^{commit}' % build_ref) if not parent_sha1: parent_sha1 = system_branch_sha1 @@ -1747,7 +1745,7 @@ class BranchAndMergePlugin(cliapp.Plugin): self.app.runcmd(['git', 'add'] + changed_files, cwd=repo_dir, env=env) - self.app.status(msg='%(repo)s: Update morphologies to use ' + self.app.status(msg='%(repo)s: Updating morphologies to use ' 'build branch instead of "%(branch)s"', repo=repo, branch=system_branch) @@ -1767,10 +1765,17 @@ class BranchAndMergePlugin(cliapp.Plugin): ['git', 'hash-object', '-t', 'blob', '-w', f.name], cwd=repo_dir, env=env) - self.app.runcmd( + try: + self.app.runcmd( ['git', 'update-index', '--cacheinfo', '100644', morphology_sha1, '%s.morph' % filename], cwd=repo_dir, env=env) + except cliapp.AppException, e: + raise cliapp.AppException( + "You seem to want to build %s, but '%s.morph' " + "doesn't exist in the morphologies repository. " + "Did you forget to commit it?" % + (filename, filename)) # Create a commit message including the build UUID. This allows us # to collect all commits of a build across repositories and thereby diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index dc9f5158..825b0124 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -274,6 +274,10 @@ class DeployPlugin(cliapp.Plugin): root_repo_dir = self.other.find_repository(branch_dir, root_repo) data = self.other.load_morphology(root_repo_dir, cluster) + if data['kind'] != 'cluster': + raise cliapp.AppException( + "Error: morph deploy is only supported for cluster" + " morphologies.") for system in data['systems']: self.deploy_system(system, env_vars) @@ -461,7 +465,7 @@ class DeployPlugin(cliapp.Plugin): self.app.runcmd( [ext_filename] + args, ['sh', '-c', 'while read l; do echo `date "+%F %T"` $l; done'], - env=env, stdout=None, stderr=None) + cwd=repo_dir, env=env, stdout=None, stderr=None) if delete_ext: os.remove(ext_filename) diff --git a/morphlib/plugins/show_dependencies_plugin.py b/morphlib/plugins/show_dependencies_plugin.py index 9fdb7c60..c59cf507 100644 --- a/morphlib/plugins/show_dependencies_plugin.py +++ b/morphlib/plugins/show_dependencies_plugin.py @@ -52,9 +52,12 @@ class ShowDependenciesPlugin(cliapp.Plugin): self.app.settings['repo-alias']) lrc = morphlib.localrepocache.LocalRepoCache( self.app, cachedir, repo_resolver, tarball_base_url) - if self.app.settings['cache-server']: + + remote_url = morphlib.util.get_git_resolve_cache_server( + self.app.settings) + if remote_url: rrc = morphlib.remoterepocache.RemoteRepoCache( - self.app.settings['cache-server'], repo_resolver) + remote_url, repo_resolver) else: rrc = None diff --git a/morphlib/remoteartifactcache.py b/morphlib/remoteartifactcache.py index 17f547f2..9f6bf69e 100644 --- a/morphlib/remoteartifactcache.py +++ b/morphlib/remoteartifactcache.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -16,6 +16,7 @@ import cliapp import logging +import urllib import urllib2 import urlparse @@ -108,7 +109,8 @@ class RemoteArtifactCache(object): if not server_url.endswith('/'): server_url += '/' return urlparse.urljoin( - server_url, '/1.0/artifacts?filename=%s' % filename) + server_url, '/1.0/artifacts?filename=%s' % + urllib.quote(filename)) def __str__(self): # pragma: no cover return self.server_url diff --git a/morphlib/remoteartifactcache_tests.py b/morphlib/remoteartifactcache_tests.py index 431731b9..e7f45f58 100644 --- a/morphlib/remoteartifactcache_tests.py +++ b/morphlib/remoteartifactcache_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -157,3 +157,8 @@ class RemoteArtifactCacheTests(unittest.TestCase): self.runtime_artifact.source, self.runtime_artifact.cache_key, 'non-existent-meta') + + def test_escapes_pluses_in_request_urls(self): + returned_url = self.cache._request_url('gtk+') + correct_url = '%s/1.0/artifacts?filename=gtk%%2B' % self.server_url + self.assertEqual(returned_url, correct_url) diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py index 7d034ff2..286e5374 100644 --- a/morphlib/stagingarea.py +++ b/morphlib/stagingarea.py @@ -156,9 +156,6 @@ class StagingArea(object): ''' - logging.debug('Installing artifact %s' % - getattr(handle, 'name', 'unknown name')) - chunk_cache_dir = os.path.join(self._app.settings['tempdir'], 'chunks') unpacked_artifact = os.path.join( chunk_cache_dir, os.path.basename(handle.name) + '.d') diff --git a/morphlib/stagingarea_tests.py b/morphlib/stagingarea_tests.py index 7cd4a5c3..27f85dff 100644 --- a/morphlib/stagingarea_tests.py +++ b/morphlib/stagingarea_tests.py @@ -57,6 +57,9 @@ class FakeApplication(object): def runcmd_unchecked(self, *args, **kwargs): return cliapp.runcmd_unchecked(*args, **kwargs) + def status(self, msg): + pass + class StagingAreaTests(unittest.TestCase): diff --git a/morphlib/util.py b/morphlib/util.py index 19c0046f..04df0633 100644 --- a/morphlib/util.py +++ b/morphlib/util.py @@ -81,6 +81,22 @@ def create_cachedir(settings): # pragma: no cover return cachedir +def get_artifact_cache_server(settings): # pragma: no cover + if settings['artifact-cache-server']: + return settings['artifact-cache-server'] + if settings['cache-server']: + return settings['cache-server'] + return None + + +def get_git_resolve_cache_server(settings): # pragma: no cover + if settings['git-resolve-cache-server']: + return settings['git-resolve-cache-server'] + if settings['cache-server']: + return settings['cache-server'] + return None + + def new_artifact_caches(settings): # pragma: no cover '''Create new objects for local and remote artifact caches. @@ -95,7 +111,7 @@ def new_artifact_caches(settings): # pragma: no cover lac = morphlib.localartifactcache.LocalArtifactCache(artifact_cachedir) - rac_url = settings['cache-server'] + rac_url = get_artifact_cache_server(settings) if rac_url: rac = morphlib.remoteartifactcache.RemoteArtifactCache(rac_url) else: @@ -162,7 +178,7 @@ def new_repo_caches(app): # pragma: no cover lrc = morphlib.localrepocache.LocalRepoCache( app, gits_dir, repo_resolver, tarball_base_url=tarball_base_url) - url = app.settings['cache-server'] + url = get_git_resolve_cache_server(app.settings) if url: rrc = morphlib.remoterepocache.RemoteRepoCache(url, repo_resolver) else: @@ -171,20 +187,22 @@ def new_repo_caches(app): # pragma: no cover return lrc, rrc -def log_dict_diff(cur, pre): # pragma: no cover +def log_dict_diff(app, cur, pre): # pragma: no cover '''Log the differences between two dicts to debug log''' dictA = cur dictB = pre for key in dictA.keys(): if key not in dictB: - logging.debug("New environment: %s = %s" % (key, dictA[key])) + app.status(msg="New environment: %(key)s = %(value)s", + key=key, value=dictA[key], chatty=True) elif dictA[key] != dictB[key]: - logging.debug( - "Environment changed: %(key)s = %(valA)s to %(key)s = %(valB)s" - % {"key": key, "valA": dictA[key], "valB": dictB[key]}) + app.status(msg="Environment changed: \ + %(key)s = %(valA)s to %(key)s = %(valB)s", + key=key, valA=dictA[key], valB=dictB[key], chatty=True) for key in dictB.keys(): if key not in dictA: - logging.debug("Environment removed: %s = %s" % (key, dictB[key])) + app.status(msg="Environment removed: %(key)s = %(value)s", + key=key, value=dictB[key], chatty=True) # This acquired from rdiff-backup which is GPLv2+ and a patch from 2011 diff --git a/scripts/setup-3rd-party-strata b/scripts/setup-3rd-party-strata index f2ea2a4c..fc263f96 100644 --- a/scripts/setup-3rd-party-strata +++ b/scripts/setup-3rd-party-strata @@ -97,9 +97,7 @@ cat <<EOF > "hello-system.morph" { "name": "hello-system", "kind": "system", - "system-kind": "rootfs-tarball", "arch": "x86_64", - "disk-size": "1G", "strata": [ { "morph": "hello-stratum", diff --git a/tests.as-root/archless-system-fails.script b/tests.as-root/archless-system-fails.script index eda797f1..2fdeb018 100755 --- a/tests.as-root/archless-system-fails.script +++ b/tests.as-root/archless-system-fails.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2011, 2012 Codethink Limited +# Copyright (C) 2011-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -26,7 +26,6 @@ cat <<EOF >archless-system.morph { "name": "archless-system", "kind": "system", - "disk-size": "1G", "strata": [ { "morph": "hello-stratum", diff --git a/tests.as-root/metadata-includes-morph-version.setup b/tests.as-root/metadata-includes-morph-version.setup index 2284cfb9..f0aefb3c 100755 --- a/tests.as-root/metadata-includes-morph-version.setup +++ b/tests.as-root/metadata-includes-morph-version.setup @@ -27,7 +27,6 @@ cat <<EOF > hello-tarball.morph { "name": "hello-tarball", "kind": "system", - "system-kind": "rootfs-tarball", "arch": "$(uname -m)", "strata": [ { diff --git a/tests.as-root/metadata-includes-repo-alias.setup b/tests.as-root/metadata-includes-repo-alias.setup index 2284cfb9..f0aefb3c 100755 --- a/tests.as-root/metadata-includes-repo-alias.setup +++ b/tests.as-root/metadata-includes-repo-alias.setup @@ -27,7 +27,6 @@ cat <<EOF > hello-tarball.morph { "name": "hello-tarball", "kind": "system", - "system-kind": "rootfs-tarball", "arch": "$(uname -m)", "strata": [ { diff --git a/tests.as-root/rootfs-tarball-builds-rootfs-and-kernel.script b/tests.as-root/rootfs-tarball-builds-rootfs-and-kernel.script index 8229127d..93cd135f 100755 --- a/tests.as-root/rootfs-tarball-builds-rootfs-and-kernel.script +++ b/tests.as-root/rootfs-tarball-builds-rootfs-and-kernel.script @@ -16,9 +16,6 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -## A system-kind of rootfs-tarball should build both kernel image and -## a tarball with the root filesystem. - set -eu # Disable test on versions of Python before 2.7. diff --git a/tests.as-root/setup b/tests.as-root/setup index eea9e061..2ab1cd94 100755 --- a/tests.as-root/setup +++ b/tests.as-root/setup @@ -134,9 +134,7 @@ git add tools-stratum.morph cat <<EOF > hello-system.morph name: hello-system kind: system -system-kind: rootfs-tarball arch: `uname -m` -disk-size: 1G strata: - morph: hello-stratum repo: test:morphs @@ -163,9 +161,7 @@ git add linux-stratum.morph cat <<EOF > linux-system.morph name: linux-system kind: system -system-kind: rootfs-tarball arch: `uname -m` -disk-size: 1G strata: - morph: hello-stratum repo: test:morphs diff --git a/tests.as-root/system-overlap.script b/tests.as-root/system-overlap.script index 6e6ef2ac..41ff7536 100755 --- a/tests.as-root/system-overlap.script +++ b/tests.as-root/system-overlap.script @@ -31,9 +31,7 @@ cat <<EOF >overlap-system.morph { "name": "overlap-system", "kind": "system", - "system-kind": "rootfs-tarball", "arch": "$(uname -m)", - "disk-size": "1G", "strata": [ { "morph": "foo-baz-stratum", diff --git a/tests.as-root/tarball-image-is-sensible.setup b/tests.as-root/tarball-image-is-sensible.setup index fa904c2c..5fd7b283 100755 --- a/tests.as-root/tarball-image-is-sensible.setup +++ b/tests.as-root/tarball-image-is-sensible.setup @@ -46,7 +46,6 @@ cat <<EOF > hello-tarball.morph { "name": "hello-tarball", "kind": "system", - "system-kind": "rootfs-tarball", "arch": "$(uname -m)", "strata": [ { diff --git a/tests.branching.disabled/workflow-petrify.stdout b/tests.branching.disabled/workflow-petrify.stdout index 9f0cfb0c..a0ce82f4 100644 --- a/tests.branching.disabled/workflow-petrify.stdout +++ b/tests.branching.disabled/workflow-petrify.stdout @@ -2,9 +2,7 @@ test/petrify after petrifying: { "name": "hello-system", "kind": "system", - "system-kind": "rootfs-tarball", "arch": "x86_64", - "disk-size": "1G", "strata": [ { "morph": "hello-stratum", @@ -56,9 +54,7 @@ test/petrify after editing a chunk: { "name": "hello-system", "kind": "system", - "system-kind": "rootfs-tarball", "arch": "x86_64", - "disk-size": "1G", "strata": [ { "morph": "hello-stratum", @@ -109,9 +105,7 @@ test/unpetrify after unpetrifying: { "name": "hello-system", "kind": "system", - "system-kind": "rootfs-tarball", "arch": "x86_64", - "disk-size": "1G", "strata": [ { "morph": "hello-stratum", diff --git a/tests.branching/edit-updates-stratum-build-depends.stdout b/tests.branching/edit-updates-stratum-build-depends.stdout index 1c6eb8e3..7120ef50 100644 --- a/tests.branching/edit-updates-stratum-build-depends.stdout +++ b/tests.branching/edit-updates-stratum-build-depends.stdout @@ -13,10 +13,10 @@ index 73ed482..475fe0f 100644 kind: stratum name: hello-stratum diff --git a/hello-system.morph b/hello-system.morph -index 721473c..1537f53 100644 +index 3f7b4d3..199c924 100644 --- a/hello-system.morph +++ b/hello-system.morph -@@ -3,9 +3,9 @@ kind: system +@@ -3,5 +3,6 @@ kind: system name: hello-system strata: - morph: hello-stratum @@ -24,27 +24,3 @@ index 721473c..1537f53 100644 + ref: newbranch repo: test:morphs + unpetrify-ref: master - - morph: xyzzy-stratum - ref: master - repo: test:morphs --system-kind: rootfs-tarball -diff --git a/xyzzy-stratum.morph b/xyzzy-stratum.morph -index e302037..bcf5b57 100644 ---- a/xyzzy-stratum.morph -+++ b/xyzzy-stratum.morph -@@ -1,11 +1,13 @@ - build-depends: - - morph: hello-stratum -- ref: master -+ ref: newbranch - repo: test:morphs -+ unpetrify-ref: master - chunks: - - build-depends: [] - name: hello -- ref: master -+ ref: newbranch - repo: test:hello -+ unpetrify-ref: master - kind: stratum - name: xyzzy-stratum diff --git a/tests.branching/edit-updates-stratum.stdout b/tests.branching/edit-updates-stratum.stdout index 32eb820d..7120ef50 100644 --- a/tests.branching/edit-updates-stratum.stdout +++ b/tests.branching/edit-updates-stratum.stdout @@ -13,15 +13,14 @@ index 73ed482..475fe0f 100644 kind: stratum name: hello-stratum diff --git a/hello-system.morph b/hello-system.morph -index b0fed3b..199c924 100644 +index 3f7b4d3..199c924 100644 --- a/hello-system.morph +++ b/hello-system.morph -@@ -3,6 +3,6 @@ kind: system +@@ -3,5 +3,6 @@ kind: system name: hello-system strata: - morph: hello-stratum - ref: master + ref: newbranch repo: test:morphs --system-kind: rootfs-tarball + unpetrify-ref: master diff --git a/tests.branching/foreach-handles-full-urls.stdout b/tests.branching/foreach-handles-full-urls.stdout index 3abae62c..cee2f70a 100644 --- a/tests.branching/foreach-handles-full-urls.stdout +++ b/tests.branching/foreach-handles-full-urls.stdout @@ -1,4 +1,4 @@ file://TMP/morphs # On branch master -nothing to commit (working directory clean) +nothing to commit, working directory clean diff --git a/tests.branching/setup b/tests.branching/setup index 589f19ed..1263e3b6 100755 --- a/tests.branching/setup +++ b/tests.branching/setup @@ -55,7 +55,6 @@ strata: - morph: hello-stratum ref: master repo: test:morphs -system-kind: rootfs-tarball EOF cat <<EOF > "$DATADIR/morphs/hello-stratum.morph" diff --git a/tests.branching/tag-creates-commit-and-tag.stdout b/tests.branching/tag-creates-commit-and-tag.stdout index 598e28bf..b6098eb5 100644 --- a/tests.branching/tag-creates-commit-and-tag.stdout +++ b/tests.branching/tag-creates-commit-and-tag.stdout @@ -5,7 +5,7 @@ Date: Tue Jul 31 16:51:54 2012 +0000 Message -commit 9509c9c379f8ba643b2ad9a6ec50ecf96993cbb5 +commit 6895ed63425bedb3dccaea3f258c705b1600f6be Author: developer <developer@example.com> Date: Tue Jul 31 16:51:54 2012 +0000 @@ -26,10 +26,10 @@ index 73ed482..2218f63 100644 kind: stratum name: hello-stratum diff --git a/hello-system.morph b/hello-system.morph -index b0fed3b..4c4ee3e 100644 +index 3f7b4d3..d909347 100644 --- a/hello-system.morph +++ b/hello-system.morph -@@ -3,6 +3,7 @@ kind: system +@@ -3,5 +3,6 @@ kind: system name: hello-system strata: - morph: hello-stratum @@ -37,10 +37,9 @@ index b0fed3b..4c4ee3e 100644 + ref: example-tag repo: test:morphs + unpetrify-ref: master - system-kind: rootfs-tarball test:morphs -commit 9509c9c379f8ba643b2ad9a6ec50ecf96993cbb5 +commit 6895ed63425bedb3dccaea3f258c705b1600f6be Author: developer <developer@example.com> AuthorDate: Tue Jul 31 16:51:54 2012 +0000 Commit: developer <developer@example.com> @@ -48,7 +47,7 @@ CommitDate: Tue Jul 31 16:51:54 2012 +0000 Message -commit 1fe013ee284724848d65096d4d88f612fae56fc6 +commit e11a36aa9e4c998c41a3ec3209324b9318e484ae Author: developer <developer@example.com> AuthorDate: Tue Jul 31 16:51:54 2012 +0000 Commit: developer <developer@example.com> diff --git a/tests.branching/tag-tag-works-as-expected.stdout b/tests.branching/tag-tag-works-as-expected.stdout index 242b2d4e..98a3be81 100644 --- a/tests.branching/tag-tag-works-as-expected.stdout +++ b/tests.branching/tag-tag-works-as-expected.stdout @@ -9,7 +9,7 @@ Date: Tue Jul 31 16:51:54 2012 +0000 Second -commit ca7594f436da55bda2cfff2c6484d11aa0ea4cbc +commit 476e4ff4b19c38eb64ad3a151b7c58a7ab95c9ee Author: developer <developer@example.com> Date: Tue Jul 31 16:51:54 2012 +0000 @@ -30,10 +30,10 @@ index 73ed482..2218f63 100644 kind: stratum name: hello-stratum diff --git a/hello-system.morph b/hello-system.morph -index b0fed3b..875d73a 100644 +index 3f7b4d3..431e15d 100644 --- a/hello-system.morph +++ b/hello-system.morph -@@ -3,6 +3,7 @@ kind: system +@@ -3,5 +3,6 @@ kind: system name: hello-system strata: - morph: hello-stratum @@ -41,10 +41,9 @@ index b0fed3b..875d73a 100644 + ref: tagged-tag repo: test:morphs + unpetrify-ref: master - system-kind: rootfs-tarball test:morphs -commit ca7594f436da55bda2cfff2c6484d11aa0ea4cbc +commit 476e4ff4b19c38eb64ad3a151b7c58a7ab95c9ee Author: developer <developer@example.com> AuthorDate: Tue Jul 31 16:51:54 2012 +0000 Commit: developer <developer@example.com> @@ -52,7 +51,7 @@ CommitDate: Tue Jul 31 16:51:54 2012 +0000 Second -commit 1fe013ee284724848d65096d4d88f612fae56fc6 +commit e11a36aa9e4c998c41a3ec3209324b9318e484ae Author: developer <developer@example.com> AuthorDate: Tue Jul 31 16:51:54 2012 +0000 Commit: developer <developer@example.com> diff --git a/tests.branching/tag-works-with-multiple-morphs-repos.script b/tests.branching/tag-works-with-multiple-morphs-repos.script index 38d73852..f6ecbf32 100755 --- a/tests.branching/tag-works-with-multiple-morphs-repos.script +++ b/tests.branching/tag-works-with-multiple-morphs-repos.script @@ -32,9 +32,7 @@ mkdir "$DATADIR/morphs1" cat <<EOF > "$DATADIR/morphs1/test-system.morph" name: test-system kind: system -system-kind: rootfs-tarball arch: $(uname -m) -disk-size: 1G strata: - morph: stratum1 ref: master diff --git a/tests.branching/tag-works-with-multiple-morphs-repos.stdout b/tests.branching/tag-works-with-multiple-morphs-repos.stdout index 938e41ff..81fbf20d 100644 --- a/tests.branching/tag-works-with-multiple-morphs-repos.stdout +++ b/tests.branching/tag-works-with-multiple-morphs-repos.stdout @@ -4,7 +4,7 @@ Date: Tue Jul 31 16:51:54 2012 +0000 create tag -commit 7253f9b1471f1983e07823d2b84582dde9fb108d +commit 7323ed5dcc47715e7303fd36d537aef98a04df9a Author: developer <developer@example.com> Date: Tue Jul 31 16:51:54 2012 +0000 @@ -67,11 +67,11 @@ index 0000000..610fae6 + repo: test:hello + unpetrify-ref: master diff --git a/test-system.morph b/test-system.morph -index 44d5ae5..08f9adc 100644 +index 62d3b08..edb5745 100644 --- a/test-system.morph +++ b/test-system.morph -@@ -5,8 +5,11 @@ arch: x86_64 - disk-size: 1G +@@ -3,8 +3,11 @@ kind: system + arch: x86_64 strata: - morph: stratum1 - ref: master @@ -85,7 +85,7 @@ index 44d5ae5..08f9adc 100644 + repo: test:morphs1 + unpetrify-ref: master + unpetrify-repo: test:morphs2 -commit 7253f9b1471f1983e07823d2b84582dde9fb108d +commit 7323ed5dcc47715e7303fd36d537aef98a04df9a Author: developer <developer@example.com> AuthorDate: Tue Jul 31 16:51:54 2012 +0000 Commit: developer <developer@example.com> @@ -93,10 +93,10 @@ CommitDate: Tue Jul 31 16:51:54 2012 +0000 create tag --- - stratum1.morph | 9 ++++++--- - stratum2.morph | 14 ++++++++++++++ - stratum3.morph | 9 +++++++++ - test-system.morph | 9 ++++++--- + stratum1.morph | 9 ++++++--- + stratum2.morph | 14 ++++++++++++++ + stratum3.morph | 9 +++++++++ + test-system.morph | 9 ++++++--- 4 files changed, 35 insertions(+), 6 deletions(-) diff --git a/stratum1.morph b/stratum1.morph @@ -156,11 +156,11 @@ index 0000000..610fae6 + repo: test:hello + unpetrify-ref: master diff --git a/test-system.morph b/test-system.morph -index 44d5ae5..08f9adc 100644 +index 62d3b08..edb5745 100644 --- a/test-system.morph +++ b/test-system.morph -@@ -5,8 +5,11 @@ arch: x86_64 - disk-size: 1G +@@ -3,8 +3,11 @@ kind: system + arch: x86_64 strata: - morph: stratum1 - ref: master diff --git a/tests.build/setup b/tests.build/setup index 499dbb21..563482e6 100755 --- a/tests.build/setup +++ b/tests.build/setup @@ -108,8 +108,6 @@ cat <<EOF > hello-system.morph "name": "hello-system", "kind": "system", "arch": "$(uname -m)", - "system-kind": "rootfs-tarball", - "disk-size": "1G", "strata": [ { "morph": "hello-stratum", diff --git a/tests.deploy/deploy-cluster.script b/tests.deploy/deploy-cluster.script index 917ac717..0efc8d3c 100755 --- a/tests.deploy/deploy-cluster.script +++ b/tests.deploy/deploy-cluster.script @@ -18,7 +18,6 @@ # Test "morph deploy" by deploying the systems in cluster morphology. - set -eu @@ -35,12 +34,13 @@ cd "$DATADIR/workspace/branch1" linux-system-2.HOSTNAME="baserock-rocks-even-more" \ > /dev/null -test -e hello-system.img -test -e linux-system-1.tar -test -e linux-system-2.tar +outputdir=test:morphs +test -e $outputdir/hello-system.img +test -e $outputdir/linux-system-1.tar +test -e $outputdir/linux-system-2.tar -hostname1=$(tar -xf linux-system-1.tar ./etc/hostname -O) -hostname2=$(tar -xf linux-system-2.tar ./etc/hostname -O) +hostname1=$(tar -xf $outputdir/linux-system-1.tar ./etc/hostname -O) +hostname2=$(tar -xf $outputdir/linux-system-2.tar ./etc/hostname -O) [ "$hostname1" = baserock-rocks ] [ "$hostname2" = baserock-rocks-even-more ] diff --git a/tests.deploy/deploy-rawdisk.script b/tests.deploy/deploy-rawdisk.script index ebda50c7..257ef0dd 100755 --- a/tests.deploy/deploy-rawdisk.script +++ b/tests.deploy/deploy-rawdisk.script @@ -27,5 +27,7 @@ cd "$DATADIR/workspace/branch1" "$SRCDIR/scripts/test-morph" build linux-system "$SRCDIR/scripts/test-morph" --log "$DATADIR/deploy.log" \ deploy rawdisk_test_cluster > /dev/null -test -e disk.img + +outputdir="test:morphs" +test -e $outputdir/disk.img diff --git a/tests.deploy/setup b/tests.deploy/setup index 88488a91..5f83747e 100755 --- a/tests.deploy/setup +++ b/tests.deploy/setup @@ -112,7 +112,6 @@ cat <<EOF > hello-system.morph { "name": "hello-system", "kind": "system", - "system-kind": "rootfs-tarball", "arch": "$(uname -m)", "strata": [ { @@ -153,7 +152,6 @@ cat <<EOF > linux-system.morph { "name": "linux-system", "kind": "system", - "system-kind": "rootfs-tarball", "arch": "$(uname -m)", "strata": [ { diff --git a/tests.merging/setup b/tests.merging/setup index 11fdf0f1..2fdf9db0 100755 --- a/tests.merging/setup +++ b/tests.merging/setup @@ -51,9 +51,7 @@ cat <<EOF > "$DATADIR/morphs/hello-system.morph" { "name": "hello-system", "kind": "system", - "system-kind": "rootfs-tarball", "arch": "$(uname -m)", - "disk-size": "1G", "strata": [ { "morph": "hello-stratum", diff --git a/tests/setup b/tests/setup index 91a30236..07643ddc 100755 --- a/tests/setup +++ b/tests/setup @@ -107,8 +107,6 @@ cat <<EOF > hello-system.morph { "name": "hello-system", "kind": "system", - "system-kind": "rootfs-tarball", - "disk-size": "1G", "strata": [ { "morph": "hello-stratum", diff --git a/tests/show-dependencies.setup b/tests/show-dependencies.setup index 9c983d8a..edb9b6ab 100755 --- a/tests/show-dependencies.setup +++ b/tests/show-dependencies.setup @@ -349,7 +349,6 @@ cat <<EOF > xfce-system.morph { "name": "xfce-system", "kind": "system", - "system-kind": "rootfs-tarball", "arch": "$(uname -m)", "strata": [ { diff --git a/yarns/architecture.yarn b/yarns/architecture.yarn new file mode 100644 index 00000000..038492cd --- /dev/null +++ b/yarns/architecture.yarn @@ -0,0 +1,11 @@ +Morph Cross-Building Tests +========================== + + SCENARIO building a system for a different architecture + GIVEN a workspace + AND a git server + AND a system called base-system-testarch for architecture testarch in the git server + WHEN the user checks out the system branch called master + AND the user attempts to build the system base-system-testarch in branch master + THEN morph failed + AND the build error message includes the string "Are you trying to cross-build?" diff --git a/yarns/branches-workspaces.yarn b/yarns/branches-workspaces.yarn index f523ebcd..b928f628 100644 --- a/yarns/branches-workspaces.yarn +++ b/yarns/branches-workspaces.yarn @@ -12,14 +12,14 @@ The first thing a user needs to do is create a workspace. SCENARIO create and initialise a new workspace GIVEN no workspace - WHEN morph initialises a workspace + WHEN the user initialises a workspace THEN an empty workspace exists The workspace directory may exist, if it is empty. SCENARIO initialise an empty workspace directory GIVEN an empty workspace directory - WHEN morph initialises a workspace + WHEN the user initialises a workspace THEN an empty workspace exists However, the directory must really be empty. It must not be @@ -27,8 +27,8 @@ an empty, but initialised workspace. SCENARIO initialise an existing, empty workspace directory GIVEN no workspace - WHEN morph initialises a workspace - AND morph attempts to initialise a workspace + WHEN the user initialises a workspace + AND the user attempts to initialise a workspace THEN morph failed Likewise, if the directory exists, and is non-empty, but isn't an @@ -36,7 +36,7 @@ existing workspace, initialising it should fail. SCENARIO initialise a non-empty workspace directory GIVEN a non-empty workspace directory - WHEN morph attempts to initialise a workspace + WHEN the user attempts to initialise a workspace THEN morph failed Checking out or branching system branches @@ -47,7 +47,7 @@ Once we have a workspace, we can check out a system branch. SCENARIO check out an existing system branch GIVEN a workspace AND a git server - WHEN checking out the master system branch + WHEN the user checks out the system branch called master THEN the system branch master is checked out Checking out a system branch should fail, if the branch doesn't exist. @@ -55,7 +55,7 @@ Checking out a system branch should fail, if the branch doesn't exist. SCENARIO checking out a system branch that doesn't exist GIVEN a workspace AND a git server - WHEN morph attempts to check out system branch foo + WHEN the user attempts to check out the system branch called foo THEN morph failed Checking out a system branch should also fail if the repository does @@ -64,7 +64,7 @@ not contain any system morphologies. SCENARIO checking out a system branch with no systems GIVEN a workspace AND a git server - WHEN morph attempts to check out a repository with no systems + WHEN the user attempts to check out from a repository with no systems THEN morph failed We can, instead, create a new system branch, off master. @@ -72,7 +72,7 @@ We can, instead, create a new system branch, off master. SCENARIO branch off master GIVEN a workspace AND a git server - WHEN creating system branch foo + WHEN the user creates a system branch called foo THEN the system branch foo is checked out We can also branch off another system branch. However, we need to first @@ -82,9 +82,9 @@ to check for that locally. SCENARIO branch off non-master GIVEN a workspace AND a git server - WHEN creating system branch foo - AND pushing system branch foo to git server - AND creating system branch bar, based on foo + WHEN the user creates a system branch called foo + AND the user pushes the system branch called foo to the git server + AND the user creates a system branch called bar, based on foo THEN the system branch bar is checked out Query commands in workspaces @@ -95,8 +95,8 @@ directory, regardless of where the user is. There's a few cases. SCENARIO morph workspace works at root of empty workspace GIVEN a workspace - WHEN morph reports workspace in . - THEN workspace is reported correctly + WHEN the user reports the workspace from the directory . + THEN the workspace is reported correctly Also check it in the root of a system branch checkout, and inside a git checkout inside that. @@ -104,22 +104,22 @@ a git checkout inside that. SCENARIO morph workspace works in system branch checkouts GIVEN a workspace AND a git server - WHEN checking out the master system branch - AND morph reports workspace in master - THEN workspace is reported correctly + WHEN the user checks out the system branch called master + AND the user reports the workspace from the directory master + THEN the workspace is reported correctly We leak a little bit of the implementation here, to keep things simple: the (mocked) git server the implementation sets up has the `test:morphs` repository, which is the system branch root repository. - WHEN morph reports workspace in master/test:morphs - THEN workspace is reported correctly + WHEN the user reports the workspace from the directory master/test:morphs + THEN the workspace is reported correctly However, running it outside a workspace should fail. SCENARIO morph fails outside workspace GIVEN no workspace - WHEN morph attempts to report workspace + WHEN the user attempts to report the workspace from a non-workspace directory THEN morph failed `morph show-system-branch` should report the name of the system @@ -130,15 +130,15 @@ current working directory, it will find it and report it correctly. SCENARIO morph reports system branch GIVEN a workspace AND a git server - WHEN checking out the master system branch - AND reporting system branch in master - THEN system branch is reported as master + WHEN the user checks out the system branch called master + AND the user reports the system branch from the directory master + THEN the system branch is reported as master - WHEN reporting system branch in master/test:morphs - THEN system branch is reported as master + WHEN the user reports the system branch from the directory master/test:morphs + THEN the system branch is reported as master - WHEN reporting system branch in . - THEN system branch is reported as master + WHEN the user reports the system branch from the directory . + THEN the system branch is reported as master However, if there's two system branches checked out below the current directory, things should fail. @@ -146,9 +146,9 @@ current directory, things should fail. SCENARIO morph fails to report system branch with two checked out GIVEN a workspace AND a git server - WHEN checking out the master system branch - AND creating system branch foo - AND attempting to report system branch in . + WHEN the user checks out the system branch called master + AND the user creates a system branch called foo + AND the user attempts to report the system branch from the directory . THEN morph failed `morph show-branch-root` reports the URL (possibly aliases) of the @@ -158,12 +158,12 @@ somewhere outside a checkout, where exactly one checkout exists below. SCENARIO morph reports system branch root repository GIVEN a workspace AND a git server - WHEN checking out the master system branch - AND reporting system branch root repository in master - THEN root repository is reported as test:morphs + WHEN the user checks out the system branch called master + AND the user reports the system branch root repository from the directory master + THEN the system branch root repository is reported as test:morphs - WHEN reporting system branch root repository in . - THEN root repository is reported as test:morphs + WHEN the user reports the system branch root repository from the directory . + THEN the system branch root repository is reported as test:morphs However, it fails if run outside a checkout and there's no system branches checked out. @@ -171,7 +171,7 @@ branches checked out. SCENARIO morph fails to report system branch with none checked out GIVEN a workspace AND a git server - WHEN attempting to report system branch root repository in . + WHEN the user attempts to report the system branch root repository from the directory . THEN morph failed Editing components @@ -186,23 +186,33 @@ all the refs are unchanged. SCENARIO morph branch does not edit refs GIVEN a workspace AND a git server - WHEN creating system branch foo + WHEN the user creates a system branch called foo THEN in branch foo, system test-system refs test-stratum in master AND in branch foo, stratum test-stratum refs test-chunk in master Then edit the stratum. - WHEN editing stratum test-stratum in system test-system in branch foo + WHEN the user edits the stratum test-stratum in the system test-system in branch foo THEN in branch foo, system test-system refs test-stratum in foo Edit the chunk. We make use of special knowledge here: `test:test-chunk` is a chunk repository created in the mocked git server, for testing purposes. - WHEN editing chunk test-chunk in test-stratum in test-system in branch foo + WHEN the user edits the chunk test-chunk in the stratum test-stratum in the system test-system in branch foo THEN in branch foo, system test-system refs test-stratum in foo AND in branch foo, stratum test-stratum refs test-chunk in foo - AND edited chunk test:test-chunk has git branch foo + AND the edited chunk test:test-chunk has git branch foo + +Morph edit should only work with a system argument. + + SCENARIO morph edit errors when supplied only a stratum and chunk as arguments + GIVEN a workspace + AND a git server + WHEN the user checks out the system branch called master + AND the user edits the chunk test-chunk in the stratum test-stratum with no system specified in branch master + THEN morph failed + AND the edit error message includes the string "is not a system" Status of system branch checkout -------------------------------- @@ -214,13 +224,13 @@ repositories referenced in the system branch. SCENARIO morph status reports changes correctly GIVEN a workspace AND a git server - WHEN creating system branch foo + WHEN the user creates a system branch called foo THEN morph reports no outstanding changes in foo - WHEN editing stratum test-stratum in system test-system in branch foo + WHEN the user edits the stratum test-stratum in the system test-system in branch foo THEN morph reports changes in foo in test:morphs only - WHEN editing chunk test-chunk in test-stratum in test-system in branch foo + WHEN the user edits the chunk test-chunk in the stratum test-stratum in the system test-system in branch foo THEN morph reports changes in foo in test:morphs only WHEN creating file foo in test:test-chunk in branch foo @@ -244,8 +254,8 @@ branch checkout. SCENARIO morph foreach runs command in each git repo GIVEN a workspace AND a git server - WHEN creating system branch foo - AND editing chunk test-chunk in test-stratum in test-system in branch foo + WHEN the user creates a system branch called foo + AND the user edits the chunk test-chunk in the stratum test-stratum in the system test-system in branch foo AND running shell command in each repo in foo THEN morph ran command in test:morphs in foo AND morph ran command in test:test-chunk in foo @@ -261,8 +271,8 @@ unpetrify and verify that we have all the same refs as before. SCENARIO morph petrifies and unpetrifies GIVEN a workspace AND a git server - WHEN creating system branch foo - AND pushing system branch foo to git server + WHEN the user creates a system branch called foo + AND the user pushes the system branch called foo to the git server AND remembering all refs in foo AND petrifying foo THEN foo is petrified @@ -279,15 +289,15 @@ system branch itself, only the tag. SCENARIO morph tags a system branch GIVEN a workspace AND a git server - WHEN creating system branch foo - AND tagging system branch foo as test123 + WHEN the user creates a system branch called foo + AND the user tags the system branch called foo as test123 THEN morph tag test123 in foo is an annotated git tag AND morph tag test123 in foo refers to a petrified commit AND foo is not petrified Creating a tag twice should fail. - WHEN attempting to tag system branch foo as test123 + WHEN the user attempts to tag the system branch called foo as test123 THEN morph failed Working with null repositories and refs @@ -303,7 +313,7 @@ These can be checked out like normal system branches. GIVEN a workspace AND a git server AND null refs for local strata - WHEN checking out the master system branch + WHEN the user checks out the system branch called master THEN the system branch master is checked out Likewise we can also create new system branches from these, and we @@ -314,7 +324,7 @@ wouldn't need to worry about changing the system branch ref. GIVEN a workspace AND a git server AND null refs for local strata - WHEN creating system branch foo + WHEN the user creates a system branch called foo THEN the system branch foo is checked out When we edit a morphology with null refs, they stay null. @@ -326,20 +336,20 @@ When we edit a morphology with null refs, they stay null. When creating the branch, the refs remain null. - WHEN creating system branch foo + WHEN the user creates a system branch called foo THEN in branch foo, system test-system refs test-stratum in None After editing the stratum they remain null. - WHEN editing stratum test-stratum in system test-system in branch foo + WHEN the user edits the stratum test-stratum in the system test-system in branch foo THEN in branch foo, system test-system refs test-stratum in None Refs to chunks are still altered as usual - WHEN editing chunk test-chunk in test-stratum in test-system in branch foo + WHEN the user edits the chunk test-chunk in the stratum test-stratum in the system test-system in branch foo THEN in branch foo, system test-system refs test-stratum in None AND in branch foo, stratum test-stratum refs test-chunk in foo - AND edited chunk test:test-chunk has git branch foo + AND the edited chunk test:test-chunk has git branch foo Petrifying also leaves null refs unmolested @@ -347,8 +357,16 @@ Petrifying also leaves null refs unmolested GIVEN a workspace AND a git server AND null refs for local strata - WHEN creating system branch foo - AND pushing system branch foo to git server + WHEN the user creates a system branch called foo + AND the user pushes the system branch called foo to the git server AND remembering all refs in foo AND petrifying foo THEN in branch foo, system test-system refs test-stratum in None + +Generating a manifest works + + SCENARIO morph generates a manifest + GIVEN a workspace + AND a system artifact + WHEN morph generates a manifest + THEN the manifest is generated diff --git a/yarns/building.yarn b/yarns/building.yarn new file mode 100644 index 00000000..b78c69cd --- /dev/null +++ b/yarns/building.yarn @@ -0,0 +1,11 @@ +Morph Building Tests +====================== + + SCENARIO attempting to build a system morphology which has never been committed + GIVEN a workspace + AND a git server + WHEN the user checks out the system branch called master + AND the user creates an uncommitted system morphology called base-system-testarch for architecture testarch in system branch master + AND the user attempts to build the system base-system-testarch in branch master + THEN morph failed + AND the build error message includes the string "Did you forget to commit it?" diff --git a/yarns/deployment.yarn b/yarns/deployment.yarn new file mode 100644 index 00000000..35f933b7 --- /dev/null +++ b/yarns/deployment.yarn @@ -0,0 +1,10 @@ +Morph Deployment Tests +====================== + + SCENARIO deploying a non-cluster morphology + GIVEN a workspace + AND a git server + WHEN the user checks out the system branch called master + AND the user attempts to deploy the system test-system in branch master + THEN morph failed + AND the deploy error message includes the string "morph deploy is only supported for cluster morphologies" diff --git a/yarns/fstab-configure.yarn b/yarns/fstab-configure.yarn new file mode 100644 index 00000000..cd7e7438 --- /dev/null +++ b/yarns/fstab-configure.yarn @@ -0,0 +1,62 @@ +`fstab.configure` +================= + +The `fstab.configure` extension appends text to the `/etc/fstab` from +environment variables beginning with `FSTAB_`. It also sets the +ownership and permissions of the file. + +The first thing to test is that the extension doesn't write anything +if not requested to do so, but does create the file if it doesn't +exist. + + SCENARIO fstab.configure does nothing by default + GIVEN a directory called tree/etc + WHEN fstab.configure is run against tree + THEN file tree/etc/fstab exists + AND file tree/etc/fstab has permissions -rw-r--r-- + AND file tree/etc/fstab is owned by uid 0 + AND file tree/etc/fstab is owned by gid 0 + AND file tree/etc/fstab is empty + +Append a something to the file, and verify the contents are exactly +correct. + + SCENARIO fstab.configure appends requested lines + GIVEN a directory called tree/etc + AND an environment variable FSTAB_FOO containing "foo" + WHEN fstab.configure is run against tree + THEN file tree/etc/fstab exists + AND file tree/etc/fstab has permissions -rw-r--r-- + AND file tree/etc/fstab is owned by uid 0 + AND file tree/etc/fstab is owned by gid 0 + AND file tree/etc/fstab contains "foo\n" + +Append something to an existing file, with wrong ownership and +permission. + + SCENARIO fstab.configure appends to existing file + GIVEN a directory called tree/etc + AND a file called tree/etc/fstab containing "# comment\n" + AND tree/etc/fstab is owned by uid 1 + AND tree/etc/fstab is owned by gid 1 + AND tree/etc/fstab has permissions 0600 + AND an environment variable FSTAB_FOO containing "foo" + WHEN fstab.configure is run against tree + THEN file tree/etc/fstab exists + AND file tree/etc/fstab has permissions -rw-r--r-- + AND file tree/etc/fstab is owned by uid 0 + AND file tree/etc/fstab is owned by gid 0 + AND file tree/etc/fstab contains "# comment\nfoo\n" + +Implement running `fstab.configure` +----------------------------------- + +When we actually run `fstab.configure`, we source `$DATADIR/env` to +get the desired environment variables. + + IMPLEMENTS WHEN fstab.configure is run against (\S+) + if [ -e "$DATADIR/env" ] + then + . "$DATADIR/env" + fi + "$SRCDIR/morphlib/exts/fstab.configure" "$DATADIR/$MATCH_1" diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn index e35e4219..6491b38e 100644 --- a/yarns/implementations.yarn +++ b/yarns/implementations.yarn @@ -20,11 +20,10 @@ We run `morph init` in two different ways: either the simple way, letting yarn catch errors, or in a way that catches the error so we can test it later in a THEN step. - IMPLEMENTS WHEN morph initialises a workspace - run_morph init "$DATADIR/workspace" - - IMPLEMENTS WHEN morph attempts to initialise a workspace - attempt_morph init "$DATADIR/workspace" + IMPLEMENTS WHEN the user (attempts to initialise|initialises) a workspace + set init "$DATADIR/workspace" + if [ $MATCH_1 == "initialises" ]; then run_morph "$@" + else attempt_morph "$@"; fi IMPLEMENTS THEN morph failed case $(cat "$DATADIR/morph-exit") in @@ -76,6 +75,13 @@ another to hold a chunk. morph: test-stratum EOF + cat << EOF > "$DATADIR/gits/morphs/simple-system.morph" + name: simple-system + kind: system + arch: $arch + strata: [] + EOF + cat << EOF > "$DATADIR/gits/morphs/test-stratum.morph" name: test-stratum kind: stratum @@ -91,6 +97,7 @@ another to hold a chunk. run_in "$DATADIR/gits/morphs" git init . run_in "$DATADIR/gits/morphs" git add . run_in "$DATADIR/gits/morphs" git commit -m Initial. + run_in "$DATADIR/gits/morphs" git tag -a "test-tag" -m "Tagging test-tag" # Create the chunk repository. @@ -112,7 +119,25 @@ another to hold a chunk. cat << EOF > "$DATADIR/morph.conf" [config] repo-alias = test=file://$DATADIR/gits/%s#file://$DATADIR/gits/%s + cachedir = $DATADIR/cache + tempdir = $DATADIR/tmp + trove-host= [] + EOF + + mkdir "$DATADIR/cache" + mkdir "$DATADIR/tmp" + + IMPLEMENTS GIVEN a system called (\S+) for architecture (\S+) in the git server + cat << EOF > "$DATADIR/gits/morphs/$MATCH_1.morph" + arch: $MATCH_2 + configuration-extensions: [] + description: A system called $MATCH_1 for architectures $MATCH_2 + kind: system + name: $MATCH_1 + strata: [] EOF + run_in "$DATADIR/gits/morphs" git add "$MATCH_1.morph" + run_in "$DATADIR/gits/morphs" git commit -m "Added $MATCH_1 morphology." Morphologies need to support having a null ref, which means look for the stratum in the same repository and ref. Testing this requires different @@ -132,19 +157,15 @@ Implementation sections for system branch operations Checkout out an existing system branch. We parameterise this so the same phrase can be used to check out any system branch. - IMPLEMENTS WHEN checking out the (\S+) system branch - cd "$DATADIR/workspace" - run_morph checkout test:morphs "$MATCH_1" - -Attempt to check out a system branch, and remember if it failed. - - IMPLEMENTS WHEN morph attempts to check out system branch (\S+) + IMPLEMENTS WHEN the user (attempts to check|checks) out the system (branch|tag) called (\S+) cd "$DATADIR/workspace" - attempt_morph checkout test:morphs "$MATCH_1" + set checkout test:morphs "$MATCH_3" + if [ $MATCH_1 == "checks" ]; then run_morph "$@" + else attempt_morph "$@"; fi Attempt to check out a system branch from a root that has no systems. - IMPLEMENTS WHEN morph attempts to check out a repository with no systems + IMPLEMENTS WHEN the user attempts to check out from a repository with no systems cd "$DATADIR/workspace" attempt_morph checkout test:test-chunk master @@ -157,100 +178,106 @@ We also need to verify that a system branch has been checked out. We can create a new branch, off master. - IMPLEMENTS WHEN creating system branch (\S+) + IMPLEMENTS WHEN the user (attempts to create|creates) a system branch called (\S+) cd "$DATADIR/workspace" - run_morph branch test:morphs "$MATCH_1" + set branch test:morphs "$MATCH_2" + if [ $MATCH_1 == "creates"]; then run morph "$@" + else attempt_morph "$@"; fi We can create a new branch, off another system branch. - IMPLEMENTS WHEN creating system branch (\S+), based on (\S+) + IMPLEMENTS WHEN the user creates a system branch called (\S+), based on (\S+) cd "$DATADIR/workspace" run_morph branch test:morphs "$MATCH_1" "$MATCH_2" Attempt to branch a system branch from a root that had no systems. - IMPLEMENTS WHEN morph attempts to branch a repository with no systems + IMPLEMENTS WHEN the user attempts to branch a repository with no systems cd "$DATADIR/workspace" attempt_morph branch test:test-chunk foo Pushing all changes in a system branch checkout to the git server. - IMPLEMENTS WHEN pushing system branch (\S+) to git server + IMPLEMENTS WHEN the user pushes the system branch called (\S+) to the git server # FIXME: For now, this is just the morphs checkout. run_in "$DATADIR/workspace/$MATCH_1/test:morphs" git push origin HEAD Report workspace path. - IMPLEMENTS WHEN morph reports workspace in (\S+) + IMPLEMENTS WHEN the user reports the workspace from the directory (\S+) cd "$DATADIR/workspace/$MATCH_1" run_morph workspace > "$DATADIR/workspace-reported" - IMPLEMENTS THEN workspace is reported correctly + IMPLEMENTS THEN the workspace is reported correctly assert_equal $(cat "$DATADIR/workspace-reported") "$DATADIR/workspace" - IMPLEMENTS WHEN morph attempts to report workspace + IMPLEMENTS WHEN the user attempts to report the workspace from a non-workspace directory cd "$DATADIR" attempt_morph workspace Report system branch name: - IMPLEMENTS WHEN reporting system branch in (\S+) - cd "$DATADIR/workspace/$MATCH_1" - run_morph show-system-branch > "$DATADIR/system-branch.reported" + IMPLEMENTS WHEN the user (attempts to report|reports) the system branch from the directory (\S+) + cd "$DATADIR/workspace/$MATCH_2" + set $DATADIR/system-branch.reported + if [ $MATCH_1 == reports ]; then run_morph show-system-branch > "$@" + else attempt_morph show-system-branch > "$@"; fi - IMPLEMENTS THEN system branch is reported as (.*) + IMPLEMENTS THEN the system branch is reported as (.*) echo "$MATCH_1" > "$DATADIR/system-branch.actual" diff -u "$DATADIR/system-branch.actual" "$DATADIR/system-branch.reported" - IMPLEMENTS WHEN attempting to report system branch in (.*) - cd "$DATADIR/workspace/$MATCH_1" - attempt_morph show-system-branch - Report system branch root repository. - IMPLEMENTS WHEN reporting system branch root repository in (.*) - cd "$DATADIR/workspace/$MATCH_1" - run_morph show-branch-root > "$DATADIR/branch-root.reported" + IMPLEMENTS WHEN the user (attempts to report|reports) the system branch root repository from the directory (.*) + cd "$DATADIR/workspace/$MATCH_2" + set $DATADIR/branch-root.reported + if [ $MATCH_1 == "reports" ]; then run_morph show-branch-root > "$@" + else attempt_morph show-branch-root > "$@"; fi - IMPLEMENTS THEN root repository is reported as (.*) + IMPLEMENTS THEN the system branch root repository is reported as (.*) echo "$MATCH_1" > "$DATADIR/branch-root.actual" diff -u "$DATADIR/branch-root.actual" "$DATADIR/branch-root.reported" - IMPLEMENTS WHEN attempting to report system branch root repository in (.*) - cd "$DATADIR/workspace/$MATCH_1" - attempt_morph show-branch-root - Editing morphologies with `morph edit`. - IMPLEMENTS THEN in branch (\S+), system (\S+) refs (\S+) in (\S+) - "$SRCDIR/scripts/yaml-extract" \ - "$DATADIR/workspace/$MATCH_1/test:morphs/$MATCH_2.morph" \ - strata name="$MATCH_3" ref > "$DATADIR/ref.actual" - echo "$MATCH_4" > "$DATADIR/ref.wanted" - diff -u "$DATADIR/ref.wanted" "$DATADIR/ref.actual" - - IMPLEMENTS THEN in branch (\S+), stratum (\S+) refs (\S+) in (\S+) + IMPLEMENTS THEN in branch (\S+), (system|stratum) (\S+) refs (\S+) in (\S+) + if [ $MATCH_2 == system ]; then set strata; else set chunks; fi "$SRCDIR/scripts/yaml-extract" \ - "$DATADIR/workspace/$MATCH_1/test:morphs/$MATCH_2.morph" \ - chunks name="$MATCH_3" ref > "$DATADIR/ref.actual" - echo "$MATCH_4" > "$DATADIR/ref.wanted" + "$DATADIR/workspace/$MATCH_1/test:morphs/$MATCH_3.morph" \ + $@ name="$MATCH_4" ref > "$DATADIR/ref.actual" + echo "$MATCH_5" > "$DATADIR/ref.wanted" diff -u "$DATADIR/ref.wanted" "$DATADIR/ref.actual" - IMPLEMENTS WHEN editing stratum (\S+) in system (\S+) in branch (\S+) + IMPLEMENTS WHEN the user edits the stratum (\S+) in the system (\S+) in branch (\S+) cd "$DATADIR/workspace/$MATCH_3/test:morphs" run_morph edit "$MATCH_2" "$MATCH_1" - IMPLEMENTS WHEN editing chunk (\S+) in (\S+) in (\S+) in branch (\S+) + IMPLEMENTS WHEN the user edits the chunk (\S+) in the stratum (\S+) in the system (\S+) in branch (\S+) cd "$DATADIR/workspace/$MATCH_4/test:morphs" run_morph edit "$MATCH_3" "$MATCH_2" "$MATCH_1" - IMPLEMENTS THEN edited chunk (\S+) has git branch (\S+) + IMPLEMENTS THEN the edited chunk (\S+) has git branch (\S+) ls -l "$DATADIR/workspace/$MATCH_2" cd "$DATADIR/workspace/$MATCH_2/$MATCH_1" git branch | awk '$1 == "*" { print $2 }' > "$DATADIR/git-branch.actual" echo "$MATCH_2" > "$DATADIR/git-branch.wanted" diff -u "$DATADIR/git-branch.wanted" "$DATADIR/git-branch.actual" + IMPLEMENTS WHEN the user edits the chunk (\S+) in the stratum (\S+) with no system specified in branch (\S+) + cd "$DATADIR/workspace/$MATCH_3" + attempt_morph edit "$MATCH_2" "$MATCH_1" + + IMPLEMENTS WHEN the user creates an uncommitted system morphology called (\S+) for architecture (\S+) in system branch (\S+) + cat << EOF > "$DATADIR/workspace/$MATCH_3/test:morphs/$MATCH_1.morph" + arch: $MATCH_2 + configuration-extensions: [] + description: A system called $MATCH_1 for architectures $MATCH_2 + kind: system + name: $MATCH_1 + strata: [] + EOF + Reporting status of checked out repositories: IMPLEMENTS THEN morph reports no outstanding changes in (\S+) @@ -357,11 +384,12 @@ Petrification and unpetrification: Tagging. - IMPLEMENTS WHEN tagging system branch (\S+) as (\S+) + IMPLEMENTS WHEN the user tags the system branch called (\S+) as (\S+) cd "$DATADIR/workspace/$MATCH_1/test:morphs" + set tag "$MATCH_2" -- -m "testing morph tagging" run_morph tag "$MATCH_2" -- -m "testing morph tagging" - IMPLEMENTS WHEN attempting to tag system branch (\S+) as (\S+) + IMPLEMENTS WHEN the user attempts to tag the system branch called (\S+) as (\S+) cd "$DATADIR/workspace/$MATCH_1/test:morphs" attempt_morph tag "$MATCH_2" -- -m "testing morph tagging" @@ -382,3 +410,148 @@ Tagging. assert_morphologies_are_petrified "$MATCH_1" temptemptemp done +Generating a manifest. + + IMPLEMENTS GIVEN a system artifact + mkdir "$DATADIR/hello_world" + + git init "$DATADIR/hello_world" + touch "$DATADIR/hello_world/configure.ac" + run_in "$DATADIR/hello_world" git add configure.ac + run_in "$DATADIR/hello_world" git commit -m 'Add configure.ac' + + mkdir "$DATADIR/baserock" + run_in "$DATADIR/hello_world" cat << EOF \ + > "$DATADIR/baserock/hello_world.meta" + { + "artifact-name": "hello_world", + "cache-key": + "ab8d00a80298a842446ce23507cea6b4d0e34c7ddfa05c67f460318b04d21308", + "kind": "chunk", + "morphology": "hello_world.morph", + "original_ref": "$(run_in "$DATADIR/hello_world" git rev-parse HEAD)", + "repo": "file://$DATADIR/hello_world", + "repo-alias": "upstream:hello_world", + "sha1": "$(run_in "$DATADIR/hello_world" git rev-parse HEAD)", + "source-name": "hello_world" + } + EOF + run_in "$DATADIR" tar -c baserock > "$DATADIR/artifact.tar" + + IMPLEMENTS WHEN morph generates a manifest + run_morph generate-manifest "$DATADIR/artifact.tar" > "$DATADIR/manifest" + + IMPLEMENTS THEN the manifest is generated + + # Generated manifest should contain the name of the repository + if ! grep -q hello_world "$DATADIR/manifest"; then + die "Output isn't what we expect" + fi + +Implementation sections for building +==================================== + + IMPLEMENTS WHEN the user (attempts to build|builds) the system (\S+) in branch (\S+) + cd "$DATADIR/workspace/$MATCH_3" + set build "$MATCH_2" + if [ $MATCH_1 == "builds" ]; then run_morph "$@" + else attempt_morph "$@"; fi + +Implementation sections for deployment +====================================== + + IMPLEMENTS WHEN the user (attempts to deploy|deploys) the (system|cluster) (\S+) in branch (\S+) + cd "$DATADIR/workspace/$MATCH_4" + set build "$MATCH_3" + if [ $MATCH_1 == "deploys" ]; then run_morph "$@" + else attempt_morph deploy "$MATCH_3"; fi + +Implementations sections for reading error messages +=================================================== + + IMPLEMENTS THEN the (branch|build|checkout|deploy|edit|init) error message includes the string "(.*)" + grep "$MATCH_2" "$DATADIR/result-$MATCH_1" + +IMPLEMENTS for test file and directory handling +=============================================== + +The IMPLEMENTS sections in this chapter create files and directories +for use as test data, and set and test their contents and permissions +and ownerships. + +Create a directory +------------------ + + IMPLEMENTS GIVEN a directory called (\S+) + mkdir -p "$DATADIR/$MATCH_1" + +Create a file +------------- + +The file contents is used as a `printf`(1) format string. + + IMPLEMENTS GIVEN a file called (\S+) containing "(.*)" + printf "$MATCH_2" > "$DATADIR/$MATCH_1" + +Set attributes on a file or directory +------------------------------------- + + IMPLEMENTS GIVEN (\S+) is owned by uid (\S+) + chown "$MATCH_2" "$DATADIR/$MATCH_1" + + IMPLEMENTS GIVEN (\S+) is owned by gid (\S+) + chgrp "$MATCH_2" "$DATADIR/$MATCH_1" + + IMPLEMENTS GIVEN (\S+) has permissions (\S+) + chmod "$MATCH_2" "$DATADIR/$MATCH_1" + +Check attributes of a file on the filesystem +-------------------------------------------- + + IMPLEMENTS THEN file (\S+) exists + test -e "$DATADIR/$MATCH_1" + + IMPLEMENTS THEN file (\S+) has permissions (\S+) + stat -c %A "$DATADIR/$MATCH_1" | grep -Fx -e "$MATCH_2" + + IMPLEMENTS THEN file (\S+) is owned by uid (\d+) + stat -c %u "$DATADIR/$MATCH_1" | grep -Fx -e "$MATCH_2" + + IMPLEMENTS THEN file (\S+) is owned by gid (\d+) + stat -c %g "$DATADIR/$MATCH_1" | grep -Fx -e "$MATCH_2" + + IMPLEMENTS THEN file (\S+) is empty + stat -c %s "$DATADIR/$MATCH_1" | grep -Fx 0 + +Check contents of a file +------------------------ + +We treat the contents of the file in the step as a `printf`(1) format +string, to allow newlines and other such stuff to be expressed. + + IMPLEMENTS THEN file (\S+) contains "(.*)" + printf "$MATCH_2" | diff - "$DATADIR/$MATCH_1" + + +IMPLEMENTS for running programs +=============================== + +This chapter contains IMPLEMENTS sections for running programs. It is +currently a bit of a placeholder. + +Remember environment variables to set when running +-------------------------------------------------- + +We need to manage the environment. We store the extra environment +variables in `$DATADIR/env`. We treat the value as a format string for +`printf`(1) so that newlines etc can be used. + + IMPLEMENTS GIVEN an environment variable (\S+) containing "(.*)" + printf "export $MATCH_1=$MATCH_2" >> "$DATADIR/env" + +Implementations for building systems +------------------------------------ + + IMPLEMENTS THEN morph build the system (\S+) of the (branch|tag) (\S+) of the repo (\S+) + cd "$DATADIR/workspace/$MATCH_3/$MATCH_4" + run_morph build "$MATCH_1" diff --git a/yarns/morph.shell-lib b/yarns/morph.shell-lib index 448c60ce..b11ddab1 100644 --- a/yarns/morph.shell-lib +++ b/yarns/morph.shell-lib @@ -31,7 +31,9 @@ run_morph() { "${SRCDIR:-.}"/morph \ - --no-default-config --config "$DATADIR/morph.conf" "$@" + --cachedir-min-space=0 --tempdir-min-space=0 \ + --no-default-config --config "$DATADIR/morph.conf" "$@" \ + 2> "$DATADIR/result-$1" } @@ -156,3 +158,22 @@ if ! env | grep '^SRCDIR=' > /dev/null then export SRCDIR="$(pwd)" fi + + +# Added until it's fixed in upstream. +# It's a solution to create an empty home directory each execution +export HOME="$DATADIR/home" +if [ ! -d "$HOME" ] +then + mkdir "$HOME" +fi + +# Generating a default git user to run the tests +if ! test -r "$HOME/.gitconfig" +then + cat > "$HOME/.gitconfig" <<EOF +[user] + name = Tomjon Codethinker + email = tomjon@codethink.co.uk +EOF +fi diff --git a/yarns/print-architecture.yarn b/yarns/print-architecture.yarn index e45d7d1b..c2496147 100644 --- a/yarns/print-architecture.yarn +++ b/yarns/print-architecture.yarn @@ -19,6 +19,7 @@ Oh, and the one line should contain no spaces, either. AND stderr is empty IMPLEMENTS WHEN morph print-architecture is run + set +x run_morph print-architecture > "$DATADIR/stdout" 2> "$DATADIR/stderr" IMPLEMENTS THEN stdout contains a single line diff --git a/yarns/regression.yarn b/yarns/regression.yarn new file mode 100644 index 00000000..a17d2f87 --- /dev/null +++ b/yarns/regression.yarn @@ -0,0 +1,34 @@ +"regression" tests +================== + +Tests for check we don't introduce some bugs again. + + +Testing if we can build after checking out from a tag. + + SCENARIO morph build works after checkout from a tag + GIVEN a workspace + AND a git server + WHEN the user checks out the system tag called test-tag + THEN morph build the system simple-system of the tag test-tag of the repo test:morphs + + +Running `morph branch` when the branch directory exists doesn't +remove the existing directory. + + SCENARIO re-running 'morph branch' fails, original branch untouched + GIVEN a workspace + AND a git server + WHEN the user creates a system branch called foo + THEN the system branch foo is checked out + +The branch is checked out correctly, now it should fail if the user executes +`morph branch` with the same branch name. + + WHEN the user attempts to create a system branch called foo + THEN morph failed + AND the branch error message includes the string "File exists" + +The branch still checked out. + + AND the system branch foo is checked out |