diff options
47 files changed, 664 insertions, 277 deletions
@@ -0,0 +1,9 @@ +NEWS for Morph +============== + +This file contains high-level summaries of what's changed in each release. + +Version 11, released 2013-10-01 +------------------------------- + +* NEWS file added. diff --git a/morphlib/__init__.py b/morphlib/__init__.py index b1e3c7c3..7eb3f975 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -78,6 +78,7 @@ import sourcepool import stagingarea import stopwatch import sysbranchdir +import systemmetadatadir import tempdir import util import workspace diff --git a/morphlib/app.py b/morphlib/app.py index 08b020ff..0c4f3455 100644 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -20,10 +20,17 @@ import logging import os import sys import time +import urlparse import warnings import morphlib +class InvalidUrlError(cliapp.AppException): + + def __init__(self, parameter, url): + cliapp.AppException.__init__( + self, 'Value %s for argument %s is not a url' % + (url, parameter)) defaults = { 'trove-host': 'git.baserock.org', @@ -200,6 +207,13 @@ class Morph(cliapp.Application): tmpdir = os.path.join(tmpdir_base, 'morph_tmp') self.settings['tempdir'] = tmpdir + if self.settings['tarball-server']: + url_split = urlparse.urlparse(self.settings['tarball-server']) + if not (url_split.netloc and + url_split.scheme in ('http', 'https', 'file')): + raise InvalidUrlError('tarball-server', + self.settings['tarball-server']) + if 'MORPH_DUMP_PROCESSED_CONFIG' in os.environ: self.settings.dump_config(sys.stdout) sys.exit(0) @@ -318,11 +332,15 @@ class Morph(cliapp.Application): visit(reponame, ref, filename, absref, tree, morphology) if morphology['kind'] == 'system': - queue.extend((s['repo'], s['ref'], '%s.morph' % s['morph']) + queue.extend((s['repo'] or reponame, + s['ref'] or ref, + '%s.morph' % s['morph']) for s in morphology['strata']) elif morphology['kind'] == 'stratum': if morphology['build-depends']: - queue.extend((s['repo'], s['ref'], '%s.morph' % s['morph']) + queue.extend((s['repo'] or reponame, + s['ref'] or ref, + '%s.morph' % s['morph']) for s in morphology['build-depends']) queue.extend((c['repo'], c['ref'], '%s.morph' % c['morph']) for c in morphology['chunks']) diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py index 186d5357..17f038a2 100644 --- a/morphlib/artifactresolver.py +++ b/morphlib/artifactresolver.py @@ -155,8 +155,8 @@ class ArtifactResolver(object): for info in source.morphology['strata']: stratum_source = self._source_pool.lookup( - info['repo'], - info['ref'], + info['repo'] or source.repo_name, + info['ref'] or source.original_ref, '%s.morph' % info['morph']) stratum_name = stratum_source.morphology.builds_artifacts[0] @@ -178,8 +178,8 @@ class ArtifactResolver(object): if stratum.source.morphology['build-depends']: for stratum_info in stratum.source.morphology['build-depends']: other_source = self._source_pool.lookup( - stratum_info['repo'], - stratum_info['ref'], + stratum_info['repo'] or stratum.source.repo_name, + stratum_info['ref'] or stratum.source.original_ref, '%s.morph' % stratum_info['morph']) other_stratum = self._get_artifact( diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index d7007233..b587aa9a 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -47,7 +47,7 @@ class BuildCommand(object): for repo_name, ref, filename in self.app.itertriplets(args): self.app.status(msg='Building %(repo_name)s %(ref)s %(filename)s', repo_name=repo_name, ref=ref, filename=filename) - self.app.status(msg='Figuring out the right build order') + self.app.status(msg='Deciding on task order') srcpool = self.create_source_pool(repo_name, ref, filename) root_artifact = self.resolve_artifacts(srcpool) self.build_in_order(root_artifact) @@ -163,8 +163,8 @@ class BuildCommand(object): def _validate_cross_refs_for_xxx(self, src, srcpool, specs, wanted): for spec in specs: - repo_name = spec['repo'] - ref = spec['ref'] + repo_name = spec['repo'] or src.repo_name + ref = spec['ref'] or src.original_ref filename = '%s.morph' % spec['morph'] logging.debug( 'Validating cross ref to %s:%s:%s' % diff --git a/morphlib/morph2.py b/morphlib/morph2.py index a733ce77..0e0d9201 100644 --- a/morphlib/morph2.py +++ b/morphlib/morph2.py @@ -58,10 +58,11 @@ class Morphology(object): ('strata', []), ('description', ''), ('arch', None), - ('system-kind', None), ('configuration-extensions', []), ], - 'cluster': [] + 'cluster': [ + ('description', ''), + ], } @staticmethod @@ -169,10 +170,6 @@ class Morphology(object): self._set_default_value(self._dict, 'max-jobs', int(self['max-jobs'])) - if 'disk-size' in self: - self._set_default_value(self._dict, 'disk-size', - self._parse_size(self['disk-size'])) - for name, value in self.static_defaults[self['kind']]: if name not in self._dict: self._set_default_value(self._dict, name, value) @@ -207,17 +204,6 @@ class Morphology(object): 'deploy', dict()) - def _parse_size(self, size): - if isinstance(size, basestring): - size = size.lower() - if size.endswith('g'): - return int(size[:-1]) * 1024 ** 3 - elif size.endswith('m'): # pragma: no cover - return int(size[:-1]) * 1024 ** 2 - elif size.endswith('k'): # pragma: no cover - return int(size[:-1]) * 1024 - return int(size) # pragma: no cover - def lookup_child_by_name(self, name): '''Find child reference by its name. @@ -252,7 +238,8 @@ class Morphology(object): continue value = self._apply_changes_for_key(key, live_dict, original_dict) - if value is not None: + # VILE HACK to preserve nulls in repo/ref fields + if value is not None or key in ('repo', 'ref'): output_dict[key] = value return output_dict @@ -284,10 +271,7 @@ class Morphology(object): # Simple values. Use original value unless it has been changed from # the default in memmory. if live_dict[key] == live_dict.get('_orig_' + key, None): - if key in original_dict: - result = original_dict[key] - else: - result = None + result = original_dict.get(key, None) else: result = live_dict[key] return result diff --git a/morphlib/morph2_tests.py b/morphlib/morph2_tests.py index bf32d3c2..aaa1d1cc 100644 --- a/morphlib/morph2_tests.py +++ b/morphlib/morph2_tests.py @@ -96,29 +96,16 @@ class MorphologyTests(unittest.TestCase): self.assertEqual(m['chunks'][0]['morph'], 'le-chunk') self.assertEqual(m['chunks'][0]['build-depends'], None) - def test_parses_system_disk_size(self): - m = Morphology(''' - { - "name": "foo", - "kind": "system", - "disk-size": "1g" - } - ''') - - self.assertEqual(m['disk-size'], 1024 ** 3) - def test_returns_dict_keys(self): m = Morphology(''' { "name": "foo", "kind": "system", - "disk-size": "1g" } ''') self.assertTrue('name' in m.keys()) self.assertTrue('kind' in m.keys()) - self.assertTrue('disk-size' in m.keys()) def test_system_indexes_strata(self): m = Morphology(''' @@ -307,32 +294,9 @@ class MorphologyTests(unittest.TestCase): system_text = '''{ "kind": "system", - "disk-size": "1g", "arch": "x86_64", - "system-kind": "rootfs-tarball" }''' - def test_writing_preserves_disk_size(self): - text_lines = self.system_text.splitlines() - morphology = Morphology(self.system_text) - - output = StringIO.StringIO() - morphology.update_text(self.system_text, output) - output_lines = output.getvalue().splitlines() - self.assertEqual(text_lines, output_lines) - - def test_writing_updates_disk_size(self): - text_lines = self.system_text.splitlines() - text_lines[2] = ' "disk-size": 512,' - - morphology = Morphology(self.system_text) - morphology._dict['disk-size'] = 512 - - output = StringIO.StringIO() - morphology.update_text(self.system_text, output) - output_lines = output.getvalue().splitlines() - self.assertEqual(text_lines, output_lines) - def test_nested_dict(self): # Real morphologies don't trigger this code path, so we test manually original_dict = { diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py index c94078f9..702a330c 100644 --- a/morphlib/morphloader.py +++ b/morphlib/morphloader.py @@ -54,6 +54,12 @@ class InvalidFieldError(morphlib.Error): self.msg = ( 'Field %s not allowed in morphology %s' % (field, morphology)) +class ObsoleteFieldsError(morphlib.Error): + + def __init__(self, fields, morphology): + self.msg = ( + 'Morphology %s uses obsolete fields: %s' % + (morphology, ' '.join(fields))) class UnknownArchitectureError(morphlib.Error): @@ -62,14 +68,6 @@ class UnknownArchitectureError(morphlib.Error): 'Unknown architecture %s in morphology %s' % (arch, morphology)) -class InvalidSystemKindError(morphlib.Error): - - def __init__(self, system_kind, morphology): - self.msg = ( - 'system-kind %s not allowed (must be rootfs-tarball), in %s' % - (system_kind, morphology)) - - class NoBuildDependenciesError(morphlib.Error): def __init__(self, stratum_name, chunk_name, morphology): @@ -115,6 +113,13 @@ class MorphologyLoader(object): ], } + _obsolete_fields = { + 'system': [ + 'system-kind', + 'disk-size', + ], + } + _static_defaults = { 'chunk': { 'description': '', @@ -144,11 +149,11 @@ class MorphologyLoader(object): 'strata': [], 'description': '', 'arch': None, - 'system-kind': 'rootfs-tarball', 'configuration-extensions': [], - 'disk-size': '1G', }, - 'cluster': {}, + 'cluster': { + 'description': '', + }, } def parse_morphology_text(self, text, whence): @@ -229,8 +234,10 @@ class MorphologyLoader(object): raise UnknownKindError(morph['kind'], morph.filename) required = ['kind'] + self._required_fields[kind] + obsolete = self._obsolete_fields.get(kind, []) allowed = self._static_defaults[kind].keys() self._require_fields(required, morph) + self._deny_obsolete_fields(obsolete, morph) self._deny_unknown_fields(required + allowed, morph) if kind == 'system': @@ -260,12 +267,6 @@ class MorphologyLoader(object): if morph['arch'] not in morphlib.valid_archs: raise UnknownArchitectureError(morph['arch'], morph.filename) - # If system-kind is present, it must be rootfs-tarball. - if 'system-kind' in morph: - if morph['system-kind'] not in (None, 'rootfs-tarball'): - raise InvalidSystemKindError( - morph['system-kind'], morph.filename) - def _validate_stratum(self, morph): # Require at least one chunk. if len(morph.get('chunks', [])) == 0: @@ -308,6 +309,11 @@ class MorphologyLoader(object): for field in fields: self._require_field(field, morphology) + def _deny_obsolete_fields(self, fields, morphology): + obsolete_ones = [x for x in morphology if x in fields] + if obsolete_ones: + raise ObsoleteFieldsError(obsolete_ones, morphology.filename) + def _deny_unknown_fields(self, allowed, morphology): for field in morphology: if field not in allowed: diff --git a/morphlib/morphloader_tests.py b/morphlib/morphloader_tests.py index ac0fef53..f38d58e8 100644 --- a/morphlib/morphloader_tests.py +++ b/morphlib/morphloader_tests.py @@ -94,6 +94,26 @@ build-system: dummy self.assertRaises( morphlib.morphloader.InvalidFieldError, self.loader.validate, m) + def test_fails_to_validate_system_with_obsolete_system_kind_field(self): + m = morphlib.morph3.Morphology({ + 'kind': 'system', + 'name': 'foo', + 'arch': 'x86_64', + 'system-kind': 'foo', + }) + self.assertRaises( + morphlib.morphloader.ObsoleteFieldsError, self.loader.validate, m) + + def test_fails_to_validate_system_with_obsolete_disk_size_field(self): + m = morphlib.morph3.Morphology({ + 'kind': 'system', + 'name': 'foo', + 'arch': 'x86_64', + 'disk-size': 'over 9000', + }) + self.assertRaises( + morphlib.morphloader.ObsoleteFieldsError, self.loader.validate, m) + def test_fails_to_validate_system_with_no_fields(self): m = morphlib.morph3.Morphology({ 'kind': 'system', @@ -182,22 +202,6 @@ build-system: dummy self.loader.validate(m) self.assertEqual(m['arch'], 'armv7l') - def test_validate_requires_system_kind_to_be_tarball_if_present(self): - m = morphlib.morph3.Morphology( - { - "kind": "system", - "name": "foo", - "arch": "armv7l", - "strata": [], - "system-kind": "blah", - }) - - self.assertRaises( - morphlib.morphloader.InvalidSystemKindError, - self.loader.validate, m) - m['system-kind'] = 'rootfs-tarball' - self.loader.validate(m) - def test_validate_requires_build_deps_for_chunks_in_strata(self): m = morphlib.morph3.Morphology( { @@ -468,13 +472,11 @@ name: foo dict(m), { 'kind': 'system', - 'system-kind': 'rootfs-tarball', 'name': 'foo', 'description': '', 'arch': 'x86_64', 'strata': [], 'configuration-extensions': [], - 'disk-size': '1G', }) def test_unsets_defaults_for_system(self): diff --git a/morphlib/morphologyfactory.py b/morphlib/morphologyfactory.py index ae5a4332..5afafefb 100644 --- a/morphlib/morphologyfactory.py +++ b/morphlib/morphologyfactory.py @@ -140,19 +140,6 @@ class MorphologyFactory(object): (morphology['arch'], ', '.join(morphlib.valid_archs))) - kind = morphology['system-kind'] - if kind == 'rootfs-tarball': # pragma: no cover - self._app.status( - msg='WARNING: Obsolete field system-kind used in morphology ' - '(it is harmless, but should be removed)') - elif kind: - raise morphlib.Error( - 'System kind %s is not supported (anymore), ' - 'the whole system-kind field is deprecated. ' - 'Please remove system-kind from your system ' - 'morphologies and morph deploy to create ' - 'the desired output format.' % kind) - name = morphology['name'] morphology.builds_artifacts = [name + '-rootfs'] diff --git a/morphlib/morphologyfactory_tests.py b/morphlib/morphologyfactory_tests.py index 06489085..6e1e67d3 100644 --- a/morphlib/morphologyfactory_tests.py +++ b/morphlib/morphologyfactory_tests.py @@ -118,7 +118,6 @@ class FakeLocalRepo(object): 'system.morph': '''{ "name": "system", "kind": "system", - "system-kind": "%(system_kind)s", "arch": "%(arch)s" }''', 'parse-error.morph': '''{ "name"''', @@ -130,13 +129,11 @@ class FakeLocalRepo(object): def __init__(self): self.arch = 'x86_64' - self.system_kind = '' def cat(self, sha1, filename): if filename in self.morphologies: values = { 'arch': self.arch, - 'system_kind': self.system_kind, } return self.morphologies[filename] % values elif filename.endswith('.morph'): @@ -308,11 +305,6 @@ class MorphologyFactoryTests(unittest.TestCase): morph = self.mf.get_morphology('reponame', 'sha1', 'system.morph') self.assertEqual(morph['arch'], 'armv7l') - def test_fails_if_system_define_system_kind_that_is_not_tarball(self): - self.lr.system_kind = 'blahblah' - self.assertRaises(morphlib.Error, self.mf.get_morphology, - 'reponame', 'sha1', 'system.morph') - def test_fails_on_parse_error(self): self.assertRaises(morphlib.Error, self.mf.get_morphology, 'reponame', 'sha1', 'parse-error.morph') diff --git a/morphlib/morphset.py b/morphlib/morphset.py index 3c07d58e..9ef1e804 100644 --- a/morphlib/morphset.py +++ b/morphlib/morphset.py @@ -95,9 +95,11 @@ class MorphologySet(object): repo_url, ref, morph = self._find_spec( system_morph['strata'], stratum_name) - if repo_url is None: + if (repo_url, ref, morph) == (None, None, None): raise StratumNotInSystemError(system_morph['name'], stratum_name) - m = self._get_morphology(repo_url, ref, '%s.morph' % morph) + m = self._get_morphology(repo_url or system_morph.repo_url, + ref or system_morph.ref, + '%s.morph' % morph) if m is None: raise StratumNotInSetError(stratum_name) return m @@ -118,7 +120,7 @@ class MorphologySet(object): repo_url, ref, morph = self._find_spec( stratum_morph['chunks'], chunk_name) - if repo_url is None: + if (repo_url, ref, morph) == (None, None, None): raise ChunkNotInStratumError(stratum_morph['name'], chunk_name) return repo_url, ref, morph diff --git a/morphlib/plugins/artifact_inspection_plugin.py b/morphlib/plugins/artifact_inspection_plugin.py index 6c321abb..9181d488 100644 --- a/morphlib/plugins/artifact_inspection_plugin.py +++ b/morphlib/plugins/artifact_inspection_plugin.py @@ -78,7 +78,8 @@ class AutotoolsVersionGuesser(ProjectVersionGuesser): break # Then, try running autoconf against the configure script - version = self._check_autoconf_package_version(filename, data) + version = self._check_autoconf_package_version( + repo, ref, filename, data) if version: self.app.status( msg='%(repo)s: Version of %(ref)s detected ' @@ -107,7 +108,7 @@ class AutotoolsVersionGuesser(ProjectVersionGuesser): return version return None - def _check_autoconf_package_version(self, filename, data): + def _check_autoconf_package_version(self, repo, ref, filename, data): tempdir = morphlib.tempdir.Tempdir(self.app.settings['tempdir']) with open(tempdir.join(filename), 'w') as f: f.write(data) diff --git a/morphlib/plugins/branch_and_merge_new_plugin.py b/morphlib/plugins/branch_and_merge_new_plugin.py index 39552ef0..0a43b918 100644 --- a/morphlib/plugins/branch_and_merge_new_plugin.py +++ b/morphlib/plugins/branch_and_merge_new_plugin.py @@ -15,6 +15,7 @@ import cliapp +import contextlib import glob import logging import os @@ -55,6 +56,15 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): arg_synopsis='-- COMMAND [ARGS...]') self.app.add_subcommand('status', self.status, arg_synopsis='') + self.app.add_subcommand('branch-from-image', self.branch_from_image, + arg_synopsis='BRANCH') + group_branch = 'Branching Options' + self.app.settings.string(['metadata-dir'], + 'Set metadata location for branch-from-image' + ' (default: /baserock)', + metavar='DIR', + default='/baserock', + group=group_branch) def disable(self): pass @@ -97,6 +107,40 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): ws = morphlib.workspace.open('.') self.app.output.write('%s\n' % ws.root) + # TODO: Move this somewhere nicer + @contextlib.contextmanager + def _initializing_system_branch(self, ws, root_url, system_branch, + cached_repo, base_ref): + '''A context manager for system branches under construction. + + The purpose of this context manager is to factor out the branch + cleanup code for if an exception occurs while a branch is being + constructed. + + This could be handled by a higher order function which takes + a function to initialize the branch as a parameter, but with + statements look nicer and are more obviously about resource + cleanup. + + ''' + root_dir = ws.get_default_system_branch_directory_name(system_branch) + try: + sb = morphlib.sysbranchdir.create( + root_dir, root_url, system_branch) + gd = sb.clone_cached_repo(cached_repo, base_ref) + + yield (sb, gd) + + gd.update_submodules(self.app) + gd.update_remotes() + + except BaseException as e: + # Oops. Clean up. + logging.error('Caught exception: %s' % str(e)) + logging.info('Removing half-finished branch %s' % system_branch) + self._remove_branch_dir_safe(ws.root, root_dir) + raise + def checkout(self, args): '''Check out an existing system branch. @@ -124,6 +168,7 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): root_url = args[0] system_branch = args[1] + base_ref = system_branch self._require_git_user_config() @@ -139,27 +184,12 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): # Check the git branch exists. cached_repo.resolve_ref(system_branch) - root_dir = ws.get_default_system_branch_directory_name(system_branch) - - try: - # Create the system branch directory. This doesn't yet clone - # the root repository there. - sb = morphlib.sysbranchdir.create( - root_dir, root_url, system_branch) - - gd = sb.clone_cached_repo(cached_repo, system_branch) + with self._initializing_system_branch( + ws, root_url, system_branch, cached_repo, base_ref) as (sb, gd): if not self._checkout_has_systems(gd): - raise BranchRootHasNoSystemsError(root_url, system_branch) + raise BranchRootHasNoSystemsError(base_ref) - gd.update_submodules(self.app) - gd.update_remotes() - except BaseException as e: - # Oops. Clean up. - logging.error('Caught exception: %s' % str(e)) - logging.info('Removing half-finished branch %s' % system_branch) - self._remove_branch_dir_safe(ws.root, root_dir) - raise def branch(self, args): '''Create a new system branch. @@ -212,29 +242,14 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): # Make sure the base_ref exists. cached_repo.resolve_ref(base_ref) - root_dir = ws.get_default_system_branch_directory_name(system_branch) + with self._initializing_system_branch( + ws, root_url, system_branch, cached_repo, base_ref) as (sb, gd): - try: - # Create the system branch directory. This doesn't yet clone - # the root repository there. - sb = morphlib.sysbranchdir.create( - root_dir, root_url, system_branch) - - gd = sb.clone_cached_repo(cached_repo, base_ref) gd.branch(system_branch, base_ref) gd.checkout(system_branch) if not self._checkout_has_systems(gd): - raise BranchRootHasNoSystemsError(root_url, base_ref) - - gd.update_submodules(self.app) - gd.update_remotes() - except BaseException as e: - # Oops. Clean up. - logging.error('Caught exception: %s' % str(e)) - logging.info('Removing half-finished branch %s' % system_branch) - self._remove_branch_dir_safe(ws.root, root_dir) - raise + raise BranchRootHasNoSystemsError(base_ref) def _save_dirty_morphologies(self, loader, sb, morphs): logging.debug('Saving dirty morphologies: start') @@ -266,7 +281,9 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): # of triplets (repo url, ref, filename). return [ - (spec['repo'], spec['ref'], '%s.morph' % spec['morph']) + (spec['repo'] or morph.repo_url, + spec['ref'] or morph.ref, + '%s.morph' % spec['morph']) for spec in specs ] @@ -417,16 +434,6 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): sb = morphlib.sysbranchdir.open_from_within('.') loader = morphlib.morphloader.MorphologyLoader() - # FIXME: The old "morph edit" code did its own morphology validation, - # which was much laxer than what MorphologyFactory does, or the - # new MorphologyLoader does. This new "morph edit" uses - # MorphologyLoader, and the stricter validation breaks the test - # suite. However, I want to keep the test suite as untouched as - # possible, until all the old code is gone (after which the test - # suite will be refactored). Thus, to work around the test suite - # breaking, we disable morphology validation for now. - loader.validate = lambda *args: None - # Load the system morphology, and all stratum morphologies, including # all the strata that are being build-depended on. @@ -679,6 +686,9 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): #TODO: Stop using app.resolve_ref def resolve_refs(morphs): for repo, ref in morphs.list_refs(): + # You can't resolve null refs, so don't attempt to. + if repo is None or ref is None: + continue # TODO: Handle refs that are only in workspace in general if (repo == sb.root_repository_url and ref == sb.system_branch_name): @@ -784,3 +794,98 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): if not has_uncommitted_changes: self.app.output.write("\nNo repos have outstanding changes.\n") + + def branch_from_image(self, args): + '''Produce a branch of an existing system image. + + Given the metadata specified by --metadata-dir, create a new + branch then petrify it to the state of the commits at the time + the system was built. + + If --metadata-dir is not specified, it defaults to your currently + running system. + + ''' + if len(args) != 1: + raise cliapp.AppException( + "branch-from-image needs exactly 1 argument " + "of the new system branch's name") + system_branch = args[0] + metadata_path = self.app.settings['metadata-dir'] + alias_resolver = morphlib.repoaliasresolver.RepoAliasResolver( + self.app.settings['repo-alias']) + + self._require_git_user_config() + + ws = morphlib.workspace.open('.') + + system, metadata = self._load_system_metadata(metadata_path) + resolved_refs = dict(self._resolve_refs_from_metadata(alias_resolver, + metadata)) + logging.debug('Resolved refs: %r' % resolved_refs) + base_ref = system['sha1'] + # The previous version would fall back to deducing this from the repo + # url and the repo alias resolver, but this does not always work, and + # new systems always have repo-alias in the metadata + root_url = system['repo-alias'] + + lrc, rrc = morphlib.util.new_repo_caches(self.app) + cached_repo = lrc.get_updated_repo(root_url) + + + with self._initializing_system_branch( + ws, root_url, system_branch, cached_repo, base_ref) as (sb, gd): + + # TODO: It's nasty to clone to a sha1 then create a branch + # of that sha1 then check it out, a nicer API may be the + # initial clone not checking out a branch at all, then + # the user creates and checks out their own branches + gd.branch(system_branch, base_ref) + gd.checkout(system_branch) + + loader = morphlib.morphloader.MorphologyLoader() + morphs = self._load_all_sysbranch_morphologies(sb, loader) + + morphs.repoint_refs(sb.root_repository_url, + sb.system_branch_name) + + morphs.petrify_chunks(resolved_refs) + + self._save_dirty_morphologies(loader, sb, morphs.morphologies) + + @staticmethod + def _load_system_metadata(path): + '''Load all metadata in `path` corresponding to a single System. + ''' + + smd = morphlib.systemmetadatadir.SystemMetadataDir(path) + metadata = smd.values() + systems = [md for md in metadata + if 'kind' in md and md['kind'] == 'system'] + + if not systems: + raise cliapp.AppException( + 'Metadata directory does not contain any systems.') + if len(systems) > 1: + raise cliapp.AppException( + 'Metadata directory contains multiple systems.') + system_metadatum = systems[0] + + metadata_cache_id_lookup = dict((md['cache-key'], md) + for md in metadata) + + return system_metadatum, metadata_cache_id_lookup + + @staticmethod + def _resolve_refs_from_metadata(alias_resolver, metadata): + '''Pre-resolve a set of refs from existing metadata. + + Given the metadata, generate a mapping of all the (repo, ref) + pairs defined in the metadata and the commit id they resolved to. + + ''' + for md in metadata.itervalues(): + repourls = set((md['repo-alias'], md['repo'])) + repourls.update(alias_resolver.aliases_from_url(md['repo'])) + for repourl in repourls: + yield ((repourl, md['original_ref']), md['sha1']) diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py index 37d5e40c..fec16415 100644 --- a/morphlib/plugins/branch_and_merge_plugin.py +++ b/morphlib/plugins/branch_and_merge_plugin.py @@ -68,15 +68,9 @@ class BranchAndMergePlugin(cliapp.Plugin): self.app.add_subcommand('build', self.build, arg_synopsis='SYSTEM') self.app.add_subcommand('old-status', self.status) - self.app.add_subcommand('branch-from-image', self.branch_from_image, - arg_synopsis='REPO BRANCH') - group_branch = 'Branching Options' - self.app.settings.string(['metadata-dir'], - 'Set metadata location for branch-from-image' - ' (default: /baserock)', - metavar='DIR', - default='/baserock', - group=group_branch) + self.app.add_subcommand('old-branch-from-image', + self.branch_from_image, + arg_synopsis='REPO BRANCH') # Advanced commands self.app.add_subcommand('old-foreach', self.foreach, @@ -339,7 +333,6 @@ class BranchAndMergePlugin(cliapp.Plugin): required = { 'system': [ 'name', - 'system-kind', 'arch', 'strata', ], @@ -360,8 +353,6 @@ class BranchAndMergePlugin(cliapp.Plugin): 'system': [ 'kind', 'description', - 'disk-size', - '_disk-size', 'configuration-extensions', ], 'stratum': [ diff --git a/morphlib/systemmetadatadir.py b/morphlib/systemmetadatadir.py new file mode 100644 index 00000000..eac5b446 --- /dev/null +++ b/morphlib/systemmetadatadir.py @@ -0,0 +1,87 @@ +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# =*= License: GPL-2 =*= + + +import collections +import glob +import json +import os + + +class SystemMetadataDir(collections.MutableMapping): + + '''An abstraction over the /baserock metadata directory. + + This allows methods of iterating over it, and accessing it like + a dict. + + The /baserock metadata directory contains information about all of + the chunks in a built system. It exists to provide traceability from + the input sources to the output. + + If you create the object with smd = SystemMetadataDir('/baserock') + data = smd['key'] will read /baserock/key.meta and return its JSON + encoded contents as native python objects. + + smd['key'] = data will write data to /baserock/key.meta as JSON + + The key may not have '\0' characters in it since the underlying + system calls don't support embedded NUL bytes. + + The key may not have '/' characters in it since we do not support + morphologies with slashes in their names. + + ''' + + def __init__(self, metadata_path): + collections.MutableMapping.__init__(self) + self._metadata_path = metadata_path + + def _join_path(self, *args): + return os.path.join(self._metadata_path, *args) + + def _raw_path_iter(self): + return glob.iglob(self._join_path('*.meta')) + + @staticmethod + def _check_key(key): + if any(c in key for c in "\0/"): + raise KeyError(key) + + def __getitem__(self, key): + self._check_key(key) + try: + with open(self._join_path('%s.meta' % key), 'r') as f: + return json.load(f) + except IOError: + raise KeyError(key) + + def __setitem__(self, key, value): + self._check_key(key) + with open(self._join_path('%s.meta' % key), 'w') as f: + json.dump(value, f, indent=4, sort_keys=True) + + def __delitem__(self, key): + self._check_key(key) + os.unlink(self._join_path('%s.meta' % key)) + + def __iter__(self): + return (os.path.basename(fn)[:-len('.meta')] + for fn in self._raw_path_iter()) + + def __len__(self): + return len(list(self._raw_path_iter())) diff --git a/morphlib/systemmetadatadir_tests.py b/morphlib/systemmetadatadir_tests.py new file mode 100644 index 00000000..0126f862 --- /dev/null +++ b/morphlib/systemmetadatadir_tests.py @@ -0,0 +1,75 @@ +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# =*= License: GPL-2 =*= + + +import operator +import os +import shutil +import tempfile +import unittest + +import morphlib + + +class SystemMetadataDirTests(unittest.TestCase): + + def setUp(self): + self.tempdir = tempfile.mkdtemp() + self.metadatadir = os.path.join(self.tempdir, 'baserock') + os.mkdir(self.metadatadir) + self.smd = morphlib.systemmetadatadir.SystemMetadataDir( + self.metadatadir) + + def tearDown(self): + shutil.rmtree(self.tempdir) + + def test_add_new(self): + self.smd['key'] = {'foo': 'bar'} + self.assertEqual(self.smd['key']['foo'], 'bar') + + def test_replace(self): + self.smd['key'] = {'foo': 'bar'} + self.smd['key'] = {'foo': 'baz'} + self.assertEqual(self.smd['key']['foo'], 'baz') + + def test_remove(self): + self.smd['key'] = {'foo': 'bar'} + del self.smd['key'] + self.assertTrue('key' not in self.smd) + + def test_iterate(self): + self.smd['build-essential'] = "Some data" + self.smd['core'] = "More data" + self.smd['foundation'] = "Yet more data" + self.assertEqual(sorted(self.smd.keys()), + ['build-essential', 'core', 'foundation']) + self.assertEqual(dict(self.smd.iteritems()), + { + 'build-essential': "Some data", + 'core': "More data", + 'foundation': "Yet more data", + }) + + def test_raises_KeyError(self): + self.assertRaises(KeyError, operator.getitem, self.smd, 'key') + + def test_validates_keys(self): + for key in ('foo/bar', 'baz\0quux'): + self.assertRaises(KeyError, operator.getitem, self.smd, key) + self.assertRaises(KeyError, operator.setitem, + self.smd, key, 'value') + self.assertRaises(KeyError, operator.delitem, self.smd, key) diff --git a/scripts/nullify-local-refs b/scripts/nullify-local-refs new file mode 100755 index 00000000..5db5c587 --- /dev/null +++ b/scripts/nullify-local-refs @@ -0,0 +1,18 @@ +#!/usr/bin/python + +import yaml, sys +repo = sys.argv[1] +ref = sys.argv[2] +for filename in sys.argv[3:]: + with open(filename, "r") as f: + d = yaml.load(f) + if "strata" in d: + for spec in d["strata"]: + if spec["repo"] == repo and spec["ref"] == ref: + spec["repo"] = spec["ref"] = None + if "build-depends" in d: + for spec in d["build-depends"]: + if spec["repo"] == repo and spec["ref"] == ref: + spec["repo"] = spec["ref"] = None + with open(filename, "w") as f: + yaml.dump(d, f) diff --git a/scripts/setup-3rd-party-strata b/scripts/setup-3rd-party-strata index f2ea2a4c..fc263f96 100644 --- a/scripts/setup-3rd-party-strata +++ b/scripts/setup-3rd-party-strata @@ -97,9 +97,7 @@ cat <<EOF > "hello-system.morph" { "name": "hello-system", "kind": "system", - "system-kind": "rootfs-tarball", "arch": "x86_64", - "disk-size": "1G", "strata": [ { "morph": "hello-stratum", diff --git a/tests.as-root/archless-system-fails.script b/tests.as-root/archless-system-fails.script index eda797f1..2fdeb018 100755 --- a/tests.as-root/archless-system-fails.script +++ b/tests.as-root/archless-system-fails.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2011, 2012 Codethink Limited +# Copyright (C) 2011-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -26,7 +26,6 @@ cat <<EOF >archless-system.morph { "name": "archless-system", "kind": "system", - "disk-size": "1G", "strata": [ { "morph": "hello-stratum", diff --git a/tests.as-root/branch-from-image-works.script b/tests.as-root/branch-from-image-works.script index 9f82f629..942301e8 100755 --- a/tests.as-root/branch-from-image-works.script +++ b/tests.as-root/branch-from-image-works.script @@ -48,8 +48,7 @@ git commit --quiet -m 'Make hello say goodbye' workspace="$DATADIR/workspace" "$SRCDIR/scripts/test-morph" init "$workspace" cd "$workspace" -"$SRCDIR/scripts/test-morph" branch-from-image \ - test:morphs mybranch \ +"$SRCDIR/scripts/test-morph" branch-from-image mybranch \ --metadata-dir="$extracted/baserock" cd mybranch/test:morphs grep -qFe "$hello_chunk_commit" hello-stratum.morph diff --git a/tests.as-root/metadata-includes-morph-version.setup b/tests.as-root/metadata-includes-morph-version.setup index 2284cfb9..f0aefb3c 100755 --- a/tests.as-root/metadata-includes-morph-version.setup +++ b/tests.as-root/metadata-includes-morph-version.setup @@ -27,7 +27,6 @@ cat <<EOF > hello-tarball.morph { "name": "hello-tarball", "kind": "system", - "system-kind": "rootfs-tarball", "arch": "$(uname -m)", "strata": [ { diff --git a/tests.as-root/metadata-includes-repo-alias.setup b/tests.as-root/metadata-includes-repo-alias.setup index 2284cfb9..f0aefb3c 100755 --- a/tests.as-root/metadata-includes-repo-alias.setup +++ b/tests.as-root/metadata-includes-repo-alias.setup @@ -27,7 +27,6 @@ cat <<EOF > hello-tarball.morph { "name": "hello-tarball", "kind": "system", - "system-kind": "rootfs-tarball", "arch": "$(uname -m)", "strata": [ { diff --git a/tests.as-root/rootfs-tarball-builds-rootfs-and-kernel.script b/tests.as-root/rootfs-tarball-builds-rootfs-and-kernel.script index 8229127d..93cd135f 100755 --- a/tests.as-root/rootfs-tarball-builds-rootfs-and-kernel.script +++ b/tests.as-root/rootfs-tarball-builds-rootfs-and-kernel.script @@ -16,9 +16,6 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -## A system-kind of rootfs-tarball should build both kernel image and -## a tarball with the root filesystem. - set -eu # Disable test on versions of Python before 2.7. diff --git a/tests.as-root/setup b/tests.as-root/setup index eea9e061..2ab1cd94 100755 --- a/tests.as-root/setup +++ b/tests.as-root/setup @@ -134,9 +134,7 @@ git add tools-stratum.morph cat <<EOF > hello-system.morph name: hello-system kind: system -system-kind: rootfs-tarball arch: `uname -m` -disk-size: 1G strata: - morph: hello-stratum repo: test:morphs @@ -163,9 +161,7 @@ git add linux-stratum.morph cat <<EOF > linux-system.morph name: linux-system kind: system -system-kind: rootfs-tarball arch: `uname -m` -disk-size: 1G strata: - morph: hello-stratum repo: test:morphs diff --git a/tests.as-root/system-overlap.script b/tests.as-root/system-overlap.script index 6e6ef2ac..41ff7536 100755 --- a/tests.as-root/system-overlap.script +++ b/tests.as-root/system-overlap.script @@ -31,9 +31,7 @@ cat <<EOF >overlap-system.morph { "name": "overlap-system", "kind": "system", - "system-kind": "rootfs-tarball", "arch": "$(uname -m)", - "disk-size": "1G", "strata": [ { "morph": "foo-baz-stratum", diff --git a/tests.as-root/tarball-image-is-sensible.setup b/tests.as-root/tarball-image-is-sensible.setup index fa904c2c..5fd7b283 100755 --- a/tests.as-root/tarball-image-is-sensible.setup +++ b/tests.as-root/tarball-image-is-sensible.setup @@ -46,7 +46,6 @@ cat <<EOF > hello-tarball.morph { "name": "hello-tarball", "kind": "system", - "system-kind": "rootfs-tarball", "arch": "$(uname -m)", "strata": [ { diff --git a/tests.branching.disabled/workflow-petrify.stdout b/tests.branching.disabled/workflow-petrify.stdout index 9f0cfb0c..a0ce82f4 100644 --- a/tests.branching.disabled/workflow-petrify.stdout +++ b/tests.branching.disabled/workflow-petrify.stdout @@ -2,9 +2,7 @@ test/petrify after petrifying: { "name": "hello-system", "kind": "system", - "system-kind": "rootfs-tarball", "arch": "x86_64", - "disk-size": "1G", "strata": [ { "morph": "hello-stratum", @@ -56,9 +54,7 @@ test/petrify after editing a chunk: { "name": "hello-system", "kind": "system", - "system-kind": "rootfs-tarball", "arch": "x86_64", - "disk-size": "1G", "strata": [ { "morph": "hello-stratum", @@ -109,9 +105,7 @@ test/unpetrify after unpetrifying: { "name": "hello-system", "kind": "system", - "system-kind": "rootfs-tarball", "arch": "x86_64", - "disk-size": "1G", "strata": [ { "morph": "hello-stratum", diff --git a/tests.branching/edit-updates-stratum-build-depends.stdout b/tests.branching/edit-updates-stratum-build-depends.stdout index 1c6eb8e3..7120ef50 100644 --- a/tests.branching/edit-updates-stratum-build-depends.stdout +++ b/tests.branching/edit-updates-stratum-build-depends.stdout @@ -13,10 +13,10 @@ index 73ed482..475fe0f 100644 kind: stratum name: hello-stratum diff --git a/hello-system.morph b/hello-system.morph -index 721473c..1537f53 100644 +index 3f7b4d3..199c924 100644 --- a/hello-system.morph +++ b/hello-system.morph -@@ -3,9 +3,9 @@ kind: system +@@ -3,5 +3,6 @@ kind: system name: hello-system strata: - morph: hello-stratum @@ -24,27 +24,3 @@ index 721473c..1537f53 100644 + ref: newbranch repo: test:morphs + unpetrify-ref: master - - morph: xyzzy-stratum - ref: master - repo: test:morphs --system-kind: rootfs-tarball -diff --git a/xyzzy-stratum.morph b/xyzzy-stratum.morph -index e302037..bcf5b57 100644 ---- a/xyzzy-stratum.morph -+++ b/xyzzy-stratum.morph -@@ -1,11 +1,13 @@ - build-depends: - - morph: hello-stratum -- ref: master -+ ref: newbranch - repo: test:morphs -+ unpetrify-ref: master - chunks: - - build-depends: [] - name: hello -- ref: master -+ ref: newbranch - repo: test:hello -+ unpetrify-ref: master - kind: stratum - name: xyzzy-stratum diff --git a/tests.branching/edit-updates-stratum.stdout b/tests.branching/edit-updates-stratum.stdout index 32eb820d..7120ef50 100644 --- a/tests.branching/edit-updates-stratum.stdout +++ b/tests.branching/edit-updates-stratum.stdout @@ -13,15 +13,14 @@ index 73ed482..475fe0f 100644 kind: stratum name: hello-stratum diff --git a/hello-system.morph b/hello-system.morph -index b0fed3b..199c924 100644 +index 3f7b4d3..199c924 100644 --- a/hello-system.morph +++ b/hello-system.morph -@@ -3,6 +3,6 @@ kind: system +@@ -3,5 +3,6 @@ kind: system name: hello-system strata: - morph: hello-stratum - ref: master + ref: newbranch repo: test:morphs --system-kind: rootfs-tarball + unpetrify-ref: master diff --git a/tests.branching/foreach-handles-full-urls.stdout b/tests.branching/foreach-handles-full-urls.stdout index 3abae62c..cee2f70a 100644 --- a/tests.branching/foreach-handles-full-urls.stdout +++ b/tests.branching/foreach-handles-full-urls.stdout @@ -1,4 +1,4 @@ file://TMP/morphs # On branch master -nothing to commit (working directory clean) +nothing to commit, working directory clean diff --git a/tests.branching/setup b/tests.branching/setup index 589f19ed..1263e3b6 100755 --- a/tests.branching/setup +++ b/tests.branching/setup @@ -55,7 +55,6 @@ strata: - morph: hello-stratum ref: master repo: test:morphs -system-kind: rootfs-tarball EOF cat <<EOF > "$DATADIR/morphs/hello-stratum.morph" diff --git a/tests.branching/tag-creates-commit-and-tag.stdout b/tests.branching/tag-creates-commit-and-tag.stdout index 598e28bf..b6098eb5 100644 --- a/tests.branching/tag-creates-commit-and-tag.stdout +++ b/tests.branching/tag-creates-commit-and-tag.stdout @@ -5,7 +5,7 @@ Date: Tue Jul 31 16:51:54 2012 +0000 Message -commit 9509c9c379f8ba643b2ad9a6ec50ecf96993cbb5 +commit 6895ed63425bedb3dccaea3f258c705b1600f6be Author: developer <developer@example.com> Date: Tue Jul 31 16:51:54 2012 +0000 @@ -26,10 +26,10 @@ index 73ed482..2218f63 100644 kind: stratum name: hello-stratum diff --git a/hello-system.morph b/hello-system.morph -index b0fed3b..4c4ee3e 100644 +index 3f7b4d3..d909347 100644 --- a/hello-system.morph +++ b/hello-system.morph -@@ -3,6 +3,7 @@ kind: system +@@ -3,5 +3,6 @@ kind: system name: hello-system strata: - morph: hello-stratum @@ -37,10 +37,9 @@ index b0fed3b..4c4ee3e 100644 + ref: example-tag repo: test:morphs + unpetrify-ref: master - system-kind: rootfs-tarball test:morphs -commit 9509c9c379f8ba643b2ad9a6ec50ecf96993cbb5 +commit 6895ed63425bedb3dccaea3f258c705b1600f6be Author: developer <developer@example.com> AuthorDate: Tue Jul 31 16:51:54 2012 +0000 Commit: developer <developer@example.com> @@ -48,7 +47,7 @@ CommitDate: Tue Jul 31 16:51:54 2012 +0000 Message -commit 1fe013ee284724848d65096d4d88f612fae56fc6 +commit e11a36aa9e4c998c41a3ec3209324b9318e484ae Author: developer <developer@example.com> AuthorDate: Tue Jul 31 16:51:54 2012 +0000 Commit: developer <developer@example.com> diff --git a/tests.branching/tag-tag-works-as-expected.stdout b/tests.branching/tag-tag-works-as-expected.stdout index 242b2d4e..98a3be81 100644 --- a/tests.branching/tag-tag-works-as-expected.stdout +++ b/tests.branching/tag-tag-works-as-expected.stdout @@ -9,7 +9,7 @@ Date: Tue Jul 31 16:51:54 2012 +0000 Second -commit ca7594f436da55bda2cfff2c6484d11aa0ea4cbc +commit 476e4ff4b19c38eb64ad3a151b7c58a7ab95c9ee Author: developer <developer@example.com> Date: Tue Jul 31 16:51:54 2012 +0000 @@ -30,10 +30,10 @@ index 73ed482..2218f63 100644 kind: stratum name: hello-stratum diff --git a/hello-system.morph b/hello-system.morph -index b0fed3b..875d73a 100644 +index 3f7b4d3..431e15d 100644 --- a/hello-system.morph +++ b/hello-system.morph -@@ -3,6 +3,7 @@ kind: system +@@ -3,5 +3,6 @@ kind: system name: hello-system strata: - morph: hello-stratum @@ -41,10 +41,9 @@ index b0fed3b..875d73a 100644 + ref: tagged-tag repo: test:morphs + unpetrify-ref: master - system-kind: rootfs-tarball test:morphs -commit ca7594f436da55bda2cfff2c6484d11aa0ea4cbc +commit 476e4ff4b19c38eb64ad3a151b7c58a7ab95c9ee Author: developer <developer@example.com> AuthorDate: Tue Jul 31 16:51:54 2012 +0000 Commit: developer <developer@example.com> @@ -52,7 +51,7 @@ CommitDate: Tue Jul 31 16:51:54 2012 +0000 Second -commit 1fe013ee284724848d65096d4d88f612fae56fc6 +commit e11a36aa9e4c998c41a3ec3209324b9318e484ae Author: developer <developer@example.com> AuthorDate: Tue Jul 31 16:51:54 2012 +0000 Commit: developer <developer@example.com> diff --git a/tests.branching/tag-works-with-multiple-morphs-repos.script b/tests.branching/tag-works-with-multiple-morphs-repos.script index 38d73852..f6ecbf32 100755 --- a/tests.branching/tag-works-with-multiple-morphs-repos.script +++ b/tests.branching/tag-works-with-multiple-morphs-repos.script @@ -32,9 +32,7 @@ mkdir "$DATADIR/morphs1" cat <<EOF > "$DATADIR/morphs1/test-system.morph" name: test-system kind: system -system-kind: rootfs-tarball arch: $(uname -m) -disk-size: 1G strata: - morph: stratum1 ref: master diff --git a/tests.branching/tag-works-with-multiple-morphs-repos.stdout b/tests.branching/tag-works-with-multiple-morphs-repos.stdout index 938e41ff..81fbf20d 100644 --- a/tests.branching/tag-works-with-multiple-morphs-repos.stdout +++ b/tests.branching/tag-works-with-multiple-morphs-repos.stdout @@ -4,7 +4,7 @@ Date: Tue Jul 31 16:51:54 2012 +0000 create tag -commit 7253f9b1471f1983e07823d2b84582dde9fb108d +commit 7323ed5dcc47715e7303fd36d537aef98a04df9a Author: developer <developer@example.com> Date: Tue Jul 31 16:51:54 2012 +0000 @@ -67,11 +67,11 @@ index 0000000..610fae6 + repo: test:hello + unpetrify-ref: master diff --git a/test-system.morph b/test-system.morph -index 44d5ae5..08f9adc 100644 +index 62d3b08..edb5745 100644 --- a/test-system.morph +++ b/test-system.morph -@@ -5,8 +5,11 @@ arch: x86_64 - disk-size: 1G +@@ -3,8 +3,11 @@ kind: system + arch: x86_64 strata: - morph: stratum1 - ref: master @@ -85,7 +85,7 @@ index 44d5ae5..08f9adc 100644 + repo: test:morphs1 + unpetrify-ref: master + unpetrify-repo: test:morphs2 -commit 7253f9b1471f1983e07823d2b84582dde9fb108d +commit 7323ed5dcc47715e7303fd36d537aef98a04df9a Author: developer <developer@example.com> AuthorDate: Tue Jul 31 16:51:54 2012 +0000 Commit: developer <developer@example.com> @@ -93,10 +93,10 @@ CommitDate: Tue Jul 31 16:51:54 2012 +0000 create tag --- - stratum1.morph | 9 ++++++--- - stratum2.morph | 14 ++++++++++++++ - stratum3.morph | 9 +++++++++ - test-system.morph | 9 ++++++--- + stratum1.morph | 9 ++++++--- + stratum2.morph | 14 ++++++++++++++ + stratum3.morph | 9 +++++++++ + test-system.morph | 9 ++++++--- 4 files changed, 35 insertions(+), 6 deletions(-) diff --git a/stratum1.morph b/stratum1.morph @@ -156,11 +156,11 @@ index 0000000..610fae6 + repo: test:hello + unpetrify-ref: master diff --git a/test-system.morph b/test-system.morph -index 44d5ae5..08f9adc 100644 +index 62d3b08..edb5745 100644 --- a/test-system.morph +++ b/test-system.morph -@@ -5,8 +5,11 @@ arch: x86_64 - disk-size: 1G +@@ -3,8 +3,11 @@ kind: system + arch: x86_64 strata: - morph: stratum1 - ref: master diff --git a/tests.build/build-system-with-null-refs.script b/tests.build/build-system-with-null-refs.script new file mode 100755 index 00000000..e23dcafa --- /dev/null +++ b/tests.build/build-system-with-null-refs.script @@ -0,0 +1,24 @@ +#!/bin/sh +# +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +## Test building a system with null refs + +set -eu + +"$SRCDIR/scripts/test-morph" build-morphology \ + test:morphs-repo master hello-system diff --git a/tests.build/build-system-with-null-refs.setup b/tests.build/build-system-with-null-refs.setup new file mode 100755 index 00000000..cbf53076 --- /dev/null +++ b/tests.build/build-system-with-null-refs.setup @@ -0,0 +1,23 @@ +#!/bin/sh +# +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +set -eu + +cd "$DATADIR/morphs-repo" +"$SRCDIR/scripts/nullify-local-refs" test:morphs master *.morph +git add *.morph +git commit --quiet -m "Nullify all refs" diff --git a/tests.build/setup b/tests.build/setup index 499dbb21..563482e6 100755 --- a/tests.build/setup +++ b/tests.build/setup @@ -108,8 +108,6 @@ cat <<EOF > hello-system.morph "name": "hello-system", "kind": "system", "arch": "$(uname -m)", - "system-kind": "rootfs-tarball", - "disk-size": "1G", "strata": [ { "morph": "hello-stratum", diff --git a/tests.deploy/deploy-with-null-refs.script b/tests.deploy/deploy-with-null-refs.script new file mode 100755 index 00000000..c283debf --- /dev/null +++ b/tests.deploy/deploy-with-null-refs.script @@ -0,0 +1,35 @@ +#!/bin/bash +# +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + + +set -eu + + +. "$SRCDIR/tests.deploy/setup-build" + +cd "$DATADIR/workspace/branch1" +"$SRCDIR/scripts/nullify-local-refs" test:morphs master test:morphs/*.morph + +"$SRCDIR/scripts/test-morph" build hello-system + +"$SRCDIR/scripts/test-morph" build linux-system + +"$SRCDIR/scripts/test-morph" --log "$DATADIR/deploy.log" \ + deploy test_cluster \ + linux-system-2.HOSTNAME="baserock-rocks-even-more" \ + > /dev/null diff --git a/tests.deploy/setup b/tests.deploy/setup index 88488a91..5f83747e 100755 --- a/tests.deploy/setup +++ b/tests.deploy/setup @@ -112,7 +112,6 @@ cat <<EOF > hello-system.morph { "name": "hello-system", "kind": "system", - "system-kind": "rootfs-tarball", "arch": "$(uname -m)", "strata": [ { @@ -153,7 +152,6 @@ cat <<EOF > linux-system.morph { "name": "linux-system", "kind": "system", - "system-kind": "rootfs-tarball", "arch": "$(uname -m)", "strata": [ { diff --git a/tests.merging/setup b/tests.merging/setup index 11fdf0f1..2fdf9db0 100755 --- a/tests.merging/setup +++ b/tests.merging/setup @@ -51,9 +51,7 @@ cat <<EOF > "$DATADIR/morphs/hello-system.morph" { "name": "hello-system", "kind": "system", - "system-kind": "rootfs-tarball", "arch": "$(uname -m)", - "disk-size": "1G", "strata": [ { "morph": "hello-stratum", diff --git a/tests/setup b/tests/setup index 91a30236..07643ddc 100755 --- a/tests/setup +++ b/tests/setup @@ -107,8 +107,6 @@ cat <<EOF > hello-system.morph { "name": "hello-system", "kind": "system", - "system-kind": "rootfs-tarball", - "disk-size": "1G", "strata": [ { "morph": "hello-stratum", diff --git a/tests/show-dependencies.setup b/tests/show-dependencies.setup index 9c983d8a..edb9b6ab 100755 --- a/tests/show-dependencies.setup +++ b/tests/show-dependencies.setup @@ -349,7 +349,6 @@ cat <<EOF > xfce-system.morph { "name": "xfce-system", "kind": "system", - "system-kind": "rootfs-tarball", "arch": "$(uname -m)", "strata": [ { diff --git a/yarns/branches-workspaces.yarn b/yarns/branches-workspaces.yarn index 5273f396..cd3f7a0e 100644 --- a/yarns/branches-workspaces.yarn +++ b/yarns/branches-workspaces.yarn @@ -290,3 +290,73 @@ Creating a tag twice should fail. WHEN attempting to tag system branch foo as test123 THEN morph failed +Working with null repositories and refs +--------------------------------------- + +It is convenient to not explicitly name the repository and branch of +a stratum morphology, instead assuming it is the same as the current +morphology. + +These can be checked out like normal system branches. + + SCENARIO check out an existing system branch with null refs + GIVEN a workspace + AND a git server + AND null refs for local strata + WHEN checking out the master system branch + THEN the system branch master is checked out + +Likewise we can also create new system branches from these, and we +wouldn't need to worry about changing the system branch ref. + + + SCENARIO branch off a system branch with null refs + GIVEN a workspace + AND a git server + AND null refs for local strata + WHEN creating system branch foo + THEN the system branch foo is checked out + +When we edit a morphology with null refs, they stay null. + + SCENARIO editing with null refs + GIVEN a workspace + AND a git server + AND null refs for local strata + +When creating the branch, the refs remain null. + + WHEN creating system branch foo + THEN in branch foo, system test-system refs test-stratum in None + +After editing the stratum they remain null. + + WHEN editing stratum test-stratum in system test-system in branch foo + THEN in branch foo, system test-system refs test-stratum in None + +Refs to chunks are still altered as usual + + WHEN editing chunk test-chunk in test-stratum in test-system in branch foo + THEN in branch foo, system test-system refs test-stratum in None + AND in branch foo, stratum test-stratum refs test-chunk in foo + AND edited chunk test:test-chunk has git branch foo + +Petrifying also leaves null refs unmolested + + SCENARIO morph petrifies null refs + GIVEN a workspace + AND a git server + AND null refs for local strata + WHEN creating system branch foo + AND pushing system branch foo to git server + AND remembering all refs in foo + AND petrifying foo + THEN in branch foo, system test-system refs test-stratum in None + +Generating a manifest works + + SCENARIO morph generates a manifest + GIVEN a workspace + AND a system artifact + WHEN morph generates a manifest + THEN the manifest is generated diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn index cfb744f7..6e1fae18 100644 --- a/yarns/implementations.yarn +++ b/yarns/implementations.yarn @@ -114,6 +114,18 @@ another to hold a chunk. repo-alias = test=file://$DATADIR/gits/%s#file://$DATADIR/gits/%s EOF +Morphologies need to support having a null ref, which means look for the +stratum in the same repository and ref. Testing this requires different +morphologies. + + IMPLEMENTS GIVEN null refs for local strata + nullify_local_refs test:morphs master \ + "$DATADIR/gits/morphs/test-system.morph" \ + "$DATADIR/gits/morphs/test-stratum.morph" + run_in "$DATADIR/gits/morphs" git add . + run_in "$DATADIR/gits/morphs" git commit -m "Use null refs." + + Implementation sections for system branch operations ---------------------------------------------------- @@ -370,3 +382,40 @@ Tagging. assert_morphologies_are_petrified "$MATCH_1" temptemptemp done +Generating a manifest. + + IMPLEMENTS GIVEN a system artifact + mkdir "$DATADIR/hello_world" + + git init "$DATADIR/hello_world" + touch "$DATADIR/hello_world/configure.ac" + run_in "$DATADIR/hello_world" git add configure.ac + run_in "$DATADIR/hello_world" git commit -m 'Add configure.ac' + + mkdir "$DATADIR/baserock" + run_in "$DATADIR/hello_world" cat << EOF \ + > "$DATADIR/baserock/hello_world.meta" + { + "artifact-name": "hello_world", + "cache-key": + "ab8d00a80298a842446ce23507cea6b4d0e34c7ddfa05c67f460318b04d21308", + "kind": "chunk", + "morphology": "hello_world.morph", + "original_ref": "$(run_in "$DATADIR/hello_world" git rev-parse HEAD)", + "repo": "file://$DATADIR/hello_world", + "repo-alias": "upstream:hello_world", + "sha1": "$(run_in "$DATADIR/hello_world" git rev-parse HEAD)", + "source-name": "hello_world" + } + EOF + run_in "$DATADIR" tar -c baserock > "$DATADIR/artifact.tar" + + IMPLEMENTS WHEN morph generates a manifest + run_morph generate-manifest "$DATADIR/artifact.tar" > "$DATADIR/manifest" + + IMPLEMENTS THEN the manifest is generated + + # Generated manifest should contain the name of the repository + if ! grep -q hello_world "$DATADIR/manifest"; then + die "Output isn't what we expect" + fi diff --git a/yarns/morph.shell-lib b/yarns/morph.shell-lib index 4fb1eb10..448c60ce 100644 --- a/yarns/morph.shell-lib +++ b/yarns/morph.shell-lib @@ -144,6 +144,11 @@ assert_morphologies_are_petrified() } +nullify_local_refs() +{ + "$SRCDIR/scripts/nullify-local-refs" "$@" +} + # Currently, yarn isn't setting $SRCDIR to point at the project source # directory. We simulate this here. |