diff options
Diffstat (limited to 'morphlib')
| -rw-r--r-- | morphlib/app.py | 10 | ||||
| -rw-r--r-- | morphlib/artifactresolver.py | 8 | ||||
| -rw-r--r-- | morphlib/buildbranch.py | 4 | ||||
| -rw-r--r-- | morphlib/buildcommand.py | 4 | ||||
| -rw-r--r-- | morphlib/morphloader.py | 57 | ||||
| -rw-r--r-- | morphlib/morphloader_tests.py | 90 | ||||
| -rw-r--r-- | morphlib/morphologyfactory.py | 10 | ||||
| -rw-r--r-- | morphlib/morphologyfactory_tests.py | 2 | ||||
| -rw-r--r-- | morphlib/morphset.py | 37 | ||||
| -rw-r--r-- | morphlib/plugins/branch_and_merge_new_plugin.py | 6 | ||||
| -rw-r--r-- | morphlib/plugins/branch_and_merge_plugin.py | 40 |
11 files changed, 195 insertions, 73 deletions
diff --git a/morphlib/app.py b/morphlib/app.py index aecee42b..7fb71c7b 100644 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2013 Codethink Limited +# Copyright (C) 2011-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -346,14 +346,14 @@ class Morph(cliapp.Application): visit(reponame, ref, filename, absref, tree, morphology) if morphology['kind'] == 'system': - queue.extend((s['repo'] or reponame, - s['ref'] or ref, + queue.extend((s.get('repo') or reponame, + s.get('ref') or ref, '%s.morph' % s['morph']) for s in morphology['strata']) elif morphology['kind'] == 'stratum': if morphology['build-depends']: - queue.extend((s['repo'] or reponame, - s['ref'] or ref, + queue.extend((s.get('repo') or reponame, + s.get('ref') or ref, '%s.morph' % s['morph']) for s in morphology['build-depends']) queue.extend((c['repo'], c['ref'], '%s.morph' % c['morph']) diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py index ae0cfcf5..00976eb7 100644 --- a/morphlib/artifactresolver.py +++ b/morphlib/artifactresolver.py @@ -140,8 +140,8 @@ class ArtifactResolver(object): for info in source.morphology['strata']: stratum_source = self._source_pool.lookup( - info['repo'] or source.repo_name, - info['ref'] or source.original_ref, + info.get('repo') or source.repo_name, + info.get('ref') or source.original_ref, '%s.morph' % info['morph']) stratum_name = stratum_source.morphology['name'] @@ -165,8 +165,8 @@ class ArtifactResolver(object): for stratum_info in source.morphology.get('build-depends') or []: other_source = self._source_pool.lookup( - stratum_info['repo'] or source.repo_name, - stratum_info['ref'] or source.original_ref, + stratum_info.get('repo') or source.repo_name, + stratum_info.get('ref') or source.original_ref, '%s.morph' % stratum_info['morph']) # Make every stratum artifact this stratum source produces diff --git a/morphlib/buildbranch.py b/morphlib/buildbranch.py index d4426afb..546a29d5 100644 --- a/morphlib/buildbranch.py +++ b/morphlib/buildbranch.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -125,7 +125,7 @@ class BuildBranch(object): sb_info[repo, ref] = (gd, build_ref) def filter(m, kind, spec): - return (spec['repo'], spec['ref']) in sb_info + return (spec.get('repo'), spec.get('ref')) in sb_info def process(m, kind, spec): repo, ref = spec['repo'], spec['ref'] gd, build_ref = sb_info[repo, ref] diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 6485f510..137c63b4 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -228,8 +228,8 @@ class BuildCommand(object): def _validate_cross_refs_for_xxx(self, src, srcpool, specs, wanted): for spec in specs: - repo_name = spec['repo'] or src.repo_name - ref = spec['ref'] or src.original_ref + repo_name = spec.get('repo') or src.repo_name + ref = spec.get('ref') or src.original_ref filename = '%s.morph' % spec['morph'] logging.debug( 'Validating cross ref to %s:%s:%s' % diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py index 32b5b40b..e4367fa1 100644 --- a/morphlib/morphloader.py +++ b/morphlib/morphloader.py @@ -18,11 +18,27 @@ import collections import logging +import warnings import yaml import morphlib +class MorphologyObsoleteFieldWarning(UserWarning): + + def __init__(self, morphology, spec, field): + self.kind = morphology['kind'] + self.morphology_name = morphology.get('name', '<unknown>') + self.stratum_name = spec.get('alias', spec['morph']) + self.field = field + + def __str__(self): + format_string = ('%(kind)s morphology %(morphology_name)s refers to ' + 'stratum %(stratum_name)s with the %(field)s field. ' + 'Defaulting to null.') + return format_string % self.__dict__ + + class MorphologySyntaxError(morphlib.Error): def __init__(self, morphology): @@ -366,6 +382,9 @@ class MorphologyLoader(object): raise DuplicateStratumError(morph['name'], name) names.add(name) + # Validate stratum spec fields + self._validate_stratum_specs_fields(morph, 'strata') + # We allow the ARMv7 little-endian architecture to be specified # as armv7 and armv7l. Normalise. if morph['arch'] == 'armv7': @@ -409,6 +428,9 @@ class MorphologyLoader(object): raise NoStratumBuildDependenciesError( morph['name'], morph.filename) + # Validate build-dependencies if specified + self._validate_stratum_specs_fields(morph, 'build-depends') + # Require build-dependencies for each chunk. for spec in morph['chunks']: if 'build-depends' not in spec: @@ -496,6 +518,18 @@ class MorphologyLoader(object): type(pattern), morphology_name) errors.append(e) + @classmethod + def _warn_obsolete_field(cls, morphology, spec, field): + warnings.warn(MorphologyObsoleteFieldWarning(morphology, spec, field), + stacklevel=2) + + @classmethod + def _validate_stratum_specs_fields(cls, morphology, specs_field): + for spec in morphology.get(specs_field, None) or []: + for obsolete_field in ('repo', 'ref'): + if obsolete_field in spec: + cls._warn_obsolete_field(morphology, spec, obsolete_field) + def _require_field(self, field, morphology): if field not in morphology: raise MissingFieldError(field, morphology.filename) @@ -542,9 +576,23 @@ class MorphologyLoader(object): if key in morphology and morphology[key] == defaults[key]: del morphology[key] - if kind in ('stratum', 'cluster'): + if kind in ('system', 'stratum', 'cluster'): getattr(self, '_unset_%s_defaults' % kind)(morphology) + @classmethod + def _set_stratum_specs_defaults(cls, morphology, specs_field): + for spec in morphology.get(specs_field, None) or []: + for obsolete_field in ('repo', 'ref'): + if obsolete_field in spec: + del spec[obsolete_field] + + @classmethod + def _unset_stratum_specs_defaults(cls, morphology, specs_field): + for spec in morphology.get(specs_field, []): + for obsolete_field in ('repo', 'ref'): + if obsolete_field in spec: + del spec[obsolete_field] + def _set_cluster_defaults(self, morph): for system in morph.get('systems', []): if 'deploy-defaults' not in system: @@ -560,7 +608,10 @@ class MorphologyLoader(object): del system['deploy'] def _set_system_defaults(self, morph): - pass + self._set_stratum_specs_defaults(morph, 'strata') + + def _unset_system_defaults(self, morph): + self._unset_stratum_specs_defaults(morph, 'strata') def _set_stratum_defaults(self, morph): for spec in morph['chunks']: @@ -568,6 +619,7 @@ class MorphologyLoader(object): spec['repo'] = spec['name'] if 'morph' not in spec: spec['morph'] = spec['name'] + self._set_stratum_specs_defaults(morph, 'build-depends') def _unset_stratum_defaults(self, morph): for spec in morph['chunks']: @@ -575,6 +627,7 @@ class MorphologyLoader(object): del spec['repo'] if 'morph' in spec and spec['morph'] == spec['name']: del spec['morph'] + self._unset_stratum_specs_defaults(morph, 'strata') def _set_chunk_defaults(self, morph): if morph['max-jobs'] is not None: diff --git a/morphlib/morphloader_tests.py b/morphlib/morphloader_tests.py index 39923348..b8738804 100644 --- a/morphlib/morphloader_tests.py +++ b/morphlib/morphloader_tests.py @@ -16,12 +16,15 @@ # =*= License: GPL-2 =*= +import contextlib import os import shutil import tempfile import unittest +import warnings import morphlib +from morphlib.morphloader import MorphologyObsoleteFieldWarning class MorphologyLoaderTests(unittest.TestCase): @@ -382,8 +385,6 @@ build-system: dummy m['build-depends'] = [ { - "repo": "foo", - "ref": "foo", "morph": "foo", }, ] @@ -650,22 +651,27 @@ name: foo name='foo', arch='testarch', strata=[ - {'morph': 'bar'}, + { + 'morph': 'bar', + 'repo': 'obsolete', + 'ref': 'obsolete', + }, ]) self.loader.set_defaults(m) - self.loader.validate(m) self.assertEqual( - dict(m), { 'kind': 'system', 'name': 'foo', 'description': '', 'arch': 'testarch', 'strata': [ - {'morph': 'bar'}, + { + 'morph': 'bar', + }, ], 'configuration-extensions': [], - }) + }, + dict(m)) def test_unsets_defaults_for_system(self): m = morphlib.morph3.Morphology( @@ -675,7 +681,11 @@ name: foo 'name': 'foo', 'arch': 'testarch', 'strata': [ - {'morph': 'bar'}, + { + 'morph': 'bar', + 'repo': None, + 'ref': None, + }, ], 'configuration-extensions': [], }) @@ -788,3 +798,67 @@ name: foo self.assertEqual(m['name'], 'foo') self.assertEqual(m['kind'], 'cluster') self.assertEqual(m['systems'][0]['morph'], 'bar') + + @contextlib.contextmanager + def catch_warnings(*warning_classes): + with warnings.catch_warnings(record=True) as caught_warnings: + warnings.resetwarnings() + for warning_class in warning_classes: + warnings.simplefilter("always", warning_class) + yield caught_warnings + + def test_warns_when_systems_refer_to_strata_with_repo_or_ref(self): + for obsolete_field in ('repo', 'ref'): + m = morphlib.morph3.Morphology( + name="foo", + kind="system", + arch="testarch", + strata=[ + { + 'morph': 'bar', + obsolete_field: 'obsolete', + }]) + + with self.catch_warnings(MorphologyObsoleteFieldWarning) \ + as caught_warnings: + + self.loader.validate(m) + self.assertEqual(len(caught_warnings), 1) + warning = caught_warnings[0].message + self.assertEqual(warning.kind, 'system') + self.assertEqual(warning.morphology_name, 'foo') + self.assertEqual(warning.stratum_name, 'bar') + self.assertEqual(warning.field, obsolete_field) + + def test_warns_when_strata_refer_to_build_depends_with_repo_or_ref(self): + for obsolete_field in ('repo', 'ref'): + m = morphlib.morph3.Morphology( + { + 'name': 'foo', + 'kind': 'stratum', + 'build-depends': [ + { + 'morph': 'bar', + obsolete_field: 'obsolete' + }, + ], + 'chunks': [ + { + 'morph': 'chunk', + 'name': 'chunk', + 'build-mode': 'test', + 'build-depends': [], + }, + ], + }) + + with self.catch_warnings(MorphologyObsoleteFieldWarning) \ + as caught_warnings: + + self.loader.validate(m) + self.assertEqual(len(caught_warnings), 1) + warning = caught_warnings[0].message + self.assertEqual(warning.kind, 'stratum') + self.assertEqual(warning.morphology_name, 'foo') + self.assertEqual(warning.stratum_name, 'bar') + self.assertEqual(warning.field, obsolete_field) diff --git a/morphlib/morphologyfactory.py b/morphlib/morphologyfactory.py index 3462dd36..8a0b047a 100644 --- a/morphlib/morphologyfactory.py +++ b/morphlib/morphologyfactory.py @@ -145,6 +145,11 @@ class MorphologyFactory(object): morphology.needs_artifact_metadata_cached = False + morphlib.morphloader.MorphologyLoader._validate_stratum_specs_fields( + morphology, 'strata') + morphlib.morphloader.MorphologyLoader._set_stratum_specs_defaults( + morphology, 'strata') + def _check_and_tweak_stratum(self, morphology, reponame, sha1, filename): '''Check and tweak a stratum morphology.''' @@ -164,6 +169,11 @@ class MorphologyFactory(object): morphology.builds_artifacts = [morphology['name']] morphology.needs_artifact_metadata_cached = True + morphlib.morphloader.MorphologyLoader._validate_stratum_specs_fields( + morphology, 'build-depends') + morphlib.morphloader.MorphologyLoader._set_stratum_specs_defaults( + morphology, 'build-depends') + def _check_and_tweak_chunk(self, morphology, reponame, sha1, filename): '''Check and tweak a chunk morphology.''' diff --git a/morphlib/morphologyfactory_tests.py b/morphlib/morphologyfactory_tests.py index 30504e00..39092857 100644 --- a/morphlib/morphologyfactory_tests.py +++ b/morphlib/morphologyfactory_tests.py @@ -103,8 +103,6 @@ class FakeLocalRepo(object): "kind": "stratum", "build-depends": [ { - "repo": "test:repo", - "ref": "sha1", "morph": "stratum" } ], diff --git a/morphlib/morphset.py b/morphlib/morphset.py index 6aabbde5..dedbabd5 100644 --- a/morphlib/morphset.py +++ b/morphlib/morphset.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -81,7 +81,7 @@ class MorphologySet(object): for spec in specs: name = spec.get('morph', spec.get('name')) if name == wanted_name: - return spec['repo'], spec['ref'], name + return spec.get('repo'), spec.get('ref'), name return None, None, None def get_stratum_in_system(self, system_morph, stratum_name): @@ -160,7 +160,8 @@ class MorphologySet(object): specs = m[kind] for spec in specs: if cb_filter(m, kind, spec): - orig_spec = (spec['repo'], spec['ref'], spec['morph']) + orig_spec = (spec.get('repo'), spec.get('ref'), + spec['morph']) dirtied = cb_process(m, kind, spec) if dirtied: m.dirty = True @@ -177,11 +178,11 @@ class MorphologySet(object): tup = (m.repo_url, m.ref, m.filename[:-len('.morph')]) if tup in altered_references: spec = altered_references[tup] - if m.ref != spec['ref']: - m.ref = spec['ref'] + if m.ref != spec.get('ref'): + m.ref = spec.get('ref') m.dirty = True assert (m.filename == spec['morph'] + '.morph' - or m.repo_url == spec['repo']), \ + or m.repo_url == spec.get('repo')), \ 'Moving morphologies is not supported.' def change_ref(self, repo_url, orig_ref, morph_filename, new_ref): @@ -193,12 +194,12 @@ class MorphologySet(object): ''' def wanted_spec(m, kind, spec): - return (spec['repo'] == repo_url and - spec['ref'] == orig_ref and + return (spec.get('repo') == repo_url and + spec.get('ref') == orig_ref and spec['morph'] + '.morph' == morph_filename) def process_spec(m, kind, spec): - spec['unpetrify-ref'] = spec['ref'] + spec['unpetrify-ref'] = spec.get('ref') spec['ref'] = new_ref return True @@ -214,10 +215,10 @@ class MorphologySet(object): known = set() def wanted_spec(m, kind, spec): - return (spec['repo'], spec['ref']) not in known + return (spec.get('repo'), spec.get('ref')) not in known def process_spec(m, kind, spec): - known.add((spec['repo'], spec['ref'])) + known.add((spec.get('repo'), spec.get('ref'))) return False self.traverse_specs(process_spec, wanted_spec) @@ -234,11 +235,11 @@ class MorphologySet(object): ''' def wanted_spec(m, kind, spec): - return spec['repo'] == repo_url + return spec.get('repo') == repo_url def process_spec(m, kind, spec): if 'unpetrify-ref' not in spec: - spec['unpetrify-ref'] = spec['ref'] + spec['unpetrify-ref'] = spec.get('ref') spec['ref'] = new_ref return True @@ -262,13 +263,13 @@ class MorphologySet(object): # the details are tricky. if not (m['kind'] == 'stratum' and kind == 'chunks'): return - ref = spec['ref'] + ref = spec.get('ref') return (not morphlib.git.is_valid_sha1(ref) - and (spec['repo'], ref) in resolutions) + and (spec.get('repo'), ref) in resolutions) def process_chunk_spec(m, kind, spec): - tup = (spec['repo'], spec['ref']) - spec['unpetrify-ref'] = spec['ref'] + tup = (spec.get('repo'), spec.get('ref')) + spec['unpetrify-ref'] = spec.get('ref') spec['ref'] = resolutions[tup] return True @@ -281,7 +282,7 @@ class MorphologySet(object): def wanted_spec(m, kind, spec): return ('unpetrify-ref' in spec and - morphlib.git.is_valid_sha1(spec['ref'])) + morphlib.git.is_valid_sha1(spec.get('ref'))) def process_spec(m, kind, spec): spec['ref'] = spec.pop('unpetrify-ref') return True diff --git a/morphlib/plugins/branch_and_merge_new_plugin.py b/morphlib/plugins/branch_and_merge_new_plugin.py index f3b47468..94b2381c 100644 --- a/morphlib/plugins/branch_and_merge_new_plugin.py +++ b/morphlib/plugins/branch_and_merge_new_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012,2013 Codethink Limited +# Copyright (C) 2012,2013,2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -284,8 +284,8 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): # of triplets (repo url, ref, filename). return [ - (spec['repo'] or morph.repo_url, - spec['ref'] or morph.ref, + (spec.get('repo') or morph.repo_url, + spec.get('ref') or morph.ref, '%s.morph' % spec['morph']) for spec in specs ] diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py index 4666ea96..d268decf 100644 --- a/morphlib/plugins/branch_and_merge_plugin.py +++ b/morphlib/plugins/branch_and_merge_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012,2013 Codethink Limited +# Copyright (C) 2012,2013,2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -610,10 +610,10 @@ class BranchAndMergePlugin(cliapp.Plugin): root_repo_dir): '''Check out the morphology that 'spec' refers to, for editing''' - if spec['repo'] == root_repo: + if spec.get('repo') in (None, root_repo): # This is only possible for stratum morphologies repo_dir = root_repo_dir - if spec['ref'] != branch: + if spec.get('ref') not in (None, root_repo): # Bring the morphology forward from its ref to the current HEAD repo = self.lrc.get_repo(root_repo) m = repo.load_morphology(spec['ref'], spec['morph']) @@ -1138,11 +1138,8 @@ class BranchAndMergePlugin(cliapp.Plugin): if 'strata' in morphology and morphology['strata']: strata += morphology['strata'] for info in strata: - # Obtain the commit SHA1 this stratum would be built from. - commit = self.resolve_info(info, resolved_refs) stratum_repo_dir = self.make_available( info, branch, branch_dir, repo, repo_dir) - info['ref'] = branch # Load the stratum morphology and petrify it recursively if # that hasn't happened yet. @@ -1150,24 +1147,13 @@ class BranchAndMergePlugin(cliapp.Plugin): if not stratum in petrified_morphologies: self.petrify_morphology(branch, branch_dir, branch_root, branch_root_dir, - info['repo'], stratum_repo_dir, - tagref, info['morph'], stratum, + info.get('repo') or branch_root, + stratum_repo_dir, tagref, + info['morph'], stratum, petrified_morphologies, resolved_refs, env, update_working_tree) - # Change the ref for this morphology to the tag we're creating. - if info['ref'] != tagref: - info['unpetrify-ref'] = info['ref'] - info['ref'] = tagref - - # We'll be copying all systems/strata into the tag commit - # in the branch root repo, so make sure to note what repos - # they all came from - if info['repo'] != branch_root: - info['unpetrify-repo'] = info['repo'] - info['repo'] = branch_root - # If this morphology is a stratum, resolve the refs of all its # chunks into SHA1s. if morphology['kind'] == 'stratum': @@ -1202,7 +1188,7 @@ class BranchAndMergePlugin(cliapp.Plugin): def resolve_info(self, info, resolved_refs): '''Takes a morphology info and resolves its ref with cache support.''' - key = (info['repo'], info['ref']) + key = (info.get('repo'), info.get('ref')) if not key in resolved_refs: commit_sha1, tree_sha1 = self.app.resolve_ref( self.lrc, self.rrc, info['repo'], info['ref'], @@ -1292,10 +1278,10 @@ class BranchAndMergePlugin(cliapp.Plugin): def component_key(info): # This function needs only to be stable and reproducible - key = info['repo'] + '|' + info['morph'] if 'name' in info: - key += '|' + info['name'] - return key + return (info.get('repo'), info['morph'], info['name']) + else: + return (info.get('repo'), info['morph']) if from_morph['name'] != to_morph['name']: # We should enforce name == filename in load_morphology() @@ -1471,12 +1457,12 @@ class BranchAndMergePlugin(cliapp.Plugin): changed = False edited_strata = [si for si in from_morph['strata'] - if si['ref'] == from_branch] + if si.get('ref') == from_branch] for si in edited_strata: for old_si in to_morph['strata']: # We make no attempt at rename / move detection if (old_si['morph'] == si['morph'] - and old_si['repo'] == si['repo']): + and old_si.get('repo') == si.get('repo')): break else: raise cliapp.AppException( @@ -1484,7 +1470,7 @@ class BranchAndMergePlugin(cliapp.Plugin): 'subsequently edited. This is not yet supported: ' 'refusing to merge.' % si['morph']) changed = True - si['ref'] = old_si['ref'] + si['ref'] = old_si.get('ref') merge_stratum(name, old_si, si) if changed: self.update_morphology(to_root_dir, name, to_morph) |