From 8c8ea75f681fa86d4c29a07419aae3bbcb3b8582 Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Fri, 8 Aug 2014 07:51:04 +0000 Subject: Remove morph2 and its tests This commit removes the now unneeded morph2 and its associated tests. --- morphlib/__init__.py | 1 - morphlib/morph2.py | 313 ------------------------------------- morphlib/morph2_tests.py | 391 ----------------------------------------------- 3 files changed, 705 deletions(-) delete mode 100644 morphlib/morph2.py delete mode 100644 morphlib/morph2_tests.py (limited to 'morphlib') diff --git a/morphlib/__init__.py b/morphlib/__init__.py index 0c928fd3..a31cf04c 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -68,7 +68,6 @@ import gitindex import localartifactcache import localrepocache import mountableimage -import morph2 import morphologyfactory import morphologyfinder import morph3 diff --git a/morphlib/morph2.py b/morphlib/morph2.py deleted file mode 100644 index b49c0f73..00000000 --- a/morphlib/morph2.py +++ /dev/null @@ -1,313 +0,0 @@ -# Copyright (C) 2012-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import re - -import morphlib -from morphlib.util import OrderedDict, json - -class Morphology(object): - - '''An in-memory representation of a morphology. - - This is a parsed version of the morphology, with rules for default - values applied. No other processing. - - ''' - - static_defaults = { - 'chunk': [ - ('description', ''), - ('pre-configure-commands', None), - ('configure-commands', None), - ('post-configure-commands', None), - ('pre-build-commands', None), - ('build-commands', None), - ('post-build-commands', None), - ('pre-test-commands', None), - ('test-commands', None), - ('post-test-commands', None), - ('pre-install-commands', None), - ('install-commands', None), - ('post-install-commands', None), - ('devices', None), - ('products', []), - ('max-jobs', None), - ('build-system', 'manual') - ], - 'stratum': [ - ('chunks', []), - ('description', ''), - ('build-depends', None), - ], - 'system': [ - ('strata', []), - ('description', ''), - ('arch', None), - ('configuration-extensions', []), - ], - 'cluster': [ - ('description', ''), - ], - } - - @staticmethod - def _load_json(text): - return json.loads(text, object_pairs_hook=OrderedDict, - encoding='unicode-escape') - - @staticmethod - def _dump_json(obj, f): - text = json.dumps(obj, indent=4, encoding='unicode-escape') - text = re.sub(" \n", "\n", text) - f.write(text) - f.write('\n') - - def __init__(self, text): - self._dict, self._dumper = self._load_morphology_dict(text) - self._set_defaults() - self._validate_children() - - def __getitem__(self, key): - return self._dict[key] - - def __contains__(self, key): - return key in self._dict - - # Not covered by tests, since it's trivial, morph2 is going away - # and only exists so the new morphology validation code can use it. - def get(self, key, default=None): # pragma: no cover - try: - return self[key] - except KeyError: - return default - - def get_commands(self, which): - '''Return the commands to run from a morphology or the build system''' - if self[which] is None: - attr = '_'.join(which.split('-')) - bs = morphlib.buildsystem.lookup_build_system(self['build-system']) - return getattr(bs, attr) - else: - return self[which] - - def keys(self): - return self._dict.keys() - - def _load_morphology_dict(self, text): - '''Load morphology, identifying whether it is JSON or YAML''' - - try: - data = self._load_json(text) - dumper = self._dump_json - except ValueError as e: # pragma: no cover - data = morphlib.yamlparse.load(text) - dumper = morphlib.yamlparse.dump - - if data is None: - raise morphlib.YAMLError("Morphology is empty") - if type(data) not in [dict, OrderedDict]: - raise morphlib.YAMLError("Morphology did not parse as a dict") - - return data, dumper - - def _validate_children(self): - if self['kind'] == 'system': - names = set() - for info in self['strata']: - name = info.get('alias', info['morph']) - if name in names: - raise ValueError('Duplicate stratum "%s"' % name) - names.add(name) - elif self['kind'] == 'stratum': - names = set() - for info in self['chunks']: - name = info.get('alias', info['name']) - if name in names: - raise ValueError('Duplicate chunk "%s"' % name) - names.add(name) - elif self['kind'] == 'cluster': - if not 'systems' in self: - raise KeyError('"systems" not found') - if not self['systems']: - raise ValueError('"systems" is empty') - for system in self['systems']: - if 'morph' not in system: - raise KeyError('"morph" not found') - if 'deploy-defaults' in system: - if not isinstance(system['deploy-defaults'], dict): - raise ValueError('deploy defaults for morph "%s" ' - 'are not a mapping: %r' - % (system['morph'], - system['deploy-defaults'])) - if 'deploy' in system: - for system_id, deploy_params in system['deploy'].items(): - if not isinstance(deploy_params, dict): - raise ValueError('deployment parameters for ' - 'system "%s" are not a mapping:' - ' %r' - % (system_id, deploy_params)) - - def _set_default_value(self, target_dict, key, value): - '''Change a value in the in-memory representation of the morphology - - Record the default value separately, so that when writing out the - morphology we can determine whether the change from the on-disk value - was done at load time, or later on (we want to only write back out - the later, deliberate changes). - - ''' - target_dict[key] = value - target_dict['_orig_' + key] = value - - def _set_defaults(self): - if 'max-jobs' in self: - self._set_default_value(self._dict, 'max-jobs', - int(self['max-jobs'])) - - for name, value in self.static_defaults[self['kind']]: - if name not in self._dict: - self._set_default_value(self._dict, name, value) - - if self['kind'] == 'stratum': - self._set_stratum_defaults() - elif self['kind'] == 'cluster': - self._set_cluster_defaults() - - def _set_stratum_defaults(self): - for source in self['chunks']: - if 'repo' not in source: - self._set_default_value(source, 'repo', source['name']) - if 'build-depends' not in source: - self._set_default_value(source, 'build-depends', None) - if 'build-mode' not in source: - self._set_default_value(source, 'build-mode', 'staging') - if 'prefix' not in source: - self._set_default_value(source, 'prefix', '/usr') - - def _set_cluster_defaults(self): - if 'systems' in self and self['systems']: - for system in self['systems']: - if 'deploy-defaults' not in system: - self._set_default_value(system, - 'deploy-defaults', - dict()) - if 'deploy' not in system: - self._set_default_value(system, - 'deploy', - dict()) - - def lookup_child_by_name(self, name): - '''Find child reference by its name. - - This lookup honors aliases. - - ''' - - if self['kind'] == 'system': - for info in self['strata']: - source_name = info.get('alias', info['morph']) - if source_name == name: - return info - elif self['kind'] == 'stratum': - for info in self['chunks']: - source_name = info.get('alias', info['name']) - if source_name == name: - return info - raise KeyError('"%s" not found' % name) - - def _apply_changes(self, live_dict, original_dict): - '''Returns a new dict updated with changes from the in-memory object - - This allows us to write out a morphology including only the changes - that were done after the morphology was loaded -- not the changes done - to set default values during construction. - - ''' - output_dict = {} - - for key in live_dict.keys(): - if key.startswith('_orig_'): - continue - - value = self._apply_changes_for_key(key, live_dict, original_dict) - # VILE HACK to preserve nulls in repo/ref fields - if value is not None or key in ('repo', 'ref'): - output_dict[key] = value - return output_dict - - def _apply_changes_for_key(self, key, live_dict, original_dict): - '''Return value to write out for one key, recursing if necessary''' - - live_value = live_dict.get(key, None) - orig_value = original_dict.get(key, None) - - if type(live_value) in [dict, OrderedDict] and orig_value is not None: - # Recursively apply changes for dict - result = self._apply_changes(live_value, orig_value) - elif type(live_value) is list and orig_value is not None: - # Recursively apply changes for list (existing, then new items). - result = [] - for i in range(0, min(len(orig_value), len(live_value))): - if type(live_value[i]) in [dict, OrderedDict]: - item = self._apply_changes(live_value[i], orig_value[i]) - else: - item = live_value[i] - result.append(item) - for i in range(len(orig_value), len(live_value)): - if type(live_value[i]) in [dict, OrderedDict]: - item = self._apply_changes(live_value[i], {}) - else: - item = live_value[i] - result.append(item) - else: - # Simple values. Use original value unless it has been changed from - # the default in memmory. - if live_dict[key] == live_dict.get('_orig_' + key, None): - result = original_dict.get(key, None) - else: - result = live_dict[key] - return result - - def update_text(self, text, output_fd, convert_to=None): - '''Write out in-memory changes to loaded morphology text - - Similar in function to update_file(). - - ''' - original_dict, dumper = self._load_morphology_dict(text) - - if convert_to == 'json': # pragma: no cover - dumper = self._dump_json - elif convert_to == 'yaml': # pragma: no cover - dumper = morphlib.yamlparse.dump - - output_dict = self._apply_changes(self._dict, original_dict) - dumper(output_dict, output_fd) - - def update_file(self, filename, output_fd=None, **kws): # pragma: no cover - '''Write out in-memory changes to on-disk morphology file - - This function reads the original morphology text from 'filename', so - that it can avoid writing out properties that are set in memory - to their default value but weren't specified by the user at all. - - ''' - with open(filename, 'r') as f: - text = f.read() - - with output_fd or morphlib.savefile.SaveFile(filename, 'w') as f: - self.update_text(text, f, **kws) diff --git a/morphlib/morph2_tests.py b/morphlib/morph2_tests.py deleted file mode 100644 index c9957ad5..00000000 --- a/morphlib/morph2_tests.py +++ /dev/null @@ -1,391 +0,0 @@ -# Copyright (C) 2012-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import copy -import json -import StringIO -import unittest - -import yaml - -import morphlib -from morphlib.morph2 import Morphology - - -class MorphologyTests(unittest.TestCase): - - def test_parses_simple_json_chunk(self): - m = Morphology(''' - { - "name": "foo", - "kind": "chunk", - "build-system": "manual" - } - ''') - - self.assertEqual(m['name'], 'foo') - self.assertEqual(m['kind'], 'chunk') - self.assertEqual(m['build-system'], 'manual') - self.assertEqual(m['pre-configure-commands'], None) - self.assertEqual(m['configure-commands'], None) - self.assertEqual(m['post-configure-commands'], None) - self.assertEqual(m['pre-build-commands'], None) - self.assertEqual(m['build-commands'], None) - self.assertEqual(m['post-build-commands'], None) - self.assertEqual(m['pre-test-commands'], None) - self.assertEqual(m['test-commands'], None) - self.assertEqual(m['post-test-commands'], None) - self.assertEqual(m['pre-install-commands'], None) - self.assertEqual(m['install-commands'], None) - self.assertEqual(m['post-install-commands'], None) - self.assertEqual(m['max-jobs'], None) - self.assertEqual(m['products'], []) - - if morphlib.got_yaml: - def test_parses_simple_yaml_chunk(self): - m = Morphology(''' - name: foo - kind: chunk - build-system: manual - ''') - - self.assertEqual(m['name'], 'foo') - self.assertEqual(m['kind'], 'chunk') - self.assertEqual(m['build-system'], 'manual') - self.assertEqual(m['pre-configure-commands'], None) - self.assertEqual(m['configure-commands'], None) - self.assertEqual(m['post-configure-commands'], None) - self.assertEqual(m['pre-build-commands'], None) - self.assertEqual(m['build-commands'], None) - self.assertEqual(m['post-build-commands'], None) - self.assertEqual(m['pre-test-commands'], None) - self.assertEqual(m['test-commands'], None) - self.assertEqual(m['post-test-commands'], None) - self.assertEqual(m['pre-install-commands'], None) - self.assertEqual(m['install-commands'], None) - self.assertEqual(m['post-install-commands'], None) - self.assertEqual(m['max-jobs'], None) - self.assertEqual(m['products'], []) - - def test_sets_stratum_chunks_repo_and_morph_from_name(self): - m = Morphology(''' - { - "name": "foo", - "kind": "stratum", - "chunks": [ - { - "name": "le-chunk", - "ref": "ref" - } - ] - } - ''') - - self.assertEqual(m['chunks'][0]['repo'], 'le-chunk') - self.assertEqual(m['chunks'][0]['build-depends'], None) - - def test_returns_dict_keys(self): - m = Morphology(''' - { - "name": "foo", - "kind": "system", - } - ''') - - self.assertTrue('name' in m.keys()) - self.assertTrue('kind' in m.keys()) - - def test_system_indexes_strata(self): - m = Morphology(''' - { - "kind": "system", - "strata": [ - { - "morph": "stratum1", - "repo": "repo", - "ref": "ref" - }, - { - "alias": "aliased-stratum", - "morph": "stratum2", - "repo": "repo", - "ref": "ref" - } - ] - } - ''') - self.assertEqual(m.lookup_child_by_name('stratum1'), - {'morph': 'stratum1', 'repo': 'repo', 'ref': 'ref' }) - self.assertEqual(m.lookup_child_by_name('aliased-stratum'), - {'alias': 'aliased-stratum', 'morph': 'stratum2', - 'repo': 'repo', 'ref': 'ref'}) - - def test_stratum_indexes_chunks(self): - m = Morphology(''' - { - "kind": "stratum", - "chunks": [ - { - "name": "chunk", - "repo": "repo", - "ref": "ref" - } - ] - } - ''') - - child = m.lookup_child_by_name('chunk') - self.assertEqual(child['name'], 'chunk') - self.assertEqual(child['repo'], 'repo') - self.assertEqual(child['ref'], 'ref') - - def test_raises_error_when_child_lookup_fails(self): - m = Morphology(''' - { - "kind": "stratum", - "chunks": [ - { - "name": "chunk", - "repo": "repo", - "ref": "ref" - } - ] - } - ''') - - self.assertRaises(KeyError, m.lookup_child_by_name, 'foo') - - ## Validation tests - - def test_not_empty(self): - self.assertRaises(morphlib.YAMLError, Morphology, '') - - def test_is_dict(self): - self.assertRaises(morphlib.YAMLError, Morphology, 'foo') - - def test_makes_max_jobs_be_an_integer(self): - m = Morphology(''' - { - "name": "foo", - "kind": "chunk", - "max-jobs": "42" - } - ''') - self.assertEqual(m['max-jobs'], 42) - - def test_stratum_names_must_be_unique_within_a_system(self): - text = ''' - { - "kind": "system", - "strata": [ - { - "morph": "stratum", - "repo": "test1", - "ref": "ref" - }, - { - "morph": "stratum", - "repo": "test2", - "ref": "ref" - } - ] - } - ''' - self.assertRaises(ValueError, - Morphology, - text) - - def test_chunk_names_must_be_unique_within_a_stratum(self): - text = ''' - { - "kind": "stratum", - "chunks": [ - { - "name": "chunk", - "repo": "test1", - "ref": "ref" - }, - { - "name": "chunk", - "repo": "test2", - "ref": "ref" - } - ] - } - ''' - self.assertRaises(ValueError, - Morphology, - text) - - ## Writing tests - - stratum_text = '''{ - "kind": "stratum", - "chunks": [ - { - "name": "foo", - "repo": "morphs", - "ref": "ref", - "build-depends": [] - }, - { - "name": "bar", - "repo": "morphs", - "ref": "ref", - "build-depends": [ - "foo" - ] - } - ] -}''' - - def test_writing_handles_added_chunks(self): - text_lines = self.stratum_text.splitlines() - text_lines = text_lines[0:16] + text_lines[8:17] + text_lines[17:] - text_lines[18] = ' "name": "baz",' - - # Add a new chunk to the list - morphology = Morphology(self.stratum_text) - morphology['chunks'].append(copy.copy(morphology['chunks'][1])) - morphology['chunks'][2]['name'] = 'baz' - - output = StringIO.StringIO() - morphology.update_text(self.stratum_text, output) - d = yaml.load(output.getvalue()) - self.assertEqual(d['chunks'][2]['name'], 'baz') - - def test_writing_handles_deleted_chunks(self): - text_lines = self.stratum_text.splitlines() - text_lines = text_lines[0:3] + text_lines[9:] - - # Delete a chunk - morphology = Morphology(self.stratum_text) - del morphology['chunks'][0] - - output = StringIO.StringIO() - morphology.update_text(self.stratum_text, output) - d = yaml.load(output.getvalue()) - self.assertEqual(len(d['chunks']), 1) - - system_text = '''{ - "kind": "system", - "arch": "x86_64", -}''' - - def test_nested_dict(self): - # Real morphologies don't trigger this code path, so we test manually - original_dict = { - 'dict': { '1': 'fee', '2': 'fie', '3': 'foe', '4': 'foo' } - } - live_dict = copy.deepcopy(original_dict) - live_dict['_orig_dict'] = live_dict['dict'] - - dummy = Morphology(self.stratum_text) - output_dict = dummy._apply_changes(live_dict, original_dict) - self.assertEqual(original_dict, output_dict) - - def test_uses_morphology_commands_when_given(self): - m = Morphology(''' - { - 'name': 'foo', - 'kind': 'chunk', - 'build-system': 'dummy', - 'build-commands': ['build-it'] - } - ''') - cmds = m.get_commands('build-commands') - self.assertEqual(cmds, ['build-it']) - - def test_uses_build_system_commands_when_morphology_doesnt(self): - m = Morphology(''' - { - 'name': 'foo', - 'kind': 'chunk', - 'build-system': 'dummy', - } - ''') - cmds = m.get_commands('build-commands') - self.assertEqual(cmds, ['echo dummy build']) - - def test_uses_morphology_commands_when_morphology_has_empty_list(self): - m = Morphology(''' - { - 'name': 'foo', - 'kind': 'chunk', - 'build-system': 'dummy', - 'build-commands': [] - } - ''') - cmds = m.get_commands('build-commands') - self.assertEqual(cmds, []) - - ## Cluster morphologies tests - - def test_parses_simple_cluster_morph(self): - m = Morphology(''' - name: foo - kind: cluster - systems: - - morph: bar - ''') - self.assertEqual(m['name'], 'foo') - self.assertEqual(m['kind'], 'cluster') - self.assertEqual(m['systems'][0]['morph'], 'bar') - - def test_fails_without_systems(self): - text = ''' - name: foo - kind: cluster - ''' - self.assertRaises(KeyError, Morphology, text) - - def test_fails_with_empty_systems(self): - text = ''' - name: foo - kind: cluster - systems: - ''' - self.assertRaises(ValueError, Morphology, text) - - def test_fails_without_morph(self): - text = ''' - name: foo - kind: cluster - systems: - - deploy: - ''' - self.assertRaises(KeyError, Morphology, text) - - def test_fails_with_invalid_deploy_defaults(self): - text = ''' - name: foo - kind: cluster - systems: - - morph: bar - deploy-defaults: ooops_i_am_not_a_mapping - ''' - self.assertRaises(ValueError, Morphology, text) - - def test_fails_with_invalid_deployment_params(self): - text = ''' - name: foo - kind: cluster - systems: - - morph: bar - deploy: - qux: ooops_i_am_not_a_mapping - ''' - self.assertRaises(ValueError, Morphology, text) -- cgit v1.2.1