Merge branch 'jjardon/yaml-v2'

10 files changed, 247 insertions, 26 deletions

diff --git a/morphlib/__init__.py b/morphlib/__init__.py
index 2d2f68a2..aba7a4d6 100644
--- a/morphlib/__init__.py
+++ b/morphlib/__init__.py
@@ -57,5 +57,6 @@ import stagingarea
 import stopwatch
 import tempdir
 import util
+import yamlparse
 
 import app  # this needs to be last
diff --git a/morphlib/morph2.py b/morphlib/morph2.py
index 4fdf7ba4..edf7bb31 100644
--- a/morphlib/morph2.py
+++ b/morphlib/morph2.py
@@ -17,16 +17,8 @@
 import copy
 import re
 
-# It is intentional that if collections does not have OrderedDict that
-# simplejson is also used in preference to json, as OrderedDict became
-# a member of collections in the same release json got its object_pairs_hook
-try: # pragma: no cover
-    from collections import OrderedDict
-    import json
-except ImportError: # pragma: no cover
-    from ordereddict import OrderedDict
-    import simplejson as json
-
+import morphlib
+from morphlib.util import OrderedDict, json
 
 class Morphology(object):
 
@@ -61,8 +53,26 @@ class Morphology(object):
         ]
     }
 
+    @staticmethod
+    def _load_json(text):
+        return json.loads(text, object_pairs_hook=OrderedDict)
+
+    @staticmethod
+    def _dump_json(obj, f):
+        text = json.dumps(obj, indent=4)
+        text = re.sub(" \n", "\n", text)
+        f.write(text)
+        f.write('\n')
+
     def __init__(self, text):
-        self._dict = json.loads(text, object_pairs_hook=OrderedDict)
+        # Load as JSON first, then try YAML, so morphologies
+        # that read as JSON are dumped as JSON, likewise with YAML.
+        try:
+            self._dict = self._load_json(text)
+            self._dumper = self._dump_json
+        except Exception, e:
+            self._dict = morphlib.yamlparse.load(text)
+            self._dumper = morphlib.yamlparse.dump
         self._set_defaults()
         self._validate_children()
 
@@ -165,7 +175,4 @@ class Morphology(object):
                 value = self[key]
             if value and key[0] != '_':
                 as_dict[key] = value
-        text = json.dumps(as_dict, indent=4)
-        text = re.sub(" \n", "\n", text)
-        f.write(text)
-        f.write('\n')
+        self._dumper(as_dict, f)
diff --git a/morphlib/morph2_tests.py b/morphlib/morph2_tests.py
index 756873a0..34df4657 100644
--- a/morphlib/morph2_tests.py
+++ b/morphlib/morph2_tests.py
@@ -22,7 +22,7 @@ from morphlib.morph2 import Morphology
 
 class MorphologyTests(unittest.TestCase):
 
-    def test_parses_simple_chunk(self):
+    def test_parses_simple_json_chunk(self):
         m = Morphology('''
             {
                 "name": "foo",
@@ -41,6 +41,23 @@ class MorphologyTests(unittest.TestCase):
         self.assertEqual(m['max-jobs'], None)
         self.assertEqual(m['chunks'], [])
 
+    def test_parses_simple_yaml_chunk(self):
+        m = Morphology('''
+            name: foo
+            kind: chunk
+            build-system: manual
+        ''')
+
+        self.assertEqual(m['name'], 'foo')
+        self.assertEqual(m['kind'], 'chunk')
+        self.assertEqual(m['build-system'], 'manual')
+        self.assertEqual(m['configure-commands'], None)
+        self.assertEqual(m['build-commands'], None)
+        self.assertEqual(m['test-commands'], None)
+        self.assertEqual(m['install-commands'], None)
+        self.assertEqual(m['max-jobs'], None)
+        self.assertEqual(m['chunks'], [])
+
     def test_sets_stratum_chunks_repo_and_morph_from_name(self):
         m = Morphology('''
             {
diff --git a/morphlib/morphologyfactory.py b/morphlib/morphologyfactory.py
index a219ed9b..261dc908 100644
--- a/morphlib/morphologyfactory.py
+++ b/morphlib/morphologyfactory.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012  Codethink Limited
+# Copyright (C) 2012-2013  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -13,6 +13,8 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
+import yaml
+
 import morphlib
 import cliapp
 
@@ -81,7 +83,7 @@ class MorphologyFactory(object):
 
         try:
             morphology = morphlib.morph2.Morphology(text)
-        except ValueError as e:
+        except yaml.YAMLError as e:
             raise morphlib.Error("Error parsing %s: %s" %
                                  (filename, str(e)))
 
diff --git a/morphlib/morphologyfactory_tests.py b/morphlib/morphologyfactory_tests.py
index b8c89d2a..56c6fc57 100644
--- a/morphlib/morphologyfactory_tests.py
+++ b/morphlib/morphologyfactory_tests.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012  Codethink Limited
+# Copyright (C) 2012-2013  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -65,9 +65,7 @@ class FakeLocalRepo(object):
                 "system-kind": "%(system_kind)s",
                 "arch": "%(arch)s"
             }''',
-        'parse-error.morph': '''{
-                "name"
-            }''',
+        'parse-error.morph': '''{ "name"''',
     }
 
     def __init__(self):
diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py
index 60e8d4ef..14dc9782 100644
--- a/morphlib/plugins/branch_and_merge_plugin.py
+++ b/morphlib/plugins/branch_and_merge_plugin.py
@@ -17,7 +17,6 @@
 import cliapp
 import copy
 import glob
-import json
 import logging
 import os
 import shutil
diff --git a/morphlib/plugins/tarball-systembuilder_plugin.py b/morphlib/plugins/tarball-systembuilder_plugin.py
index 14807924..02622067 100644
--- a/morphlib/plugins/tarball-systembuilder_plugin.py
+++ b/morphlib/plugins/tarball-systembuilder_plugin.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012  Codethink Limited
+# Copyright (C) 2012,2013  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -14,7 +14,6 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 
-import json
 import logging
 import os
 from os.path import relpath
diff --git a/morphlib/util.py b/morphlib/util.py
index 100f4b6b..e171714a 100644
--- a/morphlib/util.py
+++ b/morphlib/util.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2011-2012  Codethink Limited
+# Copyright (C) 2011-2013  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -20,6 +20,16 @@ import morphlib
 '''Utility functions for morph.'''
 
 
+# It is intentional that if collections does not have OrderedDict that
+# simplejson is also used in preference to json, as OrderedDict became
+# a member of collections in the same release json got its object_pairs_hook
+try: # pragma: no cover
+    from collections import OrderedDict
+    import json
+except ImportError: # pragma: no cover
+    from ordereddict import OrderedDict
+    import simplejson as json
+
 try:
     from multiprocessing import cpu_count
 except NotImplementedError:  # pragma: no cover
diff --git a/morphlib/yamlparse.py b/morphlib/yamlparse.py
new file mode 100644
index 00000000..7f8b00e5
--- /dev/null
+++ b/morphlib/yamlparse.py
@@ -0,0 +1,119 @@
+# Copyright (C) 2013  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import yaml
+import yaml.constructor
+
+from morphlib.util import OrderedDict
+
+class OrderedDictYAMLLoader(yaml.Loader):
+    """A YAML loader that loads mappings into ordered dictionaries.
+
+       When YAML is loaded with this Loader, it loads mappings as ordered
+       dictionaries, so the order the keys were written in is maintained.
+
+       When combined with the OrderedDictYAMLDumper, this allows yaml documents
+       to be written out in a similar format to they were read.
+
+    """
+
+    def __init__(self, *args, **kwargs):
+        yaml.Loader.__init__(self, *args, **kwargs)
+
+        # When YAML encounters a mapping (which YAML identifies with
+        # the given tag), it will use construct_yaml_map to read it as
+        # an OrderedDict.
+        self.add_constructor(u'tag:yaml.org,2002:map',
+                             type(self).construct_yaml_map)
+
+    def construct_yaml_map(self, node):
+        data = OrderedDict()
+        yield data
+        value = self.construct_mapping(node)
+        data.update(value)
+
+    def construct_mapping(self, node, deep=False):
+        if isinstance(node, yaml.MappingNode):
+            self.flatten_mapping(node)
+        else:
+            raise yaml.constructor.ConstructorError(
+                None, None,
+                'expected a mapping node, but found %s' % node.id,
+                node.start_mark)
+
+        mapping = OrderedDict()
+        for key_node, value_node in node.value:
+            key = self.construct_object(key_node, deep=deep)
+            try:
+                hash(key)
+            except TypeError, exc:
+                raise yaml.constructor.ConstructorError(
+                    'while constructing a mapping', node.start_mark,
+                    'found unacceptable key (%s)' % exc, key_node.start_mark)
+            value = self.construct_object(value_node, deep=deep)
+            mapping[key] = value
+        return mapping
+
+class OrderedDictYAMLDumper(yaml.Dumper):
+    """A YAML dumper that will dump OrderedDicts as mappings.
+
+       When YAML is dumped with this Dumper, it dumps OrderedDicts as
+       mappings, preserving the key order, so the order the keys were
+       written in is maintained.
+
+       When combined with the OrderedDictYAMLDumper, this allows yaml documents
+       to be written out in a similar format to they were read.
+
+    """
+
+    def __init__(self, *args, **kwargs):
+        yaml.Dumper.__init__(self, *args, **kwargs)
+
+        # When YAML sees an OrderedDict, use represent_ordered_dict to dump it
+        self.add_representer(OrderedDict,
+                             type(self).represent_ordered_dict)
+
+    def represent_ordered_dict(self, odict):
+        return self.represent_ordered_mapping(u'tag:yaml.org,2002:map', odict)
+
+    def represent_ordered_mapping(self, tag, omap):
+        value = []
+        node = yaml.MappingNode(tag, value)
+        if self.alias_key is not None:
+            self.represented_objects[self.alias_key] = node
+        best_style = True
+        for item_key, item_value in omap.iteritems():
+            node_key = self.represent_data(item_key)
+            node_value = self.represent_data(item_value)
+            if not (isinstance(node_key, yaml.ScalarNode) and
+                    not node_key.style):
+                best_style = False # pragma: no cover
+            if not (isinstance(node_value, yaml.ScalarNode) and
+                    not node_value.style):
+                best_style = False # pragma: no cover
+            value.append((node_key, node_value))
+        if self.default_flow_style is not None:
+            node.flow_style = self.default_flow_style
+        else:
+            node.flow_style = best_style # pragma: no cover
+        return node
+
+def load(*args, **kwargs):
+    return yaml.load(Loader=OrderedDictYAMLLoader, *args, **kwargs)
+
+def dump(*args, **kwargs):
+    if 'default_flow_style' not in kwargs:
+        kwargs['default_flow_style'] = False
+    return yaml.dump(Dumper=OrderedDictYAMLDumper, *args, **kwargs)
diff --git a/morphlib/yamlparse_tests.py b/morphlib/yamlparse_tests.py
new file mode 100644
index 00000000..cb658e15
--- /dev/null
+++ b/morphlib/yamlparse_tests.py
@@ -0,0 +1,69 @@
+# Copyright (C) 2013  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import unittest
+
+try:
+    from collections import OrderedDict
+except ImportError:
+    from ordereddict import OrderedDict
+import yaml
+
+import morphlib.yamlparse as yamlparse
+
+
+class YAMLParseTests(unittest.TestCase):
+
+    example_text = '''\
+name: foo
+kind: chunk
+build-system: manual
+'''
+
+    example_dict = OrderedDict([
+        ('name', 'foo'),
+        ('kind', 'chunk'),
+        ('build-system', 'manual'),
+    ])
+
+    def test_loads_as_ordered_dict(self):
+        m = yamlparse.load(self.example_text)
+        self.assertEqual(type(m), OrderedDict)
+
+    def test_dumps_ordered_dicts(self):
+        self.assertEqual(self.example_text,
+                         yamlparse.dump(self.example_dict))
+
+    def test_non_map_raises(self):
+        incorrect_type = '''\
+!!map
+- foo
+- bar
+'''
+        self.assertRaises(yaml.YAMLError, yamlparse.load, incorrect_type)
+
+    def test_complex_key_fails_KNOWNFAILURE(self):
+        complex_key = '? { foo: bar, baz: qux }: True'
+        self.assertRaises(yaml.YAMLError, yamlparse.load, complex_key)
+
+    def test_represents_non_scalar_nodes(self):
+        self.assertTrue(
+            yamlparse.dump(
+                {
+                    ('a', 'b'): {
+                        "foo": 1,
+                        "bar": 2,
+                    }
+                }, default_flow_style=None))