- bump minor version

- preserve hand-crafted anchor/reference names (non `idNNN`)
- preserve merges http://yaml.org/type/merge.html

12 files changed, 409 insertions, 29 deletions

diff --git a/README.rst b/README.rst
index 0ea4283..82ad3cd 100644
--- a/README.rst
+++ b/README.rst
@@ -30,6 +30,8 @@ Major differences with PyYAML 3.11:
   parent container, like comments).
 - RoundTrip preservation of flow style sequences ( 'a: b, c, d') (based
   on request and test by Anthony Sottile)
+- anchors names that are hand-crafted (not of the form``idNNN``), are preserved
+- `merges <http://yaml.org/type/merge.html>`_ in dictionaries are preserved
 - adding/replacing of comments on block style sequences and mappings
   with smart column positioning
 - collection objects (when read in via RoundTripParser) have an ``lc``
@@ -50,9 +52,9 @@ collections (mappings/sequences resuting in Python dict/list). The basic
 for for this is::
 
   from __future__ import print_function
-
+  
   import ruamel.yaml
-
+  
   inp = """\
   abc:
     - a     # comment 1
@@ -64,14 +66,14 @@ for for this is::
     e: 5
     f: 6 # comment 3
   """
-
+  
   data = ruamel.yaml.load(inp, ruamel.yaml.RoundTripLoader)
   data['abc'].append('b')
   data['abc'].yaml_add_eol_comment('comment 4', 1)  # takes column of comment 1
   data['xyz'].yaml_add_eol_comment('comment 5', 'c')  # takes column of comment 2
   data['xyz'].yaml_add_eol_comment('comment 6', 'e')  # takes column of comment 3
   data['xyz'].yaml_add_eol_comment('comment 7', 'd', column=20)
-
+  
   print(ruamel.yaml.dump(data, Dumper=ruamel.yaml.RoundTripDumper), end='')
 
 .. example code add_comment.py
@@ -172,9 +174,9 @@ Basic round trip of parsing YAML to Python objects, modifying
 and generating YAML::
 
   from __future__ import print_function
-
+  
   import ruamel.yaml
-
+  
   inp = """\
   # example
   name:
@@ -182,10 +184,10 @@ and generating YAML::
     family: Smith   # very common
     given: Alice    # one of the siblings
   """
-
+  
   code = ruamel.yaml.load(inp, ruamel.yaml.RoundTripLoader)
   code['name']['given'] = 'Bob'
-
+  
   print(ruamel.yaml.dump(code, Dumper=ruamel.yaml.RoundTripDumper), end='')
 
 .. example code small.py
@@ -201,6 +203,44 @@ Resulting in ::
 
 .. example output small.py
 
+
+YAML handcrafted anchors and references as well as key merging 
+is preserved. The merged keys can transparently be accessed
+using ``[]`` and ``.get()``::
+
+  import ruamel.yaml
+  
+  inp = """\
+  - &CENTER {x: 1, y: 2}
+  - &LEFT {x: 0, y: 2}
+  - &BIG {r: 10}
+  - &SMALL {r: 1}
+  # All the following maps are equal:
+  # Explicit keys
+  - x: 1
+    y: 2
+    r: 10
+    label: center/big
+  # Merge one map
+  - <<: *CENTER
+    r: 10
+    label: center/big
+  # Merge multiple maps
+  - <<: [*CENTER, *BIG]
+    label: center/big
+  # Override
+  - <<: [*BIG, *LEFT, *SMALL]
+    x: 1
+    label: center/big
+  """
+  
+  data = ruamel.yaml.load(inp, ruamel.yaml.RoundTripLoader)
+  assert data[7]['y'] == 2
+  
+
+.. example code anchor_merge.py
+
+
 Optional requirements
 =====================
 
@@ -238,7 +278,9 @@ A utility name  ``yaml`` is included and allows for basic operations on files:
 - ``yaml ini <file_name>`` for conversion of an INI/config file (ConfigObj
   comment and nested sections supported) to a YAML block style document.
   This requires ``configobj`` to be installed (``pip install configobj``)
-- ``yaml html <file_name>`` for conversion of the basic structure in a YAML
+- ``yaml from-csv <file_name>`` for conversion CSV to a YAML
+  file to a a table in an HTML file. 
+- ``yaml htmltable <file_name>`` for conversion of the basic structure in a YAML
   file to a a table in an HTML file. The YAML file::
 
     title:
diff --git a/example/anchor_merge.py b/example/anchor_merge.py
new file mode 100644
index 0000000..caeddba
--- /dev/null
+++ b/example/anchor_merge.py
@@ -0,0 +1,29 @@
+import ruamel.yaml
+
+inp = """\
+- &CENTER {x: 1, y: 2}
+- &LEFT {x: 0, y: 2}
+- &BIG {r: 10}
+- &SMALL {r: 1}
+# All the following maps are equal:
+# Explicit keys
+- x: 1
+  y: 2
+  r: 10
+  label: center/big
+# Merge one map
+- <<: *CENTER
+  r: 10
+  label: center/big
+# Merge multiple maps
+- <<: [*CENTER, *BIG]
+  label: center/big
+# Override
+- <<: [*BIG, *LEFT, *SMALL]
+  x: 1
+  label: center/big
+"""
+
+data = ruamel.yaml.load(inp, ruamel.yaml.RoundTripLoader)
+assert data[7]['y'] == 2
+
diff --git a/py/__init__.py b/py/__init__.py
index 8610e08..ebe529f 100644
--- a/py/__init__.py
+++ b/py/__init__.py
@@ -21,7 +21,7 @@ def _convert_version(tup):
     return ret_val
 
 
-version_info = (0, 9, 8)
+version_info = (0, 10)
 __version__ = _convert_version(version_info)
 
 del _convert_version
diff --git a/py/comments.py b/py/comments.py
index 6003f5b..577b984 100644
--- a/py/comments.py
+++ b/py/comments.py
@@ -4,7 +4,7 @@ from __future__ import absolute_import
 from __future__ import print_function
 
 __all__ = ["CommentedSeq", "CommentedMap", "CommentedOrderedMap",
-           "CommentedSet", 'comment_attrib']
+           "CommentedSet", 'comment_attrib', 'merge_attrib']
 
 """
 stuff to deal with comments and formatting on dict/list/ordereddict/set
@@ -19,7 +19,8 @@ from .compat import ordereddict
 comment_attrib = '_yaml_comment'
 format_attrib = '_yaml_format'
 line_col_attrib = '_yaml_line_col'
-
+anchor_attrib = '_yaml_anchor'
+merge_attrib = '_yaml_merge'
 
 class Comment(object):
     # sys.getsize tested the Comment objects, __slots__ make them bigger
@@ -98,6 +99,11 @@ class LineCol(object):
         self.line = None
         self.col = None
 
+class Anchor(object):
+    attrib = anchor_attrib
+
+    def __init__(self):
+        self.value = None
 
 class CommentedBase(object):
     @property
@@ -167,6 +173,21 @@ class CommentedBase(object):
         self.lc.line = line
         self.lc.col = col
 
+    @property
+    def anchor(self):
+        if not hasattr(self, Anchor.attrib):
+            setattr(self, Anchor.attrib, Anchor())
+        return getattr(self, Anchor.attrib)
+
+    def yaml_anchor(self):
+        if not hasattr(self, Anchor.attrib):
+            return None
+        return self.anchor.value
+
+    def set_yaml_anchor(self, value):
+        self.anchor.value = value
+
+
 
 class CommentedSeq(list, CommentedBase):
     __slots__ = [Comment.attrib, ]
@@ -282,6 +303,31 @@ class CommentedMap(ordereddict, CommentedBase):
                 Raise
             return default
 
+    def __getitem__(self, key):
+        try:
+            return ordereddict.__getitem__(self, key)
+        except KeyError:
+            for merged in getattr(self, merge_attrib, []):
+                if key in merged[1]:
+                    return merged[1][key]
+            raise
+
+    def get(self, key, default=None):
+        try:
+            return self.__getitem__(key)
+        except:
+            return default
+
+    @property
+    def merge(self):
+        if not hasattr(self, merge_attrib):
+            setattr(self, merge_attrib, [])
+        return getattr(self, merge_attrib)
+
+    def add_yaml_merge(self, value):
+        self.merge.extend(value)
+
+
 
 class CommentedOrderedMap(CommentedMap):
     __slots__ = [Comment.attrib, ]
diff --git a/py/composer.py b/py/composer.py
index dd739dc..13436c8 100644
--- a/py/composer.py
+++ b/py/composer.py
@@ -68,15 +68,15 @@ class Composer(object):
     def compose_node(self, parent, index):
         if self.check_event(AliasEvent):
             event = self.get_event()
-            anchor = event.anchor
-            if anchor not in self.anchors:
+            alias = event.anchor
+            if alias not in self.anchors:
                 raise ComposerError(
                     None, None, "found undefined alias %r"
-                    % utf8(anchor), event.start_mark)
-            return self.anchors[anchor]
+                    % utf8(alias), event.start_mark)
+            return self.anchors[alias]
         event = self.peek_event()
         anchor = event.anchor
-        if anchor is not None:
+        if anchor is not None:  # have an anchor
             if anchor in self.anchors:
                 raise ComposerError(
                     "found duplicate anchor %r; first occurence"
@@ -112,7 +112,7 @@ class Composer(object):
         node = SequenceNode(tag, [],
                             start_event.start_mark, None,
                             flow_style=start_event.flow_style,
-                            comment=start_event.comment)
+                            comment=start_event.comment, anchor=anchor)
         if anchor is not None:
             self.anchors[anchor] = node
         index = 0
@@ -137,7 +137,7 @@ class Composer(object):
         node = MappingNode(tag, [],
                            start_event.start_mark, None,
                            flow_style=start_event.flow_style,
-                           comment=start_event.comment)
+                           comment=start_event.comment, anchor=anchor)
         if anchor is not None:
             self.anchors[anchor] = node
         while not self.check_event(MappingEndEvent):
diff --git a/py/constructor.py b/py/constructor.py
index 0b9491c..274aa22 100644
--- a/py/constructor.py
+++ b/py/constructor.py
@@ -841,6 +841,58 @@ class RoundTripConstructor(SafeConstructor):
                 seqtyp._yaml_add_comment(child.comment, key=idx)
         return ret_val
 
+    def flatten_mapping(self, node):
+        """
+        This implements the merge key feature http://yaml.org/type/merge.html
+        by inserting keys from the merge dict/list of dicts if not yet
+        available in this node
+        """
+        #merge = []
+        merge_map_list = []
+        index = 0
+        while index < len(node.value):
+            key_node, value_node = node.value[index]
+            if key_node.tag == u'tag:yaml.org,2002:merge':
+                del node.value[index]
+                if isinstance(value_node, MappingNode):
+                    # such an anchor node is already constructed
+                    assert value_node in self.constructed_objects
+                    merge_map_list.append(
+                        (index, self.constructed_objects[value_node]))
+                    #self.flatten_mapping(value_node)
+                    #merge.extend(value_node.value)
+                elif isinstance(value_node, SequenceNode):
+                    #submerge = []
+                    for subnode in value_node.value:
+                        if not isinstance(subnode, MappingNode):
+                            raise ConstructorError(
+                                "while constructing a mapping",
+                                node.start_mark,
+                                "expected a mapping for merging, but found %s"
+                                % subnode.id, subnode.start_mark)
+                        merge_map_list.append(
+                            (index, self.constructed_objects[subnode]))
+                    #    self.flatten_mapping(subnode)
+                    #    submerge.append(subnode.value)
+                    #submerge.reverse()
+                    #for value in submerge:
+                    #    merge.extend(value)
+                else:
+                    raise ConstructorError(
+                        "while constructing a mapping", node.start_mark,
+                        "expected a mapping or list of mappings for merging, "
+                        "but found %s"
+                        % value_node.id, value_node.start_mark)
+            elif key_node.tag == u'tag:yaml.org,2002:value':
+                key_node.tag = u'tag:yaml.org,2002:str'
+                index += 1
+            else:
+                index += 1
+        #print ('merge_map_list', merge_map_list)
+        return merge_map_list
+        #if merge:
+        #    node.value = merge + node.value
+
     def construct_mapping(self, node, maptyp, deep=False):
         if not isinstance(node, MappingNode):
             raise ConstructorError(
@@ -848,12 +900,18 @@ class RoundTripConstructor(SafeConstructor):
                 "expected a mapping node, but found %s" % node.id,
                 node.start_mark)
         if isinstance(node, MappingNode):
-            self.flatten_mapping(node)
+            merge_map = self.flatten_mapping(node)
+            if merge_map:
+                maptyp.add_yaml_merge(merge_map)
         # mapping = {}
         if node.comment:
             maptyp._yaml_add_comment(node.comment[:2])
             if len(node.comment) > 2:
                 maptyp.yaml_end_comment_extend(node.comment[2], clear=True)
+        if node.anchor:
+            from ruamel.yaml.serializer import templated_id
+            if not templated_id(node.anchor):
+                maptyp.set_yaml_anchor(node.anchor)
         for key_node, value_node in node.value:
             # keys can be list -> deep
             key = self.construct_object(key_node, deep=True)
diff --git a/py/nodes.py b/py/nodes.py
index e7869c9..208e250 100644
--- a/py/nodes.py
+++ b/py/nodes.py
@@ -8,6 +8,7 @@ class Node(object):
         self.start_mark = start_mark
         self.end_mark = end_mark
         self.comment = comment
+        self.anchor = None
 
     def __repr__(self):
         value = self.value
@@ -69,9 +70,10 @@ class ScalarNode(Node):
 
 class CollectionNode(Node):
     def __init__(self, tag, value, start_mark=None, end_mark=None,
-                 flow_style=None, comment=None):
+                 flow_style=None, comment=None, anchor=None):
         Node.__init__(self, tag, value, start_mark, end_mark, comment=comment)
         self.flow_style = flow_style
+        self.anchor = anchor
 
 
 class SequenceNode(CollectionNode):
diff --git a/py/representer.py b/py/representer.py
index 6e55cd4..63c0d0d 100644
--- a/py/representer.py
+++ b/py/representer.py
@@ -567,7 +567,7 @@ Representer.add_multi_representer(object,
 
 
 from .comments import CommentedMap, CommentedOrderedMap, CommentedSeq, \
-    CommentedSet, comment_attrib
+    CommentedSet, comment_attrib, merge_attrib
 
 
 class RoundTripRepresenter(SafeRepresenter):
@@ -601,7 +601,11 @@ class RoundTripRepresenter(SafeRepresenter):
             flow_style = sequence.fa.flow_style(flow_style)
         except AttributeError:
             flow_style = flow_style
-        node = SequenceNode(tag, value, flow_style=flow_style)
+        try:
+            anchor = sequence.yaml_anchor()
+        except AttributeError:
+            anchor = None
+        node = SequenceNode(tag, value, flow_style=flow_style, anchor=anchor)
         if self.alias_key is not None:
             self.represented_objects[self.alias_key] = node
         best_style = True
@@ -634,7 +638,11 @@ class RoundTripRepresenter(SafeRepresenter):
             flow_style = mapping.fa.flow_style(flow_style)
         except AttributeError:
             flow_style = flow_style
-        node = MappingNode(tag, value, flow_style=flow_style)
+        try:
+            anchor = mapping.yaml_anchor()
+        except AttributeError:
+            anchor = None
+        node = MappingNode(tag, value, flow_style=flow_style, anchor=anchor)
         if self.alias_key is not None:
             self.represented_objects[self.alias_key] = node
         best_style = True
@@ -673,6 +681,17 @@ class RoundTripRepresenter(SafeRepresenter):
                 node.flow_style = self.default_flow_style
             else:
                 node.flow_style = best_style
+        merge_list = [m[1] for m in getattr(mapping, merge_attrib, [])]
+        if merge_list:
+            # because of the call to represent_data here, the anchors
+            # are marked as being used and thereby created
+            if len(merge_list) == 1:
+                arg = self.represent_data(merge_list[0])
+            else:
+                arg = self.represent_data(merge_list)
+                arg.flow_style = True
+            value.insert(0,
+                         (ScalarNode(u'tag:yaml.org,2002:merge', '<<'), arg))
         return node
 
     def represent_omap(self, tag, omap, flow_style=None):
@@ -681,7 +700,11 @@ class RoundTripRepresenter(SafeRepresenter):
             flow_style = omap.fa.flow_style(flow_style)
         except AttributeError:
             flow_style = flow_style
-        node = SequenceNode(tag, value, flow_style=flow_style)
+        try:
+            anchor = omap.yaml_anchor()
+        except AttributeError:
+            anchor = None
+        node = SequenceNode(tag, value, flow_style=flow_style, anchor=anchor)
         if self.alias_key is not None:
             self.represented_objects[self.alias_key] = node
         best_style = True
@@ -728,7 +751,11 @@ class RoundTripRepresenter(SafeRepresenter):
         # return self.represent_mapping(tag, value)
         value = []
         flow_style = setting.fa.flow_style(flow_style)
-        node = MappingNode(tag, value, flow_style=flow_style)
+        try:
+            anchor = setting.yaml_anchor()
+        except AttributeError:
+            anchor = None
+        node = MappingNode(tag, value, flow_style=flow_style, anchor=anchor)
         if self.alias_key is not None:
             self.represented_objects[self.alias_key] = node
         best_style = True
diff --git a/py/serializer.py b/py/serializer.py
index 7e891e1..30c80ba 100644
--- a/py/serializer.py
+++ b/py/serializer.py
@@ -2,6 +2,8 @@ from __future__ import absolute_import
 
 __all__ = ['Serializer', 'SerializerError']
 
+import re
+
 from .error import YAMLError
 from .events import *
 from .nodes import *
@@ -15,7 +17,10 @@ class SerializerError(YAMLError):
 
 class Serializer(object):
 
+    # 'id' and 3+ numbers, but not 000
     ANCHOR_TEMPLATE = u'id%03d'
+    ANCHOR_RE = re.compile(u'id(?!000$)\\d{3,}')
+
 
     def __init__(self, encoding=None, explicit_start=None, explicit_end=None,
                  version=None, tags=None):
@@ -28,6 +33,7 @@ class Serializer(object):
         self.anchors = {}
         self.last_anchor_id = 0
         self.closed = None
+        self._templated_id = None
 
     def open(self):
         if self.closed is None:
@@ -81,8 +87,14 @@ class Serializer(object):
                     self.anchor_node(value)
 
     def generate_anchor(self, node):
-        self.last_anchor_id += 1
-        return self.ANCHOR_TEMPLATE % self.last_anchor_id
+        try:
+            anchor = node.anchor
+        except:
+            anchor = None
+        if anchor is None:
+            self.last_anchor_id += 1
+            return self.ANCHOR_TEMPLATE % self.last_anchor_id
+        return anchor
 
     def serialize_node(self, node, parent, index):
         alias = self.anchors[node]
@@ -143,3 +155,6 @@ class Serializer(object):
                     self.serialize_node(value, node, key)
                 self.emit(MappingEndEvent(comment=[map_comment, end_comment]))
             self.ascend_resolver()
+
+def templated_id(s):
+    return Serializer.ANCHOR_RE.match(s)
\ No newline at end of file
diff --git a/test/test_anchor.py b/test/test_anchor.py
new file mode 100644
index 0000000..477c8ee
--- /dev/null
+++ b/test/test_anchor.py
@@ -0,0 +1,159 @@
+# coding: utf-8
+
+"""
+testing of anchors and the aliases referring to them
+"""
+
+import pytest
+from textwrap import dedent
+
+import ruamel.yaml
+from roundtrip import round_trip, dedent, round_trip_load, round_trip_dump
+
+def load(s):
+    return round_trip_load(dedent(s))
+
+def compare(data, s):
+    assert round_trip_dump(data) == dedent(s)
+
+
+class TestAnchorsAliases:
+    def test_anchor_id_renumber(self):
+        from ruamel.yaml.serializer import Serializer
+        assert Serializer.ANCHOR_TEMPLATE == 'id%03d'
+        data = load("""
+        a: &id002
+          b: 1
+          c: 2
+        d: *id002
+        """)
+        compare(data, """
+        a: &id001
+          b: 1
+          c: 2
+        d: *id001
+        """)
+
+    def test_template_matcher(self):
+        """test if id matches the anchor template"""
+        from ruamel.yaml.serializer import templated_id
+        assert templated_id(u'id001')
+        assert templated_id(u'id999')
+        assert templated_id(u'id1000')
+        assert templated_id(u'id0001')
+        assert templated_id(u'id0000')
+        assert not templated_id(u'id02')
+        assert not templated_id(u'id000')
+        assert not templated_id(u'x000')
+
+    #def test_re_matcher(self):
+    #    import re
+    #    assert re.compile(u'id(?!000)\\d{3,}').match('id001')
+    #    assert not re.compile(u'id(?!000\\d*)\\d{3,}').match('id000')
+    #    assert re.compile(u'id(?!000$)\\d{3,}').match('id0001')
+
+    def test_anchor_assigned(self):
+        from ruamel.yaml.comments import CommentedMap
+        data = load("""
+        a: &id002
+          b: 1
+          c: 2
+        d: *id002
+        e: &etemplate
+          b: 1
+          c: 2
+        f: *etemplate
+        """)
+        d = data['d']
+        assert isinstance(d, CommentedMap)
+        assert d.yaml_anchor() is None  # got dropped as it matches pattern
+        e = data['e']
+        assert isinstance(e, CommentedMap)
+        assert e.yaml_anchor() == 'etemplate'
+
+    #@pytest.mark.xfail
+    def test_anchor_id_retained(self):
+        data = load("""
+        a: &id002
+          b: 1
+          c: 2
+        d: *id002
+        e: &etemplate
+          b: 1
+          c: 2
+        f: *etemplate
+        """)
+        compare(data, """
+        a: &id001
+          b: 1
+          c: 2
+        d: *id001
+        e: &etemplate
+          b: 1
+          c: 2
+        f: *etemplate
+        """)
+
+    def test_alias_before_anchor(self):
+        from ruamel.yaml.composer import ComposerError
+        with pytest.raises(ComposerError):
+            data = load("""
+            d: *id002
+            a: &id002
+              b: 1
+              c: 2
+            """)
+
+
+    merge_yaml = dedent("""
+        - &CENTER {x: 1, y: 2}
+        - &LEFT {x: 0, y: 2}
+        - &BIG {r: 10}
+        - &SMALL {r: 1}
+        # All the following maps are equal:
+        # Explicit keys
+        - x: 1
+          y: 2
+          r: 10
+          label: center/big
+        # Merge one map
+        - <<: *CENTER
+          r: 10
+          label: center/big
+        # Merge multiple maps
+        - <<: [*CENTER, *BIG]
+          label: center/big
+        # Override
+        - <<: [*BIG, *LEFT, *SMALL]
+          x: 1
+          label: center/big
+        """)
+
+    def test_merge_00(self):
+        data = load(self.merge_yaml)
+        d = data[4]
+        ok = True
+        for k in d:
+            for o in [5, 6, 7]:
+                if d.get(k) != data[o].get(k):
+                    ok = False
+                    print('key', k, d.get(k), data[o].get(k))
+        assert ok
+
+    def test_merge_accessible(self):
+        from ruamel.yaml.comments import CommentedMap, merge_attrib
+        data = load("""
+        k: &level_2 { a: 1, b2 }
+        l: &level_1 { a: 10, c: 3 }
+        m:
+          << : *level_1
+          c: 30
+          d: 40
+        """)
+        d = data['m']
+        assert isinstance(d, CommentedMap)
+        assert hasattr(d, merge_attrib)
+
+    def test_merge_01(self):
+        data = load(self.merge_yaml)
+        compare(data, self.merge_yaml)
\ No newline at end of file
diff --git a/test/test_comment_manipulation.py b/test/test_comment_manipulation.py
index f2537e1..128cade 100644
--- a/test/test_comment_manipulation.py
+++ b/test/test_comment_manipulation.py
@@ -10,11 +10,12 @@ from roundtrip import round_trip, dedent, round_trip_load, round_trip_dump
 def load(s):
     return round_trip_load(dedent(s))
 
+
 def compare(data, s):
     assert round_trip_dump(data) == dedent(s)
-
 #@pytest.mark.xfail
 
+
 class TestCommentsManipulation:
 
     # list
diff --git a/test/test_comments.py b/test/test_comments.py
index f9e140f..b37f53e 100644
--- a/test/test_comments.py
+++ b/test/test_comments.py
@@ -183,6 +183,7 @@ class TestComments:
         x = x.replace(': secret          ', ': deleted password')
         assert round_trip_dump(data) == x
 
+
     def test_set_comment(self):
         round_trip("""
         !!set