scalars with preserved newlines

6 files changed, 141 insertions, 10 deletions

diff --git a/CHANGES b/CHANGES
index b081a3c..5e69e70 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,5 +1,6 @@
 
 0.6 2015-03-XX
+- basic support for scalars with preserved newlines
 - html option for yaml command
 - check if yaml C library is available before trying to compile C extension
 - include unreleased change in PyYAML dd 20141128
diff --git a/README.rst b/README.rst
index 1ced965..343113d 100644
--- a/README.rst
+++ b/README.rst
@@ -21,6 +21,11 @@ Major differences with PyYAML 3.11:
   do round trip editing on comments. It can be used for speeded up normal
   processing (so you don't need to install ``ruamel.yaml`` and ``PyYaml``).
   See the section *Optional requirements*.
+- Basic support for multiline strings with preserved newlines and
+  chomping ( '``|``', '``|+``', '``|-``' ). As this subclasses the string type
+  the information is lost on reassignment. (This might be changed
+  in the future so that the preservation/folding/chomping is part of the
+  parent container, like comments).
 
 
 Round trip including comments
@@ -64,7 +69,7 @@ Extending
 
 There are normally 6 files involved when extending the roundtrip
 capabilities: the reader, parser, composer and constructor to go from YAML to
-Python and the resolver, representer, sberializer and Emitter to go the other
+Python and the resolver, representer, serializer and emitter to go the other
 way.
 
 Extending involves keeping extra data around for the next process step,
diff --git a/py/constructor.py b/py/constructor.py
index c32e61d..6e3a7ac 100644
--- a/py/constructor.py
+++ b/py/constructor.py
@@ -14,7 +14,8 @@ import types
 
 from .error import *
 from .nodes import *
-from .compat import utf8, builtins_module, to_str, PY2, PY3, ordereddict
+from .compat import (utf8, builtins_module, to_str, PY2, PY3, ordereddict,
+                     text_type)
 from .comments import *
 from .scalarstring import *
 
@@ -799,11 +800,21 @@ class RoundTripConstructor(SafeConstructor):
                 "expected a scalar node, but found %s" % node.id,
                 node.start_mark)
 
-        if node.style == '|' and isinstance(node.value, unicode):
-            print('value', repr(node.value), repr(node.style))
-            return ScalarString(node.value)
+        if node.style == '|' and isinstance(node.value, text_type):
+            return PreservedScalarString(node.value)
         return node.value
 
+    def construct_yaml_str(self, node):
+        value = self.construct_scalar(node)
+        if isinstance(value, ScalarString):
+            return value
+        if PY3:
+            return value
+        try:
+            return value.encode('ascii')
+        except UnicodeEncodeError:
+            return value
+
     def construct_sequence(self, node, seqtyp, deep=False):
         if not isinstance(node, SequenceNode):
             raise ConstructorError(
diff --git a/py/representer.py b/py/representer.py
index 3b4e039..74f1578 100644
--- a/py/representer.py
+++ b/py/representer.py
@@ -8,6 +8,7 @@ from .error import *
 from .nodes import *
 from .compat import text_type, binary_type, to_unicode, PY2, PY3, \
     ordereddict, nprint
+from .scalarstring import *
 
 import datetime
 import sys
@@ -583,6 +584,14 @@ class RoundTripRepresenter(SafeRepresenter):
         return self.represent_scalar(u'tag:yaml.org,2002:null',
                                      u'')
 
+    def represent_preserved_scalarstring(self, data):
+        tag = None
+        style = '|'
+        if PY2 and not isinstance(data, unicode):
+            data = unicode(data, 'ascii')
+        tag = u'tag:yaml.org,2002:str'
+        return self.represent_scalar(tag, data, style=style)
+
     def represent_sequence(self, tag, sequence, flow_style=None):
         value = []
         node = SequenceNode(tag, value, flow_style=flow_style)
@@ -738,6 +747,10 @@ class RoundTripRepresenter(SafeRepresenter):
 RoundTripRepresenter.add_representer(type(None),
                                      RoundTripRepresenter.represent_none)
 
+RoundTripRepresenter.add_representer(
+    PreservedScalarString,
+    RoundTripRepresenter.represent_preserved_scalarstring)
+
 RoundTripRepresenter.add_representer(CommentedSeq,
                                      RoundTripRepresenter.represent_list)
 
diff --git a/py/scalarstring.py b/py/scalarstring.py
index bbd1b2d..e2e2778 100644
--- a/py/scalarstring.py
+++ b/py/scalarstring.py
@@ -1,13 +1,16 @@
 from __future__ import absolute_import
 from __future__ import print_function
 
-__all__ = ["ScalarString"]
+__all__ = ["ScalarString", "PreservedScalarString"]
 
+from .compat import text_type
 
-class ScalarString(str):
-    pass
+
+class ScalarString(text_type):
+    def __new__(cls, *args, **kw):
+        return text_type.__new__(cls, *args, **kw)
 
 
 class PreservedScalarString(ScalarString):
-    def __init__(self, value):
-        ScalarString.__init__(self, value)
\ No newline at end of file
+    def __new__(cls, value):
+        return ScalarString.__new__(cls, value)
diff --git a/test/test_string.py b/test/test_string.py
new file mode 100644
index 0000000..c962f1e
--- /dev/null
+++ b/test/test_string.py
@@ -0,0 +1,98 @@
+
+from __future__ import print_function
+
+"""
+various test cases for string scalars in YAML files
+'|' for preserved newlines
+'>' for folded (newlines become spaces)
+
+and the chomping modifiers:
+'-' for stripping: final line break and any trailing empty lines are excluded
+'+' for keeping: final line break and empty lines are preserved
+''  for clipping: final line break preserved, empty lines at end not
+    included in content (no modifier)
+
+"""
+
+import pytest
+
+import ruamel.yaml
+from ruamel.yaml.compat import ordereddict
+from roundtrip import round_trip, dedent, round_trip_load, round_trip_dump
+
+
+class TestYAML:
+    def test_basic_string(self):
+        round_trip("""
+        a: abcdefg
+        """, )
+
+    def test_quoted_string(self):
+        round_trip("""
+        a: '12345'
+        """)
+
+    def test_preserve_string(self):
+        round_trip("""
+            a: |
+              abc
+              def
+            """, intermediate=dict(a='abc\ndef\n'))
+
+    def test_preserve_string_strip(self):
+        s = """
+            a: |-
+              abc
+              def
+
+            """
+        o = dedent(s).rstrip() + '\n'
+        round_trip(s, outp=o, intermediate=dict(a='abc\ndef'))
+
+    def test_preserve_string_keep(self):
+            # with pytest.raises(AssertionError) as excinfo:
+            round_trip("""
+            a: |+
+              ghi
+              jkl
+
+
+            b: x
+            """, intermediate=dict(a='ghi\njkl\n\n\n', b='x'))
+
+    def test_preserve_string_keep_at_end(self):
+        # with pytest.raises(AssertionError) as excinfo:
+        round_trip("""
+            a: |+
+              ghi
+              jkl
+
+            ...
+            """, intermediate=dict(a='ghi\njkl\n\n'))
+
+    def test_fold_string(self):
+        with pytest.raises(AssertionError) as excinfo:
+            round_trip("""
+            a: >
+              abc
+              def
+
+            """, intermediate=dict(a='abc def\n'))
+
+    def test_fold_string_strip(self):
+        with pytest.raises(AssertionError) as excinfo:
+            round_trip("""
+            a: >-
+              abc
+              def
+
+            """, intermediate=dict(a='abc def'))
+
+    def test_fold_string_keep(self):
+        with pytest.raises(AssertionError) as excinfo:
+            round_trip("""
+            a: >+
+              abc
+              def
+
+            """, intermediate=dict(a='abc def\n\n'))