ignore unknown directives %YAML:1.1 (without colon)

7 files changed, 64 insertions, 5 deletions

diff --git a/py/__init__.py b/py/__init__.py
index 0e8b561..d164c2e 100644
--- a/py/__init__.py
+++ b/py/__init__.py
@@ -21,7 +21,7 @@ def _convert_version(tup):
     return ret_val
 
 
-version_info = (0, 10, 2)
+version_info = (0, 10, 4)
 __version__ = _convert_version(version_info)
 
 del _convert_version
diff --git a/py/convert/html.py b/py/convert/html.py
index 2fbd605..ed92f9d 100644
--- a/py/convert/html.py
+++ b/py/convert/html.py
@@ -29,7 +29,8 @@ class HTML2YAML(object):
         import ruamel.yaml
         return ruamel.yaml.dump(
             d,
-            Dumper=ruamel.yaml.RoundTripDumper
+            Dumper=ruamel.yaml.RoundTripDumper,
+            allow_unicode=True,
         )
 
     def html_to_data(self, html):
diff --git a/py/nodes.py b/py/nodes.py
index 208e250..382b492 100644
--- a/py/nodes.py
+++ b/py/nodes.py
@@ -57,7 +57,7 @@ class ScalarNode(Node):
       ? -> set() ? key, no value
       " -> double quoted
       ' -> single quoted
-      | ->
+      | -> literal style
       > ->
     """
     id = 'scalar'
diff --git a/py/scalarstring.py b/py/scalarstring.py
index e2e2778..a628396 100644
--- a/py/scalarstring.py
+++ b/py/scalarstring.py
@@ -14,3 +14,31 @@ class ScalarString(text_type):
 class PreservedScalarString(ScalarString):
     def __new__(cls, value):
         return ScalarString.__new__(cls, value)
+
+def preserve_literal(s):
+    return PreservedScalarString(s.replace('\r\n', '\n').replace('\r', '\n'))
+
+
+def walk_tree(base):
+    """
+    the routine here walks over a simple yaml tree (recursing in
+    dict values and list items) and converts strings that
+    have multiple lines to literal scalars
+    """
+    from ruamel.yaml.compat import string_types
+
+
+    if isinstance(base, dict):
+        for k in base:
+            v = base[k]
+            if isinstance(v, string_types) and '\n' in v:
+                base[k] = preserve_literal(v)
+            else:
+                walk_tree(v)
+    elif isinstance(base, list):
+        for idx, elem in enumerate(base):
+            if isinstance(elem, string_types) and '\n' in elem:
+                print(elem)
+                base[idx] = preserve_literal(elem)
+            else:
+                walk_tree(elem)
diff --git a/py/scanner.py b/py/scanner.py
index 721437c..69c750d 100644
--- a/py/scanner.py
+++ b/py/scanner.py
@@ -819,7 +819,7 @@ class Scanner(object):
         length = 0
         ch = self.peek(length)
         while u'0' <= ch <= u'9' or u'A' <= ch <= u'Z' or u'a' <= ch <= u'z' \
-                or ch in u'-_':
+                or ch in u'-_:.':
             length += 1
             ch = self.peek(length)
         if not length:
diff --git a/py/yaml.py b/py/yaml.py
index 2f64332..6eda0ab 100644
--- a/py/yaml.py
+++ b/py/yaml.py
@@ -216,7 +216,7 @@ class YAML:
         # test end
 
     def from_json(self):
-        # use roundtrip to preserver order
+        # use roundtrip to preserve order
         errors = 0
         docs = []
         for file_name in self._args.file:
@@ -226,6 +226,10 @@ class YAML:
                 inp = open(file_name).read()
             loader = ruamel.yaml.Loader # RoundTripLoader
             docs.append(ruamel.yaml.load(inp, loader))
+        #if self._args.literal:
+        #    from ruamel.yaml.convert.literal import walk_tree
+        #    for doc in docs:
+        #        walk_tree(doc)
         dumper = ruamel.yaml.RoundTripDumper
         print(ruamel.yaml.dump_all(
             docs, Dumper=dumper,
@@ -443,6 +447,8 @@ class YAML_Cmd(ProgramBase):
     )
     @option('--flow', action='store_true',
             help='use flow instead of block style')
+    #@option('--literal', action='store_true',
+    #        help='convert scalars with newlines to literal block style')
     @option('file', nargs='+')
     def json(self):
         return self._yaml.from_json()
diff --git a/test/test_comments.py b/test/test_comments.py
index b37f53e..8d87f0e 100644
--- a/test/test_comments.py
+++ b/test/test_comments.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 
 """
 comment testing is all about roundtrips
@@ -228,6 +229,29 @@ class TestComments:
         - d: 4
         """)
 
+    @pytest.mark.xfail
+    def test_non_ascii_comment(self):
+        round_trip("""
+        verbosity: 1                  # 0 is minimal output, -1 none
+        base_url: http://gopher.net
+        special_indices: [1, 5, 8]
+        also_special:
+        - a
+        - 19
+        - 32
+        asia and europe: &asia_europe
+            Turkey: Ankara
+            Russia: Moscow
+        countries:
+            Asia:
+                <<: *asia_europe
+                Japan: Tokyo # 東京
+            Europe:
+                <<: *asia_europe
+                Spain: Madrid
+                Italy: Rome
+        """)
+
 
 class TestMultiLevelGet:
     def test_mlget_00(self):