32 bit unicode not supported on MacOS, fixes issue #560.12.9

reported by (David Tagatac)

5 files changed, 31 insertions, 10 deletions

diff --git a/README.rst b/README.rst
index b381532..3f3811f 100644
--- a/README.rst
+++ b/README.rst
@@ -18,7 +18,11 @@ ChangeLog
 
 ::
 
-  0.12.8 (2016-09-xx):
+  0.12.9 (2016-09-05):
+   - Make previous fix depend on unicode char width (32 bit unicode support
+     is a problem on MacOS reported by David Tagatac)
+
+  0.12.8 (2016-09-05):
      - To be ignored Unicode characters were not properly regex matched
        (no specific tests, PR by Haraguroicha Hsu)
 
diff --git a/__init__.py b/__init__.py
index 728eee9..2fe742f 100644
--- a/__init__.py
+++ b/__init__.py
@@ -9,7 +9,7 @@ from __future__ import absolute_import
 
 _package_data = dict(
     full_package_name="ruamel.yaml",
-    version_info=(0, 12, 8),
+    version_info=(0, 12, 9),
     author="Anthon van der Neut",
     author_email="a.van.der.neut@ruamel.eu",
     description="ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order",  # NOQA
diff --git a/_test/test_comments.py b/_test/test_comments.py
index 0e52031..fc91943 100644
--- a/_test/test_comments.py
+++ b/_test/test_comments.py
@@ -594,6 +594,8 @@ class TestEmptyLines:
 
 
 class TestUnicodeComments:
+    @pytest.mark.skipif(ruamel.yaml.compat.UNICODE_SIZE < 4,
+                        reason="unicode not wide enough")
     def test_issue_55(self):  # reported by Haraguroicha Hsu
         round_trip("""\
         name: TEST
diff --git a/compat.py b/compat.py
index 23cd968..3a4af90 100644
--- a/compat.py
+++ b/compat.py
@@ -7,6 +7,7 @@ from __future__ import print_function
 import sys
 import os
 import types
+import sysconfig
 
 try:
     from ruamel.ordereddict import ordereddict  # type: ignore
@@ -36,6 +37,12 @@ except:
 PY2 = sys.version_info[0] == 2
 PY3 = sys.version_info[0] == 3
 
+# 4 if 32 bit unicode supported, 2 e.g. on MacOS (issue 56)
+UNICODE_SIZE = sysconfig.get_config_vars().get(
+    'SIZEOF_WCHAR_T',
+    sysconfig.get_config_vars().get('Py_UNICODE_SIZE', 4))
+
+
 if PY3:
     def utf8(s):
         return s
diff --git a/reader.py b/reader.py
index fe6f505..fc52d2e 100644
--- a/reader.py
+++ b/reader.py
@@ -24,7 +24,7 @@ import codecs
 import re
 
 from ruamel.yaml.error import YAMLError, Mark
-from ruamel.yaml.compat import text_type, binary_type, PY3
+from ruamel.yaml.compat import text_type, binary_type, PY3, UNICODE_SIZE
 
 __all__ = ['Reader', 'ReaderError']
 
@@ -144,13 +144,21 @@ class Reader(object):
                 self.encoding = 'utf-8'
         self.update(1)
 
-    NON_PRINTABLE = re.compile(
-        u'[^\x09\x0A\x0D\x20-\x7E\x85'
-        u'\xA0-\uD7FF'
-        u'\uE000-\uFFFD'
-        u'\U00010000-\U0010FFFF'
-        u']'
-    )
+    if UNICODE_SIZE == 4:
+        NON_PRINTABLE = re.compile(
+            u'[^\x09\x0A\x0D\x20-\x7E\x85'
+            u'\xA0-\uD7FF'
+            u'\uE000-\uFFFD'
+            u'\U00010000-\U0010FFFF'
+            u']'
+        )
+    else:
+        NON_PRINTABLE = re.compile(
+            u'[^\x09\x0A\x0D\x20-\x7E\x85'
+            u'\xA0-\uD7FF'
+            u'\uE000-\uFFFD'
+            u']'
+        )
 
     def check_printable(self, data):
         match = self.NON_PRINTABLE.search(data)