Added code to determine the input encoding from data: encoding declarations or the presence of byte order marks (UTF-8 & UTF-16).

git-svn-id: http://svn.code.sf.net/p/docutils/code/trunk/docutils@4506 929543f6-e4f2-0310-98a6-ba3bd3dd1d04

1 files changed, 36 insertions, 0 deletions

diff --git a/test/test_io.py b/test/test_io.py
index b82911e44..c8388ab99 100755
--- a/test/test_io.py
+++ b/test/test_io.py
@@ -27,6 +27,42 @@ class InputTests(unittest.TestCase):
         # Assert BOMs are still there.
         self.assertEquals(input.read(), u'\ufeff foo \ufeff bar')
 
+    def test_coding_slug(self):
+        input = io.StringInput(source="""\
+.. -*- coding: ascii -*-
+data
+blah
+""")
+        data = input.read()
+        self.assertEquals(input.successful_encoding, 'ascii')
+        input = io.StringInput(source="""\
+#! python
+# -*- coding: ascii -*-
+print "hello world"
+""")
+        data = input.read()
+        self.assertEquals(input.successful_encoding, 'ascii')
+        input = io.StringInput(source="""\
+#! python
+# extraneous comment; prevents coding slug from being read
+# -*- coding: ascii -*-
+print "hello world"
+""")
+        data = input.read()
+        self.assertNotEquals(input.successful_encoding, 'ascii')
+
+    def test_bom_detection(self):
+        source = u'\ufeffdata\nblah\n'
+        input = io.StringInput(source=source.encode('utf-16-be'))
+        data = input.read()
+        self.assertEquals(input.successful_encoding, 'utf-16-be')
+        input = io.StringInput(source=source.encode('utf-16-le'))
+        data = input.read()
+        self.assertEquals(input.successful_encoding, 'utf-16-le')
+        input = io.StringInput(source=source.encode('utf-8'))
+        data = input.read()
+        self.assertEquals(input.successful_encoding, 'utf-8')
+
 
 if __name__ == '__main__':
     unittest.main()