Port test_ucn and test_unicodedata to PyUnit. Add a few tests for error

cases increasing coverage in unicodedata.c from 87% to 95%
(when the normalization tests are run). From SF patch #662807.

1 files changed, 199 insertions, 111 deletions

diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py
index 70b20d59c5..95077c302e 100644
--- a/Lib/test/test_unicodedata.py
+++ b/Lib/test/test_unicodedata.py
@@ -5,121 +5,209 @@
     (c) Copyright CNRI, All Rights Reserved. NO WARRANTY.
 
 """#"
-from test.test_support import verify, verbose
+import unittest, test.test_support
 import sha
 
 encoding = 'utf-8'
 
-def test_methods():
-
-    h = sha.sha()
-    for i in range(65536):
-        char = unichr(i)
-        data = [
-
-            # Predicates (single char)
-            char.isalnum() and u'1' or u'0',
-            char.isalpha() and u'1' or u'0',
-            char.isdecimal() and u'1' or u'0',
-            char.isdigit() and u'1' or u'0',
-            char.islower() and u'1' or u'0',
-            char.isnumeric() and u'1' or u'0',
-            char.isspace() and u'1' or u'0',
-            char.istitle() and u'1' or u'0',
-            char.isupper() and u'1' or u'0',
-
-            # Predicates (multiple chars)
-            (char + u'abc').isalnum() and u'1' or u'0',
-            (char + u'abc').isalpha() and u'1' or u'0',
-            (char + u'123').isdecimal() and u'1' or u'0',
-            (char + u'123').isdigit() and u'1' or u'0',
-            (char + u'abc').islower() and u'1' or u'0',
-            (char + u'123').isnumeric() and u'1' or u'0',
-            (char + u' \t').isspace() and u'1' or u'0',
-            (char + u'abc').istitle() and u'1' or u'0',
-            (char + u'ABC').isupper() and u'1' or u'0',
-
-            # Mappings (single char)
-            char.lower(),
-            char.upper(),
-            char.title(),
-
-            # Mappings (multiple chars)
-            (char + u'abc').lower(),
-            (char + u'ABC').upper(),
-            (char + u'abc').title(),
-            (char + u'ABC').title(),
-
-            ]
-        h.update(u''.join(data).encode(encoding))
-    return h.hexdigest()
-
-def test_unicodedata():
-
-    h = sha.sha()
-    for i in range(65536):
-        char = unichr(i)
-        data = [
-            # Properties
-            str(unicodedata.digit(char, -1)),
-            str(unicodedata.numeric(char, -1)),
-            str(unicodedata.decimal(char, -1)),
-            unicodedata.category(char),
-            unicodedata.bidirectional(char),
-            unicodedata.decomposition(char),
-            str(unicodedata.mirrored(char)),
-            str(unicodedata.combining(char)),
-            ]
-        h.update(''.join(data))
-    return h.hexdigest()
 
 ### Run tests
 
-print 'Testing Unicode Database...'
-print 'Methods:',
-print test_methods()
-
-# In case unicodedata is not available, this will raise an ImportError,
-# but still test the above cases...
-import unicodedata
-print 'Functions:',
-print test_unicodedata()
-
-# Some additional checks of the API:
-print 'API:',
-
-verify(unicodedata.digit(u'A',None) is None)
-verify(unicodedata.digit(u'9') == 9)
-verify(unicodedata.digit(u'\u215b',None) is None)
-verify(unicodedata.digit(u'\u2468') == 9)
-
-verify(unicodedata.numeric(u'A',None) is None)
-verify(unicodedata.numeric(u'9') == 9)
-verify(unicodedata.numeric(u'\u215b') == 0.125)
-verify(unicodedata.numeric(u'\u2468') == 9.0)
-
-verify(unicodedata.decimal(u'A',None) is None)
-verify(unicodedata.decimal(u'9') == 9)
-verify(unicodedata.decimal(u'\u215b',None) is None)
-verify(unicodedata.decimal(u'\u2468',None) is None)
-
-verify(unicodedata.category(u'\uFFFE') == 'Cn')
-verify(unicodedata.category(u'a') == 'Ll')
-verify(unicodedata.category(u'A') == 'Lu')
-
-verify(unicodedata.bidirectional(u'\uFFFE') == '')
-verify(unicodedata.bidirectional(u' ') == 'WS')
-verify(unicodedata.bidirectional(u'A') == 'L')
-
-verify(unicodedata.decomposition(u'\uFFFE') == '')
-verify(unicodedata.decomposition(u'\u00bc') == '<fraction> 0031 2044 0034')
-
-verify(unicodedata.mirrored(u'\uFFFE') == 0)
-verify(unicodedata.mirrored(u'a') == 0)
-verify(unicodedata.mirrored(u'\u2201') == 1)
-
-verify(unicodedata.combining(u'\uFFFE') == 0)
-verify(unicodedata.combining(u'a') == 0)
-verify(unicodedata.combining(u'\u20e1') == 230)
-
-print 'ok'
+class UnicodeMethodsTest(unittest.TestCase):
+
+    # update this, if the database changes
+    expectedchecksum = 'a37276dc2c158bef6dfd908ad34525c97180fad9'
+
+    def test_method_checksum(self):
+        h = sha.sha()
+        for i in range(65536):
+            char = unichr(i)
+            data = [
+                # Predicates (single char)
+                u"01"[char.isalnum()],
+                u"01"[char.isalpha()],
+                u"01"[char.isdecimal()],
+                u"01"[char.isdigit()],
+                u"01"[char.islower()],
+                u"01"[char.isnumeric()],
+                u"01"[char.isspace()],
+                u"01"[char.istitle()],
+                u"01"[char.isupper()],
+
+                # Predicates (multiple chars)
+                u"01"[(char + u'abc').isalnum()],
+                u"01"[(char + u'abc').isalpha()],
+                u"01"[(char + u'123').isdecimal()],
+                u"01"[(char + u'123').isdigit()],
+                u"01"[(char + u'abc').islower()],
+                u"01"[(char + u'123').isnumeric()],
+                u"01"[(char + u' \t').isspace()],
+                u"01"[(char + u'abc').istitle()],
+                u"01"[(char + u'ABC').isupper()],
+
+                # Mappings (single char)
+                char.lower(),
+                char.upper(),
+                char.title(),
+
+                # Mappings (multiple chars)
+                (char + u'abc').lower(),
+                (char + u'ABC').upper(),
+                (char + u'abc').title(),
+                (char + u'ABC').title(),
+
+                ]
+            h.update(u''.join(data).encode(encoding))
+        result = h.hexdigest()
+        self.assertEqual(result, self.expectedchecksum)
+
+class UnicodeDatabaseTest(unittest.TestCase):
+
+    def setUp(self):
+        # In case unicodedata is not available, this will raise an ImportError,
+        # but the other test cases will still be run
+        import unicodedata
+        self.db = unicodedata
+
+    def tearDown(self):
+        del self.db
+
+class UnicodeFunctionsTest(UnicodeDatabaseTest):
+
+    # update this, if the database changes
+    expectedchecksum = 'cfe20a967a450ebc82ca68c3e4eed344164e11af'
+
+    def test_function_checksum(self):
+        data = []
+        h = sha.sha()
+
+        for i in range(0x10000):
+            char = unichr(i)
+            data = [
+                # Properties
+                str(self.db.digit(char, -1)),
+                str(self.db.numeric(char, -1)),
+                str(self.db.decimal(char, -1)),
+                self.db.category(char),
+                self.db.bidirectional(char),
+                self.db.decomposition(char),
+                str(self.db.mirrored(char)),
+                str(self.db.combining(char)),
+            ]
+            h.update(''.join(data))
+        result = h.hexdigest()
+        self.assertEqual(result, self.expectedchecksum)
+
+    def test_digit(self):
+        self.assertEqual(self.db.digit(u'A', None), None)
+        self.assertEqual(self.db.digit(u'9'), 9)
+        self.assertEqual(self.db.digit(u'\u215b', None), None)
+        self.assertEqual(self.db.digit(u'\u2468'), 9)
+
+        self.assertRaises(TypeError, self.db.digit)
+        self.assertRaises(TypeError, self.db.digit, u'xx')
+        self.assertRaises(ValueError, self.db.digit, u'x')
+
+    def test_numeric(self):
+        self.assertEqual(self.db.numeric(u'A',None), None)
+        self.assertEqual(self.db.numeric(u'9'), 9)
+        self.assertEqual(self.db.numeric(u'\u215b'), 0.125)
+        self.assertEqual(self.db.numeric(u'\u2468'), 9.0)
+
+        self.assertRaises(TypeError, self.db.numeric)
+        self.assertRaises(TypeError, self.db.numeric, u'xx')
+        self.assertRaises(ValueError, self.db.numeric, u'x')
+
+    def test_decimal(self):
+        self.assertEqual(self.db.decimal(u'A',None), None)
+        self.assertEqual(self.db.decimal(u'9'), 9)
+        self.assertEqual(self.db.decimal(u'\u215b', None), None)
+        self.assertEqual(self.db.decimal(u'\u2468', None), None)
+
+        self.assertRaises(TypeError, self.db.decimal)
+        self.assertRaises(TypeError, self.db.decimal, u'xx')
+        self.assertRaises(ValueError, self.db.decimal, u'x')
+
+    def test_category(self):
+        self.assertEqual(self.db.category(u'\uFFFE'), 'Cn')
+        self.assertEqual(self.db.category(u'a'), 'Ll')
+        self.assertEqual(self.db.category(u'A'), 'Lu')
+
+        self.assertRaises(TypeError, self.db.category)
+        self.assertRaises(TypeError, self.db.category, u'xx')
+
+    def test_bidirectional(self):
+        self.assertEqual(self.db.bidirectional(u'\uFFFE'), '')
+        self.assertEqual(self.db.bidirectional(u' '), 'WS')
+        self.assertEqual(self.db.bidirectional(u'A'), 'L')
+
+        self.assertRaises(TypeError, self.db.bidirectional)
+        self.assertRaises(TypeError, self.db.bidirectional, u'xx')
+
+    def test_decomposition(self):
+        self.assertEqual(self.db.decomposition(u'\uFFFE'),'')
+        self.assertEqual(self.db.decomposition(u'\u00bc'), '<fraction> 0031 2044 0034')
+
+        self.assertRaises(TypeError, self.db.decomposition)
+        self.assertRaises(TypeError, self.db.decomposition, u'xx')
+
+    def test_mirrored(self):
+        self.assertEqual(self.db.mirrored(u'\uFFFE'), 0)
+        self.assertEqual(self.db.mirrored(u'a'), 0)
+        self.assertEqual(self.db.mirrored(u'\u2201'), 1)
+
+        self.assertRaises(TypeError, self.db.mirrored)
+        self.assertRaises(TypeError, self.db.mirrored, u'xx')
+
+    def test_combining(self):
+        self.assertEqual(self.db.combining(u'\uFFFE'), 0)
+        self.assertEqual(self.db.combining(u'a'), 0)
+        self.assertEqual(self.db.combining(u'\u20e1'), 230)
+
+        self.assertRaises(TypeError, self.db.combining)
+        self.assertRaises(TypeError, self.db.combining, u'xx')
+
+    def test_normalize(self):
+        self.assertRaises(TypeError, self.db.normalize)
+        self.assertRaises(ValueError, self.db.normalize, 'unknown', u'xx')
+        # The rest can be found in test_normalization.py
+        # which requires an external file.
+
+
+class UnicodeMiscTest(UnicodeDatabaseTest):
+
+    def test_decimal_numeric_consistent(self):
+        # Test that decimal and numeric are consistent,
+        # i.e. if a character has a decimal value,
+        # it's numeric value should be the same.
+        count = 0
+        for i in xrange(0x10000):
+            c = unichr(i)
+            dec = self.db.decimal(c, -1)
+            if dec != -1:
+                self.assertEqual(dec, self.db.numeric(c))
+                count += 1
+        self.assert_(count >= 10) # should have tested at least the ASCII digits
+
+    def test_digit_numeric_consistent(self):
+        # Test that digit and numeric are consistent,
+        # i.e. if a character has a digit value, 
+        # it's numeric value should be the same.
+        count = 0
+        for i in xrange(0x10000):
+            c = unichr(i)
+            dec = self.db.digit(c, -1)
+            if dec != -1:
+                self.assertEqual(dec, self.db.numeric(c))
+                count += 1
+        self.assert_(count >= 10) # should have tested at least the ASCII digits
+
+def test_main():
+    suite = unittest.TestSuite()
+    suite.addTest(unittest.makeSuite(UnicodeMiscTest))
+    suite.addTest(unittest.makeSuite(UnicodeMethodsTest))
+    suite.addTest(unittest.makeSuite(UnicodeFunctionsTest))
+    test.test_support.run_suite(suite)
+
+if __name__ == "__main__":
+    test_main()