Bring CJKCodecs 1.1 into trunk. This completely reorganizes source

and installed layouts to make maintenance simple and easy.  And it
also adds four new codecs; big5hkscs, euc-jis-2004, shift-jis-2004
and iso2022-jp-2004.

13 files changed, 79 insertions, 51 deletions

diff --git a/Lib/test/cjkencodings_test.py b/Lib/test/cjkencodings_test.py
index 8bd0ee7933..f5f9a3779a 100644
--- a/Lib/test/cjkencodings_test.py
+++ b/Lib/test/cjkencodings_test.py
@@ -63,6 +63,9 @@ teststring = {
 "\x88\x91\xe5\x80\x91\xe6\x89\x80\x0a\xe8\xa6\x81\xe8\xa8\x8e\xe8"
 "\xab\x96\xe7\x9a\x84\xe5\x95\x8f\xe9\xa1\x8c\xe5\xb0\xb1\xe6\x98"
 "\xaf\x3a\x0a\x0a"),
+'big5hkscs': (
+"\x88\x45\x88\x5c\x8a\x73\x8b\xda\x8d\xd8\x0a",
+"\xf0\xa0\x84\x8c\xc4\x9a\xe9\xb5\xae\xe7\xbd\x93\xe6\xb4\x86\x0a"),
 'cp949': (
 "\x8c\x63\xb9\xe6\xb0\xa2\xc7\xcf\x20\xbc\x84\xbd\xc3\xc4\xdd\xb6"
 "\xf3\x0a\x0a\xa8\xc0\xa8\xc0\xb3\xb3\x21\x21\x20\xec\xd7\xce\xfa"
diff --git a/Lib/test/test_codecencodings_cn.py b/Lib/test/test_codecencodings_cn.py
index 269e00ae48..6d71218972 100644
--- a/Lib/test/test_codecencodings_cn.py
+++ b/Lib/test/test_codecencodings_cn.py
@@ -3,7 +3,7 @@
 # test_codecencodings_cn.py
 #   Codec encoding tests for PRC encodings.
 #
-# $CJKCodecs: test_codecencodings_cn.py,v 1.1 2003/12/19 03:00:05 perky Exp $
+# $CJKCodecs: test_codecencodings_cn.py,v 1.2 2004/06/19 06:09:55 perky Exp $
 
 from test import test_support
 from test import test_multibytecodec_support
@@ -27,8 +27,8 @@ class Test_GBK(test_multibytecodec_support.TestBase, unittest.TestCase):
     tstring = test_multibytecodec_support.load_teststring('gbk')
     codectests = (
         # invalid bytes
-        ("abc\x80\x80\xc1\xc4", "strict",  None),
-        ("abc\xc8", "strict",  None),
+        ("abc\x80\x80\xc1\xc4", "strict",  None),  
+        ("abc\xc8", "strict",  None),  
         ("abc\x80\x80\xc1\xc4", "replace", u"abc\ufffd\u804a"),
         ("abc\x80\x80\xc1\xc4\xc8", "replace", u"abc\ufffd\u804a\ufffd"),
         ("abc\x80\x80\xc1\xc4", "ignore",  u"abc\u804a"),
diff --git a/Lib/test/test_codecencodings_hk.py b/Lib/test/test_codecencodings_hk.py
new file mode 100644
index 0000000000..e7fad90888
--- /dev/null
+++ b/Lib/test/test_codecencodings_hk.py
@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+#
+# test_codecencodings_hk.py
+#   Codec encoding tests for HongKong encodings.
+#
+# $CJKCodecs: test_codecencodings_hk.py,v 1.1 2004/07/10 17:35:20 perky Exp $
+
+from test import test_support
+from test import test_multibytecodec_support
+import unittest
+
+class Test_Big5HKSCS(test_multibytecodec_support.TestBase, unittest.TestCase):
+    encoding = 'big5hkscs'
+    tstring = test_multibytecodec_support.load_teststring('big5hkscs')
+    codectests = (
+        # invalid bytes
+        ("abc\x80\x80\xc1\xc4", "strict",  None),
+        ("abc\xc8", "strict",  None),
+        ("abc\x80\x80\xc1\xc4", "replace", u"abc\ufffd\u8b10"),
+        ("abc\x80\x80\xc1\xc4\xc8", "replace", u"abc\ufffd\u8b10\ufffd"),
+        ("abc\x80\x80\xc1\xc4", "ignore",  u"abc\u8b10"),
+    )
+
+def test_main():
+    suite = unittest.TestSuite()
+    suite.addTest(unittest.makeSuite(Test_Big5HKSCS))
+    test_support.run_suite(suite)
+
+if __name__ == "__main__":
+    test_main()
diff --git a/Lib/test/test_codecencodings_jp.py b/Lib/test/test_codecencodings_jp.py
index f5745116e7..483b7dbe12 100644
--- a/Lib/test/test_codecencodings_jp.py
+++ b/Lib/test/test_codecencodings_jp.py
@@ -3,7 +3,7 @@
 # test_codecencodings_jp.py
 #   Codec encoding tests for Japanese encodings.
 #
-# $CJKCodecs: test_codecencodings_jp.py,v 1.2 2004/01/06 09:25:37 perky Exp $
+# $CJKCodecs: test_codecencodings_jp.py,v 1.3 2004/06/19 06:09:55 perky Exp $
 
 from test import test_support
 from test import test_multibytecodec_support
@@ -64,16 +64,6 @@ class Test_EUC_JP_COMPAT(test_multibytecodec_support.TestBase,
         (u"\u203e", "strict", "\x7e"),
     )
 
-class Test_EUC_JP_STRICT(test_multibytecodec_support.TestBase,
-                         unittest.TestCase):
-    encoding = 'euc_jp_strict'
-    tstring = test_multibytecodec_support.load_teststring('euc_jp')
-    codectests = eucjp_commontests + (
-        ("\xa1\xc0\\", "strict", u"\\\\"),
-        (u"\xa5", "strict", None),
-        (u"\u203e", "strict", None),
-    )
-
 shiftjis_commonenctests = (
     ("abc\x80\x80\x82\x84", "strict",  None),
     ("abc\xf8", "strict",  None),
@@ -90,14 +80,6 @@ class Test_SJIS_COMPAT(test_multibytecodec_support.TestBase, unittest.TestCase):
         ("\x81\x5f\x81\x61\x81\x7c", "strict", u"\uff3c\u2016\u2212"),
     )
 
-class Test_SJIS_STRICT(test_multibytecodec_support.TestBase, unittest.TestCase):
-    encoding = 'shift_jis_strict'
-    tstring = test_multibytecodec_support.load_teststring('shift_jis')
-    codectests = shiftjis_commonenctests + (
-        ("\\\x7e", "replace", u"\xa5\u203e"),
-        ("\x81\x5f\x81\x61\x81\x7c", "replace", u"\x5c\u2016\u2212"),
-    )
-
 class Test_SJISX0213(test_multibytecodec_support.TestBase, unittest.TestCase):
     encoding = 'shift_jisx0213'
     tstring = test_multibytecodec_support.load_teststring('shift_jisx0213')
@@ -123,9 +105,6 @@ def test_main():
     suite.addTest(unittest.makeSuite(Test_EUC_JISX0213))
     suite.addTest(unittest.makeSuite(Test_EUC_JP_COMPAT))
     suite.addTest(unittest.makeSuite(Test_SJIS_COMPAT))
-    if test_multibytecodec_support.__cjkcodecs__:
-        suite.addTest(unittest.makeSuite(Test_EUC_JP_STRICT))
-        suite.addTest(unittest.makeSuite(Test_SJIS_STRICT))
     suite.addTest(unittest.makeSuite(Test_SJISX0213))
     test_support.run_suite(suite)
 
diff --git a/Lib/test/test_codecencodings_kr.py b/Lib/test/test_codecencodings_kr.py
index 449237d3b1..489c9f1d7e 100644
--- a/Lib/test/test_codecencodings_kr.py
+++ b/Lib/test/test_codecencodings_kr.py
@@ -3,7 +3,7 @@
 # test_codecencodings_kr.py
 #   Codec encoding tests for ROK encodings.
 #
-# $CJKCodecs: test_codecencodings_kr.py,v 1.1 2003/12/19 03:00:06 perky Exp $
+# $CJKCodecs: test_codecencodings_kr.py,v 1.2 2004/06/19 06:09:55 perky Exp $
 
 from test import test_support
 from test import test_multibytecodec_support
diff --git a/Lib/test/test_codecencodings_tw.py b/Lib/test/test_codecencodings_tw.py
index 897f060605..fb8a4d0c84 100644
--- a/Lib/test/test_codecencodings_tw.py
+++ b/Lib/test/test_codecencodings_tw.py
@@ -3,7 +3,7 @@
 # test_codecencodings_tw.py
 #   Codec encoding tests for ROC encodings.
 #
-# $CJKCodecs: test_codecencodings_tw.py,v 1.1 2003/12/19 03:00:06 perky Exp $
+# $CJKCodecs: test_codecencodings_tw.py,v 1.2 2004/06/19 06:09:55 perky Exp $
 
 from test import test_support
 from test import test_multibytecodec_support
diff --git a/Lib/test/test_codecmaps_cn.py b/Lib/test/test_codecmaps_cn.py
index 337ac912ea..31871e2a2e 100644
--- a/Lib/test/test_codecmaps_cn.py
+++ b/Lib/test/test_codecmaps_cn.py
@@ -3,7 +3,7 @@
 # test_codecmaps_cn.py
 #   Codec mapping tests for PRC encodings
 #
-# $CJKCodecs: test_codecmaps_cn.py,v 1.2 2004/01/17 12:47:19 perky Exp $
+# $CJKCodecs: test_codecmaps_cn.py,v 1.3 2004/06/19 06:09:55 perky Exp $
 
 from test import test_support
 from test import test_multibytecodec_support
diff --git a/Lib/test/test_codecmaps_hk.py b/Lib/test/test_codecmaps_hk.py
new file mode 100644
index 0000000000..43137b643a
--- /dev/null
+++ b/Lib/test/test_codecmaps_hk.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+#
+# test_codecmaps_hk.py
+#   Codec mapping tests for HongKong encodings
+#
+# $CJKCodecs: test_codecmaps_hk.py,v 1.1 2004/07/10 17:35:20 perky Exp $
+
+from test import test_support
+from test import test_multibytecodec_support
+import unittest
+
+class TestBig5HKSCSMap(test_multibytecodec_support.TestBase_Mapping,
+                       unittest.TestCase):
+    encoding = 'big5hkscs'
+    mapfilename = 'BIG5HKSCS.TXT'
+    mapfileurl = 'http://people.freebsd.org/~perky/i18n/BIG5HKSCS.TXT'
+
+def test_main():
+    suite = unittest.TestSuite()
+    suite.addTest(unittest.makeSuite(TestBig5HKSCSMap))
+    test_support.run_suite(suite)
+
+test_multibytecodec_support.register_skip_expected(TestBig5HKSCSMap)
+if __name__ == "__main__":
+    test_main()
diff --git a/Lib/test/test_codecmaps_jp.py b/Lib/test/test_codecmaps_jp.py
index 9519da2e5e..014a952578 100644
--- a/Lib/test/test_codecmaps_jp.py
+++ b/Lib/test/test_codecmaps_jp.py
@@ -3,7 +3,7 @@
 # test_codecmaps_jp.py
 #   Codec mapping tests for Japanese encodings
 #
-# $CJKCodecs: test_codecmaps_jp.py,v 1.2 2004/01/17 12:47:19 perky Exp $
+# $CJKCodecs: test_codecmaps_jp.py,v 1.3 2004/06/19 06:09:55 perky Exp $
 
 from test import test_support
 from test import test_multibytecodec_support
@@ -48,15 +48,6 @@ class TestSJISCOMPATMap(test_multibytecodec_support.TestBase_Mapping,
         ('\x81_', u'\\'),
     ]
 
-
-class TestSJISSTRICTMap(test_multibytecodec_support.TestBase_Mapping,
-                        unittest.TestCase):
-    encoding = 'shift_jis_strict'
-    mapfilename = 'SHIFTJIS.TXT'
-    mapfileurl = 'http://www.unicode.org/Public/MAPPINGS/OBSOLETE' \
-                 '/EASTASIA/JIS/SHIFTJIS.TXT'
-
-
 class TestEUCJISX0213Map(test_multibytecodec_support.TestBase_Mapping,
                          unittest.TestCase):
     encoding = 'euc_jisx0213'
@@ -76,8 +67,6 @@ def test_main():
     suite.addTest(unittest.makeSuite(TestCP932Map))
     suite.addTest(unittest.makeSuite(TestEUCJPCOMPATMap))
     suite.addTest(unittest.makeSuite(TestSJISCOMPATMap))
-    if test_multibytecodec_support.__cjkcodecs__:
-        suite.addTest(unittest.makeSuite(TestSJISSTRICTMap))
     suite.addTest(unittest.makeSuite(TestEUCJISX0213Map))
     suite.addTest(unittest.makeSuite(TestSJISX0213Map))
     test_support.run_suite(suite)
@@ -85,5 +74,6 @@ def test_main():
 test_multibytecodec_support.register_skip_expected(TestCP932Map,
     TestEUCJPCOMPATMap, TestSJISCOMPATMap, TestEUCJISX0213Map,
     TestSJISX0213Map)
+
 if __name__ == "__main__":
     test_main()
diff --git a/Lib/test/test_codecmaps_kr.py b/Lib/test/test_codecmaps_kr.py
index 9a71939158..be198113e9 100644
--- a/Lib/test/test_codecmaps_kr.py
+++ b/Lib/test/test_codecmaps_kr.py
@@ -3,7 +3,7 @@
 # test_codecmaps_kr.py
 #   Codec mapping tests for ROK encodings
 #
-# $CJKCodecs: test_codecmaps_kr.py,v 1.2 2004/01/17 12:47:19 perky Exp $
+# $CJKCodecs: test_codecmaps_kr.py,v 1.3 2004/06/19 06:09:55 perky Exp $
 
 from test import test_support
 from test import test_multibytecodec_support
diff --git a/Lib/test/test_codecmaps_tw.py b/Lib/test/test_codecmaps_tw.py
index 2f416605c7..61d83cd5fa 100644
--- a/Lib/test/test_codecmaps_tw.py
+++ b/Lib/test/test_codecmaps_tw.py
@@ -3,7 +3,7 @@
 # test_codecmaps_tw.py
 #   Codec mapping tests for ROC encodings
 #
-# $CJKCodecs: test_codecmaps_tw.py,v 1.2 2004/01/17 12:47:19 perky Exp $
+# $CJKCodecs: test_codecmaps_tw.py,v 1.3 2004/06/19 06:09:55 perky Exp $
 
 from test import test_support
 from test import test_multibytecodec_support
diff --git a/Lib/test/test_multibytecodec.py b/Lib/test/test_multibytecodec.py
index e96afba229..acf333e91e 100644
--- a/Lib/test/test_multibytecodec.py
+++ b/Lib/test/test_multibytecodec.py
@@ -3,7 +3,7 @@
 # test_multibytecodec.py
 #   Unit test for multibytecodec itself
 #
-# $CJKCodecs: test_multibytecodec.py,v 1.5 2004/01/06 02:26:28 perky Exp $
+# $CJKCodecs: test_multibytecodec.py,v 1.8 2004/06/19 06:09:55 perky Exp $
 
 from test import test_support
 from test import test_multibytecodec_support
@@ -69,6 +69,12 @@ class Test_StreamWriter(unittest.TestCase):
     def test_str_decode(self):
         self.assertEqual('abcd'.encode('gb18030'), 'abcd')
 
+    def test_streamwriter_strwrite(self):
+        s = StringIO.StringIO()
+        wr = codecs.getwriter('gb18030')(s)
+        wr.write('abcd')
+        self.assertEqual(s.getvalue(), 'abcd')
+
 def test_main():
     suite = unittest.TestSuite()
     suite.addTest(unittest.makeSuite(Test_StreamWriter))
diff --git a/Lib/test/test_multibytecodec_support.py b/Lib/test/test_multibytecodec_support.py
index f471adf965..22b52f7066 100644
--- a/Lib/test/test_multibytecodec_support.py
+++ b/Lib/test/test_multibytecodec_support.py
@@ -3,7 +3,7 @@
 # test_multibytecodec_support.py
 #   Common Unittest Routines for CJK codecs
 #
-# $CJKCodecs: test_multibytecodec_support.py,v 1.5 2004/01/17 12:47:19 perky Exp $
+# $CJKCodecs: test_multibytecodec_support.py,v 1.6 2004/06/19 06:09:55 perky Exp $
 
 import sys, codecs, os.path
 import unittest
@@ -164,13 +164,8 @@ class TestBase_Mapping(unittest.TestCase):
     def __init__(self, *args, **kw):
         unittest.TestCase.__init__(self, *args, **kw)
         if not os.path.exists(self.mapfilename):
-            parent = os.path.join(os.pardir, self.mapfilename)
-            if not os.path.exists(parent):
-                format = '%s not found, download from %s'
-                raise test_support.TestSkipped(format %
-                        (self.mapfilename, self.mapfileurl))
-            else:
-                self.mapfilename = parent
+            raise test_support.TestSkipped('%s not found, download from %s' %
+                    (self.mapfilename, self.mapfileurl))
 
     def test_mapping_file(self):
         unichrs = lambda s: u''.join(map(unichr, map(eval, s.split('+'))))