Support character sets.maurits/character-sets

sr@Latn and sr@Cyrl will be added to language sr.
See https://github.com/collective/plone.app.locales/issues/326

11 files changed, 124 insertions, 0 deletions

diff --git a/.meta.toml b/.meta.toml
index 877be7c..53d84cb 100644
--- a/.meta.toml
+++ b/.meta.toml
@@ -41,4 +41,7 @@ ignore-bad-ideas = [
     "src/zope/i18n/tests/locale2/en/LC_MESSAGES/zope-i18n.mo",
     "src/zope/i18n/tests/locale3/en/LC_MESSAGES/zope-i18n.mo",
     "src/zope/i18n/tests/pl-default.mo",
+    "src/zope/i18n/tests/sr-default.mo",
+    "src/zope/i18n/tests/sr@Cyrl-default.mo",
+    "src/zope/i18n/tests/sr@Latn-default.mo",
     ]
diff --git a/CHANGES.rst b/CHANGES.rst
index c554ca2..2dad4eb 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -5,6 +5,12 @@
 4.7.1 (unreleased)
 ==================
 
+- Support character sets.
+  Example: ``sr@Latn`` and ``sr@Cyrl`` will be added to language ``sr`` (Serbian).
+  See https://github.com/collective/plone.app.locales/issues/326
+  You can choose which one to use by setting either ``sr@Latn`` or ``sr@Cyrl``
+  in environment variable ``zope_i18n_allowed_languages``.
+
 - Support and test Python 3.8 and 3.9.
   Full supported list is now: 2.7, 3.5, 3.6, 3.7, 3.8, 3.9, PyPy, PyPy3.
 
diff --git a/setup.cfg b/setup.cfg
index 43050e9..e159e42 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -21,3 +21,6 @@ ignore-bad-ideas =
     src/zope/i18n/tests/locale2/en/LC_MESSAGES/zope-i18n.mo
     src/zope/i18n/tests/locale3/en/LC_MESSAGES/zope-i18n.mo
     src/zope/i18n/tests/pl-default.mo
+    src/zope/i18n/tests/sr-default.mo
+    src/zope/i18n/tests/sr@Cyrl-default.mo
+    src/zope/i18n/tests/sr@Latn-default.mo
diff --git a/src/zope/i18n/tests/sr-default.mo b/src/zope/i18n/tests/sr-default.mo
new file mode 100644
index 0000000..28aba76
--- /dev/null
+++ b/src/zope/i18n/tests/sr-default.mo
diff --git a/src/zope/i18n/tests/sr-default.po b/src/zope/i18n/tests/sr-default.po
new file mode 100644
index 0000000..c8366f2
--- /dev/null
+++ b/src/zope/i18n/tests/sr-default.po
@@ -0,0 +1,12 @@
+msgid ""
+msgstr ""
+"Project-Id-Version: Zope 5\n"
+"PO-Revision-Date: 2021/09/02\n"
+"Last-Translator: Zope 3 contributors\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Plural-Forms: nplurals=2; plural=n != 1;\n"
+
+msgid "short_greeting"
+msgstr "Hello in Serbian Standard!"
diff --git a/src/zope/i18n/tests/sr@Cyrl-default.mo b/src/zope/i18n/tests/sr@Cyrl-default.mo
new file mode 100644
index 0000000..fef8012
--- /dev/null
+++ b/src/zope/i18n/tests/sr@Cyrl-default.mo
diff --git a/src/zope/i18n/tests/sr@Cyrl-default.po b/src/zope/i18n/tests/sr@Cyrl-default.po
new file mode 100644
index 0000000..7ffcaf7
--- /dev/null
+++ b/src/zope/i18n/tests/sr@Cyrl-default.po
@@ -0,0 +1,12 @@
+msgid ""
+msgstr ""
+"Project-Id-Version: Zope 5\n"
+"PO-Revision-Date: 2021/09/02\n"
+"Last-Translator: Zope 3 contributors\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Plural-Forms: nplurals=2; plural=n != 1;\n"
+
+msgid "short_greeting"
+msgstr "Hello in српски!"
diff --git a/src/zope/i18n/tests/sr@Latn-default.mo b/src/zope/i18n/tests/sr@Latn-default.mo
new file mode 100644
index 0000000..51173d7
--- /dev/null
+++ b/src/zope/i18n/tests/sr@Latn-default.mo
diff --git a/src/zope/i18n/tests/sr@Latn-default.po b/src/zope/i18n/tests/sr@Latn-default.po
new file mode 100644
index 0000000..65b1c97
--- /dev/null
+++ b/src/zope/i18n/tests/sr@Latn-default.po
@@ -0,0 +1,12 @@
+msgid ""
+msgstr ""
+"Project-Id-Version: Zope 5\n"
+"PO-Revision-Date: 2021/09/02\n"
+"Last-Translator: Zope 3 contributors\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Plural-Forms: nplurals=2; plural=n != 1;\n"
+
+msgid "short_greeting"
+msgstr "Hello in Serbian Latin!"
diff --git a/src/zope/i18n/tests/test_translationdomain.py b/src/zope/i18n/tests/test_translationdomain.py
index 4687107..2018427 100644
--- a/src/zope/i18n/tests/test_translationdomain.py
+++ b/src/zope/i18n/tests/test_translationdomain.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 ##############################################################################
 #
 # Copyright (c) 2001-2008 Zope Foundation and Contributors.
@@ -226,3 +227,73 @@ class TestGlobalTranslationDomain(TestITranslationDomain, unittest.TestCase):
 
         with self.assertRaises(KeyError):
             self._domain.reloadCatalogs(('dne',))
+
+    def test_character_sets(self):
+        """Test two character sets for the same language.
+
+        Serbian can be written in Latin or Cyrillic,
+        where Latin is currently most used.
+        Interestingly, every Latin character can actually be mapped to
+        one Cyrillic character, and the other way around,
+        so you could write a script to turn one po-file into the other.
+
+        But most practical is to have two locales with names
+        sr@Latn and sr@Cyrl.  These are then two character sets
+        for the same language.  So they should end up together
+        under the same language in a translation domain.
+
+        The best way for an integrator to choose which one to use,
+        is to add an environment variable 'zope_i18n_allowed_languages'
+        and let this contain either sr@Latn or sr@Cyrl.
+
+        See https://github.com/collective/plone.app.locales/issues/326
+        """
+        standard_file = os.path.join(testdir, 'sr-default.mo')
+        latin_file = os.path.join(testdir, 'sr@Latn-default.mo')
+        cyrillic_file = os.path.join(testdir, 'sr@Cyrl-default.mo')
+        standard_catalog = GettextMessageCatalog('sr', 'char', standard_file)
+        latin_catalog = GettextMessageCatalog('sr@Latn', 'char', latin_file)
+        cyrillic_catalog = GettextMessageCatalog(
+            'sr@Cyrl', 'char', cyrillic_file
+        )
+
+        # Test the standard file.
+        domain = TranslationDomain('char')
+        domain.addCatalog(standard_catalog)
+        self.assertEqual(
+            domain.translate('short_greeting', target_language='sr'),
+            u"Hello in Serbian Standard!",
+        )
+
+        # Test the Latin file.
+        domain = TranslationDomain('char')
+        domain.addCatalog(latin_catalog)
+        self.assertEqual(
+            domain.translate('short_greeting', target_language='sr'),
+            u"Hello in Serbian Latin!",
+        )
+        # Note that sr@Latn is not recognizes as language id.
+        self.assertEqual(
+            domain.translate('short_greeting', target_language='sr@Latn'),
+            u"short_greeting",
+        )
+
+        # Test the Cyrillic file.
+        domain = TranslationDomain('char')
+        domain.addCatalog(cyrillic_catalog)
+        self.assertEqual(
+            domain.translate('short_greeting', target_language='sr'),
+            u"Hello in српски!",
+        )
+
+        # When I have all three locales, this is the order that
+        # os.listdir gives them in:
+        domain = TranslationDomain('char')
+        domain.addCatalog(latin_catalog)
+        domain.addCatalog(cyrillic_catalog)
+        domain.addCatalog(standard_catalog)
+        # The Latin one is first, so it wins.
+        self.assertEqual(
+            domain.translate('short_greeting', target_language='sr'),
+            u"Hello in Serbian Latin!",
+        )
diff --git a/src/zope/i18n/translationdomain.py b/src/zope/i18n/translationdomain.py
index 9915deb..70c1cc0 100644
--- a/src/zope/i18n/translationdomain.py
+++ b/src/zope/i18n/translationdomain.py
@@ -50,6 +50,11 @@ class TranslationDomain(object):
 
     def _registerMessageCatalog(self, language, catalog_name):
         key = language
+        if "@" in key:
+            # sr@Latn and sr@Cyrl are two character set variants of
+            # the same Serbian language.
+            # See https://github.com/collective/plone.app.locales/issues/326
+            key = key.split("@")[0]
         mc = self._catalogs.setdefault(key, [])
         mc.append(catalog_name)