Fix/update import of language modules (#385).

Docutils loads language modules from its own set or the working
directory. (The latter is an undocumented but tested feature.)
However, a Python file or a directory that is named like an
unsupported language (e.g. "tr") led to errors (#385).

* Use importlib.import_module() instead of the low-level __import__().

* Use a common class for docutils.languages.get_language() and
  docutils.parsers.rst.get_language().

* Report missing language support also for parsers.rst.get_language().

cf. https://docutils.sourceforge.io/docs/howto/i18n.html

git-svn-id: http://svn.code.sf.net/p/docutils/code/trunk@8465 929543f6-e4f2-0310-98a6-ba3bd3dd1d04

3 files changed, 74 insertions, 44 deletions

diff --git a/docutils/docutils/languages/__init__.py b/docutils/docutils/languages/__init__.py
index 2edf9d5d6..8b871f7d8 100644
--- a/docutils/docutils/languages/__init__.py
+++ b/docutils/docutils/languages/__init__.py
@@ -12,36 +12,65 @@ This package contains modules for language-dependent features of Docutils.
 __docformat__ = 'reStructuredText'
 
 import sys
+from importlib import import_module
 
 from docutils.utils import normalize_language_tag
 
+class LanguageImporter(object):
 
-_languages = {}
+    packages = ('docutils.languages.', '')
+    warn_msg = ('Language "%s" not supported: '
+                'Docutils-generated text will be in English.')
+    fallback = 'en'
+    # TODO: use a dummy module returning emtpy strings?, configurable?
 
-def get_language(language_code, reporter=None):
-    """Return module with language localizations.
+    def __init__(self):
+        self.cache = {}
 
-    `language_code` is a "BCP 47" language tag.
-    If there is no matching module, warn and fall back to English.
-    """
-    # TODO: use a dummy module returning emtpy strings?, configurable?
-    for tag in normalize_language_tag(language_code):
-        tag = tag.replace('-', '_') # '-' not valid in module names
-        if tag in _languages:
-            return _languages[tag]
+    def import_from_packages(self, name):
+        """Try loading module `name` from `self.packages`."""
         try:
-            module = __import__(tag, globals(), locals(), level=1)
-        except ImportError:
+            return self.cache[name]
+        except KeyError:
+            pass
+        for package in self.packages:
             try:
-                module = __import__(tag, globals(), locals(), level=0)
-            except ImportError:
+                module = import_module(package+name)
+                self.check_content(module)
+            except (ImportError, AttributeError, AssertionError):
                 continue
-        _languages[tag] = module
+            self.cache[name] = module
+            return module
+        return None
+
+    def check_content(self, module):
+        """Check if we got a Docutils language module."""
+        assert isinstance(module.labels, dict)
+        assert isinstance(module.bibliographic_fields, dict)
+        assert isinstance(module.author_separators, list)
+
+    def __call__(self, language_code, reporter=None):
+        try:
+            return self.cache[language_code]
+        except KeyError:
+            pass
+        for tag in normalize_language_tag(language_code):
+            tag = tag.replace('-', '_') # '-' not valid in module names
+            module = self.import_from_packages(tag)
+            if module is not None:
+                return module
+        if reporter is not None:
+            reporter.warning(self.warn_msg % language_code)
+        module = self.import_from_packages('en')
+        self.cache[tag] = module # warn only one time!
         return module
-    if reporter is not None:
-        reporter.warning(
-            'language "%s" not supported: ' % language_code +
-            'Docutils-generated text will be in English.')
-    module = __import__('en', globals(), locals(), level=1)
-    _languages[tag] = module # warn only one time!
-    return module
+
+
+get_language = LanguageImporter()
+"""Return module with language localizations
+
+from `docutils.languages` or the PYTHONPATH.
+
+The argument `language_code` is a "BCP 47" language tag.
+If there is no matching module, warn and fall back to English.
+"""
diff --git a/docutils/docutils/parsers/rst/languages/__init__.py b/docutils/docutils/parsers/rst/languages/__init__.py
index 98a9d8b8b..1227155cb 100644
--- a/docutils/docutils/parsers/rst/languages/__init__.py
+++ b/docutils/docutils/parsers/rst/languages/__init__.py
@@ -14,23 +14,24 @@ __docformat__ = 'reStructuredText'
 
 import sys
 
-from docutils.utils import normalize_language_tag
-
-
-_languages = {}
-
-def get_language(language_code):
-    for tag in normalize_language_tag(language_code):
-        tag = tag.replace('-', '_') # '-' not valid in module names
-        if tag in _languages:
-            return _languages[tag]
-        try:
-            module = __import__(tag, globals(), locals(), level=1)
-        except ImportError:
-            try:
-                module = __import__(tag, globals(), locals(), level=0)
-            except ImportError:
-                continue
-        _languages[tag] = module
-        return module
-    return None
+from docutils.languages import LanguageImporter
+
+class RstLanguageImporter(LanguageImporter):
+    packages = ('docutils.parsers.rst.languages.', '')
+    warn_msg = 'rST localisation for language "%s" not found.'
+    fallback = None
+    
+    def check_content(self, module):
+        """Check if we got an rST language module."""
+        assert isinstance(module.directives, dict)
+        assert isinstance(module.roles, dict)
+        
+get_language = RstLanguageImporter()
+"""Return module with language localizations for reStructuredText.
+
+Get translated directive and rolenames from `docutils.parsers.rst.languages`
+or the PYTHONPATH.
+
+The argument `language_code` is a "BCP 47" language tag.
+If there is no matching module, warn and return None.
+"""
diff --git a/docutils/docutils/parsers/rst/states.py b/docutils/docutils/parsers/rst/states.py
index 4e338b5a0..15ae909ee 100644
--- a/docutils/docutils/parsers/rst/states.py
+++ b/docutils/docutils/parsers/rst/states.py
@@ -151,7 +151,7 @@ class RSTStateMachine(StateMachineWS):
         run the StateMachine.
         """
         self.language = languages.get_language(
-            document.settings.language_code)
+            document.settings.language_code, document.reporter)
         self.match_titles = match_titles
         if inliner is None:
             inliner = Inliner()