Sync gettextutils from oslo-incubator 997ab277

Keystone's gettextutils package was behind oslo-incubator's
version. This syncronizes gettextutils from oslo-incubator commit
997ab277 .

bp user-locale-api

Change-Id: If9cf0c83df6d6c8db844a30c605f60e907abcbdd

4 files changed, 256 insertions, 231 deletions

diff --git a/keystone/common/wsgi.py b/keystone/common/wsgi.py
index 6e2731123..4b8a6e58f 100644
--- a/keystone/common/wsgi.py
+++ b/keystone/common/wsgi.py
@@ -23,6 +23,7 @@
 import re
 
 import routes.middleware
+import six
 import webob.dec
 import webob.exc
 
@@ -559,11 +560,18 @@ def render_response(body=None, status=None, headers=None):
 
 def render_exception(error, user_locale=None):
     """Forms a WSGI response based on the current error."""
+
+    error_message = error.args[0]
+    message = gettextutils.translate(error_message, desired_locale=user_locale)
+    if message is error_message:
+        # translate() didn't do anything because it wasn't a Message,
+        # convert to a string.
+        message = six.text_type(message)
+
     body = {'error': {
         'code': error.code,
         'title': error.title,
-        'message': unicode(gettextutils.get_localized_message(error.args[0],
-                                                              user_locale)),
+        'message': message,
     }}
     headers = []
     if isinstance(error, exception.AuthPluginException):
diff --git a/keystone/openstack/common/gettextutils.py b/keystone/openstack/common/gettextutils.py
index b248d9115..3dc17a611 100644
--- a/keystone/openstack/common/gettextutils.py
+++ b/keystone/openstack/common/gettextutils.py
@@ -24,13 +24,10 @@ Usual usage in an openstack.common module:
 
 import copy
 import gettext
-import logging
+import locale
+from logging import handlers
 import os
 import re
-try:
-    import UserString as _userString
-except ImportError:
-    import collections as _userString
 
 from babel import localedata
 import six
@@ -56,7 +53,7 @@ def enable_lazy():
 
 def _(msg):
     if USE_LAZY:
-        return Message(msg, 'keystone')
+        return Message(msg, domain='keystone')
     else:
         if six.PY3:
             return _t.gettext(msg)
@@ -88,11 +85,6 @@ def install(domain, lazy=False):
         # messages in OpenStack. We override the standard _() function
         # and % (format string) operation to build Message objects that can
         # later be translated when we have more information.
-        #
-        # Also included below is an example LocaleHandler that translates
-        # Messages to an associated locale, effectively allowing many logs,
-        # each with their own locale.
-
         def _lazy_gettext(msg):
             """Create and return a Message object.
 
@@ -103,7 +95,7 @@ def install(domain, lazy=False):
             Message encapsulates a string so that we can translate
             it later when needed.
             """
-            return Message(msg, domain)
+            return Message(msg, domain=domain)
 
         from six import moves
         moves.builtins.__dict__['_'] = _lazy_gettext
@@ -118,182 +110,158 @@ def install(domain, lazy=False):
                             unicode=True)
 
 
-class Message(_userString.UserString, object):
-    """Class used to encapsulate translatable messages."""
-    def __init__(self, msg, domain):
-        # _msg is the gettext msgid and should never change
-        self._msg = msg
-        self._left_extra_msg = ''
-        self._right_extra_msg = ''
-        self._locale = None
-        self.params = None
-        self.domain = domain
-
-    @property
-    def data(self):
-        # NOTE(mrodden): this should always resolve to a unicode string
-        # that best represents the state of the message currently
-
-        localedir = os.environ.get(self.domain.upper() + '_LOCALEDIR')
-        if self.locale:
-            lang = gettext.translation(self.domain,
-                                       localedir=localedir,
-                                       languages=[self.locale],
-                                       fallback=True)
-        else:
-            # use system locale for translations
-            lang = gettext.translation(self.domain,
-                                       localedir=localedir,
-                                       fallback=True)
+class Message(six.text_type):
+    """A Message object is a unicode object that can be translated.
+
+    Translation of Message is done explicitly using the translate() method.
+    For all non-translation intents and purposes, a Message is simply unicode,
+    and can be treated as such.
+    """
 
+    def __new__(cls, msgid, msgtext=None, params=None, domain='keystone', *args):
+        """Create a new Message object.
+
+        In order for translation to work gettext requires a message ID, this
+        msgid will be used as the base unicode text. It is also possible
+        for the msgid and the base unicode text to be different by passing
+        the msgtext parameter.
+        """
+        # If the base msgtext is not given, we use the default translation
+        # of the msgid (which is in English) just in case the system locale is
+        # not English, so that the base text will be in that locale by default.
+        if not msgtext:
+            msgtext = Message._translate_msgid(msgid, domain)
+        # We want to initialize the parent unicode with the actual object that
+        # would have been plain unicode if 'Message' was not enabled.
+        msg = super(Message, cls).__new__(cls, msgtext)
+        msg.msgid = msgid
+        msg.domain = domain
+        msg.params = params
+        return msg
+
+    def translate(self, desired_locale=None):
+        """Translate this message to the desired locale.
+
+        :param desired_locale: The desired locale to translate the message to,
+                               if no locale is provided the message will be
+                               translated to the system's default locale.
+
+        :returns: the translated message in unicode
+        """
+
+        translated_message = Message._translate_msgid(self.msgid,
+                                                      self.domain,
+                                                      desired_locale)
+        if self.params is None:
+            # No need for more translation
+            return translated_message
+
+        # This Message object may have been formatted with one or more
+        # Message objects as substitution arguments, given either as a single
+        # argument, part of a tuple, or as one or more values in a dictionary.
+        # When translating this Message we need to translate those Messages too
+        translated_params = _translate_args(self.params, desired_locale)
+
+        translated_message = translated_message % translated_params
+
+        return translated_message
+
+    @staticmethod
+    def _translate_msgid(msgid, domain, desired_locale=None):
+        if not desired_locale:
+            system_locale = locale.getdefaultlocale()
+            # If the system locale is not available to the runtime use English
+            if not system_locale[0]:
+                desired_locale = 'en_US'
+            else:
+                desired_locale = system_locale[0]
+
+        locale_dir = os.environ.get(domain.upper() + '_LOCALEDIR')
+        lang = gettext.translation(domain,
+                                   localedir=locale_dir,
+                                   languages=[desired_locale],
+                                   fallback=True)
         if six.PY3:
-            ugettext = lang.gettext
-        else:
-            ugettext = lang.ugettext
-
-        full_msg = (self._left_extra_msg +
-                    ugettext(self._msg) +
-                    self._right_extra_msg)
-
-        if self.params is not None:
-            full_msg = full_msg % self.params
-
-        return six.text_type(full_msg)
-
-    @property
-    def locale(self):
-        return self._locale
-
-    @locale.setter
-    def locale(self, value):
-        self._locale = value
-        if not self.params:
-            return
-
-        # This Message object may have been constructed with one or more
-        # Message objects as substitution parameters, given as a single
-        # Message, or a tuple or Map containing some, so when setting the
-        # locale for this Message we need to set it for those Messages too.
-        if isinstance(self.params, Message):
-            self.params.locale = value
-            return
-        if isinstance(self.params, tuple):
-            for param in self.params:
-                if isinstance(param, Message):
-                    param.locale = value
-            return
-        if isinstance(self.params, dict):
-            for param in self.params.values():
-                if isinstance(param, Message):
-                    param.locale = value
-
-    def _save_dictionary_parameter(self, dict_param):
-        full_msg = self.data
-        # look for %(blah) fields in string;
-        # ignore %% and deal with the
-        # case where % is first character on the line
-        keys = re.findall('(?:[^%]|^)?%\((\w*)\)[a-z]', full_msg)
-
-        # if we don't find any %(blah) blocks but have a %s
-        if not keys and re.findall('(?:[^%]|^)%[a-z]', full_msg):
-            # apparently the full dictionary is the parameter
-            params = copy.deepcopy(dict_param)
+            translator = lang.gettext
         else:
-            params = {}
-            for key in keys:
-                try:
-                    params[key] = copy.deepcopy(dict_param[key])
-                except TypeError:
-                    # cast uncopyable thing to unicode string
-                    params[key] = six.text_type(dict_param[key])
+            translator = lang.ugettext
 
-        return params
+        translated_message = translator(msgid)
+        return translated_message
 
-    def _save_parameters(self, other):
-        # we check for None later to see if
-        # we actually have parameters to inject,
-        # so encapsulate if our parameter is actually None
+    def __mod__(self, other):
+        # When we mod a Message we want the actual operation to be performed
+        # by the parent class (i.e. unicode()), the only thing  we do here is
+        # save the original msgid and the parameters in case of a translation
+        unicode_mod = super(Message, self).__mod__(other)
+        modded = Message(self.msgid,
+                         msgtext=unicode_mod,
+                         params=self._sanitize_mod_params(other),
+                         domain=self.domain)
+        return modded
+
+    def _sanitize_mod_params(self, other):
+        """Sanitize the object being modded with this Message.
+
+        - Add support for modding 'None' so translation supports it
+        - Trim the modded object, which can be a large dictionary, to only
+        those keys that would actually be used in a translation
+        - Snapshot the object being modded, in case the message is
+        translated, it will be used as it was when the Message was created
+        """
         if other is None:
-            self.params = (other, )
+            params = (other,)
         elif isinstance(other, dict):
-            self.params = self._save_dictionary_parameter(other)
+            params = self._trim_dictionary_parameters(other)
         else:
-            # fallback to casting to unicode,
-            # this will handle the problematic python code-like
-            # objects that cannot be deep-copied
-            try:
-                self.params = copy.deepcopy(other)
-            except TypeError:
-                self.params = six.text_type(other)
-
-        return self
-
-    # overrides to be more string-like
-    def __unicode__(self):
-        return self.data
-
-    def __str__(self):
-        if six.PY3:
-            return self.__unicode__()
-        return self.data.encode('utf-8')
+            params = self._copy_param(other)
+        return params
 
-    def __getstate__(self):
-        to_copy = ['_msg', '_right_extra_msg', '_left_extra_msg',
-                   'domain', 'params', '_locale']
-        new_dict = self.__dict__.fromkeys(to_copy)
-        for attr in to_copy:
-            new_dict[attr] = copy.deepcopy(self.__dict__[attr])
+    def _trim_dictionary_parameters(self, dict_param):
+        """Return a dict that only has matching entries in the msgid."""
+        # NOTE(luisg): Here we trim down the dictionary passed as parameters
+        # to avoid carrying a lot of unnecessary weight around in the message
+        # object, for example if someone passes in Message() % locals() but
+        # only some params are used, and additionally we prevent errors for
+        # non-deepcopyable objects by unicoding() them.
+
+        # Look for %(param) keys in msgid;
+        # Skip %% and deal with the case where % is first character on the line
+        keys = re.findall('(?:[^%]|^)?%\((\w*)\)[a-z]', self.msgid)
+
+        # If we don't find any %(param) keys but have a %s
+        if not keys and re.findall('(?:[^%]|^)%[a-z]', self.msgid):
+            # Apparently the full dictionary is the parameter
+            params = self._copy_param(dict_param)
+        else:
+            params = {}
+            for key in keys:
+                params[key] = self._copy_param(dict_param[key])
 
-        return new_dict
+        return params
 
-    def __setstate__(self, state):
-        for (k, v) in state.items():
-            setattr(self, k, v)
+    def _copy_param(self, param):
+        try:
+            return copy.deepcopy(param)
+        except TypeError:
+            # Fallback to casting to unicode this will handle the
+            # python code-like objects that can't be deep-copied
+            return six.text_type(param)
 
-    # operator overloads
     def __add__(self, other):
-        copied = copy.deepcopy(self)
-        copied._right_extra_msg += other.__str__()
-        return copied
+        msg = _('Message objects do not support addition.')
+        raise TypeError(msg)
 
     def __radd__(self, other):
-        copied = copy.deepcopy(self)
-        copied._left_extra_msg += other.__str__()
-        return copied
+        return self.__add__(other)
 
-    def __mod__(self, other):
-        # do a format string to catch and raise
-        # any possible KeyErrors from missing parameters
-        self.data % other
-        copied = copy.deepcopy(self)
-        return copied._save_parameters(other)
-
-    def __mul__(self, other):
-        return self.data * other
-
-    def __rmul__(self, other):
-        return other * self.data
-
-    def __getitem__(self, key):
-        return self.data[key]
-
-    def __getslice__(self, start, end):
-        return self.data.__getslice__(start, end)
-
-    def __getattribute__(self, name):
-        # NOTE(mrodden): handle lossy operations that we can't deal with yet
-        # These override the UserString implementation, since UserString
-        # uses our __class__ attribute to try and build a new message
-        # after running the inner data string through the operation.
-        # At that point, we have lost the gettext message id and can just
-        # safely resolve to a string instead.
-        ops = ['capitalize', 'center', 'decode', 'encode',
-               'expandtabs', 'ljust', 'lstrip', 'replace', 'rjust', 'rstrip',
-               'strip', 'swapcase', 'title', 'translate', 'upper', 'zfill']
-        if name in ops:
-            return getattr(self.data, name)
-        else:
-            return _userString.UserString.__getattribute__(self, name)
+    def __str__(self):
+        # NOTE(luisg): Logging in python 2.6 tries to str() log records,
+        # and it expects specifically a UnicodeError in order to proceed.
+        msg = _('Message objects do not support str() because they may '
+                'contain non-ascii characters. '
+                'Please use unicode() or translate() instead.')
+        raise UnicodeError(msg)
 
 
 def get_available_languages(domain):
@@ -326,46 +294,118 @@ def get_available_languages(domain):
     return copy.copy(language_list)
 
 
-def get_localized_message(message, user_locale):
-    """Gets a localized version of the given message in the given locale.
+def translate(obj, desired_locale=None):
+    """Gets the translated unicode representation of the given object.
 
-    If the message is not a Message object the message is returned as-is.
-    If the locale is None the message is translated to the default locale.
+    If the object is not translatable it is returned as-is.
+    If the locale is None the object is translated to the system locale.
 
-    :returns: the translated message in unicode, or the original message if
+    :param obj: the object to translate
+    :param desired_locale: the locale to translate the message to, if None the
+                           default system locale will be used
+    :returns: the translated object in unicode, or the original object if
               it could not be translated
     """
-    translated = message
+    message = obj
+    if not isinstance(message, Message):
+        # If the object to translate is not already translatable,
+        # let's first get its unicode representation
+        message = six.text_type(obj)
     if isinstance(message, Message):
-        original_locale = message.locale
-        message.locale = user_locale
-        translated = six.text_type(message)
-        message.locale = original_locale
-    return translated
+        # Even after unicoding() we still need to check if we are
+        # running with translatable unicode before translating
+        return message.translate(desired_locale)
+    return obj
+
+
+def _translate_args(args, desired_locale=None):
+    """Translates all the translatable elements of the given arguments object.
 
+    This method is used for translating the translatable values in method
+    arguments which include values of tuples or dictionaries.
+    If the object is not a tuple or a dictionary the object itself is
+    translated if it is translatable.
 
-class LocaleHandler(logging.Handler):
-    """Handler that can have a locale associated to translate Messages.
+    If the locale is None the object is translated to the system locale.
 
-    A quick example of how to utilize the Message class above.
-    LocaleHandler takes a locale and a target logging.Handler object
-    to forward LogRecord objects to after translating the internal Message.
+    :param args: the args to translate
+    :param desired_locale: the locale to translate the args to, if None the
+                           default system locale will be used
+    :returns: a new args object with the translated contents of the original
     """
+    if isinstance(args, tuple):
+        return tuple(translate(v, desired_locale) for v in args)
+    if isinstance(args, dict):
+        translated_dict = {}
+        for (k, v) in six.iteritems(args):
+            translated_v = translate(v, desired_locale)
+            translated_dict[k] = translated_v
+        return translated_dict
+    return translate(args, desired_locale)
 
-    def __init__(self, locale, target):
-        """Initialize a LocaleHandler
+
+class TranslationHandler(handlers.MemoryHandler):
+    """Handler that translates records before logging them.
+
+    The TranslationHandler takes a locale and a target logging.Handler object
+    to forward LogRecord objects to after translating them. This handler
+    depends on Message objects being logged, instead of regular strings.
+
+    The handler can be configured declaratively in the logging.conf as follows:
+
+        [handlers]
+        keys = translatedlog, translator
+
+        [handler_translatedlog]
+        class = handlers.WatchedFileHandler
+        args = ('/var/log/api-localized.log',)
+        formatter = context
+
+        [handler_translator]
+        class = openstack.common.log.TranslationHandler
+        target = translatedlog
+        args = ('zh_CN',)
+
+    If the specified locale is not available in the system, the handler will
+    log in the default locale.
+    """
+
+    def __init__(self, locale=None, target=None):
+        """Initialize a TranslationHandler
 
         :param locale: locale to use for translating messages
         :param target: logging.Handler object to forward
                        LogRecord objects to after translation
         """
-        logging.Handler.__init__(self)
+        # NOTE(luisg): In order to allow this handler to be a wrapper for
+        # other handlers, such as a FileHandler, and still be able to
+        # configure it using logging.conf, this handler has to extend
+        # MemoryHandler because only the MemoryHandlers' logging.conf
+        # parsing is implemented such that it accepts a target handler.
+        handlers.MemoryHandler.__init__(self, capacity=0, target=target)
         self.locale = locale
-        self.target = target
+
+    def setFormatter(self, fmt):
+        self.target.setFormatter(fmt)
 
     def emit(self, record):
-        if isinstance(record.msg, Message):
-            # set the locale and resolve to a string
-            record.msg.locale = self.locale
+        # We save the message from the original record to restore it
+        # after translation, so other handlers are not affected by this
+        original_msg = record.msg
+        original_args = record.args
+
+        try:
+            self._translate_and_log_record(record)
+        finally:
+            record.msg = original_msg
+            record.args = original_args
+
+    def _translate_and_log_record(self, record):
+        record.msg = translate(record.msg, self.locale)
+
+        # In addition to translating the message, we also need to translate
+        # arguments that were passed to the log method that were not part
+        # of the main message e.g., log.info(_('Some message %s'), this_one))
+        record.args = _translate_args(record.args, self.locale)
 
         self.target.emit(record)
diff --git a/keystone/tests/core.py b/keystone/tests/core.py
index 69180106c..79003b52b 100644
--- a/keystone/tests/core.py
+++ b/keystone/tests/core.py
@@ -469,10 +469,17 @@ class TestCase(testtools.TestCase):
         except expected_exception as exc_value:
             if isinstance(expected_regexp, basestring):
                 expected_regexp = re.compile(expected_regexp)
-            if not expected_regexp.search(str(exc_value)):
-                raise self.failureException(
-                    '"%s" does not match "%s"' %
-                    (expected_regexp.pattern, str(exc_value)))
+
+            if isinstance(exc_value.args[0], gettextutils.Message):
+                if not expected_regexp.search(unicode(exc_value)):
+                    raise self.failureException(
+                        '"%s" does not match "%s"' %
+                        (expected_regexp.pattern, unicode(exc_value)))
+            else:
+                if not expected_regexp.search(str(exc_value)):
+                    raise self.failureException(
+                        '"%s" does not match "%s"' %
+                        (expected_regexp.pattern, str(exc_value)))
         else:
             if hasattr(expected_exception, '__name__'):
                 excName = expected_exception.__name__
diff --git a/keystone/tests/test_wsgi.py b/keystone/tests/test_wsgi.py
index fcd00b191..73868b586 100644
--- a/keystone/tests/test_wsgi.py
+++ b/keystone/tests/test_wsgi.py
@@ -14,8 +14,6 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import uuid
-
 from babel import localedata
 import gettext
 
@@ -240,34 +238,6 @@ class LocalizedResponseTest(tests.TestCase):
         req = wsgi.Request.blank('/', headers={'Accept-Language': 'zh'})
         self.assertIsNone(req.best_match_language())
 
-    def test_localized_message(self):
-        # If the accept-language header is set on the request, the localized
-        # message is returned by calling get_localized_message.
-
-        LANG_ID = uuid.uuid4().hex
-        ORIGINAL_TEXT = uuid.uuid4().hex
-        TRANSLATED_TEXT = uuid.uuid4().hex
-
-        self._set_expected_languages(all_locales=[LANG_ID])
-
-        def fake_get_localized_message(message, user_locale):
-            if (user_locale == LANG_ID and
-                    message == ORIGINAL_TEXT):
-                return TRANSLATED_TEXT
-
-        self.stubs.Set(gettextutils, 'get_localized_message',
-                       fake_get_localized_message)
-
-        error = exception.NotFound(message=ORIGINAL_TEXT)
-        resp = wsgi.render_exception(error, user_locale=LANG_ID)
-        result = jsonutils.loads(resp.body)
-
-        exp = {'error': {'message': TRANSLATED_TEXT,
-                         'code': 404,
-                         'title': 'Not Found'}}
-
-        self.assertEqual(exp, result)
-
     def test_static_translated_string_is_Message(self):
         # Statically created message strings are Message objects so that they
         # are lazy-translated.