Sync with latest oslo-incubator

Last commit: 32e7f0b56f527427544050f251999f3de588ac93

This patch syncs the python-keystoneclient with olso-incubator as I
need this patch 4ef01931 which fixes a bug that's I am hitting in
another client which uses the keystoneclient.

Closes-bug: 1277565

Change-Id: I22f10f4fe27be16a6808b75c154ee342fea2bdda

8 files changed, 135 insertions, 111 deletions

diff --git a/keystoneclient/openstack/common/apiclient/base.py b/keystoneclient/openstack/common/apiclient/base.py
index 511fd73..9d7119d 100644
--- a/keystoneclient/openstack/common/apiclient/base.py
+++ b/keystoneclient/openstack/common/apiclient/base.py
@@ -32,7 +32,6 @@ from six.moves.urllib import parse
 from keystoneclient.openstack.common.apiclient import exceptions
 from keystoneclient.openstack.common.gettextutils import _
 from keystoneclient.openstack.common import strutils
-from keystoneclient.openstack.common import uuidutils
 
 
 def getid(obj):
@@ -100,12 +99,13 @@ class BaseManager(HookableMixin):
         super(BaseManager, self).__init__()
         self.client = client
 
-    def _list(self, url, response_key, obj_class=None, json=None):
+    def _list(self, url, response_key=None, obj_class=None, json=None):
         """List the collection.
 
         :param url: a partial URL, e.g., '/servers'
         :param response_key: the key to be looked up in response dictionary,
-            e.g., 'servers'
+            e.g., 'servers'. If response_key is None - all response body
+            will be used.
         :param obj_class: class for constructing the returned objects
             (self.resource_class will be used by default)
         :param json: data that will be encoded as JSON and passed in POST
@@ -119,7 +119,7 @@ class BaseManager(HookableMixin):
         if obj_class is None:
             obj_class = self.resource_class
 
-        data = body[response_key]
+        data = body[response_key] if response_key is not None else body
         # NOTE(ja): keystone returns values as list as {'values': [ ... ]}
         #           unlike other services which just return the list...
         try:
@@ -129,15 +129,17 @@ class BaseManager(HookableMixin):
 
         return [obj_class(self, res, loaded=True) for res in data if res]
 
-    def _get(self, url, response_key):
+    def _get(self, url, response_key=None):
         """Get an object from collection.
 
         :param url: a partial URL, e.g., '/servers'
         :param response_key: the key to be looked up in response dictionary,
-            e.g., 'server'
+            e.g., 'server'. If response_key is None - all response body
+            will be used.
         """
         body = self.client.get(url).json()
-        return self.resource_class(self, body[response_key], loaded=True)
+        data = body[response_key] if response_key is not None else body
+        return self.resource_class(self, data, loaded=True)
 
     def _head(self, url):
         """Retrieve request headers for an object.
@@ -147,21 +149,23 @@ class BaseManager(HookableMixin):
         resp = self.client.head(url)
         return resp.status_code == 204
 
-    def _post(self, url, json, response_key, return_raw=False):
+    def _post(self, url, json, response_key=None, return_raw=False):
         """Create an object.
 
         :param url: a partial URL, e.g., '/servers'
         :param json: data that will be encoded as JSON and passed in POST
             request (GET will be sent by default)
         :param response_key: the key to be looked up in response dictionary,
-            e.g., 'servers'
+            e.g., 'server'. If response_key is None - all response body
+            will be used.
         :param return_raw: flag to force returning raw JSON instead of
             Python object of self.resource_class
         """
         body = self.client.post(url, json=json).json()
+        data = body[response_key] if response_key is not None else body
         if return_raw:
-            return body[response_key]
-        return self.resource_class(self, body[response_key])
+            return data
+        return self.resource_class(self, data)
 
     def _put(self, url, json=None, response_key=None):
         """Update an object with PUT method.
@@ -170,7 +174,8 @@ class BaseManager(HookableMixin):
         :param json: data that will be encoded as JSON and passed in POST
             request (GET will be sent by default)
         :param response_key: the key to be looked up in response dictionary,
-            e.g., 'servers'
+            e.g., 'servers'. If response_key is None - all response body
+            will be used.
         """
         resp = self.client.put(url, json=json)
         # PUT requests may not return a body
@@ -188,7 +193,8 @@ class BaseManager(HookableMixin):
         :param json: data that will be encoded as JSON and passed in POST
             request (GET will be sent by default)
         :param response_key: the key to be looked up in response dictionary,
-            e.g., 'servers'
+            e.g., 'servers'. If response_key is None - all response body
+            will be used.
         """
         body = self.client.patch(url, json=json).json()
         if response_key is not None:
@@ -437,21 +443,6 @@ class Resource(object):
         self._info = info
         self._add_details(info)
         self._loaded = loaded
-        self._init_completion_cache()
-
-    def _init_completion_cache(self):
-        cache_write = getattr(self.manager, 'write_to_completion_cache', None)
-        if not cache_write:
-            return
-
-        # NOTE(sirp): ensure `id` is already present because if it isn't we'll
-        # enter an infinite loop of __getattr__ -> get -> __init__ ->
-        # __getattr__ -> ...
-        if 'id' in self.__dict__ and uuidutils.is_uuid_like(self.id):
-            cache_write('uuid', self.id)
-
-        if self.human_id:
-            cache_write('human_id', self.human_id)
 
     def __repr__(self):
         reprkeys = sorted(k
diff --git a/keystoneclient/openstack/common/apiclient/client.py b/keystoneclient/openstack/common/apiclient/client.py
index ed0fbc8..4e435b7 100644
--- a/keystoneclient/openstack/common/apiclient/client.py
+++ b/keystoneclient/openstack/common/apiclient/client.py
@@ -357,8 +357,7 @@ class BaseClient(object):
                     "Must be one of: %(version_map)s") % {
                         'api_name': api_name,
                         'version': version,
-                        'version_map': ', '.join(version_map.keys())
-                    }
+                        'version_map': ', '.join(version_map.keys())}
             raise exceptions.UnsupportedVersion(msg)
 
         return importutils.import_class(client_path)
diff --git a/keystoneclient/openstack/common/apiclient/exceptions.py b/keystoneclient/openstack/common/apiclient/exceptions.py
index d7e9370..7e5c2ea 100644
--- a/keystoneclient/openstack/common/apiclient/exceptions.py
+++ b/keystoneclient/openstack/common/apiclient/exceptions.py
@@ -447,8 +447,8 @@ def from_response(response, method, url):
         except ValueError:
             pass
         else:
-            if isinstance(body, dict):
-                error = list(body.values())[0]
+            if isinstance(body, dict) and isinstance(body.get("error"), dict):
+                error = body["error"]
                 kwargs["message"] = error.get("message")
                 kwargs["details"] = error.get("details")
     elif content_type.startswith("text/"):
diff --git a/keystoneclient/openstack/common/apiclient/fake_client.py b/keystoneclient/openstack/common/apiclient/fake_client.py
index 47894e3..ce7311c 100644
--- a/keystoneclient/openstack/common/apiclient/fake_client.py
+++ b/keystoneclient/openstack/common/apiclient/fake_client.py
@@ -33,7 +33,9 @@ from six.moves.urllib import parse
 from keystoneclient.openstack.common.apiclient import client
 
 
-def assert_has_keys(dct, required=[], optional=[]):
+def assert_has_keys(dct, required=None, optional=None):
+    required = required or []
+    optional = optional or []
     for k in required:
         try:
             assert k in dct
diff --git a/keystoneclient/openstack/common/gettextutils.py b/keystoneclient/openstack/common/gettextutils.py
index d57d468..55a60df 100644
--- a/keystoneclient/openstack/common/gettextutils.py
+++ b/keystoneclient/openstack/common/gettextutils.py
@@ -23,7 +23,6 @@ Usual usage in an openstack.common module:
 """
 
 import copy
-import functools
 import gettext
 import locale
 from logging import handlers
@@ -42,7 +41,7 @@ class TranslatorFactory(object):
     """Create translator functions
     """
 
-    def __init__(self, domain, lazy=False, localedir=None):
+    def __init__(self, domain, localedir=None):
         """Establish a set of translation functions for the domain.
 
         :param domain: Name of translation domain,
@@ -55,7 +54,6 @@ class TranslatorFactory(object):
         :type localedir: str
         """
         self.domain = domain
-        self.lazy = lazy
         if localedir is None:
             localedir = os.environ.get(domain.upper() + '_LOCALEDIR')
         self.localedir = localedir
@@ -75,16 +73,19 @@ class TranslatorFactory(object):
         """
         if domain is None:
             domain = self.domain
-        if self.lazy:
-            return functools.partial(Message, domain=domain)
-        t = gettext.translation(
-            domain,
-            localedir=self.localedir,
-            fallback=True,
-        )
-        if six.PY3:
-            return t.gettext
-        return t.ugettext
+        t = gettext.translation(domain,
+                                localedir=self.localedir,
+                                fallback=True)
+        # Use the appropriate method of the translation object based
+        # on the python version.
+        m = t.gettext if six.PY3 else t.ugettext
+
+        def f(msg):
+            """oslo.i18n.gettextutils translation function."""
+            if USE_LAZY:
+                return Message(msg, domain=domain)
+            return m(msg)
+        return f
 
     @property
     def primary(self):
@@ -147,19 +148,11 @@ def enable_lazy():
     your project is importing _ directly instead of using the
     gettextutils.install() way of importing the _ function.
     """
-    # FIXME(dhellmann): This function will be removed in oslo.i18n,
-    # because the TranslatorFactory makes it superfluous.
-    global _, _LI, _LW, _LE, _LC, USE_LAZY
-    tf = TranslatorFactory('keystoneclient', lazy=True)
-    _ = tf.primary
-    _LI = tf.log_info
-    _LW = tf.log_warning
-    _LE = tf.log_error
-    _LC = tf.log_critical
+    global USE_LAZY
     USE_LAZY = True
 
 
-def install(domain, lazy=False):
+def install(domain):
     """Install a _() function using the given translation domain.
 
     Given a translation domain, install a _() function using gettext's
@@ -170,26 +163,14 @@ def install(domain, lazy=False):
     a translation-domain-specific environment variable (e.g.
     NOVA_LOCALEDIR).
 
+    Note that to enable lazy translation, enable_lazy must be
+    called.
+
     :param domain: the translation domain
-    :param lazy: indicates whether or not to install the lazy _() function.
-                 The lazy _() introduces a way to do deferred translation
-                 of messages by installing a _ that builds Message objects,
-                 instead of strings, which can then be lazily translated into
-                 any available locale.
     """
-    if lazy:
-        from six import moves
-        tf = TranslatorFactory(domain, lazy=True)
-        moves.builtins.__dict__['_'] = tf.primary
-    else:
-        localedir = '%s_LOCALEDIR' % domain.upper()
-        if six.PY3:
-            gettext.install(domain,
-                            localedir=os.environ.get(localedir))
-        else:
-            gettext.install(domain,
-                            localedir=os.environ.get(localedir),
-                            unicode=True)
+    from six import moves
+    tf = TranslatorFactory(domain)
+    moves.builtins.__dict__['_'] = tf.primary
 
 
 class Message(six.text_type):
diff --git a/keystoneclient/openstack/common/jsonutils.py b/keystoneclient/openstack/common/jsonutils.py
index 3ae414a..3252588 100644
--- a/keystoneclient/openstack/common/jsonutils.py
+++ b/keystoneclient/openstack/common/jsonutils.py
@@ -38,11 +38,19 @@ import inspect
 import itertools
 import sys
 
+is_simplejson = False
 if sys.version_info < (2, 7):
     # On Python <= 2.6, json module is not C boosted, so try to use
     # simplejson module if available
     try:
         import simplejson as json
+        # NOTE(mriedem): Make sure we have a new enough version of simplejson
+        # to support the namedobject_as_tuple argument. This can be removed
+        # in the Kilo release when python 2.6 support is dropped.
+        if 'namedtuple_as_object' in inspect.getargspec(json.dumps).args:
+            is_simplejson = True
+        else:
+            import json
     except ImportError:
         import json
 else:
@@ -165,9 +173,17 @@ def to_primitive(value, convert_instances=False, convert_datetime=True,
 
 
 def dumps(value, default=to_primitive, **kwargs):
+    if is_simplejson:
+        kwargs['namedtuple_as_object'] = False
     return json.dumps(value, default=default, **kwargs)
 
 
+def dump(obj, fp, *args, **kwargs):
+    if is_simplejson:
+        kwargs['namedtuple_as_object'] = False
+    return json.dump(obj, fp, *args, **kwargs)
+
+
 def loads(s, encoding='utf-8', **kwargs):
     return json.loads(strutils.safe_decode(s, encoding), **kwargs)
 
diff --git a/keystoneclient/openstack/common/strutils.py b/keystoneclient/openstack/common/strutils.py
index 0a4c42e..fc3ef3f 100644
--- a/keystoneclient/openstack/common/strutils.py
+++ b/keystoneclient/openstack/common/strutils.py
@@ -50,6 +50,39 @@ SLUGIFY_STRIP_RE = re.compile(r"[^\w\s-]")
 SLUGIFY_HYPHENATE_RE = re.compile(r"[-\s]+")
 
 
+# NOTE(flaper87): The following globals are used by `mask_password`
+_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password']
+
+# NOTE(ldbragst): Let's build a list of regex objects using the list of
+# _SANITIZE_KEYS we already have. This way, we only have to add the new key
+# to the list of _SANITIZE_KEYS and we can generate regular expressions
+# for XML and JSON automatically.
+_SANITIZE_PATTERNS_2 = []
+_SANITIZE_PATTERNS_1 = []
+
+# NOTE(amrith): Some regular expressions have only one parameter, some
+# have two parameters. Use different lists of patterns here.
+_FORMAT_PATTERNS_1 = [r'(%(key)s\s*[=]\s*)[^\s^\'^\"]+']
+_FORMAT_PATTERNS_2 = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])',
+                      r'(%(key)s\s+[\"\']).*?([\"\'])',
+                      r'([-]{2}%(key)s\s+)[^\'^\"^=^\s]+([\s]*)',
+                      r'(<%(key)s>).*?(</%(key)s>)',
+                      r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])',
+                      r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])',
+                      r'([\'"].*?%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?'
+                      '[\'"]).*?([\'"])',
+                      r'(%(key)s\s*--?[A-z]+\s*)\S+(\s*)']
+
+for key in _SANITIZE_KEYS:
+    for pattern in _FORMAT_PATTERNS_2:
+        reg_ex = re.compile(pattern % {'key': key}, re.DOTALL)
+        _SANITIZE_PATTERNS_2.append(reg_ex)
+
+    for pattern in _FORMAT_PATTERNS_1:
+        reg_ex = re.compile(pattern % {'key': key}, re.DOTALL)
+        _SANITIZE_PATTERNS_1.append(reg_ex)
+
+
 def int_from_bool_as_string(subject):
     """Interpret a string as a boolean and return either 1 or 0.
 
@@ -237,3 +270,42 @@ def to_slug(value, incoming=None, errors="strict"):
         "ascii", "ignore").decode("ascii")
     value = SLUGIFY_STRIP_RE.sub("", value).strip().lower()
     return SLUGIFY_HYPHENATE_RE.sub("-", value)
+
+
+def mask_password(message, secret="***"):
+    """Replace password with 'secret' in message.
+
+    :param message: The string which includes security information.
+    :param secret: value with which to replace passwords.
+    :returns: The unicode value of message with the password fields masked.
+
+    For example:
+
+    >>> mask_password("'adminPass' : 'aaaaa'")
+    "'adminPass' : '***'"
+    >>> mask_password("'admin_pass' : 'aaaaa'")
+    "'admin_pass' : '***'"
+    >>> mask_password('"password" : "aaaaa"')
+    '"password" : "***"'
+    >>> mask_password("'original_password' : 'aaaaa'")
+    "'original_password' : '***'"
+    >>> mask_password("u'original_password' :   u'aaaaa'")
+    "u'original_password' :   u'***'"
+    """
+    message = six.text_type(message)
+
+    # NOTE(ldbragst): Check to see if anything in message contains any key
+    # specified in _SANITIZE_KEYS, if not then just return the message since
+    # we don't have to mask any passwords.
+    if not any(key in message for key in _SANITIZE_KEYS):
+        return message
+
+    substitute = r'\g<1>' + secret + r'\g<2>'
+    for pattern in _SANITIZE_PATTERNS_2:
+        message = re.sub(pattern, substitute, message)
+
+    substitute = r'\g<1>' + secret
+    for pattern in _SANITIZE_PATTERNS_1:
+        message = re.sub(pattern, substitute, message)
+
+    return message
diff --git a/keystoneclient/openstack/common/uuidutils.py b/keystoneclient/openstack/common/uuidutils.py
deleted file mode 100644
index 234b880..0000000
--- a/keystoneclient/openstack/common/uuidutils.py
+++ /dev/null
@@ -1,37 +0,0 @@
-# Copyright (c) 2012 Intel Corporation.
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-"""
-UUID related utilities and helper functions.
-"""
-
-import uuid
-
-
-def generate_uuid():
-    return str(uuid.uuid4())
-
-
-def is_uuid_like(val):
-    """Returns validation of a value as a UUID.
-
-    For our purposes, a UUID is a canonical form string:
-    aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa
-
-    """
-    try:
-        return str(uuid.UUID(val)) == val
-    except (TypeError, ValueError, AttributeError):
-        return False