Properly decode body before calling json.loads()

This commit fixes an issue when running the rest client and token
clients on python 3. httplib2 sets the type for the response body as
'bytes' when running on python 3, which requires that it be decoded
prior to running json.loads() on it. Additionally, in the v2 token
client a type check was done on the response body which was no longer
evaluating true because the type was no longer a 'str'. This was
fixed as part of the broader cleanup.

Change-Id: If4d496c4f10cec7d7050afc4b07f1f263de4c3e5

5 files changed, 54 insertions, 7 deletions

diff --git a/tempest_lib/common/rest_client.py b/tempest_lib/common/rest_client.py
index 6ae2348..ca0afbe 100644
--- a/tempest_lib/common/rest_client.py
+++ b/tempest_lib/common/rest_client.py
@@ -439,9 +439,16 @@ class RestClient(object):
             self._log_request_full(method, req_url, resp, secs, req_headers,
                                    req_body, resp_body, caller_name, extra)
 
+    def _json_loads(self, resp_body):
+        if isinstance(resp_body, bytes):
+            resp_body = json.loads(resp_body.decode('utf8'))
+        else:
+            resp_body = json.loads(resp_body)
+        return resp_body
+
     def _parse_resp(self, body):
         try:
-            body = json.loads(body)
+            body = self._json_loads(body)
         except ValueError:
             return body
 
diff --git a/tempest_lib/services/identity/v2/token_client.py b/tempest_lib/services/identity/v2/token_client.py
index ca128d0..1db6a93 100644
--- a/tempest_lib/services/identity/v2/token_client.py
+++ b/tempest_lib/services/identity/v2/token_client.py
@@ -85,15 +85,13 @@ class TokenClientJSON(rest_client.RestClient):
         self._log_request(method, url, resp)
 
         if resp.status in [401, 403]:
-            resp_body = json.loads(resp_body)
+            resp_body = self._json_loads(resp_body)
             raise exceptions.Unauthorized(resp_body['error']['message'])
         elif resp.status not in [200, 201]:
             raise exceptions.IdentityError(
                 'Unexpected status code {0}'.format(resp.status))
 
-        if isinstance(resp_body, str):
-            resp_body = json.loads(resp_body)
-        return resp, resp_body
+        return resp, self._json_loads(resp_body)
 
     def get_token(self, user, password, tenant, auth_data=False):
         """Returns (token id, token data) for supplied credentials."""
diff --git a/tempest_lib/services/identity/v3/token_client.py b/tempest_lib/services/identity/v3/token_client.py
index c51a1b3..db0723e 100644
--- a/tempest_lib/services/identity/v3/token_client.py
+++ b/tempest_lib/services/identity/v3/token_client.py
@@ -135,13 +135,13 @@ class V3TokenClientJSON(rest_client.RestClient):
         self._log_request(method, url, resp)
 
         if resp.status in [401, 403]:
-            resp_body = json.loads(resp_body)
+            resp_body = self._json_loads(resp_body)
             raise exceptions.Unauthorized(resp_body['error']['message'])
         elif resp.status not in [200, 201, 204]:
             raise exceptions.IdentityError(
                 'Unexpected status code {0}'.format(resp.status))
 
-        return resp, json.loads(resp_body)
+        return resp, self._json_loads(resp_body)
 
     def get_token(self, **kwargs):
         """Returns (token id, token data) for supplied credentials"""
diff --git a/tempest_lib/tests/services/identity/v2/test_token_client.py b/tempest_lib/tests/services/identity/v2/test_token_client.py
index 81943dc..15effbf 100644
--- a/tempest_lib/tests/services/identity/v2/test_token_client.py
+++ b/tempest_lib/tests/services/identity/v2/test_token_client.py
@@ -14,6 +14,7 @@
 
 import json
 
+import httplib2
 from oslotest import mockpatch
 
 from tempest_lib.common import rest_client
@@ -64,3 +65,23 @@ class TestTokenClientV2(base.TestCase):
         })
         post_mock.mock.assert_called_once_with('fake_url/tokens',
                                                body=req_dict)
+
+    def test_request_with_str_body(self):
+        token_client_v2 = token_client.TokenClientJSON('fake_url')
+        self.useFixture(mockpatch.PatchObject(
+            token_client_v2, 'raw_request', return_value=(
+                httplib2.Response({'status': '200'}),
+                str('{"access": {"token": "fake_token"}}'))))
+        resp, body = token_client_v2.request('GET', 'fake_uri')
+        self.assertIsInstance(resp, httplib2.Response)
+        self.assertIsInstance(body, dict)
+
+    def test_request_with_bytes_body(self):
+        token_client_v2 = token_client.TokenClientJSON('fake_url')
+        self.useFixture(mockpatch.PatchObject(
+            token_client_v2, 'raw_request', return_value=(
+                httplib2.Response({'status': '200'}),
+                bytes(b'{"access": {"token": "fake_token"}}'))))
+        resp, body = token_client_v2.request('GET', 'fake_uri')
+        self.assertIsInstance(resp, httplib2.Response)
+        self.assertIsInstance(body, dict)
diff --git a/tempest_lib/tests/services/identity/v3/test_token_client.py b/tempest_lib/tests/services/identity/v3/test_token_client.py
index 9b81784..f8295c4 100644
--- a/tempest_lib/tests/services/identity/v3/test_token_client.py
+++ b/tempest_lib/tests/services/identity/v3/test_token_client.py
@@ -14,6 +14,7 @@
 
 import json
 
+import httplib2
 from oslotest import mockpatch
 
 from tempest_lib.common import rest_client
@@ -79,3 +80,23 @@ class TestTokenClientV2(base.TestCase):
 
         post_mock.mock.assert_called_once_with('fake_url/auth/tokens',
                                                body=req_dict)
+
+    def test_request_with_str_body(self):
+        token_client_v3 = token_client.V3TokenClientJSON('fake_url')
+        self.useFixture(mockpatch.PatchObject(
+            token_client_v3, 'raw_request', return_value=(
+                httplib2.Response({"status": "200"}),
+                str('{"access": {"token": "fake_token"}}'))))
+        resp, body = token_client_v3.request('GET', 'fake_uri')
+        self.assertIsInstance(resp, httplib2.Response)
+        self.assertIsInstance(body, dict)
+
+    def test_request_with_bytes_body(self):
+        token_client_v3 = token_client.V3TokenClientJSON('fake_url')
+        self.useFixture(mockpatch.PatchObject(
+            token_client_v3, 'raw_request', return_value=(
+                httplib2.Response({"status": "200"}),
+                bytes(b'{"access": {"token": "fake_token"}}'))))
+        resp, body = token_client_v3.request('GET', 'fake_uri')
+        self.assertIsInstance(resp, httplib2.Response)
+        self.assertIsInstance(body, dict)