Add wrapper classes for return-request-id-to-caller

Added wrapper classes which are inherited from base data types tuple, dict
and str. Each of these wrapper classes contain a 'request_ids' attribute
which is populated with a 'x-openstack-request-id' received in a header
from a response body.

This change is required to return 'request_id' from client to log
request_id mappings of cross projects[1].

[1]: http://specs.openstack.org/openstack/openstack-specs/specs/return-request-id.html

Change-Id: I55fcba61c4efb308f575e95e154aba23e5dd5245
Implements: blueprint return-request-id-to-caller

5 files changed, 327 insertions, 23 deletions

diff --git a/doc/source/usage/library.rst b/doc/source/usage/library.rst
index dff0696..8b301e3 100644
--- a/doc/source/usage/library.rst
+++ b/doc/source/usage/library.rst
@@ -59,3 +59,13 @@ and a service endpoint URL directly.
     >>> from neutronclient.v2_0 import client
     >>> neutron = client.Client(endpoint_url='http://192.168.206.130:9696/',
     ...                         token='d3f9226f27774f338019aa2611112ef6')
+
+You can get ``X-Openstack-Request-Id`` as ``request_ids`` from the result.
+
+.. code-block:: python
+
+    >>> network = {'name': 'mynetwork', 'admin_state_up': True}
+    >>> neutron.create_network({'network':network})
+    >>> networks = neutron.list_networks(name='mynetwork')
+    >>> print networks.request_ids
+    ['req-978a0160-7ab0-44f0-8a93-08e9a4e785fa']
diff --git a/neutronclient/common/exceptions.py b/neutronclient/common/exceptions.py
index 95f54f7..6aff6d6 100644
--- a/neutronclient/common/exceptions.py
+++ b/neutronclient/common/exceptions.py
@@ -60,10 +60,19 @@ class NeutronClientException(NeutronException):
     """
 
     status_code = 0
+    req_ids_msg = _("Neutron server returns request_ids: %s")
+    request_ids = []
 
     def __init__(self, message=None, **kwargs):
+        self.request_ids = kwargs.get('request_ids')
         if 'status_code' in kwargs:
             self.status_code = kwargs['status_code']
+        if self.request_ids:
+            req_ids_msg = self.req_ids_msg % self.request_ids
+            if message:
+                message += '\n' + req_ids_msg
+            else:
+                message = req_ids_msg
         super(NeutronClientException, self).__init__(message, **kwargs)
 
 
diff --git a/neutronclient/tests/unit/test_cli20.py b/neutronclient/tests/unit/test_cli20.py
index e3a1014..5407e7b 100644
--- a/neutronclient/tests/unit/test_cli20.py
+++ b/neutronclient/tests/unit/test_cli20.py
@@ -37,6 +37,7 @@ API_VERSION = "2.0"
 FORMAT = 'json'
 TOKEN = 'testtoken'
 ENDURL = 'localurl'
+REQUEST_ID = 'test_request_id'
 
 
 @contextlib.contextmanager
@@ -65,7 +66,7 @@ class FakeStdout(object):
         return result
 
 
-class MyResp(object):
+class MyResp(requests.Response):
     def __init__(self, status_code, headers=None, reason=None):
         self.status_code = status_code
         self.headers = headers or {}
@@ -648,41 +649,46 @@ class ClientV2TestJson(CLITestV20Base):
         self.client.httpclient.auth_token = encodeutils.safe_encode(
             unicode_text)
         expected_auth_token = encodeutils.safe_encode(unicode_text)
+        resp_headers = {'x-openstack-request-id': REQUEST_ID}
 
         self.client.httpclient.request(
             end_url(expected_action, query=expect_query, format=self.format),
             'PUT', body=expect_body,
             headers=mox.ContainsKeyValue(
                 'X-Auth-Token',
-                expected_auth_token)).AndReturn((MyResp(200), expect_body))
+                expected_auth_token)).AndReturn((MyResp(200, resp_headers),
+                                                 expect_body))
 
         self.mox.ReplayAll()
-        res_body = self.client.do_request('PUT', action, body=body,
-                                          params=params)
+        result = self.client.do_request('PUT', action, body=body,
+                                        params=params)
         self.mox.VerifyAll()
         self.mox.UnsetStubs()
 
         # test response with unicode
-        self.assertEqual(body, res_body)
+        self.assertEqual(body, result)
 
     def test_do_request_error_without_response_body(self):
         self.mox.StubOutWithMock(self.client.httpclient, "request")
         params = {'test': 'value'}
         expect_query = six.moves.urllib.parse.urlencode(params)
         self.client.httpclient.auth_token = 'token'
+        resp_headers = {'x-openstack-request-id': REQUEST_ID}
 
         self.client.httpclient.request(
             MyUrlComparator(end_url(
                 '/test', query=expect_query, format=self.format), self.client),
             'PUT', body='',
             headers=mox.ContainsKeyValue('X-Auth-Token', 'token')
-        ).AndReturn((MyResp(400, reason='An error'), ''))
+        ).AndReturn((MyResp(400, headers=resp_headers, reason='An error'), ''))
 
         self.mox.ReplayAll()
         error = self.assertRaises(exceptions.NeutronClientException,
                                   self.client.do_request, 'PUT', '/test',
                                   body='', params=params)
-        self.assertEqual("An error", str(error))
+        expected_error = "An error\nNeutron server returns " \
+                         "request_ids: %s" % [REQUEST_ID]
+        self.assertEqual(expected_error, str(error))
         self.mox.VerifyAll()
         self.mox.UnsetStubs()
 
@@ -697,21 +703,126 @@ class ClientV2TestJson(CLITestV20Base):
         else:
             self.fail('Expected exception NOT raised')
 
+    def test_do_request_request_ids(self):
+        self.mox.StubOutWithMock(self.client.httpclient, "request")
+        params = {'test': 'value'}
+        expect_query = six.moves.urllib.parse.urlencode(params)
+        self.client.httpclient.auth_token = 'token'
+        body = params
+        expect_body = self.client.serialize(body)
+        resp_headers = {'x-openstack-request-id': REQUEST_ID}
+        self.client.httpclient.request(
+            MyUrlComparator(end_url(
+                '/test', query=expect_query,
+                format=self.format), self.client),
+            'PUT', body=expect_body,
+            headers=mox.ContainsKeyValue('X-Auth-Token', 'token')
+        ).AndReturn((MyResp(200, resp_headers), expect_body))
+
+        self.mox.ReplayAll()
+        result = self.client.do_request('PUT', '/test', body=body,
+                                        params=params)
+        self.mox.VerifyAll()
+        self.mox.UnsetStubs()
+
+        self.assertEqual(body, result)
+        self.assertEqual([REQUEST_ID], result.request_ids)
+
+    def test_list_request_ids_with_retrieve_all_true(self):
+        self.mox.StubOutWithMock(self.client.httpclient, "request")
+
+        path = '/test'
+        resources = 'tests'
+        fake_query = "marker=myid2&limit=2"
+        reses1 = {resources: [{'id': 'myid1', },
+                              {'id': 'myid2', }],
+                  '%s_links' % resources: [{'href': end_url(path, fake_query),
+                                            'rel': 'next'}]}
+        reses2 = {resources: [{'id': 'myid3', },
+                              {'id': 'myid4', }]}
+        resstr1 = self.client.serialize(reses1)
+        resstr2 = self.client.serialize(reses2)
+        resp_headers = {'x-openstack-request-id': REQUEST_ID}
+        self.client.httpclient.request(
+            end_url(path, "", format=self.format), 'GET',
+            body=None,
+            headers=mox.ContainsKeyValue(
+                'X-Auth-Token', TOKEN)).AndReturn((MyResp(200, resp_headers),
+                                                   resstr1))
+        self.client.httpclient.request(
+            MyUrlComparator(end_url(path, fake_query, format=self.format),
+                            self.client), 'GET',
+            body=None,
+            headers=mox.ContainsKeyValue(
+                'X-Auth-Token', TOKEN)).AndReturn((MyResp(200, resp_headers),
+                                                   resstr2))
+        self.mox.ReplayAll()
+        result = self.client.list(resources, path)
+
+        self.mox.VerifyAll()
+        self.mox.UnsetStubs()
+
+        self.assertEqual([REQUEST_ID, REQUEST_ID], result.request_ids)
+
+    def test_list_request_ids_with_retrieve_all_false(self):
+        self.mox.StubOutWithMock(self.client.httpclient, "request")
+
+        path = '/test'
+        resources = 'tests'
+        fake_query = "marker=myid2&limit=2"
+        reses1 = {resources: [{'id': 'myid1', },
+                              {'id': 'myid2', }],
+                  '%s_links' % resources: [{'href': end_url(path, fake_query),
+                                            'rel': 'next'}]}
+        reses2 = {resources: [{'id': 'myid3', },
+                              {'id': 'myid4', }]}
+        resstr1 = self.client.serialize(reses1)
+        resstr2 = self.client.serialize(reses2)
+        resp_headers = {'x-openstack-request-id': REQUEST_ID}
+        self.client.httpclient.request(
+            end_url(path, "", format=self.format), 'GET',
+            body=None,
+            headers=mox.ContainsKeyValue(
+                'X-Auth-Token', TOKEN)).AndReturn((MyResp(200, resp_headers),
+                                                   resstr1))
+        self.client.httpclient.request(
+            MyUrlComparator(end_url(path, fake_query, format=self.format),
+                            self.client), 'GET',
+            body=None,
+            headers=mox.ContainsKeyValue(
+                'X-Auth-Token', TOKEN)).AndReturn((MyResp(200, resp_headers),
+                                                   resstr2))
+        self.mox.ReplayAll()
+        result = self.client.list(resources, path, retrieve_all=False)
+        next(result)
+        self.assertEqual([REQUEST_ID], result.request_ids)
+        next(result)
+        self.assertEqual([REQUEST_ID, REQUEST_ID], result.request_ids)
+        self.mox.VerifyAll()
+        self.mox.UnsetStubs()
+
 
 class CLITestV20ExceptionHandler(CLITestV20Base):
 
     def _test_exception_handler_v20(
             self, expected_exception, status_code, expected_msg,
             error_type=None, error_msg=None, error_detail=None,
-            error_content=None):
+            request_id=None, error_content=None):
+
+        resp = MyResp(status_code, {'x-openstack-request-id': request_id})
+        if request_id is not None:
+            expected_msg += "\nNeutron server returns " \
+                            "request_ids: %s" % [request_id]
         if error_content is None:
             error_content = {'NeutronError': {'type': error_type,
                                               'message': error_msg,
                                               'detail': error_detail}}
+        expected_content = self.client._convert_into_with_meta(error_content,
+                                                               resp)
 
         e = self.assertRaises(expected_exception,
                               client.exception_handler_v20,
-                              status_code, error_content)
+                              status_code, expected_content)
         self.assertEqual(status_code, e.status_code)
         self.assertEqual(expected_exception.__name__,
                          e.__class__.__name__)
@@ -728,7 +839,7 @@ class CLITestV20ExceptionHandler(CLITestV20Base):
                    'fake-network-uuid. The IP address fake-ip is in use.')
         self._test_exception_handler_v20(
             exceptions.IpAddressInUseClient, 409, err_msg,
-            'IpAddressInUse', err_msg, '')
+            'IpAddressInUse', err_msg, '', REQUEST_ID)
 
     def test_exception_handler_v20_neutron_known_error(self):
         known_error_map = [
@@ -754,7 +865,7 @@ class CLITestV20ExceptionHandler(CLITestV20Base):
             self._test_exception_handler_v20(
                 client_exc, status_code,
                 error_msg + '\n' + error_detail,
-                server_exc, error_msg, error_detail)
+                server_exc, error_msg, error_detail, REQUEST_ID)
 
     def test_exception_handler_v20_neutron_known_error_without_detail(self):
         error_msg = 'Network not found'
@@ -762,7 +873,7 @@ class CLITestV20ExceptionHandler(CLITestV20Base):
         self._test_exception_handler_v20(
             exceptions.NetworkNotFoundClient, 404,
             error_msg,
-            'NetworkNotFound', error_msg, error_detail)
+            'NetworkNotFound', error_msg, error_detail, REQUEST_ID)
 
     def test_exception_handler_v20_unknown_error_to_per_code_exception(self):
         for status_code, client_exc in exceptions.HTTP_EXCEPTION_MAP.items():
@@ -771,7 +882,7 @@ class CLITestV20ExceptionHandler(CLITestV20Base):
             self._test_exception_handler_v20(
                 client_exc, status_code,
                 error_msg + '\n' + error_detail,
-                'UnknownError', error_msg, error_detail)
+                'UnknownError', error_msg, error_detail, [REQUEST_ID])
 
     def test_exception_handler_v20_neutron_unknown_status_code(self):
         error_msg = 'Unknown error'
@@ -779,7 +890,7 @@ class CLITestV20ExceptionHandler(CLITestV20Base):
         self._test_exception_handler_v20(
             exceptions.NeutronClientException, 501,
             error_msg + '\n' + error_detail,
-            'UnknownError', error_msg, error_detail)
+            'UnknownError', error_msg, error_detail, REQUEST_ID)
 
     def test_exception_handler_v20_bad_neutron_error(self):
         for status_code, client_exc in exceptions.HTTP_EXCEPTION_MAP.items():
@@ -787,7 +898,8 @@ class CLITestV20ExceptionHandler(CLITestV20Base):
             self._test_exception_handler_v20(
                 client_exc, status_code,
                 expected_msg="{'unknown_key': 'UNKNOWN'}",
-                error_content=error_content)
+                error_content=error_content,
+                request_id=REQUEST_ID)
 
     def test_exception_handler_v20_error_dict_contains_message(self):
         error_content = {'message': 'This is an error message'}
@@ -795,7 +907,8 @@ class CLITestV20ExceptionHandler(CLITestV20Base):
             self._test_exception_handler_v20(
                 client_exc, status_code,
                 expected_msg='This is an error message',
-                error_content=error_content)
+                error_content=error_content,
+                request_id=REQUEST_ID)
 
     def test_exception_handler_v20_error_dict_not_contain_message(self):
         # 599 is not contained in HTTP_EXCEPTION_MAP.
@@ -804,6 +917,7 @@ class CLITestV20ExceptionHandler(CLITestV20Base):
         self._test_exception_handler_v20(
             exceptions.NeutronClientException, 599,
             expected_msg=expected_msg,
+            request_id=None,
             error_content=error_content)
 
     def test_exception_handler_v20_default_fallback(self):
@@ -813,6 +927,7 @@ class CLITestV20ExceptionHandler(CLITestV20Base):
         self._test_exception_handler_v20(
             exceptions.NeutronClientException, 599,
             expected_msg=expected_msg,
+            request_id=None,
             error_content=error_content)
 
     def test_exception_status(self):
@@ -848,3 +963,60 @@ class CLITestV20ExceptionHandler(CLITestV20Base):
         self.assertIsNotNone(error.status_code)
         self.mox.VerifyAll()
         self.mox.UnsetStubs()
+
+
+class DictWithMetaTest(base.BaseTestCase):
+
+    def test_dict_with_meta(self):
+        body = {'test': 'value'}
+        resp = MyResp(200, {'x-openstack-request-id': REQUEST_ID})
+        obj = client._DictWithMeta(body, resp)
+        self.assertEqual(body, obj)
+
+        # Check request_ids attribute is added to obj
+        self.assertTrue(hasattr(obj, 'request_ids'))
+        self.assertEqual([REQUEST_ID], obj.request_ids)
+
+
+class TupleWithMetaTest(base.BaseTestCase):
+
+    def test_tuple_with_meta(self):
+        body = ('test', 'value')
+        resp = MyResp(200, {'x-openstack-request-id': REQUEST_ID})
+        obj = client._TupleWithMeta(body, resp)
+        self.assertEqual(body, obj)
+
+        # Check request_ids attribute is added to obj
+        self.assertTrue(hasattr(obj, 'request_ids'))
+        self.assertEqual([REQUEST_ID], obj.request_ids)
+
+
+class StrWithMetaTest(base.BaseTestCase):
+
+    def test_str_with_meta(self):
+        body = "test_string"
+        resp = MyResp(200, {'x-openstack-request-id': REQUEST_ID})
+        obj = client._StrWithMeta(body, resp)
+        self.assertEqual(body, obj)
+
+        # Check request_ids attribute is added to obj
+        self.assertTrue(hasattr(obj, 'request_ids'))
+        self.assertEqual([REQUEST_ID], obj.request_ids)
+
+
+class GeneratorWithMetaTest(base.BaseTestCase):
+
+    body = {'test': 'value'}
+    resp = MyResp(200, {'x-openstack-request-id': REQUEST_ID})
+
+    def _pagination(self, collection, path, **params):
+        obj = client._DictWithMeta(self.body, self.resp)
+        yield obj
+
+    def test_generator(self):
+        obj = client._GeneratorWithMeta(self._pagination, 'test_collection',
+                                        'test_path', test_args='test_args')
+        self.assertEqual(self.body, next(obj))
+
+        self.assertTrue(hasattr(obj, 'request_ids'))
+        self.assertEqual([REQUEST_ID], obj.request_ids)
diff --git a/neutronclient/v2_0/client.py b/neutronclient/v2_0/client.py
index 08a8503..9fd2c6a 100644
--- a/neutronclient/v2_0/client.py
+++ b/neutronclient/v2_0/client.py
@@ -22,6 +22,7 @@ import time
 
 import requests
 import six.moves.urllib.parse as urlparse
+from six import string_types
 
 from neutronclient._i18n import _
 from neutronclient import client
@@ -44,6 +45,7 @@ def exception_handler_v20(status_code, error_content):
     :param error_content: deserialized body of error response
     """
     error_dict = None
+    request_ids = error_content.request_ids
     if isinstance(error_content, dict):
         error_dict = error_content.get('NeutronError')
     # Find real error type
@@ -78,7 +80,8 @@ def exception_handler_v20(status_code, error_content):
         client_exc = exceptions.NeutronClientException
 
     raise client_exc(message=error_message,
-                     status_code=status_code)
+                     status_code=status_code,
+                     request_ids=request_ids)
 
 
 class APIParamsCall(object):
@@ -97,6 +100,99 @@ class APIParamsCall(object):
         return with_params
 
 
+class _RequestIdMixin(object):
+    """Wrapper class to expose x-openstack-request-id to the caller."""
+    def _request_ids_setup(self):
+        self._request_ids = []
+
+    @property
+    def request_ids(self):
+        return self._request_ids
+
+    def _append_request_ids(self, resp):
+        """Add request_ids as an attribute to the object
+
+        :param resp: Response object or list of Response objects
+        """
+        if isinstance(resp, list):
+            # Add list of request_ids if response is of type list.
+            for resp_obj in resp:
+                self._append_request_id(resp_obj)
+        elif resp is not None:
+            # Add request_ids if response contains single object.
+            self._append_request_id(resp)
+
+    def _append_request_id(self, resp):
+        if isinstance(resp, requests.Response):
+            # Extract 'x-openstack-request-id' from headers if
+            # response is a Response object.
+            request_id = resp.headers.get('x-openstack-request-id')
+        else:
+            # If resp is of type string.
+            request_id = resp
+        if request_id:
+            self._request_ids.append(request_id)
+
+
+class _DictWithMeta(dict, _RequestIdMixin):
+    def __init__(self, values, resp):
+        super(_DictWithMeta, self).__init__(values)
+        self._request_ids_setup()
+        self._append_request_ids(resp)
+
+
+class _TupleWithMeta(tuple, _RequestIdMixin):
+    def __new__(cls, values, resp):
+        return super(_TupleWithMeta, cls).__new__(cls, values)
+
+    def __init__(self, values, resp):
+        self._request_ids_setup()
+        self._append_request_ids(resp)
+
+
+class _StrWithMeta(str, _RequestIdMixin):
+    def __new__(cls, value, resp):
+        return super(_StrWithMeta, cls).__new__(cls, value)
+
+    def __init__(self, values, resp):
+        self._request_ids_setup()
+        self._append_request_ids(resp)
+
+
+class _GeneratorWithMeta(_RequestIdMixin):
+    def __init__(self, paginate_func, collection, path, **params):
+        self.paginate_func = paginate_func
+        self.collection = collection
+        self.path = path
+        self.params = params
+        self.generator = None
+        self._request_ids_setup()
+
+    def _paginate(self):
+        for r in self.paginate_func(
+                self.collection, self.path, **self.params):
+            yield r, r.request_ids
+
+    def __iter__(self):
+        return self
+
+    # Python 3 compatibility
+    def __next__(self):
+        return self.next()
+
+    def next(self):
+        if not self.generator:
+            self.generator = self._paginate()
+
+        try:
+            obj, req_id = next(self.generator)
+            self._append_request_ids(req_id)
+        except StopIteration:
+            raise StopIteration()
+
+        return obj
+
+
 class ClientBase(object):
     """Client for the OpenStack Neutron v2.0 API.
 
@@ -162,7 +258,7 @@ class ClientBase(object):
         self.action_prefix = "/v%s" % (self.version)
         self.retry_interval = 1
 
-    def _handle_fault_response(self, status_code, response_body):
+    def _handle_fault_response(self, status_code, response_body, resp):
         # Create exception with HTTP status code and message
         _logger.debug("Error message: %s", response_body)
         # Add deserialized error message to exception arguments
@@ -172,8 +268,9 @@ class ClientBase(object):
             # If unable to deserialized body it is probably not a
             # Neutron error
             des_error_body = {'message': response_body}
+        error_body = self._convert_into_with_meta(des_error_body, resp)
         # Raise the appropriate exception
-        exception_handler_v20(status_code, des_error_body)
+        exception_handler_v20(status_code, error_body)
 
     def do_request(self, method, action, body=None, headers=None, params=None):
         # Add format and tenant_id
@@ -193,11 +290,12 @@ class ClientBase(object):
                            requests.codes.created,
                            requests.codes.accepted,
                            requests.codes.no_content):
-            return self.deserialize(replybody, status_code)
+            data = self.deserialize(replybody, status_code)
+            return self._convert_into_with_meta(data, resp)
         else:
             if not replybody:
                 replybody = resp.reason
-            self._handle_fault_response(status_code, replybody)
+            self._handle_fault_response(status_code, replybody, resp)
 
     def get_auth_info(self):
         return self.httpclient.get_auth_info()
@@ -271,11 +369,14 @@ class ClientBase(object):
     def list(self, collection, path, retrieve_all=True, **params):
         if retrieve_all:
             res = []
+            request_ids = []
             for r in self._pagination(collection, path, **params):
                 res.extend(r[collection])
-            return {collection: res}
+                request_ids.extend(r.request_ids)
+            return _DictWithMeta({collection: res}, request_ids)
         else:
-            return self._pagination(collection, path, **params)
+            return _GeneratorWithMeta(self._pagination, collection,
+                                      path, **params)
 
     def _pagination(self, collection, path, **params):
         if params.get('page_reverse', False):
@@ -297,6 +398,15 @@ class ClientBase(object):
             except KeyError:
                 break
 
+    def _convert_into_with_meta(self, item, resp):
+        if item:
+            if isinstance(item, dict):
+                return _DictWithMeta(item, resp)
+            elif isinstance(item, string_types):
+                return _StrWithMeta(item, resp)
+        else:
+            return _TupleWithMeta((), resp)
+
 
 class Client(ClientBase):
 
diff --git a/releasenotes/notes/return-request-id-to-caller-15b1d23a4ddc27a3.yaml b/releasenotes/notes/return-request-id-to-caller-15b1d23a4ddc27a3.yaml
new file mode 100644
index 0000000..da7c5f8
--- /dev/null
+++ b/releasenotes/notes/return-request-id-to-caller-15b1d23a4ddc27a3.yaml
@@ -0,0 +1,3 @@
+---
+features:
+  - Neutron client returns 'x-openstack-request-id'.