Delay denial when service token is invalid

This patch modifies AuthProtocol to defer authentication
to a downstream service if an invalid service token is found
and delay_auth_decision is True. This makes the behavior for
an invalid service token similar to that for an invalid user
token.

This is required by Swift because multiple auth middlewares
may co-exist, and auth_token will currently deny a request
on detecting an invalid service token when that service token
is in fact intended to be validated by another downstream auth
middleware. This is precisely the configuration used in
devstack which configures both authtoken and tempauth in
the Swift proxy pipeline [1].

Swift support for service tokens is currently in review [2]
and functional tests will not pass using devstack without the
change proposed here.

[1] https://github.com/openstack-dev/devstack/blob/master/lib/swift#L396
[2] change I6072b4efb3a479a8e0cc2d9c11ffda5764b55e30

DocImpact
SecurityImpact
Closes-Bug: #1422389

Change-Id: Ic9402ef35ce3dd7c905d868a9eff7db5f3a4a40b

3 files changed, 79 insertions, 12 deletions

diff --git a/doc/source/middlewarearchitecture.rst b/doc/source/middlewarearchitecture.rst
index e93bc4b..e02aad4 100644
--- a/doc/source/middlewarearchitecture.rst
+++ b/doc/source/middlewarearchitecture.rst
@@ -426,6 +426,22 @@ is set to `Confirmed`. If the middleware is delegating the auth decision to the
 service, then the status is set to `Invalid` if the auth request was
 unsuccessful.
 
+An ``X-Service-Token`` header may also be included with a request. If present,
+and the value of ``X-Auth-Token`` or ``X-Storage-Token`` has not caused the
+request to be denied, then the middleware will attempt to validate the value of
+``X-Service-Token``. If valid, the authentication middleware extends the HTTP
+request with the header ``X-Service-Identity-Status`` having value `Confirmed`
+and also extends the request with additional headers representing the identity
+authenticated and authorised by the token.
+
+If ``X-Service-Token`` is present and its value is invalid and the
+``delay_auth_decision`` option is True then the value of
+``X-Service-Identity-Status`` is set to `Invalid` and no further headers are
+added. Otherwise if ``X-Service-Token`` is present and its value is invalid
+then the middleware will respond to the HTTP request with HTTPUnauthorized,
+regardless of the validity of the ``X-Auth-Token`` or ``X-Storage-Token``
+values.
+
 Extended the request with additional User Information
 -----------------------------------------------------
 
diff --git a/keystonemiddleware/auth_token/__init__.py b/keystonemiddleware/auth_token/__init__.py
index 0d4b41e..98010ed 100644
--- a/keystonemiddleware/auth_token/__init__.py
+++ b/keystonemiddleware/auth_token/__init__.py
@@ -69,10 +69,15 @@ will be added. They take the same form as the standard headers but add
 '_SERVICE_'. These headers will not exist in the environment if no
 service token is present.
 
-HTTP_X_IDENTITY_STATUS
+HTTP_X_IDENTITY_STATUS, HTTP_X_SERVICE_IDENTITY_STATUS
     'Confirmed' or 'Invalid'
     The underlying service will only see a value of 'Invalid' if the Middleware
-    is configured to run in 'delay_auth_decision' mode
+    is configured to run in 'delay_auth_decision' mode. As with all such
+    headers, HTTP_X_SERVICE_IDENTITY_STATUS will only exist in the
+    environment if a service token is presented. This is different than
+    HTTP_X_IDENTITY_STATUS which is always set even if no user token is
+    presented. This allows the underlying service to determine if a
+    denial should use 401 or 403.
 
 HTTP_X_DOMAIN_ID, HTTP_X_SERVICE_DOMAIN_ID
     Identity service managed unique identifier, string. Only present if
@@ -938,11 +943,16 @@ class AuthProtocol(object):
                     serv_headers = self._build_service_headers(serv_token_info)
                     self._add_headers(env, serv_headers)
             except exc.InvalidToken:
-                # Delayed auth not currently supported for service tokens.
-                # (Can be implemented if a use case is found.)
-                self._LOG.info(
-                    _LI('Invalid service token - rejecting request'))
-                return self._reject_request(env, start_response)
+                if self._delay_auth_decision:
+                    self._LOG.info(
+                        _LI('Invalid service token - deferring reject '
+                            'downstream'))
+                    self._add_headers(env,
+                                      {'X-Service-Identity-Status': 'Invalid'})
+                else:
+                    self._LOG.info(
+                        _LI('Invalid service token - rejecting request'))
+                    return self._reject_request(env, start_response)
 
             env['keystone.token_auth'] = _UserAuthPlugin(user_auth_ref,
                                                          serv_auth_ref)
@@ -967,6 +977,7 @@ class AuthProtocol(object):
         """
         auth_headers = ['X-Service-Catalog',
                         'X-Identity-Status',
+                        'X-Service-Identity-Status',
                         'X-Roles',
                         'X-Service-Roles']
         for key in six.iterkeys(_HEADER_TEMPLATE):
@@ -1164,6 +1175,7 @@ class AuthProtocol(object):
 
         roles = ','.join(auth_ref.role_names)
         rval = {
+            'X-Service-Identity-Status': 'Confirmed',
             'X-Service-Roles': roles,
         }
 
diff --git a/keystonemiddleware/tests/auth_token/test_auth_token_middleware.py b/keystonemiddleware/tests/auth_token/test_auth_token_middleware.py
index 99a01b1..0e4ab5e 100644
--- a/keystonemiddleware/tests/auth_token/test_auth_token_middleware.py
+++ b/keystonemiddleware/tests/auth_token/test_auth_token_middleware.py
@@ -62,6 +62,7 @@ EXPECTED_V2_DEFAULT_ENV_RESPONSE = {
 }
 
 EXPECTED_V2_DEFAULT_SERVICE_ENV_RESPONSE = {
+    'HTTP_X_SERVICE_IDENTITY_STATUS': 'Confirmed',
     'HTTP_X_SERVICE_PROJECT_ID': 'service_project_id1',
     'HTTP_X_SERVICE_PROJECT_NAME': 'service_project_name1',
     'HTTP_X_SERVICE_USER_ID': 'service_user_id1',
@@ -195,7 +196,18 @@ class FakeApp(object):
 
         resp = webob.Response()
 
-        if env['HTTP_X_IDENTITY_STATUS'] == 'Invalid':
+        if (env.get('HTTP_X_IDENTITY_STATUS') == 'Invalid'
+                and env['HTTP_X_SERVICE_IDENTITY_STATUS'] == 'Invalid'):
+            # Simulate delayed auth forbidding access with arbitrary status
+            # code to differentiate checking this code path
+            resp.status = 419
+            resp.body = FakeApp.FORBIDDEN
+        elif env.get('HTTP_X_SERVICE_IDENTITY_STATUS') == 'Invalid':
+            # Simulate delayed auth forbidding access with arbitrary status
+            # code to differentiate checking this code path
+            resp.status = 420
+            resp.body = FakeApp.FORBIDDEN
+        elif env['HTTP_X_IDENTITY_STATUS'] == 'Invalid':
             # Simulate delayed auth forbidding access
             resp.status = 403
             resp.body = FakeApp.FORBIDDEN
@@ -2315,7 +2327,8 @@ class CommonCompositeAuthTests(object):
         req.headers['X-Foo'] = 'Bar'
         body = self.middleware(req.environ, self.start_fake_response)
         for key in six.iterkeys(self.service_token_expected_env):
-            self.assertFalse(req.headers.get(key))
+            header_key = key[len('HTTP_'):].replace('_', '-')
+            self.assertFalse(req.headers.get(header_key))
         self.assertEqual('Bar', req.headers.get('X-Foo'))
         self.assertEqual(418, self.response_status)
         self.assertEqual([FakeApp.FORBIDDEN], body)
@@ -2351,14 +2364,39 @@ class CommonCompositeAuthTests(object):
 
     def test_composite_auth_delay_invalid_service_token(self):
         self.middleware._delay_auth_decision = True
+        self.purge_service_token_expected_env()
+        expected_env = {
+            'HTTP_X_SERVICE_IDENTITY_STATUS': 'Invalid',
+        }
+        self.update_expected_env(expected_env)
+
         req = webob.Request.blank('/')
         token = self.token_dict['uuid_token_default']
         service_token = 'invalid-service-token'
         req.headers['X-Auth-Token'] = token
         req.headers['X-Service-Token'] = service_token
         body = self.middleware(req.environ, self.start_fake_response)
-        self.assertEqual(401, self.response_status)
-        self.assertEqual([b'Authentication required'], body)
+        self.assertEqual(420, self.response_status)
+        self.assertEqual([FakeApp.FORBIDDEN], body)
+
+    def test_composite_auth_delay_invalid_service_and_user_tokens(self):
+        self.middleware._delay_auth_decision = True
+        self.purge_service_token_expected_env()
+        self.purge_token_expected_env()
+        expected_env = {
+            'HTTP_X_IDENTITY_STATUS': 'Invalid',
+            'HTTP_X_SERVICE_IDENTITY_STATUS': 'Invalid',
+        }
+        self.update_expected_env(expected_env)
+
+        req = webob.Request.blank('/')
+        token = 'invalid-user-token'
+        service_token = 'invalid-service-token'
+        req.headers['X-Auth-Token'] = token
+        req.headers['X-Service-Token'] = service_token
+        body = self.middleware(req.environ, self.start_fake_response)
+        self.assertEqual(419, self.response_status)
+        self.assertEqual([FakeApp.FORBIDDEN], body)
 
     def test_composite_auth_delay_no_service_token(self):
         self.middleware._delay_auth_decision = True
@@ -2376,7 +2414,8 @@ class CommonCompositeAuthTests(object):
         req.headers['X-Foo'] = 'Bar'
         body = self.middleware(req.environ, self.start_fake_response)
         for key in six.iterkeys(self.service_token_expected_env):
-            self.assertFalse(req.headers.get(key))
+            header_key = key[len('HTTP_'):].replace('_', '-')
+            self.assertFalse(req.headers.get(header_key))
         self.assertEqual('Bar', req.headers.get('X-Foo'))
         self.assertEqual(418, self.response_status)
         self.assertEqual([FakeApp.FORBIDDEN], body)