diff options
| -rw-r--r-- | keystonemiddleware/auth_token.py | 765 | ||||
| -rw-r--r-- | keystonemiddleware/openstack/common/fixture/__init__.py | 0 | ||||
| -rw-r--r-- | keystonemiddleware/openstack/common/fixture/config.py | 85 | ||||
| -rw-r--r-- | keystonemiddleware/openstack/common/gettextutils.py | 67 | ||||
| -rw-r--r-- | keystonemiddleware/openstack/common/jsonutils.py | 4 | ||||
| -rw-r--r-- | keystonemiddleware/openstack/common/strutils.py | 56 | ||||
| -rw-r--r-- | keystonemiddleware/opts.py | 49 | ||||
| -rw-r--r-- | keystonemiddleware/tests/test_auth_token_middleware.py | 3 | ||||
| -rw-r--r-- | keystonemiddleware/tests/test_opts.py | 79 | ||||
| -rw-r--r-- | openstack-common.conf | 2 | ||||
| -rw-r--r-- | requirements.txt | 2 | ||||
| -rw-r--r-- | setup.cfg | 4 | ||||
| -rw-r--r-- | setup.py | 8 | ||||
| -rw-r--r-- | test-requirements.txt | 2 |
14 files changed, 638 insertions, 488 deletions
diff --git a/keystonemiddleware/auth_token.py b/keystonemiddleware/auth_token.py index 2146f07..40ce015 100644 --- a/keystonemiddleware/auth_token.py +++ b/keystonemiddleware/auth_token.py @@ -335,7 +335,7 @@ _OPTS = [ CONF = cfg.CONF CONF.register_opts(_OPTS, group='keystone_authtoken') -_LIST_OF_VERSIONS_TO_ATTEMPT = ['v2.0', 'v3.0'] +_LIST_OF_VERSIONS_TO_ATTEMPT = ['v3.0', 'v2.0'] class _BIND_MODE: @@ -466,55 +466,34 @@ class AuthProtocol(object): (True, 'true', 't', '1', 'on', 'yes', 'y') ) - # where to find the auth service (we use this to validate tokens) - self._identity_uri = self._conf_get('identity_uri') - self._auth_uri = self._conf_get('auth_uri') - - # NOTE(jamielennox): it does appear here that our defaults arguments - # are backwards. We need to do it this way so that we can handle the - # same deprecation strategy for CONF and the conf variable. - if not self._identity_uri: - self._LOG.warning('Configuring admin URI using auth fragments. ' - 'This is deprecated, use \'identity_uri\'' - ' instead.') - - auth_host = self._conf_get('auth_host') - auth_port = int(self._conf_get('auth_port')) - auth_protocol = self._conf_get('auth_protocol') - auth_admin_prefix = self._conf_get('auth_admin_prefix') - - if netaddr.valid_ipv6(auth_host): - # Note(dzyu) it is an IPv6 address, so it needs to be wrapped - # with '[]' to generate a valid IPv6 URL, based on - # http://www.ietf.org/rfc/rfc2732.txt - auth_host = '[%s]' % auth_host - - self._identity_uri = '%s://%s:%s' % (auth_protocol, auth_host, - auth_port) - if auth_admin_prefix: - self._identity_uri = '%s/%s' % (self._identity_uri, - auth_admin_prefix.strip('/')) - else: - self._identity_uri = self._identity_uri.rstrip('/') - - if self._auth_uri is None: - self._LOG.warning( - 'Configuring auth_uri to point to the public identity ' - 'endpoint is required; clients may not be able to ' - 'authenticate against an admin endpoint') - - # FIXME(dolph): drop support for this fallback behavior as - # documented in bug 1207517. - # NOTE(jamielennox): we urljoin '/' to get just the base URI as - # this is the original behaviour. - self._auth_uri = urllib.parse.urljoin(self._identity_uri, '/') - self._auth_uri = self._auth_uri.rstrip('/') + self._include_service_catalog = self._conf_get( + 'include_service_catalog') - # SSL - self._cert_file = self._conf_get('certfile') - self._key_file = self._conf_get('keyfile') - self._ssl_ca_file = self._conf_get('cafile') - self._ssl_insecure = self._conf_get('insecure') + http_connect_timeout_cfg = self._conf_get('http_connect_timeout') + http_connect_timeout = (http_connect_timeout_cfg and + int(http_connect_timeout_cfg)) + + http_request_max_retries = self._conf_get('http_request_max_retries') + + self._identity_server = _IdentityServer( + self._LOG, include_service_catalog=self._include_service_catalog, + identity_uri=self._conf_get('identity_uri'), + auth_uri=self._conf_get('auth_uri'), + auth_host=self._conf_get('auth_host'), + auth_port=int(self._conf_get('auth_port')), + auth_protocol=self._conf_get('auth_protocol'), + auth_admin_prefix=self._conf_get('auth_admin_prefix'), + cert_file=self._conf_get('certfile'), + key_file=self._conf_get('keyfile'), + ssl_ca_file=self._conf_get('cafile'), + ssl_insecure=self._conf_get('insecure'), + admin_token=self._conf_get('admin_token'), + admin_user=self._conf_get('admin_user'), + admin_password=self._conf_get('admin_password'), + admin_tenant_name=self._conf_get('admin_tenant_name'), + http_connect_timeout=http_connect_timeout, + http_request_max_retries=http_request_max_retries, + auth_version=self._conf_get('auth_version')) # signing self._signing_dirname = self._conf_get('signing_dir') @@ -532,20 +511,6 @@ class AuthProtocol(object): val = '%s/revoked.pem' % self._signing_dirname self._revoked_file_name = val - # Credentials used to verify this component with the Auth service since - # validating tokens is a privileged call - self._admin_token = self._conf_get('admin_token') - if self._admin_token: - self._LOG.warning( - "The admin_token option in the auth_token middleware is " - "deprecated and should not be used. The admin_user and " - "admin_password options should be used instead. The " - "admin_token option may be removed in a future release.") - self._admin_token_expiry = None - self._admin_user = self._conf_get('admin_user') - self._admin_password = self._conf_get('admin_password') - self._admin_tenant_name = self._conf_get('admin_tenant_name') - memcache_security_strategy = ( self._conf_get('memcache_security_strategy')) @@ -562,15 +527,6 @@ class AuthProtocol(object): self._token_revocation_list_fetched_time_prop = None self._token_revocation_list_cache_timeout = datetime.timedelta( seconds=self._conf_get('revocation_cache_time')) - http_connect_timeout_cfg = self._conf_get('http_connect_timeout') - self._http_connect_timeout = (http_connect_timeout_cfg and - int(http_connect_timeout_cfg)) - self._auth_version = None - self._http_request_max_retries = ( - self._conf_get('http_request_max_retries')) - - self._include_service_catalog = self._conf_get( - 'include_service_catalog') self._check_revocations_for_cached = self._conf_get( 'check_revocations_for_cached') @@ -582,64 +538,6 @@ class AuthProtocol(object): else: return CONF.keystone_authtoken[name] - def _choose_api_version(self): - """Determine the api version that we should use.""" - - # If the configuration specifies an auth_version we will just - # assume that is correct and use it. We could, of course, check - # that this version is supported by the server, but in case - # there are some problems in the field, we want as little code - # as possible in the way of letting auth_token talk to the - # server. - if self._conf_get('auth_version'): - version_to_use = self._conf_get('auth_version') - self._LOG.info('Auth Token proceeding with requested %s apis', - version_to_use) - else: - version_to_use = None - versions_supported_by_server = self._get_supported_versions() - if versions_supported_by_server: - for version in _LIST_OF_VERSIONS_TO_ATTEMPT: - if version in versions_supported_by_server: - version_to_use = version - break - if version_to_use: - self._LOG.info('Auth Token confirmed use of %s apis', - version_to_use) - else: - self._LOG.error( - 'Attempted versions [%s] not in list supported by ' - 'server [%s]', - ', '.join(_LIST_OF_VERSIONS_TO_ATTEMPT), - ', '.join(versions_supported_by_server)) - raise ServiceError('No compatible apis supported by server') - return version_to_use - - def _get_supported_versions(self): - versions = [] - response, data = self._json_request('GET', '/') - if response.status_code == 501: - msg = 'Old keystone installation found...assuming v2.0' - self._LOG.warning(msg) - versions.append('v2.0') - elif response.status_code != 300: - self._LOG.error('Unable to get version info from keystone: %s', - response.status_code) - raise ServiceError('Unable to get version info from keystone') - else: - try: - for version in data['versions']['values']: - versions.append(version['id']) - except KeyError: - self._LOG.error( - 'Invalid version response format from server') - raise ServiceError('Unable to parse version response ' - 'from keystone') - - self._LOG.debug('Server reports support for api versions: %s', - ', '.join(versions)) - return versions - def __call__(self, env, start_response): """Handle incoming request. @@ -734,149 +632,12 @@ class AuthProtocol(object): :returns HTTPUnauthorized http response """ - header_val = 'Keystone uri=\'%s\'' % self._auth_uri + header_val = 'Keystone uri=\'%s\'' % self._identity_server.auth_uri headers = [('WWW-Authenticate', header_val)] resp = _MiniResp('Authentication required', env, headers) start_response('401 Unauthorized', resp.headers) return resp.body - def _get_admin_token(self): - """Return admin token, possibly fetching a new one. - - if self.admin_token_expiry is set from fetching an admin token, check - it for expiration, and request a new token is the existing token - is about to expire. - - :return admin token id - :raise ServiceError when unable to retrieve token from keystone - - """ - if self._admin_token_expiry: - if _will_expire_soon(self._admin_token_expiry): - self._admin_token = None - - if not self._admin_token: - (self._admin_token, - self._admin_token_expiry) = self._request_admin_token() - - return self._admin_token - - def _http_request(self, method, path, **kwargs): - """HTTP request helper used to make unspecified content type requests. - - :param method: http method - :param path: relative request url - :return (http response object, response body) - :raise ServerError when unable to communicate with keystone - - """ - url = '%s/%s' % (self._identity_uri, path.lstrip('/')) - - kwargs.setdefault('timeout', self._http_connect_timeout) - if self._cert_file and self._key_file: - kwargs['cert'] = (self._cert_file, self._key_file) - elif self._cert_file or self._key_file: - self._LOG.warn('Cannot use only a cert or key file. ' - 'Please provide both. Ignoring.') - - kwargs['verify'] = self._ssl_ca_file or True - if self._ssl_insecure: - kwargs['verify'] = False - - RETRIES = self._http_request_max_retries - retry = 0 - while True: - try: - response = requests.request(method, url, **kwargs) - break - except Exception as e: - if retry >= RETRIES: - self._LOG.error('HTTP connection exception: %s', e) - raise NetworkError('Unable to communicate with keystone') - # NOTE(vish): sleep 0.5, 1, 2 - self._LOG.warn('Retrying on HTTP connection exception: %s', e) - time.sleep(2.0 ** retry / 2) - retry += 1 - - return response - - def _json_request(self, method, path, body=None, additional_headers=None): - """HTTP request helper used to make json requests. - - :param method: http method - :param path: relative request url - :param body: dict to encode to json as request body. Optional. - :param additional_headers: dict of additional headers to send with - http request. Optional. - :return (http response object, response body parsed as json) - :raise ServerError when unable to communicate with keystone - - """ - kwargs = { - 'headers': { - 'Content-type': 'application/json', - 'Accept': 'application/json', - }, - } - - if additional_headers: - kwargs['headers'].update(additional_headers) - - if body: - kwargs['data'] = jsonutils.dumps(body) - - response = self._http_request(method, path, **kwargs) - - try: - data = jsonutils.loads(response.text) - except ValueError: - self._LOG.debug('Keystone did not return json-encoded body') - data = {} - - return response, data - - def _request_admin_token(self): - """Retrieve new token as admin user from keystone. - - :return token id upon success - :raises ServerError when unable to communicate with keystone - - Irrespective of the auth version we are going to use for the - user token, for simplicity we always use a v2 admin token to - validate the user token. - - """ - params = { - 'auth': { - 'passwordCredentials': { - 'username': self._admin_user, - 'password': self._admin_password, - }, - 'tenantName': self._admin_tenant_name, - } - } - - response, data = self._json_request('POST', - '/v2.0/tokens', - body=params) - - try: - token = data['access']['token']['id'] - expiry = data['access']['token']['expires'] - if not (token and expiry): - raise AssertionError('invalid token or expire') - datetime_expiry = timeutils.parse_isotime(expiry) - return (token, timeutils.normalize_time(datetime_expiry)) - except (AssertionError, KeyError): - self._LOG.warn( - 'Unexpected response from keystone service: %s', data) - raise ServiceError('invalid json response') - except (ValueError): - data['access']['token']['id'] = '<SANITIZED>' - self._LOG.warn( - 'Unable to parse expiration time from token: %s', data) - raise ServiceError('invalid json response') - def _validate_user_token(self, user_token, env, retry=True): """Authenticate user token @@ -913,7 +674,7 @@ class AuthProtocol(object): verified = self._verify_signed_token(user_token, token_ids) data = jsonutils.loads(verified) else: - data = self._verify_uuid_token(user_token, retry) + data = self._identity_server.verify_token(user_token, retry) expires = _confirm_token_not_expired(data) self._confirm_token_bind(data, env) self._token_cache.store(token_id, data, expires) @@ -1078,59 +839,6 @@ class AuthProtocol(object): 'identifier': identifier}) self._invalid_user_token() - def _verify_uuid_token(self, user_token, retry=True): - """Authenticate user token with keystone. - - :param user_token: user's token id - :param retry: flag that forces the middleware to retry - user authentication when an indeterminate - response is received. Optional. - :return: token object received from keystone on success - :raise InvalidUserToken: if token is rejected - :raise ServiceError: if unable to authenticate token - - """ - # Determine the highest api version we can use. - if not self._auth_version: - self._auth_version = self._choose_api_version() - - if self._auth_version == 'v3.0': - headers = {'X-Auth-Token': self._get_admin_token(), - 'X-Subject-Token': _safe_quote(user_token)} - path = '/v3/auth/tokens' - if not self._include_service_catalog: - # NOTE(gyee): only v3 API support this option - path = path + '?nocatalog' - response, data = self._json_request( - 'GET', - path, - additional_headers=headers) - else: - headers = {'X-Auth-Token': self._get_admin_token()} - response, data = self._json_request( - 'GET', - '/v2.0/tokens/%s' % _safe_quote(user_token), - additional_headers=headers) - - if response.status_code == 200: - return data - if response.status_code == 404: - self._LOG.warn('Authorization failed for token') - raise InvalidUserToken('Token authorization failed') - if response.status_code == 401: - self._LOG.info('Keystone rejected admin token, resetting') - self.admin_token = None - else: - self._LOG.error('Bad response code while validating token: %s', - response.status_code) - if retry: - self._LOG.info('Retrying validation') - return self._verify_uuid_token(user_token, False) - else: - self._LOG.warn('Invalid user token. Keystone response: %s', data) - - raise InvalidUserToken() - def _is_signed_token_revoked(self, token_ids): """Indicate whether the token appears in the revocation list.""" for token_id in token_ids: @@ -1280,42 +988,19 @@ class AuthProtocol(object): self._token_revocation_list_fetched_time = timeutils.utcnow() self._atomic_write_to_signing_dir(self._revoked_file_name, value) - def _fetch_revocation_list(self, retry=True): - headers = {'X-Auth-Token': self._get_admin_token()} - response, data = self._json_request('GET', '/v2.0/tokens/revoked', - additional_headers=headers) - if response.status_code == 401: - if retry: - self._LOG.info( - 'Keystone rejected admin token, resetting admin token') - self._admin_token = None - return self._fetch_revocation_list(retry=False) - if response.status_code != 200: - raise ServiceError('Unable to fetch token revocation list.') - if 'signed' not in data: - raise ServiceError('Revocation list improperly formatted.') - return self._cms_verify(data['signed']) - - def _fetch_cert_file(self, cert_file_name, cert_type): - if not self._auth_version: - self._auth_version = self._choose_api_version() - - if self._auth_version == 'v3.0': - if cert_type == 'signing': - cert_type = 'certificates' - path = '/v3/OS-SIMPLE-CERT/' + cert_type - else: - path = '/v2.0/certificates/' + cert_type - response = self._http_request('GET', path) - if response.status_code != 200: - raise exceptions.CertificateConfigError(response.text) - self._atomic_write_to_signing_dir(cert_file_name, response.text) + def _fetch_revocation_list(self): + revocation_list_data = self._identity_server.fetch_revocation_list() + return self._cms_verify(revocation_list_data) def _fetch_signing_cert(self): - self._fetch_cert_file(self._signing_cert_file_name, 'signing') + self._atomic_write_to_signing_dir( + self._signing_cert_file_name, + self._identity_server.fetch_signing_cert()) def _fetch_ca_cert(self): - self._fetch_cert_file(self._signing_ca_file_name, 'ca') + self._atomic_write_to_signing_dir( + self._signing_ca_file_name, + self._identity_server.fetch_ca_cert()) class _CachePool(list): @@ -1345,6 +1030,378 @@ class _CachePool(list): self.append(c) +class _IdentityServer(object): + """Operations on the Identity API server. + + The auth_token middleware needs to communicate with the Identity API server + to validate UUID tokens, fetch the revocation list, signing certificates, + etc. This class encapsulates the data and methods to perform these + operations. + + """ + + def __init__(self, log, include_service_catalog=None, identity_uri=None, + auth_uri=None, auth_host=None, auth_port=None, + auth_protocol=None, auth_admin_prefix=None, cert_file=None, + key_file=None, ssl_ca_file=None, ssl_insecure=None, + admin_token=None, admin_user=None, admin_password=None, + admin_tenant_name=None, http_connect_timeout=None, + http_request_max_retries=None, auth_version=None): + self._LOG = log + self._include_service_catalog = include_service_catalog + self._req_auth_version = auth_version + + # where to find the auth service (we use this to validate tokens) + self._identity_uri = identity_uri + self.auth_uri = auth_uri + + # NOTE(jamielennox): it does appear here that our defaults arguments + # are backwards. We need to do it this way so that we can handle the + # same deprecation strategy for CONF and the conf variable. + if not self._identity_uri: + self._LOG.warning('Configuring admin URI using auth fragments. ' + 'This is deprecated, use \'identity_uri\'' + ' instead.') + + if netaddr.valid_ipv6(auth_host): + # Note(dzyu) it is an IPv6 address, so it needs to be wrapped + # with '[]' to generate a valid IPv6 URL, based on + # http://www.ietf.org/rfc/rfc2732.txt + auth_host = '[%s]' % auth_host + + self._identity_uri = '%s://%s:%s' % (auth_protocol, auth_host, + auth_port) + if auth_admin_prefix: + self._identity_uri = '%s/%s' % (self._identity_uri, + auth_admin_prefix.strip('/')) + else: + self._identity_uri = self._identity_uri.rstrip('/') + + if self.auth_uri is None: + self._LOG.warning( + 'Configuring auth_uri to point to the public identity ' + 'endpoint is required; clients may not be able to ' + 'authenticate against an admin endpoint') + + # FIXME(dolph): drop support for this fallback behavior as + # documented in bug 1207517. + # NOTE(jamielennox): we urljoin '/' to get just the base URI as + # this is the original behaviour. + self.auth_uri = urllib.parse.urljoin(self._identity_uri, '/') + self.auth_uri = self.auth_uri.rstrip('/') + + # SSL + self._cert_file = cert_file + self._key_file = key_file + self._ssl_ca_file = ssl_ca_file + self._ssl_insecure = ssl_insecure + + # Credentials used to verify this component with the Auth service since + # validating tokens is a privileged call + self._admin_token = admin_token + if self._admin_token: + self._LOG.warning( + "The admin_token option in the auth_token middleware is " + "deprecated and should not be used. The admin_user and " + "admin_password options should be used instead. The " + "admin_token option may be removed in a future release.") + self._admin_token_expiry = None + self._admin_user = admin_user + self._admin_password = admin_password + self._admin_tenant_name = admin_tenant_name + + self._http_connect_timeout = http_connect_timeout + + self._auth_version = None + self._http_request_max_retries = http_request_max_retries + + def verify_token(self, user_token, retry=True): + """Authenticate user token with keystone. + + :param user_token: user's token id + :param retry: flag that forces the middleware to retry + user authentication when an indeterminate + response is received. Optional. + :return: token object received from keystone on success + :raise InvalidUserToken: if token is rejected + :raise ServiceError: if unable to authenticate token + + """ + # Determine the highest api version we can use. + if not self._auth_version: + self._auth_version = self._choose_api_version() + + if self._auth_version == 'v3.0': + headers = {'X-Auth-Token': self.get_admin_token(), + 'X-Subject-Token': _safe_quote(user_token)} + path = '/v3/auth/tokens' + if not self._include_service_catalog: + # NOTE(gyee): only v3 API support this option + path = path + '?nocatalog' + response, data = self._json_request( + 'GET', + path, + additional_headers=headers) + else: + headers = {'X-Auth-Token': self.get_admin_token()} + response, data = self._json_request( + 'GET', + '/v2.0/tokens/%s' % _safe_quote(user_token), + additional_headers=headers) + + if response.status_code == 200: + return data + if response.status_code == 404: + self._LOG.warn('Authorization failed for token') + raise InvalidUserToken('Token authorization failed') + if response.status_code == 401: + self._LOG.info( + 'Keystone rejected admin token, resetting') + self._admin_token = None + else: + self._LOG.error('Bad response code while validating token: %s', + response.status_code) + if retry: + self._LOG.info('Retrying validation') + return self.verify_token(user_token, False) + else: + self._LOG.warn('Invalid user token. Keystone response: %s', data) + + raise InvalidUserToken() + + def fetch_revocation_list(self, retry=True): + headers = {'X-Auth-Token': self.get_admin_token()} + response, data = self._json_request('GET', '/v2.0/tokens/revoked', + additional_headers=headers) + if response.status_code == 401: + if retry: + self._LOG.info( + 'Keystone rejected admin token, resetting admin token') + self._admin_token = None + return self.fetch_revocation_list(retry=False) + if response.status_code != 200: + raise ServiceError('Unable to fetch token revocation list.') + if 'signed' not in data: + raise ServiceError('Revocation list improperly formatted.') + return data['signed'] + + def fetch_signing_cert(self): + return self._fetch_cert_file('signing') + + def fetch_ca_cert(self): + return self._fetch_cert_file('ca') + + def _choose_api_version(self): + """Determine the api version that we should use.""" + + # If the configuration specifies an auth_version we will just + # assume that is correct and use it. We could, of course, check + # that this version is supported by the server, but in case + # there are some problems in the field, we want as little code + # as possible in the way of letting auth_token talk to the + # server. + if self._req_auth_version: + version_to_use = self._req_auth_version + self._LOG.info('Auth Token proceeding with requested %s apis', + version_to_use) + else: + version_to_use = None + versions_supported_by_server = self._get_supported_versions() + if versions_supported_by_server: + for version in _LIST_OF_VERSIONS_TO_ATTEMPT: + if version in versions_supported_by_server: + version_to_use = version + break + if version_to_use: + self._LOG.info('Auth Token confirmed use of %s apis', + version_to_use) + else: + self._LOG.error( + 'Attempted versions [%s] not in list supported by ' + 'server [%s]', + ', '.join(_LIST_OF_VERSIONS_TO_ATTEMPT), + ', '.join(versions_supported_by_server)) + raise ServiceError('No compatible apis supported by server') + return version_to_use + + def _get_supported_versions(self): + versions = [] + response, data = self._json_request('GET', '/') + if response.status_code == 501: + self._LOG.warning( + 'Old keystone installation found...assuming v2.0') + versions.append('v2.0') + elif response.status_code != 300: + self._LOG.error('Unable to get version info from keystone: %s', + response.status_code) + raise ServiceError('Unable to get version info from keystone') + else: + try: + for version in data['versions']['values']: + versions.append(version['id']) + except KeyError: + self._LOG.error( + 'Invalid version response format from server') + raise ServiceError('Unable to parse version response ' + 'from keystone') + + self._LOG.debug('Server reports support for api versions: %s', + ', '.join(versions)) + return versions + + def get_admin_token(self): + """Return admin token, possibly fetching a new one. + + if self._admin_token_expiry is set from fetching an admin token, check + it for expiration, and request a new token is the existing token + is about to expire. + + :return admin token id + :raise ServiceError when unable to retrieve token from keystone + + """ + if self._admin_token_expiry: + if _will_expire_soon(self._admin_token_expiry): + self._admin_token = None + + if not self._admin_token: + (self._admin_token, + self._admin_token_expiry) = self._request_admin_token() + + return self._admin_token + + def _http_request(self, method, path, **kwargs): + """HTTP request helper used to make unspecified content type requests. + + :param method: http method + :param path: relative request url + :return (http response object, response body) + :raise ServerError when unable to communicate with keystone + + """ + url = '%s/%s' % (self._identity_uri, path.lstrip('/')) + + kwargs.setdefault('timeout', self._http_connect_timeout) + if self._cert_file and self._key_file: + kwargs['cert'] = (self._cert_file, self._key_file) + elif self._cert_file or self._key_file: + self._LOG.warn('Cannot use only a cert or key file. ' + 'Please provide both. Ignoring.') + + kwargs['verify'] = self._ssl_ca_file or True + if self._ssl_insecure: + kwargs['verify'] = False + + RETRIES = self._http_request_max_retries + retry = 0 + while True: + try: + response = requests.request(method, url, **kwargs) + break + except Exception as e: + if retry >= RETRIES: + self._LOG.error('HTTP connection exception: %s', e) + raise NetworkError('Unable to communicate with keystone') + # NOTE(vish): sleep 0.5, 1, 2 + self._LOG.warn('Retrying on HTTP connection exception: %s', e) + time.sleep(2.0 ** retry / 2) + retry += 1 + + return response + + def _json_request(self, method, path, body=None, additional_headers=None): + """HTTP request helper used to make json requests. + + :param method: http method + :param path: relative request url + :param body: dict to encode to json as request body. Optional. + :param additional_headers: dict of additional headers to send with + http request. Optional. + :return (http response object, response body parsed as json) + :raise ServerError when unable to communicate with keystone + + """ + kwargs = { + 'headers': { + 'Content-type': 'application/json', + 'Accept': 'application/json', + }, + } + + if additional_headers: + kwargs['headers'].update(additional_headers) + + if body: + kwargs['data'] = jsonutils.dumps(body) + + response = self._http_request(method, path, **kwargs) + + try: + data = jsonutils.loads(response.text) + except ValueError: + self._LOG.debug('Keystone did not return json-encoded body') + data = {} + + return response, data + + def _request_admin_token(self): + """Retrieve new token as admin user from keystone. + + :return token id upon success + :raises ServerError when unable to communicate with keystone + + Irrespective of the auth version we are going to use for the + user token, for simplicity we always use a v2 admin token to + validate the user token. + + """ + params = { + 'auth': { + 'passwordCredentials': { + 'username': self._admin_user, + 'password': self._admin_password, + }, + 'tenantName': self._admin_tenant_name, + } + } + + response, data = self._json_request('POST', + '/v2.0/tokens', + body=params) + + try: + token = data['access']['token']['id'] + expiry = data['access']['token']['expires'] + if not (token and expiry): + raise AssertionError('invalid token or expire') + datetime_expiry = timeutils.parse_isotime(expiry) + return (token, timeutils.normalize_time(datetime_expiry)) + except (AssertionError, KeyError): + self._LOG.warn( + 'Unexpected response from keystone service: %s', data) + raise ServiceError('invalid json response') + except (ValueError): + data['access']['token']['id'] = '<SANITIZED>' + self._LOG.warn( + 'Unable to parse expiration time from token: %s', data) + raise ServiceError('invalid json response') + + def _fetch_cert_file(self, cert_type): + if not self._auth_version: + self._auth_version = self._choose_api_version() + + if self._auth_version == 'v3.0': + if cert_type == 'signing': + cert_type = 'certificates' + path = '/v3/OS-SIMPLE-CERT/' + cert_type + else: + path = '/v2.0/certificates/' + cert_type + response = self._http_request('GET', path) + if response.status_code != 200: + raise exceptions.CertificateConfigError(response.text) + return response.text + + class _TokenCache(object): """Encapsulates the auth_token token cache functionality. diff --git a/keystonemiddleware/openstack/common/fixture/__init__.py b/keystonemiddleware/openstack/common/fixture/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/keystonemiddleware/openstack/common/fixture/__init__.py +++ /dev/null diff --git a/keystonemiddleware/openstack/common/fixture/config.py b/keystonemiddleware/openstack/common/fixture/config.py deleted file mode 100644 index 9489b85..0000000 --- a/keystonemiddleware/openstack/common/fixture/config.py +++ /dev/null @@ -1,85 +0,0 @@ -# -# Copyright 2013 Mirantis, Inc. -# Copyright 2013 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import fixtures -from oslo.config import cfg -import six - - -class Config(fixtures.Fixture): - """Allows overriding configuration settings for the test. - - `conf` will be reset on cleanup. - - """ - - def __init__(self, conf=cfg.CONF): - self.conf = conf - - def setUp(self): - super(Config, self).setUp() - # NOTE(morganfainberg): unregister must be added to cleanup before - # reset is because cleanup works in reverse order of registered items, - # and a reset must occur before unregistering options can occur. - self.addCleanup(self._unregister_config_opts) - self.addCleanup(self.conf.reset) - self._registered_config_opts = {} - - def config(self, **kw): - """Override configuration values. - - The keyword arguments are the names of configuration options to - override and their values. - - If a `group` argument is supplied, the overrides are applied to - the specified configuration option group, otherwise the overrides - are applied to the ``default`` group. - - """ - - group = kw.pop('group', None) - for k, v in six.iteritems(kw): - self.conf.set_override(k, v, group) - - def _unregister_config_opts(self): - for group in self._registered_config_opts: - self.conf.unregister_opts(self._registered_config_opts[group], - group=group) - - def register_opt(self, opt, group=None): - """Register a single option for the test run. - - Options registered in this manner will automatically be unregistered - during cleanup. - - If a `group` argument is supplied, it will register the new option - to that group, otherwise the option is registered to the ``default`` - group. - """ - self.conf.register_opt(opt, group=group) - self._registered_config_opts.setdefault(group, set()).add(opt) - - def register_opts(self, opts, group=None): - """Register multiple options for the test run. - - This works in the same manner as register_opt() but takes a list of - options as the first argument. All arguments will be registered to the - same group if the ``group`` argument is supplied, otherwise all options - will be registered to the ``default`` group. - """ - for opt in opts: - self.register_opt(opt, group=group) diff --git a/keystonemiddleware/openstack/common/gettextutils.py b/keystonemiddleware/openstack/common/gettextutils.py index f901d3a..3525699 100644 --- a/keystonemiddleware/openstack/common/gettextutils.py +++ b/keystonemiddleware/openstack/common/gettextutils.py @@ -23,7 +23,6 @@ Usual usage in an openstack.common module: """ import copy -import functools import gettext import locale from logging import handlers @@ -42,7 +41,7 @@ class TranslatorFactory(object): """Create translator functions """ - def __init__(self, domain, lazy=False, localedir=None): + def __init__(self, domain, localedir=None): """Establish a set of translation functions for the domain. :param domain: Name of translation domain, @@ -55,7 +54,6 @@ class TranslatorFactory(object): :type localedir: str """ self.domain = domain - self.lazy = lazy if localedir is None: localedir = os.environ.get(domain.upper() + '_LOCALEDIR') self.localedir = localedir @@ -75,16 +73,19 @@ class TranslatorFactory(object): """ if domain is None: domain = self.domain - if self.lazy: - return functools.partial(Message, domain=domain) - t = gettext.translation( - domain, - localedir=self.localedir, - fallback=True, - ) - if six.PY3: - return t.gettext - return t.ugettext + t = gettext.translation(domain, + localedir=self.localedir, + fallback=True) + # Use the appropriate method of the translation object based + # on the python version. + m = t.gettext if six.PY3 else t.ugettext + + def f(msg): + """oslo.i18n.gettextutils translation function.""" + if USE_LAZY: + return Message(msg, domain=domain) + return m(msg) + return f @property def primary(self): @@ -147,19 +148,11 @@ def enable_lazy(): your project is importing _ directly instead of using the gettextutils.install() way of importing the _ function. """ - # FIXME(dhellmann): This function will be removed in oslo.i18n, - # because the TranslatorFactory makes it superfluous. - global _, _LI, _LW, _LE, _LC, USE_LAZY - tf = TranslatorFactory('keystonemiddleware', lazy=True) - _ = tf.primary - _LI = tf.log_info - _LW = tf.log_warning - _LE = tf.log_error - _LC = tf.log_critical + global USE_LAZY USE_LAZY = True -def install(domain, lazy=False): +def install(domain): """Install a _() function using the given translation domain. Given a translation domain, install a _() function using gettext's @@ -170,26 +163,14 @@ def install(domain, lazy=False): a translation-domain-specific environment variable (e.g. NOVA_LOCALEDIR). + Note that to enable lazy translation, enable_lazy must be + called. + :param domain: the translation domain - :param lazy: indicates whether or not to install the lazy _() function. - The lazy _() introduces a way to do deferred translation - of messages by installing a _ that builds Message objects, - instead of strings, which can then be lazily translated into - any available locale. """ - if lazy: - from six import moves - tf = TranslatorFactory(domain, lazy=True) - moves.builtins.__dict__['_'] = tf.primary - else: - localedir = '%s_LOCALEDIR' % domain.upper() - if six.PY3: - gettext.install(domain, - localedir=os.environ.get(localedir)) - else: - gettext.install(domain, - localedir=os.environ.get(localedir), - unicode=True) + from six import moves + tf = TranslatorFactory(domain) + moves.builtins.__dict__['_'] = tf.primary class Message(six.text_type): @@ -373,8 +354,8 @@ def get_available_languages(domain): 'zh_Hant_HK': 'zh_HK', 'zh_Hant': 'zh_TW', 'fil': 'tl_PH'} - for (locale, alias) in six.iteritems(aliases): - if locale in language_list and alias not in language_list: + for (locale_, alias) in six.iteritems(aliases): + if locale_ in language_list and alias not in language_list: language_list.append(alias) _AVAILABLE_LANGUAGES[domain] = language_list diff --git a/keystonemiddleware/openstack/common/jsonutils.py b/keystonemiddleware/openstack/common/jsonutils.py index ec9e7d9..580e646 100644 --- a/keystonemiddleware/openstack/common/jsonutils.py +++ b/keystonemiddleware/openstack/common/jsonutils.py @@ -168,6 +168,10 @@ def dumps(value, default=to_primitive, **kwargs): return json.dumps(value, default=default, **kwargs) +def dump(obj, fp, *args, **kwargs): + return json.dump(obj, fp, *args, **kwargs) + + def loads(s, encoding='utf-8', **kwargs): return json.loads(strutils.safe_decode(s, encoding), **kwargs) diff --git a/keystonemiddleware/openstack/common/strutils.py b/keystonemiddleware/openstack/common/strutils.py index df8ccdb..f0471c4 100644 --- a/keystonemiddleware/openstack/common/strutils.py +++ b/keystonemiddleware/openstack/common/strutils.py @@ -50,6 +50,28 @@ SLUGIFY_STRIP_RE = re.compile(r"[^\w\s-]") SLUGIFY_HYPHENATE_RE = re.compile(r"[-\s]+") +# NOTE(flaper87): The following 3 globals are used by `mask_password` +_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password'] + +# NOTE(ldbragst): Let's build a list of regex objects using the list of +# _SANITIZE_KEYS we already have. This way, we only have to add the new key +# to the list of _SANITIZE_KEYS and we can generate regular expressions +# for XML and JSON automatically. +_SANITIZE_PATTERNS = [] +_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])', + r'(<%(key)s>).*?(</%(key)s>)', + r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])', + r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])', + r'([\'"].*?%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?[\'"])' + '.*?([\'"])', + r'(%(key)s\s*--?[A-z]+\s*)\S+(\s*)'] + +for key in _SANITIZE_KEYS: + for pattern in _FORMAT_PATTERNS: + reg_ex = re.compile(pattern % {'key': key}, re.DOTALL) + _SANITIZE_PATTERNS.append(reg_ex) + + def int_from_bool_as_string(subject): """Interpret a string as a boolean and return either 1 or 0. @@ -237,3 +259,37 @@ def to_slug(value, incoming=None, errors="strict"): "ascii", "ignore").decode("ascii") value = SLUGIFY_STRIP_RE.sub("", value).strip().lower() return SLUGIFY_HYPHENATE_RE.sub("-", value) + + +def mask_password(message, secret="***"): + """Replace password with 'secret' in message. + + :param message: The string which includes security information. + :param secret: value with which to replace passwords. + :returns: The unicode value of message with the password fields masked. + + For example: + + >>> mask_password("'adminPass' : 'aaaaa'") + "'adminPass' : '***'" + >>> mask_password("'admin_pass' : 'aaaaa'") + "'admin_pass' : '***'" + >>> mask_password('"password" : "aaaaa"') + '"password" : "***"' + >>> mask_password("'original_password' : 'aaaaa'") + "'original_password' : '***'" + >>> mask_password("u'original_password' : u'aaaaa'") + "u'original_password' : u'***'" + """ + message = six.text_type(message) + + # NOTE(ldbragst): Check to see if anything in message contains any key + # specified in _SANITIZE_KEYS, if not then just return the message since + # we don't have to mask any passwords. + if not any(key in message for key in _SANITIZE_KEYS): + return message + + secret = r'\g<1>' + secret + r'\g<2>' + for pattern in _SANITIZE_PATTERNS: + message = re.sub(pattern, secret, message) + return message diff --git a/keystonemiddleware/opts.py b/keystonemiddleware/opts.py new file mode 100644 index 0000000..6abea9d --- /dev/null +++ b/keystonemiddleware/opts.py @@ -0,0 +1,49 @@ +# Copyright (c) 2014 OpenStack Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +__all__ = [ + 'list_auth_token_opts', +] + +import copy + +import keystonemiddleware.auth_token + + +auth_token_opts = [ + ('keystone_authtoken', keystonemiddleware.auth_token._OPTS) +] + + +def list_auth_token_opts(): + """Return a list of oslo.config options available in auth_token middleware. + + The returned list includes all oslo.config options which may be registered + at runtime by the project. + + Each element of the list is a tuple. The first element is the name of the + group under which the list of elements in the second element will be + registered. A group name of None corresponds to the [DEFAULT] group in + config files. + + This function is also discoverable via the entry point + 'keystonemiddleware.auth_token' under the 'oslo.config.opts' + namespace. + + The purpose of this is to allow tools like the Oslo sample config file + generator to discover the options exposed to users by this middleware. + + :returns: a list of (group_name, opts) tuples + """ + return [(g, copy.deepcopy(o)) for g, o in auth_token_opts] diff --git a/keystonemiddleware/tests/test_auth_token_middleware.py b/keystonemiddleware/tests/test_auth_token_middleware.py index 49a683b..66fd07e 100644 --- a/keystonemiddleware/tests/test_auth_token_middleware.py +++ b/keystonemiddleware/tests/test_auth_token_middleware.py @@ -581,7 +581,8 @@ class CommonAuthTokenMiddlewareTest(object): } self.set_middleware(conf=conf) expected_auth_uri = 'http://[2001:2013:1:f101::1]:1234' - self.assertEqual(expected_auth_uri, self.middleware._auth_uri) + self.assertEqual(expected_auth_uri, + self.middleware._identity_server.auth_uri) def assert_valid_request_200(self, token, with_catalog=True): req = webob.Request.blank('/') diff --git a/keystonemiddleware/tests/test_opts.py b/keystonemiddleware/tests/test_opts.py new file mode 100644 index 0000000..d6839b2 --- /dev/null +++ b/keystonemiddleware/tests/test_opts.py @@ -0,0 +1,79 @@ +# Copyright (c) 2014 OpenStack Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import pkg_resources +from testtools import matchers + +from keystonemiddleware import opts +from keystonemiddleware.tests import utils + + +class OptsTestCase(utils.TestCase): + + def _test_list_auth_token_opts(self, result): + self.assertThat(result, matchers.HasLength(1)) + + for group in (g for (g, _l) in result): + self.assertEqual('keystone_authtoken', group) + + expected_opt_names = [ + 'auth_admin_prefix', + 'auth_host', + 'auth_port', + 'auth_protocol', + 'auth_uri', + 'identity_uri', + 'auth_version', + 'delay_auth_decision', + 'http_connect_timeout', + 'http_request_max_retries', + 'admin_token', + 'admin_user', + 'admin_password', + 'admin_tenant_name', + 'cache', + 'certfile', + 'keyfile', + 'cafile', + 'insecure', + 'signing_dir', + 'memcached_servers', + 'token_cache_time', + 'revocation_cache_time', + 'memcache_security_strategy', + 'memcache_secret_key', + 'include_service_catalog', + 'enforce_token_bind', + 'check_revocations_for_cached', + 'hash_algorithms' + ] + opt_names = [o.name for (g, l) in result for o in l] + self.assertThat(opt_names, matchers.HasLength(len(expected_opt_names))) + + for opt in opt_names: + self.assertIn(opt, expected_opt_names) + + def test_list_auth_token_opts(self): + self._test_list_auth_token_opts(opts.list_auth_token_opts()) + + def test_entry_point(self): + result = None + for ep in pkg_resources.iter_entry_points('oslo.config.opts'): + if ep.name == 'keystonemiddleware.auth_token': + list_fn = ep.load() + result = list_fn() + break + + self.assertIsNotNone(result) + self._test_list_auth_token_opts(result) diff --git a/openstack-common.conf b/openstack-common.conf index e6f394c..2dc8fb8 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -2,10 +2,8 @@ # The list of modules to copy from oslo-incubator module=install_venv_common -module=fixture.config module=jsonutils module=memorycache -module=strutils module=timeutils # The base module to hold the copy of openstack.common diff --git a/requirements.txt b/requirements.txt index 3079862..3ca9486 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ iso8601>=0.1.9 netaddr>=0.7.6 oslo.config>=1.2.1 pbr>=0.6,!=0.7,<1.0 -PrettyTable>=0.7,<0.8 python-keystoneclient>=0.9.0 requests>=1.1 six>=1.7.0 +WebOb>=1.2.3 @@ -27,6 +27,10 @@ packages = setup-hooks = pbr.hooks.setup_hook +[entry_points] +oslo.config.opts = + keystonemiddleware.auth_token = keystonemiddleware.opts:list_auth_token_opts + [build_sphinx] source-dir = doc/source build-dir = doc/build @@ -17,6 +17,14 @@ # THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT import setuptools +# In python < 2.7.4, a lazy loading of package `pbr` will break +# setuptools if some other modules registered functions in `atexit`. +# solution from: http://bugs.python.org/issue15881#msg170215 +try: + import multiprocessing # noqa +except ImportError: + pass + setuptools.setup( setup_requires=['pbr'], pbr=True) diff --git a/test-requirements.txt b/test-requirements.txt index 93b69bc..f6a6cf9 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -7,8 +7,6 @@ mock>=1.0 mox3>=0.7.0 pycrypto>=2.6 sphinx>=1.1.2,!=1.2.0,<1.3 -stevedore>=0.14 testrepository>=0.0.18 testresources>=0.2.4 testtools>=0.9.34 -WebOb>=1.2.3 |