Remove last dependency from keystone client

EC2 tests use keystone client to generate AWS style credentials if they
are not configured (like in the gate).
Implementing the identity v2 functions to list and create AWS style
credentials in tempest's identity_client.
Replacing calls to keystoneclient with calls to identity client.

The generation of AWS credentials is moved one level up to the
client manager, because the boto client would otherwise depend
on the identity client. Client manager already has an identity
client and can prepare credentials for the boto client if needed.

Partially-implements: bp/multi-keystone-api-version-tests
Change-Id: If82ec054caec14f81679738148d702ad71c534a8

5 files changed, 88 insertions, 77 deletions

diff --git a/tempest/clients.py b/tempest/clients.py
index e5f41ebbf..809ec1596 100644
--- a/tempest/clients.py
+++ b/tempest/clients.py
@@ -228,13 +228,14 @@ class Manager(manager.Manager):
         self.negative_client = negative_rest_client.NegativeRestClient(
             self.auth_provider, service)
 
-        # TODO(andreaf) EC2 client still do their auth, v2 only
-        ec2_client_args = (self.credentials.username,
-                           self.credentials.password,
-                           CONF.identity.uri,
-                           self.credentials.tenant_name)
-        self.ec2api_client = botoclients.APIClientEC2(*ec2_client_args)
-        self.s3_client = botoclients.ObjectClientS3(*ec2_client_args)
+        # Generating EC2 credentials in tempest is only supported
+        # with identity v2
+        if CONF.identity_feature_enabled.api_v2 and \
+                CONF.identity.auth_version == 'v2':
+            # EC2 and S3 clients, if used, will check onfigured AWS credentials
+            # and generate new ones if needed
+            self.ec2api_client = botoclients.APIClientEC2(self.identity_client)
+            self.s3_client = botoclients.ObjectClientS3(self.identity_client)
 
     def _set_compute_clients(self):
         params = {
diff --git a/tempest/services/botoclients.py b/tempest/services/botoclients.py
index 1cbdb0c78..6a1af6ccc 100644
--- a/tempest/services/botoclients.py
+++ b/tempest/services/botoclients.py
@@ -20,7 +20,6 @@ import types
 import urlparse
 
 from tempest import config
-from tempest import exceptions
 
 import boto
 import boto.ec2
@@ -33,41 +32,15 @@ class BotoClientBase(object):
 
     ALLOWED_METHODS = set()
 
-    def __init__(self, username=None, password=None,
-                 auth_url=None, tenant_name=None,
-                 *args, **kwargs):
-        # FIXME(andreaf) replace credentials and auth_url with auth_provider
+    def __init__(self, identity_client):
+        self.identity_client = identity_client
 
-        insecure_ssl = CONF.identity.disable_ssl_certificate_validation
         self.ca_cert = CONF.identity.ca_certificates_file
-
         self.connection_timeout = str(CONF.boto.http_socket_timeout)
         self.num_retries = str(CONF.boto.num_retries)
         self.build_timeout = CONF.boto.build_timeout
-        self.ks_cred = {"username": username,
-                        "password": password,
-                        "auth_url": auth_url,
-                        "tenant_name": tenant_name,
-                        "insecure": insecure_ssl,
-                        "cacert": self.ca_cert}
-
-    def _keystone_aws_get(self):
-        # FIXME(andreaf) Move EC2 credentials to AuthProvider
-        import keystoneclient.v2_0.client
-
-        keystone = keystoneclient.v2_0.client.Client(**self.ks_cred)
-        ec2_cred_list = keystone.ec2.list(keystone.auth_user_id)
-        ec2_cred = None
-        for cred in ec2_cred_list:
-            if cred.tenant_id == keystone.auth_tenant_id:
-                ec2_cred = cred
-                break
-        else:
-            ec2_cred = keystone.ec2.create(keystone.auth_user_id,
-                                           keystone.auth_tenant_id)
-        if not all((ec2_cred, ec2_cred.access, ec2_cred.secret)):
-            raise lib_exc.NotFound("Unable to get access and secret keys")
-        return ec2_cred
+
+        self.connection_data = {}
 
     def _config_boto_timeout(self, timeout, retries):
         try:
@@ -105,33 +78,47 @@ class BotoClientBase(object):
     def get_connection(self):
         self._config_boto_timeout(self.connection_timeout, self.num_retries)
         self._config_boto_ca_certificates_file(self.ca_cert)
-        if not all((self.connection_data["aws_access_key_id"],
-                   self.connection_data["aws_secret_access_key"])):
-            if all([self.ks_cred.get('auth_url'),
-                    self.ks_cred.get('username'),
-                    self.ks_cred.get('tenant_name'),
-                    self.ks_cred.get('password')]):
-                ec2_cred = self._keystone_aws_get()
-                self.connection_data["aws_access_key_id"] = \
-                    ec2_cred.access
-                self.connection_data["aws_secret_access_key"] = \
-                    ec2_cred.secret
-            else:
-                raise exceptions.InvalidConfiguration(
-                    "Unable to get access and secret keys")
+
+        ec2_client_args = {'aws_access_key_id': CONF.boto.aws_access,
+                           'aws_secret_access_key': CONF.boto.aws_secret}
+        if not all(ec2_client_args.values()):
+            ec2_client_args = self.get_aws_credentials(self.identity_client)
+
+        self.connection_data.update(ec2_client_args)
         return self.connect_method(**self.connection_data)
 
+    def get_aws_credentials(self, identity_client):
+        """
+        Obtain existing, or create new AWS credentials
+        :param identity_client: identity client with embedded credentials
+        :return: EC2 credentials
+        """
+        ec2_cred_list = identity_client.list_user_ec2_credentials(
+            identity_client.user_id)
+        for cred in ec2_cred_list:
+            if cred['tenant_id'] == identity_client.tenant_id:
+                ec2_cred = cred
+                break
+        else:
+            ec2_cred = identity_client.create_user_ec2_credentials(
+                identity_client.user_id, identity_client.tenant_id)
+        if not all((ec2_cred, ec2_cred['access'], ec2_cred['secret'])):
+            raise lib_exc.NotFound("Unable to get access and secret keys")
+        else:
+            ec2_cred_aws = {}
+            ec2_cred_aws['aws_access_key_id'] = ec2_cred['access']
+            ec2_cred_aws['aws_secret_access_key'] = ec2_cred['secret']
+        return ec2_cred_aws
+
 
 class APIClientEC2(BotoClientBase):
 
     def connect_method(self, *args, **kwargs):
         return boto.connect_ec2(*args, **kwargs)
 
-    def __init__(self, *args, **kwargs):
-        super(APIClientEC2, self).__init__(*args, **kwargs)
+    def __init__(self, identity_client):
+        super(APIClientEC2, self).__init__(identity_client)
         insecure_ssl = CONF.identity.disable_ssl_certificate_validation
-        aws_access = CONF.boto.aws_access
-        aws_secret = CONF.boto.aws_secret
         purl = urlparse.urlparse(CONF.boto.ec2_url)
 
         region_name = CONF.compute.region
@@ -147,14 +134,12 @@ class APIClientEC2(BotoClientBase):
                 port = 443
         else:
             port = int(port)
-        self.connection_data = {"aws_access_key_id": aws_access,
-                                "aws_secret_access_key": aws_secret,
-                                "is_secure": purl.scheme == "https",
-                                "validate_certs": not insecure_ssl,
-                                "region": region,
-                                "host": purl.hostname,
-                                "port": port,
-                                "path": purl.path}
+        self.connection_data.update({"is_secure": purl.scheme == "https",
+                                     "validate_certs": not insecure_ssl,
+                                     "region": region,
+                                     "host": purl.hostname,
+                                     "port": port,
+                                     "path": purl.path})
 
     ALLOWED_METHODS = set(('create_key_pair', 'get_key_pair',
                            'delete_key_pair', 'import_key_pair',
@@ -207,11 +192,9 @@ class ObjectClientS3(BotoClientBase):
     def connect_method(self, *args, **kwargs):
         return boto.connect_s3(*args, **kwargs)
 
-    def __init__(self, *args, **kwargs):
-        super(ObjectClientS3, self).__init__(*args, **kwargs)
+    def __init__(self, identity_client):
+        super(ObjectClientS3, self).__init__(identity_client)
         insecure_ssl = CONF.identity.disable_ssl_certificate_validation
-        aws_access = CONF.boto.aws_access
-        aws_secret = CONF.boto.aws_secret
         purl = urlparse.urlparse(CONF.boto.s3_url)
         port = purl.port
         if port is None:
@@ -221,14 +204,12 @@ class ObjectClientS3(BotoClientBase):
                 port = 443
         else:
             port = int(port)
-        self.connection_data = {"aws_access_key_id": aws_access,
-                                "aws_secret_access_key": aws_secret,
-                                "is_secure": purl.scheme == "https",
-                                "validate_certs": not insecure_ssl,
-                                "host": purl.hostname,
-                                "port": port,
-                                "calling_format": boto.s3.connection.
-                                OrdinaryCallingFormat()}
+        self.connection_data.update({"is_secure": purl.scheme == "https",
+                                     "validate_certs": not insecure_ssl,
+                                     "host": purl.hostname,
+                                     "port": port,
+                                     "calling_format": boto.s3.connection.
+                                     OrdinaryCallingFormat()})
 
     ALLOWED_METHODS = set(('create_bucket', 'delete_bucket', 'generate_url',
                            'get_all_buckets', 'get_bucket', 'delete_key',
diff --git a/tempest/services/identity/v2/json/identity_client.py b/tempest/services/identity/v2/json/identity_client.py
index 6c4a6b4f7..039f9bbf6 100644
--- a/tempest/services/identity/v2/json/identity_client.py
+++ b/tempest/services/identity/v2/json/identity_client.py
@@ -269,3 +269,15 @@ class IdentityClientJSON(service_client.ServiceClient):
         body = json.loads(body)
         return service_client.ResponseBodyList(resp,
                                                body['extensions']['values'])
+
+    def create_user_ec2_credentials(self, user_id, tenant_id):
+        post_body = json.dumps({'tenant_id': tenant_id})
+        resp, body = self.post('/users/%s/credentials/OS-EC2' % user_id,
+                               post_body)
+        self.expected_success(200, resp.status)
+        return service_client.ResponseBody(resp, self._parse_resp(body))
+
+    def list_user_ec2_credentials(self, user_id):
+        resp, body = self.get('/users/%s/credentials/OS-EC2' % user_id)
+        self.expected_success(200, resp.status)
+        return service_client.ResponseBodyList(resp, self._parse_resp(body))
diff --git a/tempest/tests/test_tenant_isolation.py b/tempest/tests/test_tenant_isolation.py
index a18ad468f..917f63563 100644
--- a/tempest/tests/test_tenant_isolation.py
+++ b/tempest/tests/test_tenant_isolation.py
@@ -41,6 +41,7 @@ class TestTenantIsolation(base.TestCase):
                        fake_identity._fake_v2_response)
         cfg.CONF.set_default('operator_role', 'FakeRole',
                              group='object-storage')
+        self._mock_list_ec2_credentials('fake_user_id', 'fake_tenant_id')
 
     def test_tempest_client(self):
         iso_creds = isolated_creds.IsolatedCreds('test class')
@@ -102,6 +103,18 @@ class TestTenantIsolation(base.TestCase):
                           (200, [{'id': '1', 'name': 'FakeRole'}]))))
         return roles_fix
 
+    def _mock_list_ec2_credentials(self, user_id, tenant_id):
+        ec2_creds_fix = self.useFixture(mockpatch.PatchObject(
+            json_iden_client.IdentityClientJSON,
+            'list_user_ec2_credentials',
+            return_value=(service_client.ResponseBodyList
+                          (200, [{'access': 'fake_access',
+                                  'secret': 'fake_secret',
+                                  'tenant_id': tenant_id,
+                                  'user_id': user_id,
+                                  'trust_id': None}]))))
+        return ec2_creds_fix
+
     def _mock_network_create(self, iso_creds, id, name):
         net_fix = self.useFixture(mockpatch.PatchObject(
             iso_creds.network_admin_client,
diff --git a/tempest/thirdparty/boto/test.py b/tempest/thirdparty/boto/test.py
index 5b2ed70b4..8ed00d9bc 100644
--- a/tempest/thirdparty/boto/test.py
+++ b/tempest/thirdparty/boto/test.py
@@ -26,6 +26,8 @@ from boto import s3
 import keystoneclient.exceptions
 import six
 
+from tempest_lib import exceptions as lib_exc
+
 import tempest.clients
 from tempest.common.utils import file_utils
 from tempest import config
@@ -65,6 +67,8 @@ def decision_maker():
         if not secret_matcher.match(connection_data["aws_secret_access_key"]):
             raise Exception("Invalid AWS secret Key")
         raise Exception("Unknown (Authentication?) Error")
+    # NOTE(andreaf) Setting up an extra manager here is redundant,
+    # and should be removed.
     openstack = tempest.clients.Manager()
     try:
         if urlparse.urlparse(CONF.boto.ec2_url).hostname is None:
@@ -77,7 +81,7 @@ def decision_maker():
                     raise Exception("EC2 target does not looks EC2 service")
                 _cred_sub_check(ec2client.connection_data)
 
-    except keystoneclient.exceptions.Unauthorized:
+    except lib_exc.Unauthorized:
         EC2_CAN_CONNECT_ERROR = "AWS credentials not set," +\
                                 " failed to get them even by keystoneclient"
     except Exception as exc: