Add keystone v3 auth support

Enables swiftclient to authenticate using
the keystone v3 API, allowing user id's, user
domains and tenant/project domains to be
specified.

Since swiftclient imports keystoneclient, the
main changes in swiftclient/client.py are to
selectively import the correct keystoneclient
library version and pass a number of new
options to it via the get_auth() function. In
addition the get_keystoneclient_2_0 method
has been renamed get_auth_keystone to better
reflect its purpose since it now deals with
both v2 and v3 use cases.

In swiftclient/shell.py the new options are
added to the parser. To make the default help
message shorter, help for all the --os-*
options (including the existing v2 options)
is only displayed when explicitly requested
usng a new --os-help option.

A new set of unit tests is added to
test_shell.py to verify the parser. A comment
in tests/sample.conf explains how to
configure the existing functional tests to
run using keystone v3 API.

Note that to use keystone v3
with swift you will need to set
auth_version = v3.0 in the auth_token
middleware config section of
proxy-server.conf.

Change-Id: Ifda0b3263eb919a8c6a1b204ba0a1215ed6f642f

6 files changed, 555 insertions, 89 deletions

diff --git a/swiftclient/client.py b/swiftclient/client.py
index 7b50860..7ce7218 100644
--- a/swiftclient/client.py
+++ b/swiftclient/client.py
@@ -35,6 +35,10 @@ from swiftclient import version as swiftclient_version
 from swiftclient.exceptions import ClientException
 from swiftclient.utils import LengthWrapper
 
+AUTH_VERSIONS_V1 = ('1.0', '1', 1)
+AUTH_VERSIONS_V2 = ('2.0', '2', 2)
+AUTH_VERSIONS_V3 = ('3.0', '3', 3)
+
 try:
     from logging import NullHandler
 except ImportError:
@@ -275,35 +279,57 @@ def get_auth_1_0(url, user, key, snet, **kwargs):
 
 
 def get_keystoneclient_2_0(auth_url, user, key, os_options, **kwargs):
+    # this function is only here to preserve the historic 'public'
+    # interface of this module
+    kwargs.update({'auth_version': '2.0'})
+    return get_auth_keystone(auth_url, user, key, os_options, **kwargs)
+
+
+def get_auth_keystone(auth_url, user, key, os_options, **kwargs):
     """
-    Authenticate against an auth 2.0 server.
+    Authenticate against a keystone server.
 
-    We are using the keystoneclient library for our 2.0 authentication.
+    We are using the keystoneclient library for authentication.
     """
 
     insecure = kwargs.get('insecure', False)
+    auth_version = kwargs.get('auth_version', '2.0')
     debug = logger.isEnabledFor(logging.DEBUG) and True or False
 
     try:
-        from keystoneclient.v2_0 import client as ksclient
+        if auth_version in AUTH_VERSIONS_V3:
+            from keystoneclient.v3 import client as ksclient
+        else:
+            from keystoneclient.v2_0 import client as ksclient
         from keystoneclient import exceptions
     except ImportError:
         sys.exit('''
-Auth version 2.0 requires python-keystoneclient, install it or use Auth
+Auth versions 2.0 and 3 require python-keystoneclient, install it or use Auth
 version 1.0 which requires ST_AUTH, ST_USER, and ST_KEY environment
 variables to be set or overridden with -A, -U, or -K.''')
 
     try:
-        _ksclient = ksclient.Client(username=user,
-                                    password=key,
-                                    tenant_name=os_options.get('tenant_name'),
-                                    tenant_id=os_options.get('tenant_id'),
-                                    debug=debug,
-                                    cacert=kwargs.get('cacert'),
-                                    auth_url=auth_url, insecure=insecure)
+        _ksclient = ksclient.Client(
+            username=user,
+            password=key,
+            tenant_name=os_options.get('tenant_name'),
+            tenant_id=os_options.get('tenant_id'),
+            user_id=os_options.get('user_id'),
+            user_domain_name=os_options.get('user_domain_name'),
+            user_domain_id=os_options.get('user_domain_id'),
+            project_name=os_options.get('project_name'),
+            project_id=os_options.get('project_id'),
+            project_domain_name=os_options.get('project_domain_name'),
+            project_domain_id=os_options.get('project_domain_id'),
+            debug=debug,
+            cacert=kwargs.get('cacert'),
+            auth_url=auth_url, insecure=insecure)
     except exceptions.Unauthorized:
-        raise ClientException('Unauthorised. Check username, password'
-                              ' and tenant name/id')
+        msg = 'Unauthorized. Check username, password and tenant name/id.'
+        if auth_version in AUTH_VERSIONS_V3:
+            msg = 'Unauthorized. Check username/id, password, ' \
+                  + 'tenant name/id and user/tenant domain name/id.'
+        raise ClientException(msg)
     except exceptions.AuthorizationFailure as err:
         raise ClientException('Authorization Failure. %s' % err)
     service_type = os_options.get('service_type') or 'object-store'
@@ -335,13 +361,13 @@ def get_auth(auth_url, user, key, **kwargs):
 
     storage_url, token = None, None
     insecure = kwargs.get('insecure', False)
-    if auth_version in ['1.0', '1', 1]:
+    if auth_version in AUTH_VERSIONS_V1:
         storage_url, token = get_auth_1_0(auth_url,
                                           user,
                                           key,
                                           kwargs.get('snet'),
                                           insecure=insecure)
-    elif auth_version in ['2.0', '2', 2]:
+    elif auth_version in AUTH_VERSIONS_V2 + AUTH_VERSIONS_V3:
         # We are allowing to specify a token/storage-url to re-use
         # without having to re-authenticate.
         if (os_options.get('object_storage_url') and
@@ -349,10 +375,9 @@ def get_auth(auth_url, user, key, **kwargs):
             return (os_options.get('object_storage_url'),
                     os_options.get('auth_token'))
 
-        # We are handling a special use case here when we were
-        # allowing specifying the account/tenant_name with the -U
-        # argument
-        if not kwargs.get('tenant_name') and ':' in user:
+        # We are handling a special use case here where the user argument
+        # specifies both the user name and tenant name in the form tenant:user
+        if user and not kwargs.get('tenant_name') and ':' in user:
             (os_options['tenant_name'],
              user) = user.split(':')
 
@@ -361,14 +386,17 @@ def get_auth(auth_url, user, key, **kwargs):
         if kwargs.get('tenant_name'):
             os_options['tenant_name'] = kwargs['tenant_name']
 
-        if not (os_options.get('tenant_name') or os_options.get('tenant_id')):
+        if not (os_options.get('tenant_name') or os_options.get('tenant_id')
+                or os_options.get('project_name')
+                or os_options.get('project_id')):
             raise ClientException('No tenant specified')
 
         cacert = kwargs.get('cacert', None)
-        storage_url, token = get_keystoneclient_2_0(auth_url, user,
-                                                    key, os_options,
-                                                    cacert=cacert,
-                                                    insecure=insecure)
+        storage_url, token = get_auth_keystone(auth_url, user,
+                                               key, os_options,
+                                               cacert=cacert,
+                                               insecure=insecure,
+                                               auth_version=auth_version)
     else:
         raise ClientException('Unknown auth_version %s specified.'
                               % auth_version)
diff --git a/swiftclient/shell.py b/swiftclient/shell.py
index ef153c7..c0387bb 100755
--- a/swiftclient/shell.py
+++ b/swiftclient/shell.py
@@ -22,7 +22,7 @@ import logging
 
 from errno import EEXIST, ENOENT
 from hashlib import md5
-from optparse import OptionParser, SUPPRESS_HELP
+from optparse import OptionParser, OptionGroup, SUPPRESS_HELP
 from os import environ, listdir, makedirs, utime, _exit as os_exit
 from os.path import dirname, getmtime, getsize, isdir, join, \
     sep as os_path_sep
@@ -1356,9 +1356,14 @@ def parse_args(parser, args, enforce_requires=True):
     if len(args) > 0 and args[0] == 'tempurl':
         return options, args
 
-    if (not (options.auth and options.user and options.key)):
-        # Use 2.0 auth if none of the old args are present
-        options.auth_version = '2.0'
+    if options.auth_version == '3.0':
+        # tolerate sloppy auth_version
+        options.auth_version = '3'
+
+    if (not (options.auth and options.user and options.key)
+            and options.auth_version != '3'):
+            # Use keystone auth if any of the old-style args are missing
+            options.auth_version = '2.0'
 
     # Use new-style args if old ones not present
     if not options.auth and options.os_auth_url:
@@ -1370,8 +1375,15 @@ def parse_args(parser, args, enforce_requires=True):
 
     # Specific OpenStack options
     options.os_options = {
+        'user_id': options.os_user_id,
+        'user_domain_id': options.os_user_domain_id,
+        'user_domain_name': options.os_user_domain_name,
         'tenant_id': options.os_tenant_id,
         'tenant_name': options.os_tenant_name,
+        'project_id': options.os_project_id,
+        'project_name': options.os_project_name,
+        'project_domain_id': options.os_project_domain_id,
+        'project_domain_name': options.os_project_domain_name,
         'service_type': options.os_service_type,
         'endpoint_type': options.os_endpoint_type,
         'auth_token': options.os_auth_token,
@@ -1384,12 +1396,23 @@ def parse_args(parser, args, enforce_requires=True):
 
     if (options.os_options.get('object_storage_url') and
             options.os_options.get('auth_token') and
-            options.auth_version == '2.0'):
+            (options.auth_version == '2.0' or options.auth_version == '3')):
         return options, args
 
-    if enforce_requires and \
-            not (options.auth and options.user and options.key):
-        exit('''
+    if enforce_requires:
+        if options.auth_version == '3':
+            if not options.auth:
+                exit('Auth version 3 requires OS_AUTH_URL to be set or ' +
+                     'overridden with --os-auth-url')
+            if not (options.user or options.os_user_id):
+                exit('Auth version 3 requires either OS_USERNAME or ' +
+                     'OS_USER_ID to be set or overridden with ' +
+                     '--os-username or --os-user-id respectively.')
+            if not options.key:
+                exit('Auth version 3 requires OS_PASSWORD to be set or ' +
+                     'overridden with --os-password')
+        elif not (options.auth and options.user and options.key):
+            exit('''
 Auth version 1.0 requires ST_AUTH, ST_USER, and ST_KEY environment variables
 to be set or overridden with -A, -U, or -K.
 
@@ -1409,13 +1432,20 @@ def main(arguments=None):
     version = client_version
     parser = OptionParser(version='%%prog %s' % version,
                           usage='''
-usage: %%prog [--version] [--help] [--snet] [--verbose]
+usage: %%prog [--version] [--help] [--os-help] [--snet] [--verbose]
              [--debug] [--info] [--quiet] [--auth <auth_url>]
              [--auth-version <auth_version>] [--user <username>]
              [--key <api_key>] [--retries <num_retries>]
              [--os-username <auth-user-name>] [--os-password <auth-password>]
+             [--os-user-id <auth-user-id>]
+             [--os-user-domain-id <auth-user-domain-id>]
+             [--os-user-domain-name <auth-user-domain-name>]
              [--os-tenant-id <auth-tenant-id>]
              [--os-tenant-name <auth-tenant-name>]
+             [--os-project-id <auth-project-id>]
+             [--os-project-name <auth-project-name>]
+             [--os-project-domain-id <auth-project-domain-id>]
+             [--os-project-domain-name <auth-project-domain-name>]
              [--os-auth-url <auth-url>] [--os-auth-token <auth-token>]
              [--os-storage-url <storage-url>] [--os-region-name <region-name>]
              [--os-service-type <service-type>]
@@ -1449,12 +1479,25 @@ Examples:
   %%prog --os-auth-url https://api.example.com/v2.0 --os-tenant-name tenant \\
       --os-username user --os-password password list
 
+  %%prog --os-auth-url https://api.example.com/v3 --auth-version 3\\
+      --os-project-name project1 --os-project-domain-name domain1 \\
+      --os-username user --os-user-domain-name domain1 \\
+      --os-password password list
+
+  %%prog --os-auth-url https://api.example.com/v3 --auth-version 3\\
+      --os-project-id 0123456789abcdef0123456789abcdef \\
+      --os-user-id abcdef0123456789abcdef0123456789 \\
+      --os-password password list
+
   %%prog --os-auth-token 6ee5eb33efad4e45ab46806eac010566 \\
       --os-storage-url https://10.1.5.2:8080/v1/AUTH_ced809b6a4baea7aeab61a \\
       list
 
   %%prog list --lh
 '''.strip('\n') % globals())
+    parser.add_option('--os-help', action='store_true', dest='os_help',
+                      help='Show OpenStack authentication options.')
+    parser.add_option('--os_help', action='store_true', help=SUPPRESS_HELP)
     parser.add_option('-s', '--snet', action='store_true', dest='snet',
                       default=False, help='Use SERVICENET internal network.')
     parser.add_option('-v', '--verbose', action='count', dest='verbose',
@@ -1472,7 +1515,9 @@ Examples:
                       help='URL for obtaining an auth token.')
     parser.add_option('-V', '--auth-version',
                       dest='auth_version',
-                      default=environ.get('ST_AUTH_VERSION', '1.0'),
+                      default=environ.get('ST_AUTH_VERSION',
+                                          (environ.get('OS_AUTH_VERSION',
+                                                       '1.0'))),
                       type=str,
                       help='Specify a version for authentication. '
                            'Defaults to 1.0.')
@@ -1484,47 +1529,112 @@ Examples:
                       help='Key for obtaining an auth token.')
     parser.add_option('-R', '--retries', type=int, default=5, dest='retries',
                       help='The number of times to retry a failed connection.')
-    parser.add_option('--os-username',
+    default_val = config_true_value(environ.get('SWIFTCLIENT_INSECURE'))
+    parser.add_option('--insecure',
+                      action="store_true", dest="insecure",
+                      default=default_val,
+                      help='Allow swiftclient to access servers without '
+                           'having to verify the SSL certificate. '
+                           'Defaults to env[SWIFTCLIENT_INSECURE] '
+                           '(set to \'true\' to enable).')
+    parser.add_option('--no-ssl-compression',
+                      action='store_false', dest='ssl_compression',
+                      default=True,
+                      help='This option is deprecated and not used anymore. '
+                           'SSL compression should be disabled by default '
+                           'by the system SSL library.')
+
+    os_grp = OptionGroup(parser, "OpenStack authentication options")
+    os_grp.add_option('--os-username',
                       metavar='<auth-user-name>',
                       default=environ.get('OS_USERNAME'),
                       help='OpenStack username. Defaults to env[OS_USERNAME].')
-    parser.add_option('--os_username',
+    os_grp.add_option('--os_username',
+                      help=SUPPRESS_HELP)
+    os_grp.add_option('--os-user-id',
+                      metavar='<auth-user-id>',
+                      default=environ.get('OS_USER_ID'),
+                      help='OpenStack user ID. '
+                      'Defaults to env[OS_USER_ID].')
+    os_grp.add_option('--os_user_id',
+                      help=SUPPRESS_HELP)
+    os_grp.add_option('--os-user-domain-id',
+                      metavar='<auth-user-domain-id>',
+                      default=environ.get('OS_USER_DOMAIN_ID'),
+                      help='OpenStack user domain ID. '
+                      'Defaults to env[OS_USER_DOMAIN_ID].')
+    os_grp.add_option('--os_user_domain_id',
+                      help=SUPPRESS_HELP)
+    os_grp.add_option('--os-user-domain-name',
+                      metavar='<auth-user-domain-name>',
+                      default=environ.get('OS_USER_DOMAIN_NAME'),
+                      help='OpenStack user domain name. '
+                           'Defaults to env[OS_USER_DOMAIN_NAME].')
+    os_grp.add_option('--os_user_domain_name',
                       help=SUPPRESS_HELP)
-    parser.add_option('--os-password',
+    os_grp.add_option('--os-password',
                       metavar='<auth-password>',
                       default=environ.get('OS_PASSWORD'),
                       help='OpenStack password. Defaults to env[OS_PASSWORD].')
-    parser.add_option('--os_password',
+    os_grp.add_option('--os_password',
                       help=SUPPRESS_HELP)
-    parser.add_option('--os-tenant-id',
+    os_grp.add_option('--os-tenant-id',
                       metavar='<auth-tenant-id>',
                       default=environ.get('OS_TENANT_ID'),
                       help='OpenStack tenant ID. '
                       'Defaults to env[OS_TENANT_ID].')
-    parser.add_option('--os_tenant_id',
+    os_grp.add_option('--os_tenant_id',
                       help=SUPPRESS_HELP)
-    parser.add_option('--os-tenant-name',
+    os_grp.add_option('--os-tenant-name',
                       metavar='<auth-tenant-name>',
                       default=environ.get('OS_TENANT_NAME'),
                       help='OpenStack tenant name. '
                            'Defaults to env[OS_TENANT_NAME].')
-    parser.add_option('--os_tenant_name',
+    os_grp.add_option('--os_tenant_name',
                       help=SUPPRESS_HELP)
-    parser.add_option('--os-auth-url',
+    os_grp.add_option('--os-project-id',
+                      metavar='<auth-project-id>',
+                      default=environ.get('OS_PROJECT_ID'),
+                      help='OpenStack project ID. '
+                      'Defaults to env[OS_PROJECT_ID].')
+    os_grp.add_option('--os_project_id',
+                      help=SUPPRESS_HELP)
+    os_grp.add_option('--os-project-name',
+                      metavar='<auth-project-name>',
+                      default=environ.get('OS_PROJECT_NAME'),
+                      help='OpenStack project name. '
+                           'Defaults to env[OS_PROJECT_NAME].')
+    os_grp.add_option('--os_project_name',
+                      help=SUPPRESS_HELP)
+    os_grp.add_option('--os-project-domain-id',
+                      metavar='<auth-project-domain-id>',
+                      default=environ.get('OS_PROJECT_DOMAIN_ID'),
+                      help='OpenStack project domain ID. '
+                      'Defaults to env[OS_PROJECT_DOMAIN_ID].')
+    os_grp.add_option('--os_project_domain_id',
+                      help=SUPPRESS_HELP)
+    os_grp.add_option('--os-project-domain-name',
+                      metavar='<auth-project-domain-name>',
+                      default=environ.get('OS_PROJECT_DOMAIN_NAME'),
+                      help='OpenStack project domain name. '
+                           'Defaults to env[OS_PROJECT_DOMAIN_NAME].')
+    os_grp.add_option('--os_project_domain_name',
+                      help=SUPPRESS_HELP)
+    os_grp.add_option('--os-auth-url',
                       metavar='<auth-url>',
                       default=environ.get('OS_AUTH_URL'),
                       help='OpenStack auth URL. Defaults to env[OS_AUTH_URL].')
-    parser.add_option('--os_auth_url',
+    os_grp.add_option('--os_auth_url',
                       help=SUPPRESS_HELP)
-    parser.add_option('--os-auth-token',
+    os_grp.add_option('--os-auth-token',
                       metavar='<auth-token>',
                       default=environ.get('OS_AUTH_TOKEN'),
                       help='OpenStack token. Defaults to env[OS_AUTH_TOKEN]. '
                            'Used with --os-storage-url to bypass the '
                            'usual username/password authentication.')
-    parser.add_option('--os_auth_token',
+    os_grp.add_option('--os_auth_token',
                       help=SUPPRESS_HELP)
-    parser.add_option('--os-storage-url',
+    os_grp.add_option('--os-storage-url',
                       metavar='<storage-url>',
                       default=environ.get('OS_STORAGE_URL'),
                       help='OpenStack storage URL. '
@@ -1532,48 +1642,44 @@ Examples:
                            'Overrides the storage url returned during auth. '
                            'Will bypass authentication when used with '
                            '--os-auth-token.')
-    parser.add_option('--os_storage_url',
+    os_grp.add_option('--os_storage_url',
                       help=SUPPRESS_HELP)
-    parser.add_option('--os-region-name',
+    os_grp.add_option('--os-region-name',
                       metavar='<region-name>',
                       default=environ.get('OS_REGION_NAME'),
                       help='OpenStack region name. '
                            'Defaults to env[OS_REGION_NAME].')
-    parser.add_option('--os_region_name',
+    os_grp.add_option('--os_region_name',
                       help=SUPPRESS_HELP)
-    parser.add_option('--os-service-type',
+    os_grp.add_option('--os-service-type',
                       metavar='<service-type>',
                       default=environ.get('OS_SERVICE_TYPE'),
                       help='OpenStack Service type. '
                            'Defaults to env[OS_SERVICE_TYPE].')
-    parser.add_option('--os_service_type',
+    os_grp.add_option('--os_service_type',
                       help=SUPPRESS_HELP)
-    parser.add_option('--os-endpoint-type',
+    os_grp.add_option('--os-endpoint-type',
                       metavar='<endpoint-type>',
                       default=environ.get('OS_ENDPOINT_TYPE'),
                       help='OpenStack Endpoint type. '
                            'Defaults to env[OS_ENDPOINT_TYPE].')
-    parser.add_option('--os-cacert',
+    os_grp.add_option('--os_endpoint_type',
+                      help=SUPPRESS_HELP)
+    os_grp.add_option('--os-cacert',
                       metavar='<ca-certificate>',
                       default=environ.get('OS_CACERT'),
                       help='Specify a CA bundle file to use in verifying a '
                       'TLS (https) server certificate. '
                       'Defaults to env[OS_CACERT].')
-    default_val = config_true_value(environ.get('SWIFTCLIENT_INSECURE'))
-    parser.add_option('--insecure',
-                      action="store_true", dest="insecure",
-                      default=default_val,
-                      help='Allow swiftclient to access servers without '
-                           'having to verify the SSL certificate. '
-                           'Defaults to env[SWIFTCLIENT_INSECURE] '
-                           '(set to \'true\' to enable).')
-    parser.add_option('--no-ssl-compression',
-                      action='store_false', dest='ssl_compression',
-                      default=True,
-                      help='This option is deprecated and not used anymore. '
-                           'SSL compression should be disabled by default '
-                           'by the system SSL library.')
     parser.disable_interspersed_args()
+    # call parse_args before adding os options group so that -h, --help will
+    # print a condensed help message without the os options
+    (options, args) = parse_args(parser, argv[1:], enforce_requires=False)
+    parser.add_option_group(os_grp)
+    if options.os_help:
+        # if openstack option help has been explicitly requested then force
+        # help message, now that os_options group has been added to parser
+        argv = ['-h']
     (options, args) = parse_args(parser, argv[1:], enforce_requires=False)
     parser.enable_interspersed_args()
 
diff --git a/tests/sample.conf b/tests/sample.conf
index 4578ab7..3b9b03d 100644
--- a/tests/sample.conf
+++ b/tests/sample.conf
@@ -4,7 +4,8 @@ auth_host = 127.0.0.1
 auth_port = 8080
 auth_ssl = no
 auth_prefix = /auth/
-## sample config for Swift with Keystone
+## sample config for Swift with Keystone v2 API
+# For keystone v3 change auth_version to 3 and auth_prefix to /v3/
 #auth_version = 2
 #auth_host = localhost
 #auth_port = 5000
diff --git a/tests/unit/test_shell.py b/tests/unit/test_shell.py
index daf4cd8..501af9a 100644
--- a/tests/unit/test_shell.py
+++ b/tests/unit/test_shell.py
@@ -378,3 +378,281 @@ class TestSubcommandHelp(unittest.TestCase):
             self.assertRaises(SystemExit, swiftclient.shell.main, argv)
         expected = 'no help for bad_command'
         self.assertEqual(out.getvalue().strip('\n'), expected)
+
+
+class TestParsing(unittest.TestCase):
+
+    def _make_fake_command(self, result):
+        def fake_command(parser, args, thread_manager):
+            result[0], result[1] = swiftclient.shell.parse_args(parser, args)
+        return fake_command
+
+    def _make_args(self, cmd, opts, os_opts, separator='-'):
+        """
+        Construct command line arguments for given options.
+        """
+        args = [""]
+        for k, v in opts.items():
+            arg = "--" + k.replace("_", "-")
+            args = args + [arg, v]
+        for k, v in os_opts.items():
+            arg = "--os" + separator + k.replace("_", separator)
+            args = args + [arg, v]
+        args = args + [cmd]
+        return args
+
+    def _make_env(self, opts, os_opts):
+        """
+        Construct a dict of environment variables for given options.
+        """
+        env = {}
+        for k, v in opts.items():
+            key = 'ST_' + k.upper()
+            env[key] = v
+        for k, v in os_opts.items():
+            key = 'OS_' + k.upper()
+            env[key] = v
+        return env
+
+    def _verify_opts(self, actual_opts, opts, os_opts={}, os_opts_dict={}):
+        """
+        Check parsed options are correct.
+
+        :param opts: v1 style options.
+        :param os_opts: openstack style options.
+        :param os_opts_dict: openstack options that should be found in the
+                             os_options dict.
+        """
+        # check the expected opts are set
+        for key, v in opts.items():
+            actual = getattr(actual_opts, key)
+            self.assertEqual(v, actual, 'Expected %s for key %s, found %s'
+                                        % (v, key, actual))
+
+        for key, v in os_opts.items():
+            actual = getattr(actual_opts, "os_" + key)
+            self.assertEqual(v, actual, 'Expected %s for key %s, found %s'
+                                        % (v, key, actual))
+
+        # check the os_options dict values are set
+        self.assertTrue(hasattr(actual_opts, 'os_options'))
+        actual_os_opts_dict = getattr(actual_opts, 'os_options')
+        expected_os_opts_keys = ['project_name', 'region_name',
+                                 'tenant_name',
+                                 'user_domain_name', 'endpoint_type',
+                                 'object_storage_url', 'project_domain_id',
+                                 'user_id', 'user_domain_id', 'tenant_id',
+                                 'service_type', 'project_id', 'auth_token',
+                                 'project_domain_name']
+        for key in expected_os_opts_keys:
+            self.assertTrue(key in actual_os_opts_dict)
+            cli_key = key
+            if key == 'object_storage_url':
+                # exceptions to the pattern...
+                cli_key = 'storage_url'
+            if cli_key in os_opts_dict:
+                expect = os_opts_dict[cli_key]
+            else:
+                expect = None
+            actual = actual_os_opts_dict[key]
+            self.assertEqual(expect, actual, 'Expected %s for %s, got %s'
+                             % (expect, key, actual))
+        for key in actual_os_opts_dict:
+            self.assertTrue(key in expected_os_opts_keys)
+
+        # check that equivalent keys have equal values
+        equivalents = [('os_username', 'user'),
+                       ('os_auth_url', 'auth'),
+                       ('os_password', 'key')]
+        for pair in equivalents:
+            self.assertEqual(getattr(actual_opts, pair[0]),
+                             getattr(actual_opts, pair[1]))
+
+    def test_minimum_required_args_v3(self):
+        opts = {"auth_version": "3"}
+        os_opts = {"password": "secret",
+                   "username": "user",
+                   "auth_url": "http://example.com:5000/v3"}
+
+        # username with domain is sufficient in args because keystone will
+        # assume user is in default domain
+        args = self._make_args("stat", opts, os_opts, '-')
+        result = [None, None]
+        fake_command = self._make_fake_command(result)
+        with mock.patch('swiftclient.shell.st_stat', fake_command):
+            swiftclient.shell.main(args)
+        self._verify_opts(result[0], opts, os_opts, {})
+
+        # check its ok to have user_id instead of username
+        os_opts = {"password": "secret",
+                   "auth_url": "http://example.com:5000/v3"}
+        os_opts_dict = {"user_id": "user_ID"}
+        all_os_opts = os_opts.copy()
+        all_os_opts.update(os_opts_dict)
+
+        args = self._make_args("stat", opts, all_os_opts, '-')
+        result = [None, None]
+        fake_command = self._make_fake_command(result)
+        with mock.patch('swiftclient.shell.st_stat', fake_command):
+            swiftclient.shell.main(args)
+        self._verify_opts(result[0], opts, os_opts, os_opts_dict)
+
+        # check no user credentials required if token and url supplied
+        os_opts = {}
+        os_opts_dict = {"storage_url": "http://example.com:8080/v1",
+                        "auth_token": "0123abcd"}
+
+        args = self._make_args("stat", opts, os_opts_dict, '-')
+        result = [None, None]
+        fake_command = self._make_fake_command(result)
+        with mock.patch('swiftclient.shell.st_stat', fake_command):
+            swiftclient.shell.main(args)
+        self._verify_opts(result[0], opts, os_opts, os_opts_dict)
+
+    def test_args_v3(self):
+        opts = {"auth_version": "3"}
+        os_opts = {"password": "secret",
+                   "username": "user",
+                   "auth_url": "http://example.com:5000/v3"}
+        os_opts_dict = {"user_id": "user_ID",
+                        "project_id": "project_ID",
+                        "tenant_id": "tenant_ID",
+                        "project_domain_id": "project_domain_ID",
+                        "user_domain_id": "user_domain_ID",
+                        "tenant_name": "tenant",
+                        "project_name": "project",
+                        "project_domain_name": "project_domain",
+                        "user_domain_name": "user_domain",
+                        "auth_token": "token",
+                        "storage_url": "http://example.com:8080/v1",
+                        "region_name": "region",
+                        "service_type": "service",
+                        "endpoint_type": "endpoint"}
+        all_os_opts = os_opts.copy()
+        all_os_opts.update(os_opts_dict)
+
+        # check using hyphen separator
+        args = self._make_args("stat", opts, all_os_opts, '-')
+        result = [None, None]
+        fake_command = self._make_fake_command(result)
+        with mock.patch('swiftclient.shell.st_stat', fake_command):
+            swiftclient.shell.main(args)
+        self._verify_opts(result[0], opts, os_opts, os_opts_dict)
+
+        # check using underscore separator
+        args = self._make_args("stat", opts, all_os_opts, '_')
+        result = [None, None]
+        fake_command = self._make_fake_command(result)
+        with mock.patch('swiftclient.shell.st_stat', fake_command):
+            swiftclient.shell.main(args)
+        self._verify_opts(result[0], opts, os_opts, os_opts_dict)
+
+        # check using environment variables
+        args = self._make_args("stat", {}, {})
+        env = self._make_env(opts, all_os_opts)
+        result = [None, None]
+        fake_command = self._make_fake_command(result)
+        with mock.patch.dict(os.environ, env):
+            with mock.patch('swiftclient.shell.st_stat', fake_command):
+                swiftclient.shell.main(args)
+        self._verify_opts(result[0], opts, os_opts, os_opts_dict)
+
+        # check again using OS_AUTH_VERSION instead of ST_AUTH_VERSION
+        env = self._make_env({}, all_os_opts)
+        env.update({'OS_AUTH_VERSION': '3'})
+        result = [None, None]
+        fake_command = self._make_fake_command(result)
+        with mock.patch.dict(os.environ, env):
+            with mock.patch('swiftclient.shell.st_stat', fake_command):
+                swiftclient.shell.main(args)
+        self._verify_opts(result[0], opts, os_opts, os_opts_dict)
+
+    def test_command_args_v3(self):
+        result = [None, None]
+        fake_command = self._make_fake_command(result)
+        opts = {"auth_version": "3"}
+        os_opts = {"password": "secret",
+                   "username": "user",
+                   "auth_url": "http://example.com:5000/v3"}
+        args = self._make_args("stat", opts, os_opts)
+        with mock.patch('swiftclient.shell.st_stat', fake_command):
+            swiftclient.shell.main(args)
+            self.assertEqual(['stat'], result[1])
+        with mock.patch('swiftclient.shell.st_stat', fake_command):
+            args = args + ["container_name"]
+            swiftclient.shell.main(args)
+            self.assertEqual(["stat", "container_name"], result[1])
+
+    def test_insufficient_args_v3(self):
+        opts = {"auth_version": "3"}
+        os_opts = {"password": "secret",
+                   "auth_url": "http://example.com:5000/v3"}
+        args = self._make_args("stat", opts, os_opts)
+        self.assertRaises(SystemExit, swiftclient.shell.main, args)
+
+        os_opts = {"username": "user",
+                   "auth_url": "http://example.com:5000/v3"}
+        args = self._make_args("stat", opts, os_opts)
+        self.assertRaises(SystemExit, swiftclient.shell.main, args)
+
+        os_opts = {"username": "user",
+                   "password": "secret"}
+        args = self._make_args("stat", opts, os_opts)
+        self.assertRaises(SystemExit, swiftclient.shell.main, args)
+
+    def test_insufficient_env_vars_v3(self):
+        args = self._make_args("stat", {}, {})
+        opts = {"auth_version": "3"}
+        os_opts = {"password": "secret",
+                   "auth_url": "http://example.com:5000/v3"}
+        env = self._make_env(opts, os_opts)
+        with mock.patch.dict(os.environ, env):
+            self.assertRaises(SystemExit, swiftclient.shell.main, args)
+
+        os_opts = {"username": "user",
+                   "auth_url": "http://example.com:5000/v3"}
+        env = self._make_env(opts, os_opts)
+        with mock.patch.dict(os.environ, env):
+            self.assertRaises(SystemExit, swiftclient.shell.main, args)
+
+        os_opts = {"username": "user",
+                   "password": "secret"}
+        env = self._make_env(opts, os_opts)
+        with mock.patch.dict(os.environ, env):
+            self.assertRaises(SystemExit, swiftclient.shell.main, args)
+
+    def test_help(self):
+        # --help returns condensed help message
+        opts = {"help": ""}
+        os_opts = {}
+        args = self._make_args("stat", opts, os_opts)
+        mock_stdout = six.StringIO()
+        with mock.patch('sys.stdout', mock_stdout):
+            self.assertRaises(SystemExit, swiftclient.shell.main, args)
+        out = mock_stdout.getvalue()
+        self.assertTrue(out.find('[--key <api_key>]') > 0)
+        self.assertEqual(-1, out.find('--os-username=<auth-user-name>'))
+
+        # --help returns condensed help message, overrides --os-help
+        opts = {"help": ""}
+        os_opts = {"help": ""}
+                   # "password": "secret",
+                   # "username": "user",
+                   # "auth_url": "http://example.com:5000/v3"}
+        args = self._make_args("", opts, os_opts)
+        mock_stdout = six.StringIO()
+        with mock.patch('sys.stdout', mock_stdout):
+            self.assertRaises(SystemExit, swiftclient.shell.main, args)
+        out = mock_stdout.getvalue()
+        self.assertTrue(out.find('[--key <api_key>]') > 0)
+        self.assertEqual(-1, out.find('--os-username=<auth-user-name>'))
+
+        ## --os-help return os options help
+        opts = {}
+        args = self._make_args("", opts, os_opts)
+        mock_stdout = six.StringIO()
+        with mock.patch('sys.stdout', mock_stdout):
+            self.assertRaises(SystemExit, swiftclient.shell.main, args)
+        out = mock_stdout.getvalue()
+        self.assertTrue(out.find('[--key <api_key>]') > 0)
+        self.assertTrue(out.find('--os-username=<auth-user-name>') > 0)
diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py
index d473765..facd6d9 100644
--- a/tests/unit/test_swiftclient.py
+++ b/tests/unit/test_swiftclient.py
@@ -30,7 +30,7 @@ from six.moves.urllib.parse import urlparse
 from six.moves import reload_module
 
 # TODO: mock http connection class with more control over headers
-from .utils import fake_http_connect, fake_get_keystoneclient_2_0
+from .utils import fake_http_connect, fake_get_auth_keystone
 
 from swiftclient import client as c
 import swiftclient.utils
@@ -287,7 +287,9 @@ class TestGetAuth(MockHttpTest):
 
     def test_auth_v2_with_tenant_name(self):
         os_options = {'tenant_name': 'asdf'}
-        c.get_keystoneclient_2_0 = fake_get_keystoneclient_2_0(os_options)
+        req_args = {'auth_version': '2.0'}
+        c.get_auth_keystone = fake_get_auth_keystone(os_options,
+                                                     required_kwargs=req_args)
         url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf',
                                 os_options=os_options,
                                 auth_version="2.0")
@@ -296,7 +298,31 @@ class TestGetAuth(MockHttpTest):
 
     def test_auth_v2_with_tenant_id(self):
         os_options = {'tenant_id': 'asdf'}
-        c.get_keystoneclient_2_0 = fake_get_keystoneclient_2_0(os_options)
+        req_args = {'auth_version': '2.0'}
+        c.get_auth_keystone = fake_get_auth_keystone(os_options,
+                                                     required_kwargs=req_args)
+        url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf',
+                                os_options=os_options,
+                                auth_version="2.0")
+        self.assertTrue(url.startswith("http"))
+        self.assertTrue(token)
+
+    def test_auth_v2_with_project_name(self):
+        os_options = {'project_name': 'asdf'}
+        req_args = {'auth_version': '2.0'}
+        c.get_auth_keystone = fake_get_auth_keystone(os_options,
+                                                     required_kwargs=req_args)
+        url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf',
+                                os_options=os_options,
+                                auth_version="2.0")
+        self.assertTrue(url.startswith("http"))
+        self.assertTrue(token)
+
+    def test_auth_v2_with_project_id(self):
+        os_options = {'project_id': 'asdf'}
+        req_args = {'auth_version': '2.0'}
+        c.get_auth_keystone = fake_get_auth_keystone(os_options,
+                                                     required_kwargs=req_args)
         url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf',
                                 os_options=os_options,
                                 auth_version="2.0")
@@ -304,7 +330,7 @@ class TestGetAuth(MockHttpTest):
         self.assertTrue(token)
 
     def test_auth_v2_no_tenant_name_or_tenant_id(self):
-        c.get_keystoneclient_2_0 = fake_get_keystoneclient_2_0({})
+        c.get_auth_keystone = fake_get_auth_keystone({})
         self.assertRaises(c.ClientException, c.get_auth,
                           'http://www.tests.com', 'asdf', 'asdf',
                           os_options={},
@@ -313,7 +339,7 @@ class TestGetAuth(MockHttpTest):
     def test_auth_v2_with_tenant_name_none_and_tenant_id_none(self):
         os_options = {'tenant_name': None,
                       'tenant_id': None}
-        c.get_keystoneclient_2_0 = fake_get_keystoneclient_2_0(os_options)
+        c.get_auth_keystone = fake_get_auth_keystone(os_options)
         self.assertRaises(c.ClientException, c.get_auth,
                           'http://www.tests.com', 'asdf', 'asdf',
                           os_options=os_options,
@@ -321,7 +347,7 @@ class TestGetAuth(MockHttpTest):
 
     def test_auth_v2_with_tenant_user_in_user(self):
         tenant_option = {'tenant_name': 'foo'}
-        c.get_keystoneclient_2_0 = fake_get_keystoneclient_2_0(tenant_option)
+        c.get_auth_keystone = fake_get_auth_keystone(tenant_option)
         url, token = c.get_auth('http://www.test.com', 'foo:bar', 'asdf',
                                 os_options={},
                                 auth_version="2.0")
@@ -330,7 +356,7 @@ class TestGetAuth(MockHttpTest):
 
     def test_auth_v2_tenant_name_no_os_options(self):
         tenant_option = {'tenant_name': 'asdf'}
-        c.get_keystoneclient_2_0 = fake_get_keystoneclient_2_0(tenant_option)
+        c.get_auth_keystone = fake_get_auth_keystone(tenant_option)
         url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf',
                                 tenant_name='asdf',
                                 os_options={},
@@ -342,7 +368,7 @@ class TestGetAuth(MockHttpTest):
         os_options = {'service_type': 'object-store',
                       'endpoint_type': 'internalURL',
                       'tenant_name': 'asdf'}
-        c.get_keystoneclient_2_0 = fake_get_keystoneclient_2_0(os_options)
+        c.get_auth_keystone = fake_get_auth_keystone(os_options)
         url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf',
                                 os_options=os_options,
                                 auth_version="2.0")
@@ -351,7 +377,7 @@ class TestGetAuth(MockHttpTest):
 
     def test_auth_v2_with_tenant_user_in_user_no_os_options(self):
         tenant_option = {'tenant_name': 'foo'}
-        c.get_keystoneclient_2_0 = fake_get_keystoneclient_2_0(tenant_option)
+        c.get_auth_keystone = fake_get_auth_keystone(tenant_option)
         url, token = c.get_auth('http://www.test.com', 'foo:bar', 'asdf',
                                 auth_version="2.0")
         self.assertTrue(url.startswith("http"))
@@ -360,7 +386,7 @@ class TestGetAuth(MockHttpTest):
     def test_auth_v2_with_os_region_name(self):
         os_options = {'region_name': 'good-region',
                       'tenant_name': 'asdf'}
-        c.get_keystoneclient_2_0 = fake_get_keystoneclient_2_0(os_options)
+        c.get_auth_keystone = fake_get_auth_keystone(os_options)
         url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf',
                                 os_options=os_options,
                                 auth_version="2.0")
@@ -370,14 +396,14 @@ class TestGetAuth(MockHttpTest):
     def test_auth_v2_no_endpoint(self):
         os_options = {'region_name': 'unknown_region',
                       'tenant_name': 'asdf'}
-        c.get_keystoneclient_2_0 = fake_get_keystoneclient_2_0(
+        c.get_auth_keystone = fake_get_auth_keystone(
             os_options, c.ClientException)
         self.assertRaises(c.ClientException, c.get_auth,
                           'http://www.tests.com', 'asdf', 'asdf',
                           os_options=os_options, auth_version='2.0')
 
     def test_auth_v2_ks_exception(self):
-        c.get_keystoneclient_2_0 = fake_get_keystoneclient_2_0(
+        c.get_auth_keystone = fake_get_auth_keystone(
             {}, c.ClientException)
         self.assertRaises(c.ClientException, c.get_auth,
                           'http://www.tests.com', 'asdf', 'asdf',
@@ -386,7 +412,7 @@ class TestGetAuth(MockHttpTest):
 
     def test_auth_v2_cacert(self):
         os_options = {'tenant_name': 'foo'}
-        c.get_keystoneclient_2_0 = fake_get_keystoneclient_2_0(
+        c.get_auth_keystone = fake_get_auth_keystone(
             os_options, None)
 
         auth_url_secure = 'https://www.tests.com'
@@ -414,7 +440,7 @@ class TestGetAuth(MockHttpTest):
 
     def test_auth_v2_insecure(self):
         os_options = {'tenant_name': 'foo'}
-        c.get_keystoneclient_2_0 = fake_get_keystoneclient_2_0(
+        c.get_auth_keystone = fake_get_auth_keystone(
             os_options, None)
 
         auth_url_secure = 'https://www.tests.com'
@@ -439,6 +465,29 @@ class TestGetAuth(MockHttpTest):
                           os_options=os_options, auth_version='2.0',
                           insecure=False)
 
+    def test_auth_v3_with_tenant_name(self):
+        # check the correct auth version is passed to get_auth_keystone
+        os_options = {'tenant_name': 'asdf'}
+        req_args = {'auth_version': '3'}
+        c.get_auth_keystone = fake_get_auth_keystone(os_options,
+                                                     required_kwargs=req_args)
+        url, token = c.get_auth('http://www.test.com', 'asdf', 'asdf',
+                                os_options=os_options,
+                                auth_version="3")
+        self.assertTrue(url.startswith("http"))
+        self.assertTrue(token)
+
+    def test_get_keystone_client_2_0(self):
+        # check the correct auth version is passed to get_auth_keystone
+        os_options = {'tenant_name': 'asdf'}
+        req_args = {'auth_version': '2.0'}
+        c.get_auth_keystone = fake_get_auth_keystone(os_options,
+                                                     required_kwargs=req_args)
+        url, token = c.get_keystoneclient_2_0('http://www.test.com', 'asdf',
+                                              'asdf', os_options=os_options)
+        self.assertTrue(url.startswith("http"))
+        self.assertTrue(token)
+
 
 class TestGetAccount(MockHttpTest):
 
diff --git a/tests/unit/utils.py b/tests/unit/utils.py
index 914bc8f..cb671cf 100644
--- a/tests/unit/utils.py
+++ b/tests/unit/utils.py
@@ -16,11 +16,11 @@ from requests import RequestException
 from time import sleep
 
 
-def fake_get_keystoneclient_2_0(os_options, exc=None, **kwargs):
-    def fake_get_keystoneclient_2_0(auth_url,
-                                    user,
-                                    key,
-                                    actual_os_options, **actual_kwargs):
+def fake_get_auth_keystone(os_options, exc=None, **kwargs):
+    def fake_get_auth_keystone(auth_url,
+                               user,
+                               key,
+                               actual_os_options, **actual_kwargs):
         if exc:
             raise exc('test')
         if actual_os_options != os_options:
@@ -37,9 +37,13 @@ def fake_get_keystoneclient_2_0(os_options, exc=None, **kwargs):
            actual_kwargs['cacert'] is None:
             from swiftclient import client as c
             raise c.ClientException("unverified-certificate")
+        if 'required_kwargs' in kwargs:
+            for k, v in kwargs['required_kwargs'].items():
+                if v != actual_kwargs.get(k):
+                    return "", None
 
         return "http://url/", "token"
-    return fake_get_keystoneclient_2_0
+    return fake_get_auth_keystone
 
 
 def fake_http_connect(*code_iter, **kwargs):