diff options
| -rw-r--r-- | glance/store/_drivers/swift/__init__.py | 17 | ||||
| -rw-r--r-- | glance/store/_drivers/swift/store.py | 837 | ||||
| -rw-r--r-- | glance/store/_drivers/swift/utils.py | 111 | ||||
| -rw-r--r-- | glance/store/common/auth.py | 288 | ||||
| -rw-r--r-- | glance/store/openstack/common/context.py | 127 | ||||
| -rw-r--r-- | glance/store/tests/base.py | 21 | ||||
| -rw-r--r-- | glance/store/tests/etc/glance-swift.conf | 34 | ||||
| -rw-r--r-- | openstack-common.conf | 1 | ||||
| -rw-r--r-- | requirements.txt | 3 | ||||
| -rw-r--r-- | setup.cfg | 1 | ||||
| -rw-r--r-- | test-requirements.txt | 4 | ||||
| -rw-r--r-- | tests/unit/test_filesystem_store.py | 3 | ||||
| -rw-r--r-- | tests/unit/test_swift_store.py | 1127 |
13 files changed, 2567 insertions, 7 deletions
diff --git a/glance/store/_drivers/swift/__init__.py b/glance/store/_drivers/swift/__init__.py new file mode 100644 index 0000000..891b61e --- /dev/null +++ b/glance/store/_drivers/swift/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2014 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from glance.store._drivers.swift import utils # noqa +from glance.store._drivers.swift.store import * # noqa diff --git a/glance/store/_drivers/swift/store.py b/glance/store/_drivers/swift/store.py new file mode 100644 index 0000000..0fce94e --- /dev/null +++ b/glance/store/_drivers/swift/store.py @@ -0,0 +1,837 @@ +# Copyright 2010-2011 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Storage backend for SWIFT""" + +import hashlib +import httplib +import logging +import math + +from oslo.config import cfg +import six.moves.urllib.parse as urlparse +import swiftclient +import urllib + +import glance.store +from glance.store._drivers.swift import utils as sutils +from glance.store.common import auth +from glance.store import driver +from glance.store import exceptions +from glance.store import i18n +from glance.store import location +from glance.store.openstack.common import excutils + +_ = i18n._ +LOG = logging.getLogger(__name__) +_LI = i18n._LI + +DEFAULT_CONTAINER = 'glance' +DEFAULT_LARGE_OBJECT_SIZE = 5 * 1024 # 5GB +DEFAULT_LARGE_OBJECT_CHUNK_SIZE = 200 # 200M +ONE_MB = 1000 * 1024 + +_SWIFT_OPTS = [ + cfg.BoolOpt('swift_enable_snet', default=False, + help=_('Whether to use ServiceNET to communicate with the ' + 'Swift storage servers.')), + cfg.StrOpt('swift_store_auth_version', default='2', + help=_('Version of the authentication service to use. ' + 'Valid versions are 2 for keystone and 1 for swauth ' + 'and rackspace. (deprecated)')), + cfg.BoolOpt('swift_store_auth_insecure', default=False, + help=_('If True, swiftclient won\'t check for a valid SSL ' + 'certificate when authenticating.')), + cfg.StrOpt('swift_store_region', + help=_('The region of the swift endpoint to be used for ' + 'single tenant. This setting is only necessary if the ' + 'tenant has multiple swift endpoints.')), + cfg.StrOpt('swift_store_endpoint_type', default='publicURL', + help=_('A string giving the endpoint type of the swift ' + 'service to use (publicURL, adminURL or internalURL). ' + 'This setting is only used if swift_store_auth_version ' + 'is 2.')), + cfg.StrOpt('swift_store_service_type', default='object-store', + help=_('A string giving the service type of the swift service ' + 'to use. This setting is only used if ' + 'swift_store_auth_version is 2.')), + cfg.StrOpt('swift_store_container', + default=DEFAULT_CONTAINER, + help=_('Container within the account that the account should ' + 'use for storing images in Swift.')), + cfg.IntOpt('swift_store_large_object_size', + default=DEFAULT_LARGE_OBJECT_SIZE, + help=_('The size, in MB, that Glance will start chunking image ' + 'files and do a large object manifest in Swift.')), + cfg.IntOpt('swift_store_large_object_chunk_size', + default=DEFAULT_LARGE_OBJECT_CHUNK_SIZE, + help=_('The amount of data written to a temporary ' + 'disk buffer during the process of chunking ' + 'the image file.')), + cfg.BoolOpt('swift_store_create_container_on_put', default=False, + help=_('A boolean value that determines if we create the ' + 'container if it does not exist.')), + cfg.BoolOpt('swift_store_multi_tenant', default=False, + help=_('If set to True, enables multi-tenant storage ' + 'mode which causes Glance images to be stored in ' + 'tenant specific Swift accounts.')), + cfg.ListOpt('swift_store_admin_tenants', default=[], + help=_('A list of tenants that will be granted read/write ' + 'access on all Swift containers created by Glance in ' + 'multi-tenant mode.')), + cfg.BoolOpt('swift_store_ssl_compression', default=True, + help=_('If set to False, disables SSL layer compression of ' + 'https swift requests. Setting to False may improve ' + 'performance for images which are already in a ' + 'compressed format, eg qcow2.')), + cfg.IntOpt('swift_store_retry_get_count', default=0, + help=_('The number of times a Swift download will be retried ' + 'before the request fails.')) +] + +CONF = cfg.CONF + +SWIFT_STORE_REF_PARAMS = sutils.SwiftParams().params + + +def swift_retry_iter(resp_iter, length, store, location, context): + length = length if length else (resp_iter.len + if hasattr(resp_iter, 'len') else 0) + retries = 0 + bytes_read = 0 + + while retries <= store.conf.glance_store.swift_store_retry_get_count: + try: + for chunk in resp_iter: + yield chunk + bytes_read += len(chunk) + except swiftclient.ClientException as e: + LOG.warn(_(u"Swift exception raised %s") % unicode(e)) + + if bytes_read != length: + if retries == store.conf.glance_store.swift_store_retry_get_count: + # terminate silently and let higher level decide + LOG.error(_("Stopping Swift retries after %d " + "attempts") % retries) + break + else: + retries += 1 + glance_conf = store.conf.glance_store + retry_count = glance_conf.swift_store_retry_get_count + LOG.info(_("Retrying Swift connection " + "(%(retries)d/%(max_retries)d) with " + "range=%(start)d-%(end)d") % + {'retries': retries, + 'max_retries': retry_count, + 'start': bytes_read, + 'end': length}) + (resp_headers, resp_iter) = store._get_object(location, None, + bytes_read, + context=context) + else: + break + + +class StoreLocation(location.StoreLocation): + + """ + Class describing a Swift URI. A Swift URI can look like any of + the following: + + swift://user:pass@authurl.com/container/obj-id + swift://account:user:pass@authurl.com/container/obj-id + swift+http://user:pass@authurl.com/container/obj-id + swift+https://user:pass@authurl.com/container/obj-id + + When using multi-tenant a URI might look like this (a storage URL): + + swift+https://example.com/container/obj-id + + The swift+http:// URIs indicate there is an HTTP authentication URL. + The default for Swift is an HTTPS authentication URL, so swift:// and + swift+https:// are the same... + """ + + def process_specs(self): + self.scheme = self.specs.get('scheme', 'swift+https') + self.user = self.specs.get('user') + self.key = self.specs.get('key') + self.auth_or_store_url = self.specs.get('auth_or_store_url') + self.container = self.specs.get('container') + self.obj = self.specs.get('obj') + + def _get_credstring(self): + if self.user and self.key: + return '%s:%s' % (urllib.quote(self.user), urllib.quote(self.key)) + return '' + + def get_uri(self, credentials_included=True): + auth_or_store_url = self.auth_or_store_url + if auth_or_store_url.startswith('http://'): + auth_or_store_url = auth_or_store_url[len('http://'):] + elif auth_or_store_url.startswith('https://'): + auth_or_store_url = auth_or_store_url[len('https://'):] + + credstring = self._get_credstring() + auth_or_store_url = auth_or_store_url.strip('/') + container = self.container.strip('/') + obj = self.obj.strip('/') + + if not credentials_included: + #Used only in case of an add + #Get the current store from config + store = CONF.glance_store.default_swift_reference + + return '%s://%s/%s/%s' % ('swift+config', store, container, obj) + if self.scheme == 'swift+config': + if self.ssl_enabled == True: + self.scheme = 'swift+https' + else: + self.scheme = 'swift+http' + if credstring != '': + credstring = "%s@" % credstring + return '%s://%s%s/%s/%s' % (self.scheme, credstring, auth_or_store_url, + container, obj) + + def _get_conf_value_from_account_ref(self, netloc): + try: + self.user = SWIFT_STORE_REF_PARAMS[netloc]['user'] + self.key = SWIFT_STORE_REF_PARAMS[netloc]['key'] + netloc = SWIFT_STORE_REF_PARAMS[netloc]['auth_address'] + self.ssl_enabled = True + if netloc != '': + if netloc.startswith('http://'): + self.ssl_enabled = False + netloc = netloc[len('http://'):] + elif netloc.startswith('https://'): + netloc = netloc[len('https://'):] + except KeyError: + reason = _("Badly formed Swift URI. Credentials not found for " + "account reference") + LOG.info(reason) + raise exceptions.BadStoreUri(message=reason) + return netloc + + def _form_uri_parts(self, netloc, path): + if netloc != '': + # > Python 2.6.1 + if '@' in netloc: + creds, netloc = netloc.split('@') + else: + creds = None + else: + # Python 2.6.1 compat + # see lp659445 and Python issue7904 + if '@' in path: + creds, path = path.split('@') + else: + creds = None + netloc = path[0:path.find('/')].strip('/') + path = path[path.find('/'):].strip('/') + if creds: + cred_parts = creds.split(':') + if len(cred_parts) < 2: + reason = _("Badly formed credentials in Swift URI.") + LOG.info(reason) + raise exceptions.BadStoreUri(message=reason) + key = cred_parts.pop() + user = ':'.join(cred_parts) + creds = urllib.unquote(creds) + try: + self.user, self.key = creds.rsplit(':', 1) + except exceptions.BadStoreConfiguration: + self.user = urllib.unquote(user) + self.key = urllib.unquote(key) + else: + self.user = None + self.key = None + return netloc, path + + def _form_auth_or_store_url(self, netloc, path): + path_parts = path.split('/') + try: + self.obj = path_parts.pop() + self.container = path_parts.pop() + if not netloc.startswith('http'): + # push hostname back into the remaining to build full authurl + path_parts.insert(0, netloc) + self.auth_or_store_url = '/'.join(path_parts) + except IndexError: + reason = _("Badly formed Swift URI.") + LOG.info(reason) + raise exceptions.BadStoreUri(message=reason) + + def parse_uri(self, uri): + """ + Parse URLs. This method fixes an issue where credentials specified + in the URL are interpreted differently in Python 2.6.1+ than prior + versions of Python. It also deals with the peculiarity that new-style + Swift URIs have where a username can contain a ':', like so: + + swift://account:user:pass@authurl.com/container/obj + and for system created locations with account reference + swift+config://account_reference/container/obj + """ + # Make sure that URIs that contain multiple schemes, such as: + # swift://user:pass@http://authurl.com/v1/container/obj + # are immediately rejected. + if uri.count('://') != 1: + reason = _("URI cannot contain more than one occurrence " + "of a scheme. If you have specified a URI like " + "swift://user:pass@http://authurl.com/v1/container/obj" + ", you need to change it to use the " + "swift+http:// scheme, like so: " + "swift+http://user:pass@authurl.com/v1/container/obj") + LOG.info(_LI("Invalid store URI: %(reason)s"), {'reason': reason}) + raise exceptions.BadStoreUri(message=reason) + + pieces = urlparse.urlparse(uri) + assert pieces.scheme in ('swift', 'swift+http', 'swift+https', + 'swift+config') + + self.scheme = pieces.scheme + netloc = pieces.netloc + path = pieces.path.lstrip('/') + + # NOTE(Sridevi): Fix to map the account reference to the + # corresponding CONF value + if self.scheme == 'swift+config': + netloc = self._get_conf_value_from_account_ref(netloc) + else: + netloc, path = self._form_uri_parts(netloc, path) + + self._form_auth_or_store_url(netloc, path) + + @property + def swift_url(self): + """ + Creates a fully-qualified auth address that the Swift client library + can use. The scheme for the auth_address is determined using the scheme + included in the `location` field. + + HTTPS is assumed, unless 'swift+http' is specified. + """ + if self.auth_or_store_url.startswith('http'): + return self.auth_or_store_url + else: + if self.scheme == 'swift+config': + if self.ssl_enabled == True: + self.scheme = 'swift+https' + else: + self.scheme = 'swift+http' + if self.scheme in ('swift+https', 'swift'): + auth_scheme = 'https://' + else: + auth_scheme = 'http://' + + return ''.join([auth_scheme, self.auth_or_store_url]) + + +def Store(conf): + try: + conf.register_opts(_SWIFT_OPTS, group='glance_store') + except cfg.DuplicateOptError: + pass + + if conf.glance_store.swift_store_multi_tenant: + return MultiTenantStore(conf) + return SingleTenantStore(conf) + + +class BaseStore(driver.Store): + + CHUNKSIZE = 65536 + OPTIONS = _SWIFT_OPTS + + def get_schemes(self): + return ('swift+https', 'swift', 'swift+http', 'swift+config') + + def configure(self): + glance_conf = self.conf.glance_store + _obj_size = self._option_get('swift_store_large_object_size') + self.large_object_size = _obj_size * ONE_MB + _chunk_size = self._option_get('swift_store_large_object_chunk_size') + self.large_object_chunk_size = _chunk_size * ONE_MB + self.admin_tenants = glance_conf.swift_store_admin_tenants + self.region = glance_conf.swift_store_region + self.service_type = glance_conf.swift_store_service_type + self.endpoint_type = glance_conf.swift_store_endpoint_type + self.snet = glance_conf.swift_enable_snet + self.insecure = glance_conf.swift_store_auth_insecure + self.ssl_compression = glance_conf.swift_store_ssl_compression + super(BaseStore, self).configure() + + def _get_object(self, location, connection=None, start=None, context=None): + if not connection: + connection = self.get_connection(location, context=context) + headers = {} + if start is not None: + bytes_range = 'bytes=%d-' % start + headers = {'Range': bytes_range} + + try: + resp_headers, resp_body = connection.get_object( + container=location.container, obj=location.obj, + resp_chunk_size=self.CHUNKSIZE, headers=headers) + except swiftclient.ClientException as e: + if e.http_status == httplib.NOT_FOUND: + msg = _("Swift could not find object %s.") % location.obj + LOG.warn(msg) + raise exceptions.NotFound(message=msg) + else: + raise + + return (resp_headers, resp_body) + + def validate_location(self, uri): + pieces = urlparse.urlparse(uri) + if pieces.scheme in ['swift+config']: + reason = (_("Location credentials are invalid")) + raise exceptions.BadStoreUri(message=reason) + + def get(self, location, connection=None, context=None): + location = location.store_location + (resp_headers, resp_body) = self._get_object(location, connection, + context=context) + + class ResponseIndexable(glance.store.Indexable): + def another(self): + try: + return self.wrapped.next() + except StopIteration: + return '' + + length = int(resp_headers.get('content-length', 0)) + if self.conf.glance_store.swift_store_retry_get_count > 0: + resp_body = swift_retry_iter(resp_body, length, + self, location, context) + return (ResponseIndexable(resp_body, length), length) + + def get_size(self, location, connection=None, context=None): + location = location.store_location + if not connection: + connection = self.get_connection(location, context=context) + try: + resp_headers = connection.head_object( + container=location.container, obj=location.obj) + return int(resp_headers.get('content-length', 0)) + except Exception: + return 0 + + def _option_get(self, param): + result = getattr(self.conf.glance_store, param) + if not result: + reason = (_("Could not find %(param)s in configuration options.") + % param) + LOG.error(reason) + raise exceptions.BadStoreConfiguration(store_name="swift", + reason=reason) + return result + + def _delete_stale_chunks(self, connection, container, chunk_list): + for chunk in chunk_list: + LOG.debug("Deleting chunk %s" % chunk) + try: + connection.delete_object(container, chunk) + except Exception: + msg = _("Failed to delete orphaned chunk " + "%(container)s/%(chunk)s") + LOG.exception(msg % {'container': container, + 'chunk': chunk}) + + def add(self, image_id, image_file, image_size, + connection=None, context=None): + location = self.create_location(image_id, context=context) + if not connection: + connection = self.get_connection(location, context=context) + + self._create_container_if_missing(location.container, connection) + + LOG.debug("Adding image object '%(obj_name)s' " + "to Swift" % dict(obj_name=location.obj)) + try: + if image_size > 0 and image_size < self.large_object_size: + # Image size is known, and is less than large_object_size. + # Send to Swift with regular PUT. + obj_etag = connection.put_object(location.container, + location.obj, image_file, + content_length=image_size) + else: + # Write the image into Swift in chunks. + chunk_id = 1 + if image_size > 0: + total_chunks = str(int( + math.ceil(float(image_size) / + float(self.large_object_chunk_size)))) + else: + # image_size == 0 is when we don't know the size + # of the image. This can occur with older clients + # that don't inspect the payload size. + LOG.debug("Cannot determine image size. Adding as a " + "segmented object to Swift.") + total_chunks = '?' + + checksum = hashlib.md5() + written_chunks = [] + combined_chunks_size = 0 + while True: + chunk_size = self.large_object_chunk_size + if image_size == 0: + content_length = None + else: + left = image_size - combined_chunks_size + if left == 0: + break + if chunk_size > left: + chunk_size = left + content_length = chunk_size + + chunk_name = "%s-%05d" % (location.obj, chunk_id) + reader = ChunkReader(image_file, checksum, chunk_size) + try: + chunk_etag = connection.put_object( + location.container, chunk_name, reader, + content_length=content_length) + written_chunks.append(chunk_name) + except Exception: + # Delete orphaned segments from swift backend + with excutils.save_and_reraise_exception(): + LOG.exception(_("Error during chunked upload to " + "backend, deleting stale chunks")) + self._delete_stale_chunks(connection, + location.container, + written_chunks) + + bytes_read = reader.bytes_read + msg = ("Wrote chunk %(chunk_name)s (%(chunk_id)d/" + "%(total_chunks)s) of length %(bytes_read)d " + "to Swift returning MD5 of content: " + "%(chunk_etag)s" % + {'chunk_name': chunk_name, + 'chunk_id': chunk_id, + 'total_chunks': total_chunks, + 'bytes_read': bytes_read, + 'chunk_etag': chunk_etag}) + LOG.debug(msg) + + if bytes_read == 0: + # Delete the last chunk, because it's of zero size. + # This will happen if size == 0. + LOG.debug("Deleting final zero-length chunk") + connection.delete_object(location.container, + chunk_name) + break + + chunk_id += 1 + combined_chunks_size += bytes_read + + # In the case we have been given an unknown image size, + # set the size to the total size of the combined chunks. + if image_size == 0: + image_size = combined_chunks_size + + # Now we write the object manifest and return the + # manifest's etag... + manifest = "%s/%s-" % (location.container, location.obj) + headers = {'ETag': hashlib.md5("").hexdigest(), + 'X-Object-Manifest': manifest} + + # The ETag returned for the manifest is actually the + # MD5 hash of the concatenated checksums of the strings + # of each chunk...so we ignore this result in favour of + # the MD5 of the entire image file contents, so that + # users can verify the image file contents accordingly + connection.put_object(location.container, location.obj, + None, headers=headers) + obj_etag = checksum.hexdigest() + + # NOTE: We return the user and key here! Have to because + # location is used by the API server to return the actual + # image data. We *really* should consider NOT returning + # the location attribute from GET /images/<ID> and + # GET /images/details + if sutils.is_multiple_swift_store_accounts_enabled(): + include_creds = False + else: + include_creds = True + + return (location.get_uri(credentials_included=include_creds), + image_size, obj_etag, {}) + except swiftclient.ClientException as e: + if e.http_status == httplib.CONFLICT: + msg = _("Swift already has an image at this location") + raise exceptions.Duplicate(message=msg) + + msg = (_(u"Failed to add object to Swift.\n" + "Got error from Swift: %s") % unicode(e)) + LOG.error(msg) + raise glance.store.BackendException(msg) + + def delete(self, location, connection=None, context=None): + location = location.store_location + if not connection: + connection = self.get_connection(location, context=context) + + try: + # We request the manifest for the object. If one exists, + # that means the object was uploaded in chunks/segments, + # and we need to delete all the chunks as well as the + # manifest. + manifest = None + try: + headers = connection.head_object( + location.container, location.obj) + manifest = headers.get('x-object-manifest') + except swiftclient.ClientException as e: + if e.http_status != httplib.NOT_FOUND: + raise + if manifest: + # Delete all the chunks before the object manifest itself + obj_container, obj_prefix = manifest.split('/', 1) + segments = connection.get_container( + obj_container, prefix=obj_prefix)[1] + for segment in segments: + # TODO(jaypipes): This would be an easy area to parallelize + # since we're simply sending off parallelizable requests + # to Swift to delete stuff. It's not like we're going to + # be hogging up network or file I/O here... + connection.delete_object(obj_container, + segment['name']) + + # Delete object (or, in segmented case, the manifest) + connection.delete_object(location.container, location.obj) + + except swiftclient.ClientException as e: + if e.http_status == httplib.NOT_FOUND: + msg = _("Swift could not find image at URI.") + raise exceptions.NotFound(message=msg) + else: + raise + + def _create_container_if_missing(self, container, connection): + """ + Creates a missing container in Swift if the + ``swift_store_create_container_on_put`` option is set. + + :param container: Name of container to create + :param connection: Connection to swift service + """ + try: + connection.head_container(container) + except swiftclient.ClientException as e: + if e.http_status == httplib.NOT_FOUND: + if self.conf.glance_store.swift_store_create_container_on_put: + try: + msg = (_LI("Creating swift container %(container)s") % + {'container': container}) + LOG.info(msg) + connection.put_container(container) + except swiftclient.ClientException as e: + msg = (_("Failed to add container to Swift.\n" + "Got error from Swift: %(e)s") % {'e': e}) + raise glance.store.BackendException(msg) + else: + msg = (_("The container %(container)s does not exist in " + "Swift. Please set the " + "swift_store_create_container_on_put option" + "to add container to Swift automatically.") % + {'container': container}) + raise glance.store.BackendException(msg) + else: + raise + + def get_connection(self, location, context=None): + raise NotImplementedError() + + def create_location(self, image_id, context=None): + raise NotImplementedError() + + +class SingleTenantStore(BaseStore): + EXAMPLE_URL = "swift://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<FILE>" + + def configure(self): + super(SingleTenantStore, self).configure() + self.auth_version = self._option_get('swift_store_auth_version') + + def configure_add(self): + default_swift_reference = \ + SWIFT_STORE_REF_PARAMS.get( + self.conf.glance_store.default_swift_reference) + if default_swift_reference: + self.auth_address = default_swift_reference.get('auth_address') + if (not default_swift_reference) or (not self.auth_address): + reason = _("A value for swift_store_auth_address is required.") + LOG.error(reason) + raise exceptions.BadStoreConfiguration(message=reason) + + if self.auth_address.startswith('http://'): + self.scheme = 'swift+http' + else: + self.scheme = 'swift+https' + self.container = self.conf.glance_store.swift_store_container + self.user = default_swift_reference.get('user') + self.key = default_swift_reference.get('key') + + if not (self.user or self.key): + reason = _("A value for swift_store_ref_params is required.") + LOG.error(reason) + raise exceptions.BadStoreConfiguration(store_name="swift", + reason=reason) + + def create_location(self, image_id, context=None): + specs = {'scheme': self.scheme, + 'container': self.container, + 'obj': str(image_id), + 'auth_or_store_url': self.auth_address, + 'user': self.user, + 'key': self.key} + return StoreLocation(specs) + + def get_connection(self, location, context=None): + if not location.user: + reason = _("Location is missing user:password information.") + LOG.info(reason) + raise exceptions.BadStoreUri(message=reason) + + auth_url = location.swift_url + if not auth_url.endswith('/'): + auth_url += '/' + + if self.auth_version == '2': + try: + tenant_name, user = location.user.split(':') + except ValueError: + reason = (_("Badly formed tenant:user '%(user)s' in " + "Swift URI") % {'user': location.user}) + LOG.info(reason) + raise exceptions.BadStoreUri(message=reason) + else: + tenant_name = None + user = location.user + + os_options = {} + if self.region: + os_options['region_name'] = self.region + os_options['endpoint_type'] = self.endpoint_type + os_options['service_type'] = self.service_type + + return swiftclient.Connection( + auth_url, user, location.key, insecure=self.insecure, + tenant_name=tenant_name, snet=self.snet, + auth_version=self.auth_version, os_options=os_options, + ssl_compression=self.ssl_compression) + + +class MultiTenantStore(BaseStore): + EXAMPLE_URL = "swift://<SWIFT_URL>/<CONTAINER>/<FILE>" + + def _get_endpoint(self, context): + self.container = self.conf.glance_store.swift_store_container + if context is None: + reason = _("Multi-tenant Swift storage requires a context.") + raise exceptions.BadStoreConfiguration(store_name="swift", + reason=reason) + if context.service_catalog is None: + reason = _("Multi-tenant Swift storage requires " + "a service catalog.") + raise exceptions.BadStoreConfiguration(store_name="swift", + reason=reason) + self.storage_url = auth.get_endpoint( + context.service_catalog, service_type=self.service_type, + endpoint_region=self.region, endpoint_type=self.endpoint_type) + if self.storage_url.startswith('http://'): + self.scheme = 'swift+http' + else: + self.scheme = 'swift+https' + + return self.storage_url + + def delete(self, location, connection=None, context=None): + if not connection: + connection = self.get_connection(location.store_location, + context=context) + super(MultiTenantStore, self).delete(location, connection) + connection.delete_container(location.store_location.container) + + def set_acls(self, location, public=False, read_tenants=None, + write_tenants=None, connection=None, context=None): + location = location.store_location + if not connection: + connection = self.get_connection(location, context=context) + + if read_tenants is None: + read_tenants = [] + if write_tenants is None: + write_tenants = [] + + headers = {} + if public: + headers['X-Container-Read'] = ".r:*,.rlistings" + elif read_tenants: + headers['X-Container-Read'] = ','.join('%s:*' % i + for i in read_tenants) + else: + headers['X-Container-Read'] = '' + + write_tenants.extend(self.admin_tenants) + if write_tenants: + headers['X-Container-Write'] = ','.join('%s:*' % i + for i in write_tenants) + else: + headers['X-Container-Write'] = '' + + try: + connection.post_container(location.container, headers=headers) + except swiftclient.ClientException as e: + if e.http_status == httplib.NOT_FOUND: + msg = _("Swift could not find image at URI.") + raise exceptions.NotFound(message=msg) + else: + raise + + def create_location(self, image_id, context=None): + ep = self._get_endpoint(context) + specs = {'scheme': self.scheme, + 'container': self.container + '_' + str(image_id), + 'obj': str(image_id), + 'auth_or_store_url': ep} + return StoreLocation(specs) + + def get_connection(self, location, context=None): + return swiftclient.Connection( + None, context.user, None, + preauthurl=location.swift_url, + preauthtoken=context.auth_token, + tenant_name=context.tenant, + auth_version='2', snet=self.snet, insecure=self.insecure, + ssl_compression=self.ssl_compression) + + +class ChunkReader(object): + def __init__(self, fd, checksum, total): + self.fd = fd + self.checksum = checksum + self.total = total + self.bytes_read = 0 + + def read(self, i): + left = self.total - self.bytes_read + if i > left: + i = left + result = self.fd.read(i) + self.bytes_read += len(result) + self.checksum.update(result) + return result diff --git a/glance/store/_drivers/swift/utils.py b/glance/store/_drivers/swift/utils.py new file mode 100644 index 0000000..77c7a98 --- /dev/null +++ b/glance/store/_drivers/swift/utils.py @@ -0,0 +1,111 @@ +# Copyright 2014 Rackspace +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import ConfigParser +import logging + +try: + from collections import OrderedDict +except ImportError: + from ordereddict import OrderedDict + +from oslo.config import cfg + +from glance.store import exceptions +from glance.store import i18n + +swift_opts = [ + cfg.StrOpt('default_swift_reference', + default="ref1", + help=i18n._('The reference to the default swift account/backing' + ' store parameters to use for adding new images.')), + cfg.StrOpt('swift_store_auth_address', + help=i18n._('The address where the Swift authentication ' + 'service is listening.(deprecated)')), + cfg.StrOpt('swift_store_user', secret=True, + help=i18n._('The user to authenticate against the Swift ' + 'authentication service (deprecated)')), + cfg.StrOpt('swift_store_key', secret=True, + help=i18n._('Auth key for the user authenticating against the ' + 'Swift authentication service. (deprecated)')), + cfg.StrOpt('swift_store_config_file', secret=True, + help=i18n._('The config file that has the swift account(s)' + 'configs.')), +] + +# NOTE(bourke): The default dict_type is collections.OrderedDict in py27, but +# we must set manually for compatibility with py26 +CONFIG = ConfigParser.SafeConfigParser(dict_type=OrderedDict) +LOG = logging.getLogger(__name__) + + +CONF = cfg.CONF +CONF.register_opts(swift_opts, group='glance_store') + + +def is_multiple_swift_store_accounts_enabled(): + if CONF.glance_store.swift_store_config_file is None: + return False + return True + + +class SwiftParams(object): + def __init__(self): + if is_multiple_swift_store_accounts_enabled(): + self.params = self._load_config() + else: + self.params = self._form_default_params() + + def _form_default_params(self): + default = {} + + if ( + CONF.glance_store.swift_store_user and + CONF.glance_store.swift_store_key and + CONF.glance_store.swift_store_auth_address + ): + + glance_store = CONF.glance_store + default['user'] = glance_store.swift_store_user + default['key'] = glance_store.swift_store_key + default['auth_address'] = glance_store.swift_store_auth_address + return {glance_store.default_swift_reference: default} + return {} + + def _load_config(self): + try: + scf = CONF.glance_store.swift_store_config_file + conf_file = CONF.find_file(scf) + CONFIG.read(conf_file) + except Exception as e: + msg = (i18n._("swift config file " + "%(conf_file)s:%(exc)s not found") % + {'conf_file': CONF.glance_store.swift_store_config_file, + 'exc': e}) + LOG.error(msg) + raise exceptions.BadStoreConfiguration(store_name='swift', + reason=msg) + account_params = {} + account_references = CONFIG.sections() + for ref in account_references: + reference = {} + try: + reference['auth_address'] = CONFIG.get(ref, 'auth_address') + reference['user'] = CONFIG.get(ref, 'user') + reference['key'] = CONFIG.get(ref, 'key') + account_params[ref] = reference + except (ValueError, SyntaxError, ConfigParser.NoOptionError) as e: + LOG.exception(i18n._("Invalid format of swift store config" + "cfg")) + return account_params diff --git a/glance/store/common/auth.py b/glance/store/common/auth.py new file mode 100644 index 0000000..dc78cec --- /dev/null +++ b/glance/store/common/auth.py @@ -0,0 +1,288 @@ +# Copyright 2011 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +This auth module is intended to allow OpenStack client-tools to select from a +variety of authentication strategies, including NoAuth (the default), and +Keystone (an identity management system). + + > auth_plugin = AuthPlugin(creds) + + > auth_plugin.authenticate() + + > auth_plugin.auth_token + abcdefg + + > auth_plugin.management_url + http://service_endpoint/ +""" +import httplib2 +import logging + +import six.moves.urllib.parse as urlparse + +from glance.store.openstack.common import jsonutils + + +LOG = logging.getLogger(__name__) + + +class BaseStrategy(object): + def __init__(self): + self.auth_token = None + # TODO(sirp): Should expose selecting public/internal/admin URL. + self.management_url = None + + def authenticate(self): + raise NotImplementedError + + @property + def is_authenticated(self): + raise NotImplementedError + + @property + def strategy(self): + raise NotImplementedError + + +class NoAuthStrategy(BaseStrategy): + def authenticate(self): + pass + + @property + def is_authenticated(self): + return True + + @property + def strategy(self): + return 'noauth' + + +class KeystoneStrategy(BaseStrategy): + MAX_REDIRECTS = 10 + + def __init__(self, creds, insecure=False, configure_via_auth=True): + self.creds = creds + self.insecure = insecure + self.configure_via_auth = configure_via_auth + super(KeystoneStrategy, self).__init__() + + def check_auth_params(self): + # Ensure that supplied credential parameters are as required + for required in ('username', 'password', 'auth_url', + 'strategy'): + if self.creds.get(required) is None: + raise exception.MissingCredentialError(required=required) + if self.creds['strategy'] != 'keystone': + raise exception.BadAuthStrategy(expected='keystone', + received=self.creds['strategy']) + # For v2.0 also check tenant is present + if self.creds['auth_url'].rstrip('/').endswith('v2.0'): + if self.creds.get("tenant") is None: + raise exception.MissingCredentialError(required='tenant') + + def authenticate(self): + """Authenticate with the Keystone service. + + There are a few scenarios to consider here: + + 1. Which version of Keystone are we using? v1 which uses headers to + pass the credentials, or v2 which uses a JSON encoded request body? + + 2. Keystone may respond back with a redirection using a 305 status + code. + + 3. We may attempt a v1 auth when v2 is what's called for. In this + case, we rewrite the url to contain /v2.0/ and retry using the v2 + protocol. + """ + def _authenticate(auth_url): + # If OS_AUTH_URL is missing a trailing slash add one + if not auth_url.endswith('/'): + auth_url += '/' + token_url = urlparse.urljoin(auth_url, "tokens") + # 1. Check Keystone version + is_v2 = auth_url.rstrip('/').endswith('v2.0') + if is_v2: + self._v2_auth(token_url) + else: + self._v1_auth(token_url) + + self.check_auth_params() + auth_url = self.creds['auth_url'] + for _ in range(self.MAX_REDIRECTS): + try: + _authenticate(auth_url) + except exception.AuthorizationRedirect as e: + # 2. Keystone may redirect us + auth_url = e.url + except exception.AuthorizationFailure: + # 3. In some configurations nova makes redirection to + # v2.0 keystone endpoint. Also, new location does not + # contain real endpoint, only hostname and port. + if 'v2.0' not in auth_url: + auth_url = urlparse.urljoin(auth_url, 'v2.0/') + else: + # If we successfully auth'd, then memorize the correct auth_url + # for future use. + self.creds['auth_url'] = auth_url + break + else: + # Guard against a redirection loop + raise exception.MaxRedirectsExceeded(redirects=self.MAX_REDIRECTS) + + def _v1_auth(self, token_url): + creds = self.creds + + headers = {} + headers['X-Auth-User'] = creds['username'] + headers['X-Auth-Key'] = creds['password'] + + tenant = creds.get('tenant') + if tenant: + headers['X-Auth-Tenant'] = tenant + + resp, resp_body = self._do_request(token_url, 'GET', headers=headers) + + def _management_url(self, resp): + for url_header in ('x-image-management-url', + 'x-server-management-url', + 'x-glance'): + try: + return resp[url_header] + except KeyError as e: + not_found = e + raise not_found + + if resp.status in (200, 204): + try: + if self.configure_via_auth: + self.management_url = _management_url(self, resp) + self.auth_token = resp['x-auth-token'] + except KeyError: + raise exception.AuthorizationFailure() + elif resp.status == 305: + raise exception.AuthorizationRedirect(uri=resp['location']) + elif resp.status == 400: + raise exception.AuthBadRequest(url=token_url) + elif resp.status == 401: + raise exception.NotAuthenticated() + elif resp.status == 404: + raise exception.AuthUrlNotFound(url=token_url) + else: + raise Exception(_('Unexpected response: %s') % resp.status) + + def _v2_auth(self, token_url): + + creds = self.creds + + creds = { + "auth": { + "tenantName": creds['tenant'], + "passwordCredentials": { + "username": creds['username'], + "password": creds['password'] + } + } + } + + headers = {} + headers['Content-Type'] = 'application/json' + req_body = jsonutils.dumps(creds) + + resp, resp_body = self._do_request( + token_url, 'POST', headers=headers, body=req_body) + + if resp.status == 200: + resp_auth = jsonutils.loads(resp_body)['access'] + creds_region = self.creds.get('region') + if self.configure_via_auth: + endpoint = get_endpoint(resp_auth['serviceCatalog'], + endpoint_region=creds_region) + self.management_url = endpoint + self.auth_token = resp_auth['token']['id'] + elif resp.status == 305: + raise exception.RedirectException(resp['location']) + elif resp.status == 400: + raise exception.AuthBadRequest(url=token_url) + elif resp.status == 401: + raise exception.NotAuthenticated() + elif resp.status == 404: + raise exception.AuthUrlNotFound(url=token_url) + else: + raise Exception(_('Unexpected response: %s') % resp.status) + + @property + def is_authenticated(self): + return self.auth_token is not None + + @property + def strategy(self): + return 'keystone' + + def _do_request(self, url, method, headers=None, body=None): + headers = headers or {} + conn = httplib2.Http() + conn.force_exception_to_status_code = True + conn.disable_ssl_certificate_validation = self.insecure + headers['User-Agent'] = 'glance-client' + resp, resp_body = conn.request(url, method, headers=headers, body=body) + return resp, resp_body + + +def get_plugin_from_strategy(strategy, creds=None, insecure=False, + configure_via_auth=True): + if strategy == 'noauth': + return NoAuthStrategy() + elif strategy == 'keystone': + return KeystoneStrategy(creds, insecure, + configure_via_auth=configure_via_auth) + else: + raise Exception(_("Unknown auth strategy '%s'") % strategy) + + +def get_endpoint(service_catalog, service_type='image', endpoint_region=None, + endpoint_type='publicURL'): + """ + Select an endpoint from the service catalog + + We search the full service catalog for services + matching both type and region. If the client + supplied no region then any 'image' endpoint + is considered a match. There must be one -- and + only one -- successful match in the catalog, + otherwise we will raise an exception. + """ + endpoint = None + for service in service_catalog: + s_type = None + try: + s_type = service['type'] + except KeyError: + msg = _('Encountered service with no "type": %s') % s_type + LOG.warn(msg) + continue + + if s_type == service_type: + for ep in service['endpoints']: + if endpoint_region is None or endpoint_region == ep['region']: + if endpoint is not None: + # This is a second match, abort + raise exception.RegionAmbiguity(region=endpoint_region) + endpoint = ep + if endpoint and endpoint.get(endpoint_type): + return endpoint[endpoint_type] + else: + raise exception.NoServiceEndpoint() diff --git a/glance/store/openstack/common/context.py b/glance/store/openstack/common/context.py new file mode 100644 index 0000000..615cb49 --- /dev/null +++ b/glance/store/openstack/common/context.py @@ -0,0 +1,127 @@ +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Simple class that stores security context information in the web request. + +Projects should subclass this class if they wish to enhance the request +context or provide additional information in their specific WSGI pipeline. +""" + +import itertools +import uuid + + +def generate_request_id(): + return b'req-' + str(uuid.uuid4()).encode('ascii') + + +class RequestContext(object): + + """Helper class to represent useful information about a request context. + + Stores information about the security context under which the user + accesses the system, as well as additional request information. + """ + + user_idt_format = '{user} {tenant} {domain} {user_domain} {p_domain}' + + def __init__(self, auth_token=None, user=None, tenant=None, domain=None, + user_domain=None, project_domain=None, is_admin=False, + read_only=False, show_deleted=False, request_id=None, + instance_uuid=None, service_catalog=None): + self.auth_token = auth_token + self.user = user + self.tenant = tenant + self.domain = domain + self.user_domain = user_domain + self.project_domain = project_domain + self.is_admin = is_admin + self.read_only = read_only + self.show_deleted = show_deleted + self.instance_uuid = instance_uuid + self.service_catalog = service_catalog + if not request_id: + request_id = generate_request_id() + self.request_id = request_id + + def to_dict(self): + user_idt = ( + self.user_idt_format.format(user=self.user or '-', + tenant=self.tenant or '-', + domain=self.domain or '-', + user_domain=self.user_domain or '-', + p_domain=self.project_domain or '-')) + + return {'user': self.user, + 'tenant': self.tenant, + 'domain': self.domain, + 'user_domain': self.user_domain, + 'project_domain': self.project_domain, + 'is_admin': self.is_admin, + 'read_only': self.read_only, + 'show_deleted': self.show_deleted, + 'auth_token': self.auth_token, + 'request_id': self.request_id, + 'instance_uuid': self.instance_uuid, + 'user_identity': user_idt} + + @classmethod + def from_dict(cls, ctx): + return cls( + auth_token=ctx.get("auth_token"), + user=ctx.get("user"), + tenant=ctx.get("tenant"), + domain=ctx.get("domain"), + user_domain=ctx.get("user_domain"), + project_domain=ctx.get("project_domain"), + is_admin=ctx.get("is_admin", False), + read_only=ctx.get("read_only", False), + show_deleted=ctx.get("show_deleted", False), + request_id=ctx.get("request_id"), + instance_uuid=ctx.get("instance_uuid")) + + +def get_admin_context(show_deleted=False): + context = RequestContext(None, + tenant=None, + is_admin=True, + show_deleted=show_deleted) + return context + + +def get_context_from_function_and_args(function, args, kwargs): + """Find an arg of type RequestContext and return it. + + This is useful in a couple of decorators where we don't + know much about the function we're wrapping. + """ + + for arg in itertools.chain(kwargs.values(), args): + if isinstance(arg, RequestContext): + return arg + + return None + + +def is_user_context(context): + """Indicates if the request context is a normal user.""" + if not context: + return False + if context.is_admin: + return False + if not context.user_id or not context.project_id: + return False + return True diff --git a/glance/store/tests/base.py b/glance/store/tests/base.py index 73b1eb4..7ce3aca 100644 --- a/glance/store/tests/base.py +++ b/glance/store/tests/base.py @@ -14,19 +14,27 @@ # License for the specific language governing permissions and limitations # under the License. +import os +import shutil + import fixtures from oslo.config import cfg -import testtools +from oslotest import base import glance.store as store from glance.store import location -class StoreBaseTest(testtools.TestCase): +class StoreBaseTest(base.BaseTestCase): + + #NOTE(flaper87): temporary until we + # can move to a fully-local lib. + # (Swift store's fault) + _CONF = cfg.ConfigOpts() def setUp(self): super(StoreBaseTest, self).setUp() - self.conf = cfg.ConfigOpts() + self.conf = self._CONF self.conf(args=[]) store.register_opts(self.conf) @@ -36,6 +44,13 @@ class StoreBaseTest(testtools.TestCase): store.create_stores(self.conf) self.addCleanup(setattr, location, 'SCHEME_TO_CLS_MAP', dict()) self.test_dir = self.useFixture(fixtures.TempDir()).path + self.addCleanup(self.conf.reset) + + def copy_data_file(self, file_name, dst_dir): + src_file_name = os.path.join('glance/store/tests/etc', file_name) + shutil.copy(src_file_name, dst_dir) + dst_file_name = os.path.join(dst_dir, file_name) + return dst_file_name def config(self, **kw): """Override some configuration values. diff --git a/glance/store/tests/etc/glance-swift.conf b/glance/store/tests/etc/glance-swift.conf new file mode 100644 index 0000000..956433b --- /dev/null +++ b/glance/store/tests/etc/glance-swift.conf @@ -0,0 +1,34 @@ +[ref1] +user = tenant:user1 +key = key1 +auth_address = example.com + +[ref2] +user = user2 +key = key2 +auth_address = http://example.com + +[store_2] +user = tenant:user1 +key = key1 +auth_address= https://localhost:8080 + +[store_3] +user= tenant:user2 +key= key2 +auth_address= https://localhost:8080 + +[store_4] +user = tenant:user1 +key = key1 +auth_address = http://localhost:80 + +[store_5] +user = tenant:user1 +key = key1 +auth_address = http://localhost + +[store_6] +user = tenant:user1 +key = key1 +auth_address = https://localhost/v1 diff --git a/openstack-common.conf b/openstack-common.conf index 7f2fe32..0ec58ae 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -1,6 +1,7 @@ [DEFAULT] # The list of modules to copy from openstack-common +module=context module=fileutils module=gettextutils module=importutils diff --git a/requirements.txt b/requirements.txt index ac5e692..8baecb1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,9 +2,6 @@ oslo.config>=1.2.0 oslo.i18n>=0.1.0 stevedore>=0.12 -# For Swift storage backend. -python-swiftclient>=1.5 - python-cinderclient>=1.0.6 # Required by openstack.common libraries @@ -29,6 +29,7 @@ namespace_packages = glance.store.drivers = file = glance.store._drivers.filesystem:Store http = glance.store._drivers.http:Store + swift = glance.store._drivers.swift:Store vmware = glance.store._drivers.vmware_datastore:Store # TESTS ONLY diff --git a/test-requirements.txt b/test-requirements.txt index 0154dd5..33277cb 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -23,3 +23,7 @@ boto>=2.12.0,!=2.13.0 # For VMware storage backend. oslo.vmware>=0.4 # Apache-2.0 + +# Swift Backend +httplib2>=0.7.5 +python-swiftclient>=2.0.2 diff --git a/tests/unit/test_filesystem_store.py b/tests/unit/test_filesystem_store.py index 936d6a5..2c9af70 100644 --- a/tests/unit/test_filesystem_store.py +++ b/tests/unit/test_filesystem_store.py @@ -348,7 +348,8 @@ class TestStore(base.StoreBaseTest): [store_map[0] + ":100", store_map[1] + ":200"], group='glance_store') - self.store.configure_add() + + self.store.configure() """Test that we can add an image via the filesystem backend""" ChunkedFile.CHUNKSIZE = 1024 diff --git a/tests/unit/test_swift_store.py b/tests/unit/test_swift_store.py new file mode 100644 index 0000000..50cdad3 --- /dev/null +++ b/tests/unit/test_swift_store.py @@ -0,0 +1,1127 @@ +# Copyright 2011 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Tests the Swift backend store""" + +import copy +import fixtures +import hashlib +import httplib +import mock +import tempfile +import uuid + +from oslo.config import cfg +from oslotest import moxstubout +import six +import stubout +import swiftclient + +from glance.store._drivers.swift import store as swift +from glance.store._drivers.swift import utils as sutils +from glance.store import backend +from glance.store import BackendException +from glance.store.common import auth +from glance.store import exceptions +from glance.store.location import get_location_from_uri +from glance.store.openstack.common import context +from glance.store.openstack.common import units +from glance.store.tests import base + +CONF = cfg.CONF + +FAKE_UUID = lambda: str(uuid.uuid4()) + +Store = swift.Store +FIVE_KB = 5 * units.Ki +FIVE_GB = 5 * units.Gi +MAX_SWIFT_OBJECT_SIZE = FIVE_GB +SWIFT_PUT_OBJECT_CALLS = 0 +SWIFT_CONF = {'swift_store_auth_address': 'localhost:8080', + 'swift_store_container': 'glance', + 'swift_store_user': 'user', + 'swift_store_key': 'key', + 'swift_store_auth_address': 'localhost:8080', + 'swift_store_container': 'glance', + 'swift_store_retry_get_count': 1, + 'default_swift_reference': 'ref1' + } + + +# We stub out as little as possible to ensure that the code paths +# between swift and swiftclient are tested +# thoroughly +def stub_out_swiftclient(stubs, swift_store_auth_version): + fixture_containers = ['glance'] + fixture_container_headers = {} + fixture_headers = { + 'glance/%s' % FAKE_UUID: { + 'content-length': FIVE_KB, + 'etag': 'c2e5db72bd7fd153f53ede5da5a06de3' + } + } + fixture_objects = {'glance/%s' % FAKE_UUID: + six.StringIO("*" * FIVE_KB)} + + def fake_head_container(url, token, container, **kwargs): + if container not in fixture_containers: + msg = "No container %s found" % container + raise swiftclient.ClientException(msg, + http_status=httplib.NOT_FOUND) + return fixture_container_headers + + def fake_put_container(url, token, container, **kwargs): + fixture_containers.append(container) + + def fake_post_container(url, token, container, headers, http_conn=None): + for key, value in six.iteritems(headers): + fixture_container_headers[key] = value + + def fake_put_object(url, token, container, name, contents, **kwargs): + # PUT returns the ETag header for the newly-added object + # Large object manifest... + global SWIFT_PUT_OBJECT_CALLS + SWIFT_PUT_OBJECT_CALLS += 1 + CHUNKSIZE = 64 * units.Ki + fixture_key = "%s/%s" % (container, name) + if fixture_key not in fixture_headers: + if kwargs.get('headers'): + etag = kwargs['headers']['ETag'] + fixture_headers[fixture_key] = {'manifest': True, + 'etag': etag} + return etag + if hasattr(contents, 'read'): + fixture_object = six.StringIO() + chunk = contents.read(CHUNKSIZE) + checksum = hashlib.md5() + while chunk: + fixture_object.write(chunk) + checksum.update(chunk) + chunk = contents.read(CHUNKSIZE) + etag = checksum.hexdigest() + else: + fixture_object = six.StringIO(contents) + etag = hashlib.md5(fixture_object.getvalue()).hexdigest() + read_len = fixture_object.len + if read_len > MAX_SWIFT_OBJECT_SIZE: + msg = ('Image size:%d exceeds Swift max:%d' % + (read_len, MAX_SWIFT_OBJECT_SIZE)) + raise swiftclient.ClientException( + msg, http_status=httplib.REQUEST_ENTITY_TOO_LARGE) + fixture_objects[fixture_key] = fixture_object + fixture_headers[fixture_key] = { + 'content-length': read_len, + 'etag': etag} + return etag + else: + msg = ("Object PUT failed - Object with key %s already exists" + % fixture_key) + raise swiftclient.ClientException(msg, + http_status=httplib.CONFLICT) + + def fake_get_object(url, token, container, name, **kwargs): + # GET returns the tuple (list of headers, file object) + fixture_key = "%s/%s" % (container, name) + if fixture_key not in fixture_headers: + msg = "Object GET failed" + raise swiftclient.ClientException(msg, + http_status=httplib.NOT_FOUND) + + byte_range = None + headers = kwargs.get('headers', dict()) + if headers is not None: + headers = dict((k.lower(), v) for k, v in six.iteritems(headers)) + if 'range' in headers: + byte_range = headers.get('range') + + fixture = fixture_headers[fixture_key] + if 'manifest' in fixture: + # Large object manifest... we return a file containing + # all objects with prefix of this fixture key + chunk_keys = sorted([k for k in fixture_headers.keys() + if k.startswith(fixture_key) and + k != fixture_key]) + result = six.StringIO() + for key in chunk_keys: + result.write(fixture_objects[key].getvalue()) + else: + result = fixture_objects[fixture_key] + + if byte_range is not None: + start = int(byte_range.split('=')[1].strip('-')) + result = six.StringIO(result.getvalue()[start:]) + fixture_headers[fixture_key]['content-length'] = len( + result.getvalue()) + + return fixture_headers[fixture_key], result + + def fake_head_object(url, token, container, name, **kwargs): + # HEAD returns the list of headers for an object + try: + fixture_key = "%s/%s" % (container, name) + return fixture_headers[fixture_key] + except KeyError: + msg = "Object HEAD failed - Object does not exist" + raise swiftclient.ClientException(msg, + http_status=httplib.NOT_FOUND) + + def fake_delete_object(url, token, container, name, **kwargs): + # DELETE returns nothing + fixture_key = "%s/%s" % (container, name) + if fixture_key not in fixture_headers: + msg = "Object DELETE failed - Object does not exist" + raise swiftclient.ClientException(msg, + http_status=httplib.NOT_FOUND) + else: + del fixture_headers[fixture_key] + del fixture_objects[fixture_key] + + def fake_http_connection(*args, **kwargs): + return None + + def fake_get_auth(url, user, key, snet, auth_version, **kwargs): + if url is None: + return None, None + if 'http' in url and '://' not in url: + raise ValueError('Invalid url %s' % url) + # Check the auth version against the configured value + if swift_store_auth_version != auth_version: + msg = 'AUTHENTICATION failed (version mismatch)' + raise swiftclient.ClientException(msg) + return None, None + + stubs.Set(swiftclient.client, + 'head_container', fake_head_container) + stubs.Set(swiftclient.client, + 'put_container', fake_put_container) + stubs.Set(swiftclient.client, + 'post_container', fake_post_container) + stubs.Set(swiftclient.client, + 'put_object', fake_put_object) + stubs.Set(swiftclient.client, + 'delete_object', fake_delete_object) + stubs.Set(swiftclient.client, + 'head_object', fake_head_object) + stubs.Set(swiftclient.client, + 'get_object', fake_get_object) + stubs.Set(swiftclient.client, + 'get_auth', fake_get_auth) + stubs.Set(swiftclient.client, + 'http_connection', fake_http_connection) + + +class SwiftTests(object): + + @property + def swift_store_user(self): + return 'tenant:user1' + + def test_get_size(self): + """ + Test that we can get the size of an object in the swift store + """ + uri = "swift://%s:key@auth_address/glance/%s" % ( + self.swift_store_user, FAKE_UUID) + loc = get_location_from_uri(uri) + image_size = self.store.get_size(loc) + self.assertEqual(image_size, 5120) + + def test_validate_location_for_invalid_uri(self): + """ + Test that validate location raises when the location contains + any account reference. + """ + uri = "swift+config://store_1/glance/%s" + self.assertRaises(exceptions.BadStoreUri, + self.store.validate_location, + uri) + + def test_validate_location_for_valid_uri(self): + """ + Test that validate location verifies that the location does not + contain any account reference + """ + uri = "swift://user:key@auth_address/glance/%s" + try: + self.assertIsNone(self.store.validate_location(uri)) + except Exception: + self.fail('Location uri validation failed') + + def test_get_size_with_multi_tenant_on(self): + """Test that single tenant uris work with multi tenant on.""" + uri = ("swift://%s:key@auth_address/glance/%s" % + (self.swift_store_user, FAKE_UUID)) + self.config(swift_store_multi_tenant=True) + #NOTE(markwash): ensure the image is found + size = backend.get_size_from_backend(uri, context={}) + self.assertEqual(size, 5120) + + def test_get(self): + """Test a "normal" retrieval of an image in chunks""" + uri = "swift://%s:key@auth_address/glance/%s" % ( + self.swift_store_user, FAKE_UUID) + loc = get_location_from_uri(uri) + (image_swift, image_size) = self.store.get(loc) + self.assertEqual(image_size, 5120) + + expected_data = "*" * FIVE_KB + data = "" + + for chunk in image_swift: + data += chunk + self.assertEqual(expected_data, data) + + def test_get_with_retry(self): + """ + Test a retrieval where Swift does not get the full image in a single + request. + """ + uri = "swift://%s:key@auth_address/glance/%s" % ( + self.swift_store_user, FAKE_UUID) + loc = get_location_from_uri(uri) + ctxt = context.RequestContext() + (image_swift, image_size) = self.store.get(loc, context=ctxt) + resp_full = ''.join([chunk for chunk in image_swift.wrapped]) + resp_half = resp_full[:len(resp_full) / 2] + image_swift.wrapped = swift.swift_retry_iter(resp_half, image_size, + self.store, + loc.store_location, + ctxt) + self.assertEqual(image_size, 5120) + + expected_data = "*" * FIVE_KB + data = "" + + for chunk in image_swift: + data += chunk + self.assertEqual(expected_data, data) + + def test_get_with_http_auth(self): + """ + Test a retrieval from Swift with an HTTP authurl. This is + specified either via a Location header with swift+http:// or using + http:// in the swift_store_auth_address config value + """ + loc = get_location_from_uri("swift+http://%s:key@auth_address/" + "glance/%s" % + (self.swift_store_user, FAKE_UUID)) + + ctxt = context.RequestContext() + (image_swift, image_size) = self.store.get(loc, context=ctxt) + self.assertEqual(image_size, 5120) + + expected_data = "*" * FIVE_KB + data = "" + + for chunk in image_swift: + data += chunk + self.assertEqual(expected_data, data) + + def test_get_non_existing(self): + """ + Test that trying to retrieve a swift that doesn't exist + raises an error + """ + loc = get_location_from_uri("swift://%s:key@authurl/glance/noexist" % ( + self.swift_store_user)) + self.assertRaises(exceptions.NotFound, + self.store.get, + loc) + + def test_add(self): + """Test that we can add an image via the swift backend""" + sutils.is_multiple_swift_store_accounts_enabled = \ + mock.Mock(return_value=False) + reload(swift) + self.store = Store(self.conf) + expected_swift_size = FIVE_KB + expected_swift_contents = "*" * expected_swift_size + expected_checksum = hashlib.md5(expected_swift_contents).hexdigest() + expected_image_id = str(uuid.uuid4()) + loc = "swift+https://tenant%%3Auser1:key@localhost:8080/glance/%s" + expected_location = loc % (expected_image_id) + image_swift = six.StringIO(expected_swift_contents) + + global SWIFT_PUT_OBJECT_CALLS + SWIFT_PUT_OBJECT_CALLS = 0 + + location, size, checksum, _ = self.store.add(expected_image_id, + image_swift, + expected_swift_size) + + self.assertEqual(expected_location, location) + self.assertEqual(expected_swift_size, size) + self.assertEqual(expected_checksum, checksum) + # Expecting a single object to be created on Swift i.e. no chunking. + self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 1) + + loc = get_location_from_uri(expected_location) + (new_image_swift, new_image_size) = self.store.get(loc) + new_image_contents = ''.join([chunk for chunk in new_image_swift]) + new_image_swift_size = len(new_image_swift) + + self.assertEqual(expected_swift_contents, new_image_contents) + self.assertEqual(expected_swift_size, new_image_swift_size) + + def test_add_multi_store(self): + + conf = copy.deepcopy(SWIFT_CONF) + conf['default_swift_reference'] = 'store_2' + self.config(**conf) + reload(swift) + self.store = Store(self.conf) + + expected_swift_size = FIVE_KB + expected_swift_contents = "*" * expected_swift_size + expected_image_id = str(uuid.uuid4()) + image_swift = six.StringIO(expected_swift_contents) + global SWIFT_PUT_OBJECT_CALLS + SWIFT_PUT_OBJECT_CALLS = 0 + loc = 'swift+config://store_2/glance/%s' + + expected_location = loc % (expected_image_id) + + location, size, checksum, arg = self.store.add(expected_image_id, + image_swift, + expected_swift_size) + self.assertEqual(expected_location, location) + + def test_add_auth_url_variations(self): + """ + Test that we can add an image via the swift backend with + a variety of different auth_address values + """ + sutils.is_multiple_swift_store_accounts_enabled = \ + mock.Mock(return_value=True) + conf = copy.deepcopy(SWIFT_CONF) + self.config(**conf) + + variations = { + 'store_4': 'swift+config://store_4/glance/%s', + 'store_5': 'swift+config://store_5/glance/%s', + 'store_6': 'swift+config://store_6/glance/%s' + } + + for variation, expected_location in variations.items(): + image_id = str(uuid.uuid4()) + expected_location = expected_location % image_id + expected_swift_size = FIVE_KB + expected_swift_contents = "*" * expected_swift_size + expected_checksum = \ + hashlib.md5(expected_swift_contents).hexdigest() + + image_swift = six.StringIO(expected_swift_contents) + + global SWIFT_PUT_OBJECT_CALLS + SWIFT_PUT_OBJECT_CALLS = 0 + conf['default_swift_reference'] = variation + self.config(**conf) + reload(swift) + self.store = Store(self.conf) + location, size, checksum, _ = self.store.add(image_id, image_swift, + expected_swift_size) + + self.assertEqual(expected_location, location) + self.assertEqual(expected_swift_size, size) + self.assertEqual(expected_checksum, checksum) + self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 1) + + loc = get_location_from_uri(expected_location) + (new_image_swift, new_image_size) = self.store.get(loc) + new_image_contents = ''.join([chunk for chunk in new_image_swift]) + new_image_swift_size = len(new_image_swift) + + self.assertEqual(expected_swift_contents, new_image_contents) + self.assertEqual(expected_swift_size, new_image_swift_size) + + def test_add_no_container_no_create(self): + """ + Tests that adding an image with a non-existing container + raises an appropriate exception + """ + conf = copy.deepcopy(SWIFT_CONF) + conf['swift_store_user'] = 'tenant:user' + conf['swift_store_create_container_on_put'] = False + conf['swift_store_container'] = 'noexist' + self.config(**conf) + reload(swift) + + self.store = Store(self.conf) + + image_swift = six.StringIO("nevergonnamakeit") + + global SWIFT_PUT_OBJECT_CALLS + SWIFT_PUT_OBJECT_CALLS = 0 + + # We check the exception text to ensure the container + # missing text is found in it, otherwise, we would have + # simply used self.assertRaises here + exception_caught = False + try: + self.store.add(str(uuid.uuid4()), image_swift, 0) + except BackendException as e: + exception_caught = True + self.assertIn("container noexist does not exist " + "in Swift", unicode(e)) + self.assertTrue(exception_caught) + self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 0) + + def test_add_no_container_and_create(self): + """ + Tests that adding an image with a non-existing container + creates the container automatically if flag is set + """ + sutils.is_multiple_swift_store_accounts_enabled = \ + mock.Mock(return_value=True) + expected_swift_size = FIVE_KB + expected_swift_contents = "*" * expected_swift_size + expected_checksum = hashlib.md5(expected_swift_contents).hexdigest() + expected_image_id = str(uuid.uuid4()) + loc = 'swift+config://ref1/noexist/%s' + expected_location = loc % (expected_image_id) + image_swift = six.StringIO(expected_swift_contents) + + global SWIFT_PUT_OBJECT_CALLS + SWIFT_PUT_OBJECT_CALLS = 0 + conf = copy.deepcopy(SWIFT_CONF) + conf['swift_store_user'] = 'tenant:user' + conf['swift_store_create_container_on_put'] = True + conf['swift_store_container'] = 'noexist' + self.config(**conf) + reload(swift) + self.store = Store(self.conf) + location, size, checksum, _ = self.store.add(expected_image_id, + image_swift, + expected_swift_size) + + self.assertEqual(expected_location, location) + self.assertEqual(expected_swift_size, size) + self.assertEqual(expected_checksum, checksum) + self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 1) + + loc = get_location_from_uri(expected_location) + (new_image_swift, new_image_size) = self.store.get(loc) + new_image_contents = ''.join([chunk for chunk in new_image_swift]) + new_image_swift_size = len(new_image_swift) + + self.assertEqual(expected_swift_contents, new_image_contents) + self.assertEqual(expected_swift_size, new_image_swift_size) + + def test_add_large_object(self): + """ + Tests that adding a very large image. We simulate the large + object by setting store.large_object_size to a small number + and then verify that there have been a number of calls to + put_object()... + """ + sutils.is_multiple_swift_store_accounts_enabled = \ + mock.Mock(return_value=True) + expected_swift_size = FIVE_KB + expected_swift_contents = "*" * expected_swift_size + expected_checksum = hashlib.md5(expected_swift_contents).hexdigest() + expected_image_id = str(uuid.uuid4()) + loc = 'swift+config://ref1/glance/%s' + expected_location = loc % (expected_image_id) + image_swift = six.StringIO(expected_swift_contents) + + global SWIFT_PUT_OBJECT_CALLS + SWIFT_PUT_OBJECT_CALLS = 0 + + self.store = Store(self.conf) + orig_max_size = self.store.large_object_size + orig_temp_size = self.store.large_object_chunk_size + try: + self.store.large_object_size = 1024 + self.store.large_object_chunk_size = 1024 + location, size, checksum, _ = self.store.add(expected_image_id, + image_swift, + expected_swift_size) + finally: + self.store.large_object_chunk_size = orig_temp_size + self.store.large_object_size = orig_max_size + + self.assertEqual(expected_location, location) + self.assertEqual(expected_swift_size, size) + self.assertEqual(expected_checksum, checksum) + # Expecting 6 objects to be created on Swift -- 5 chunks and 1 + # manifest. + self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 6) + + loc = get_location_from_uri(expected_location) + (new_image_swift, new_image_size) = self.store.get(loc) + new_image_contents = ''.join([chunk for chunk in new_image_swift]) + new_image_swift_size = len(new_image_contents) + + self.assertEqual(expected_swift_contents, new_image_contents) + self.assertEqual(expected_swift_size, new_image_swift_size) + + def test_add_large_object_zero_size(self): + """ + Tests that adding an image to Swift which has both an unknown size and + exceeds Swift's maximum limit of 5GB is correctly uploaded. + + We avoid the overhead of creating a 5GB object for this test by + temporarily setting MAX_SWIFT_OBJECT_SIZE to 1KB, and then adding + an object of 5KB. + + Bug lp:891738 + """ + # Set up a 'large' image of 5KB + expected_swift_size = FIVE_KB + expected_swift_contents = "*" * expected_swift_size + expected_checksum = hashlib.md5(expected_swift_contents).hexdigest() + expected_image_id = str(uuid.uuid4()) + loc = 'swift+config://ref1/glance/%s' + expected_location = loc % (expected_image_id) + image_swift = six.StringIO(expected_swift_contents) + + global SWIFT_PUT_OBJECT_CALLS + SWIFT_PUT_OBJECT_CALLS = 0 + + # Temporarily set Swift MAX_SWIFT_OBJECT_SIZE to 1KB and add our image, + # explicitly setting the image_length to 0 + + self.store = Store(self.conf) + orig_max_size = self.store.large_object_size + orig_temp_size = self.store.large_object_chunk_size + global MAX_SWIFT_OBJECT_SIZE + orig_max_swift_object_size = MAX_SWIFT_OBJECT_SIZE + try: + MAX_SWIFT_OBJECT_SIZE = 1024 + self.store.large_object_size = 1024 + self.store.large_object_chunk_size = 1024 + location, size, checksum, _ = self.store.add(expected_image_id, + image_swift, 0) + finally: + self.store.large_object_chunk_size = orig_temp_size + self.store.large_object_size = orig_max_size + MAX_SWIFT_OBJECT_SIZE = orig_max_swift_object_size + + self.assertEqual(expected_location, location) + self.assertEqual(expected_swift_size, size) + self.assertEqual(expected_checksum, checksum) + # Expecting 7 calls to put_object -- 5 chunks, a zero chunk which is + # then deleted, and the manifest. Note the difference with above + # where the image_size is specified in advance (there's no zero chunk + # in that case). + self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 7) + + loc = get_location_from_uri(expected_location) + (new_image_swift, new_image_size) = self.store.get(loc) + new_image_contents = ''.join([chunk for chunk in new_image_swift]) + new_image_swift_size = len(new_image_contents) + + self.assertEqual(expected_swift_contents, new_image_contents) + self.assertEqual(expected_swift_size, new_image_swift_size) + + def test_add_already_existing(self): + """ + Tests that adding an image with an existing identifier + raises an appropriate exception + """ + image_swift = six.StringIO("nevergonnamakeit") + self.assertRaises(exceptions.Duplicate, + self.store.add, + FAKE_UUID, image_swift, 0) + + def _option_required(self, key): + conf = self.getConfig() + conf[key] = None + + try: + self.config(**conf) + self.store = Store(self.conf) + return self.store.add == self.store.add_disabled + except Exception: + return False + return False + + def test_no_store_credentials(self): + """ + Tests that options without a valid credentials disables the add method + """ + swift.SWIFT_STORE_REF_PARAMS = {'ref1': {'auth_address': + 'authurl.com', 'user': '', + 'key': ''}} + self.store = Store(self.conf) + self.assertEqual(self.store.add, self.store.add_disabled) + + def test_no_auth_address(self): + """ + Tests that options without auth address disables the add method + """ + swift.SWIFT_STORE_REF_PARAMS = {'ref1': {'auth_address': + '', 'user': 'user1', + 'key': 'key1'}} + + self.store = Store(self.conf) + self.assertEqual(self.store.add, self.store.add_disabled) + + def test_delete(self): + """ + Test we can delete an existing image in the swift store + """ + uri = "swift://%s:key@authurl/glance/%s" % ( + self.swift_store_user, FAKE_UUID) + loc = get_location_from_uri(uri) + self.store.delete(loc) + + self.assertRaises(exceptions.NotFound, self.store.get, loc) + + def test_delete_with_reference_params(self): + """ + Test we can delete an existing image in the swift store + """ + uri = "swift+config://ref1/glance/%s" % (FAKE_UUID) + loc = get_location_from_uri(uri) + self.store.delete(loc) + + self.assertRaises(exceptions.NotFound, self.store.get, loc) + + def test_delete_non_existing(self): + """ + Test that trying to delete a swift that doesn't exist + raises an error + """ + loc = get_location_from_uri("swift://%s:key@authurl/glance/noexist" % ( + self.swift_store_user)) + self.assertRaises(exceptions.NotFound, self.store.delete, loc) + + def test_read_acl_public(self): + """ + Test that we can set a public read acl. + """ + self.config(swift_store_multi_tenant=True) + store = Store(self.conf) + uri = "swift+http://storeurl/glance/%s" % FAKE_UUID + loc = get_location_from_uri(uri) + ctxt = context.RequestContext() + store.set_acls(loc, public=True, context=ctxt) + container_headers = swiftclient.client.head_container('x', 'y', + 'glance') + self.assertEqual(container_headers['X-Container-Read'], + ".r:*,.rlistings") + + def test_read_acl_tenants(self): + """ + Test that we can set read acl for tenants. + """ + self.config(swift_store_multi_tenant=True) + store = Store(self.conf) + uri = "swift+http://storeurl/glance/%s" % FAKE_UUID + loc = get_location_from_uri(uri) + read_tenants = ['matt', 'mark'] + ctxt = context.RequestContext() + store.set_acls(loc, read_tenants=read_tenants, context=ctxt) + container_headers = swiftclient.client.head_container('x', 'y', + 'glance') + self.assertEqual(container_headers['X-Container-Read'], + 'matt:*,mark:*') + + def test_write_acls(self): + """ + Test that we can set write acl for tenants. + """ + self.config(swift_store_multi_tenant=True) + store = Store(self.conf) + uri = "swift+http://storeurl/glance/%s" % FAKE_UUID + loc = get_location_from_uri(uri) + read_tenants = ['frank', 'jim'] + ctxt = context.RequestContext() + store.set_acls(loc, write_tenants=read_tenants, context=ctxt) + container_headers = swiftclient.client.head_container('x', 'y', + 'glance') + self.assertEqual(container_headers['X-Container-Write'], + 'frank:*,jim:*') + + +class TestStoreAuthV1(base.StoreBaseTest, SwiftTests): + + _CONF = cfg.CONF + + def getConfig(self): + conf = SWIFT_CONF.copy() + conf['swift_store_auth_version'] = '1' + conf['swift_store_user'] = 'tenant:user1' + return conf + + def setUp(self): + """Establish a clean test environment""" + super(TestStoreAuthV1, self).setUp() + conf = self.getConfig() + + conf_file = 'glance-swift.conf' + self.swift_config_file = self.copy_data_file(conf_file, self.test_dir) + conf.update({'swift_store_config_file': self.swift_config_file}) + + self.stubs = stubout.StubOutForTesting() + stub_out_swiftclient(self.stubs, conf['swift_store_auth_version']) + self.store = Store(self.conf) + self.config(**conf) + self.store.configure() + self.addCleanup(self.stubs.UnsetAll) + self.register_store_schemes(self.store) + swift.SWIFT_STORE_REF_PARAMS = sutils.SwiftParams().params + self.addCleanup(self.conf.reset) + + +class TestStoreAuthV2(TestStoreAuthV1): + + def getConfig(self): + conf = super(TestStoreAuthV2, self).getConfig() + conf['swift_store_auth_version'] = '2' + conf['swift_store_user'] = 'tenant:user1' + return conf + + def test_v2_with_no_tenant(self): + uri = "swift://failme:key@auth_address/glance/%s" % (FAKE_UUID) + loc = get_location_from_uri(uri) + self.assertRaises(exceptions.BadStoreUri, + self.store.get, + loc) + + def test_v2_multi_tenant_location(self): + conf = self.getConfig() + conf['swift_store_multi_tenant'] = True + uri = "swift://auth_address/glance/%s" % (FAKE_UUID) + loc = get_location_from_uri(uri) + self.assertEqual('swift', loc.store_name) + + +class FakeConnection(object): + def __init__(self, authurl, user, key, retries=5, preauthurl=None, + preauthtoken=None, snet=False, starting_backoff=1, + tenant_name=None, os_options=None, auth_version="1", + insecure=False, ssl_compression=True): + if os_options is None: + os_options = {} + + self.authurl = authurl + self.user = user + self.key = key + self.preauthurl = preauthurl + self.preauthtoken = preauthtoken + self.snet = snet + self.tenant_name = tenant_name + self.os_options = os_options + self.auth_version = auth_version + self.insecure = insecure + + +class TestSingleTenantStoreConnections(base.StoreBaseTest): + _CONF = cfg.CONF + + def setUp(self): + super(TestSingleTenantStoreConnections, self).setUp() + moxfixture = self.useFixture(moxstubout.MoxStubout()) + self.stubs = moxfixture.stubs + self.stubs.Set(swiftclient, 'Connection', FakeConnection) + self.store = swift.SingleTenantStore(self.conf) + specs = {'scheme': 'swift', + 'auth_or_store_url': 'example.com/v2/', + 'user': 'tenant:user1', + 'key': 'key1', + 'container': 'cont', + 'obj': 'object'} + self.location = swift.StoreLocation(specs) + self.addCleanup(self.conf.reset) + + def test_basic_connection(self): + connection = self.store.get_connection(self.location) + self.assertEqual(connection.authurl, 'https://example.com/v2/') + self.assertEqual(connection.auth_version, '2') + self.assertEqual(connection.user, 'user1') + self.assertEqual(connection.tenant_name, 'tenant') + self.assertFalse(connection.snet) + self.assertEqual(connection.key, 'key1') + self.assertIsNone(connection.preauthurl) + self.assertIsNone(connection.preauthtoken) + self.assertFalse(connection.insecure) + self.assertEqual(connection.os_options, + {'service_type': 'object-store', + 'endpoint_type': 'publicURL'}) + + def test_connection_with_no_trailing_slash(self): + self.location.auth_or_store_url = 'example.com/v2' + connection = self.store.get_connection(self.location) + self.assertEqual(connection.authurl, 'https://example.com/v2/') + + def test_connection_insecure(self): + self.config(swift_store_auth_insecure=True) + self.store.configure() + connection = self.store.get_connection(self.location) + self.assertTrue(connection.insecure) + + def test_connection_with_auth_v1(self): + self.config(swift_store_auth_version='1') + self.store.configure() + self.location.user = 'auth_v1_user' + connection = self.store.get_connection(self.location) + self.assertEqual(connection.auth_version, '1') + self.assertEqual(connection.user, 'auth_v1_user') + self.assertIsNone(connection.tenant_name) + + def test_connection_invalid_user(self): + self.store.configure() + self.location.user = 'invalid:format:user' + self.assertRaises(exceptions.BadStoreUri, + self.store.get_connection, self.location) + + def test_connection_missing_user(self): + self.store.configure() + self.location.user = None + self.assertRaises(exceptions.BadStoreUri, + self.store.get_connection, self.location) + + def test_connection_with_region(self): + self.config(swift_store_region='Sahara') + self.store.configure() + connection = self.store.get_connection(self.location) + self.assertEqual(connection.os_options, + {'region_name': 'Sahara', + 'service_type': 'object-store', + 'endpoint_type': 'publicURL'}) + + def test_connection_with_service_type(self): + self.config(swift_store_service_type='shoe-store') + self.store.configure() + connection = self.store.get_connection(self.location) + self.assertEqual(connection.os_options, + {'service_type': 'shoe-store', + 'endpoint_type': 'publicURL'}) + + def test_connection_with_endpoint_type(self): + self.config(swift_store_endpoint_type='internalURL') + self.store.configure() + connection = self.store.get_connection(self.location) + self.assertEqual(connection.os_options, + {'service_type': 'object-store', + 'endpoint_type': 'internalURL'}) + + def test_connection_with_snet(self): + self.config(swift_enable_snet=True) + self.store.configure() + connection = self.store.get_connection(self.location) + self.assertTrue(connection.snet) + + def test_bad_location_uri(self): + self.store.configure() + self.location.uri = 'http://bad_uri://' + self.assertRaises(exceptions.BadStoreUri, + self.location.parse_uri, + self.location.uri) + + def test_bad_location_uri_invalid_credentials(self): + self.store.configure() + self.location.uri = 'swift://bad_creds@uri/cont/obj' + self.assertRaises(exceptions.BadStoreUri, + self.location.parse_uri, + self.location.uri) + + def test_bad_location_uri_invalid_object_path(self): + self.store.configure() + self.location.uri = 'swift://user:key@uri/cont' + self.assertRaises(exceptions.BadStoreUri, + self.location.parse_uri, + self.location.uri) + + +class TestMultiTenantStoreConnections(base.StoreBaseTest): + def setUp(self): + super(TestMultiTenantStoreConnections, self).setUp() + moxfixture = self.useFixture(moxstubout.MoxStubout()) + self.stubs = moxfixture.stubs + self.stubs.Set(swiftclient, 'Connection', FakeConnection) + self.context = context.RequestContext( + user='tenant:user1', tenant='tenant', auth_token='0123') + self.store = swift.MultiTenantStore(self.conf) + specs = {'scheme': 'swift', + 'auth_or_store_url': 'example.com', + 'container': 'cont', + 'obj': 'object'} + self.location = swift.StoreLocation(specs) + self.addCleanup(self.conf.reset) + + def test_basic_connection(self): + self.store.configure() + connection = self.store.get_connection(self.location, + context=self.context) + self.assertIsNone(connection.authurl) + self.assertEqual(connection.auth_version, '2') + self.assertEqual(connection.user, 'tenant:user1') + self.assertEqual(connection.tenant_name, 'tenant') + self.assertIsNone(connection.key) + self.assertFalse(connection.snet) + self.assertEqual(connection.preauthurl, 'https://example.com') + self.assertEqual(connection.preauthtoken, '0123') + self.assertEqual(connection.os_options, {}) + + def test_connection_with_snet(self): + self.config(swift_enable_snet=True) + self.store.configure() + connection = self.store.get_connection(self.location, + context=self.context) + self.assertTrue(connection.snet) + + +class FakeGetEndpoint(object): + def __init__(self, response): + self.response = response + + def __call__(self, service_catalog, service_type=None, + endpoint_region=None, endpoint_type=None): + self.service_type = service_type + self.endpoint_region = endpoint_region + self.endpoint_type = endpoint_type + return self.response + + +class TestCreatingLocations(base.StoreBaseTest): + _CONF = cfg.CONF + + def setUp(self): + super(TestCreatingLocations, self).setUp() + moxfixture = self.useFixture(moxstubout.MoxStubout()) + self.stubs = moxfixture.stubs + conf = copy.deepcopy(SWIFT_CONF) + self.store = Store(self.conf) + self.config(**conf) + reload(swift) + self.addCleanup(self.conf.reset) + + def test_single_tenant_location(self): + conf = copy.deepcopy(SWIFT_CONF) + conf['swift_store_container'] = 'container' + conf_file = "glance-swift.conf" + self.swift_config_file = self.copy_data_file(conf_file, self.test_dir) + conf.update({'swift_store_config_file': self.swift_config_file}) + conf['default_swift_reference'] = 'ref1' + self.config(**conf) + reload(swift) + + store = swift.SingleTenantStore(self.conf) + location = store.create_location('image-id') + self.assertEqual(location.scheme, 'swift+https') + self.assertEqual(location.swift_url, 'https://example.com') + self.assertEqual(location.container, 'container') + self.assertEqual(location.obj, 'image-id') + self.assertEqual(location.user, 'tenant:user1') + self.assertEqual(location.key, 'key1') + + def test_single_tenant_location_http(self): + conf_file = "glance-swift.conf" + test_dir = self.useFixture(fixtures.TempDir()).path + self.swift_config_file = self.copy_data_file(conf_file, test_dir) + self.config(swift_store_container='container', + default_swift_reference='ref2', + swift_store_config_file=self.swift_config_file) + + swift.SWIFT_STORE_REF_PARAMS = sutils.SwiftParams().params + store = swift.SingleTenantStore(self.conf) + location = store.create_location('image-id') + self.assertEqual(location.scheme, 'swift+http') + self.assertEqual(location.swift_url, 'http://example.com') + + def test_multi_tenant_location(self): + self.config(swift_store_container='container') + fake_get_endpoint = FakeGetEndpoint('https://some_endpoint') + self.stubs.Set(auth, 'get_endpoint', fake_get_endpoint) + ctxt = context.RequestContext( + user='user', tenant='tenant', auth_token='123', + service_catalog={}) + store = swift.MultiTenantStore(self.conf) + location = store.create_location('image-id', context=ctxt) + self.assertEqual(location.scheme, 'swift+https') + self.assertEqual(location.swift_url, 'https://some_endpoint') + self.assertEqual(location.container, 'container_image-id') + self.assertEqual(location.obj, 'image-id') + self.assertIsNone(location.user) + self.assertIsNone(location.key) + self.assertEqual(fake_get_endpoint.service_type, 'object-store') + + def test_multi_tenant_location_http(self): + fake_get_endpoint = FakeGetEndpoint('http://some_endpoint') + self.stubs.Set(auth, 'get_endpoint', fake_get_endpoint) + ctxt = context.RequestContext( + user='user', tenant='tenant', auth_token='123', + service_catalog={}) + store = swift.MultiTenantStore(self.conf) + location = store.create_location('image-id', context=ctxt) + self.assertEqual(location.scheme, 'swift+http') + self.assertEqual(location.swift_url, 'http://some_endpoint') + + def test_multi_tenant_location_with_region(self): + self.config(swift_store_region='WestCarolina') + fake_get_endpoint = FakeGetEndpoint('https://some_endpoint') + self.stubs.Set(auth, 'get_endpoint', fake_get_endpoint) + ctxt = context.RequestContext( + user='user', tenant='tenant', auth_token='123', + service_catalog={}) + store = swift.MultiTenantStore(self.conf) + store._get_endpoint(ctxt) + self.assertEqual(fake_get_endpoint.endpoint_region, 'WestCarolina') + + def test_multi_tenant_location_custom_service_type(self): + self.config(swift_store_service_type='toy-store') + fake_get_endpoint = FakeGetEndpoint('https://some_endpoint') + self.stubs.Set(auth, 'get_endpoint', fake_get_endpoint) + ctxt = context.RequestContext( + user='user', tenant='tenant', auth_token='123', + service_catalog={}) + store = swift.MultiTenantStore(self.conf) + store._get_endpoint(ctxt) + self.assertEqual(fake_get_endpoint.service_type, 'toy-store') + + def test_multi_tenant_location_custom_endpoint_type(self): + self.config(swift_store_endpoint_type='InternalURL') + fake_get_endpoint = FakeGetEndpoint('https://some_endpoint') + self.stubs.Set(auth, 'get_endpoint', fake_get_endpoint) + ctxt = context.RequestContext( + user='user', tenant='tenant', auth_token='123', + service_catalog={}) + store = swift.MultiTenantStore(self.conf) + store._get_endpoint(ctxt) + self.assertEqual(fake_get_endpoint.endpoint_type, 'InternalURL') + + +class TestChunkReader(base.StoreBaseTest): + _CONF = cfg.CONF + + def setUp(self): + super(TestChunkReader, self).setUp() + conf = copy.deepcopy(SWIFT_CONF) + store = Store(self.conf) + self.config(**conf) + + def test_read_all_data(self): + """ + Replicate what goes on in the Swift driver with the + repeated creation of the ChunkReader object + """ + CHUNKSIZE = 100 + checksum = hashlib.md5() + data_file = tempfile.NamedTemporaryFile() + data_file.write('*' * units.Ki) + data_file.flush() + infile = open(data_file.name, 'rb') + bytes_read = 0 + while True: + cr = swift.ChunkReader(infile, checksum, CHUNKSIZE) + chunk = cr.read(CHUNKSIZE) + bytes_read += len(chunk) + if not chunk: + break + self.assertEqual(1024, bytes_read) + data_file.close() |