diff options
| -rw-r--r-- | .gitreview | 4 | ||||
| -rw-r--r-- | doc/source/conf.py | 65 | ||||
| -rw-r--r-- | doc/source/index.rst | 29 | ||||
| -rw-r--r-- | glance/store/_drivers/swift.py | 687 | ||||
| -rw-r--r-- | glance/store/_drivers/vmware_datastore.py | 30 | ||||
| -rw-r--r-- | requirements.txt | 2 | ||||
| -rw-r--r-- | test-requirements.txt | 20 | ||||
| -rw-r--r-- | tests/unit/test_swift_store.py | 965 |
8 files changed, 129 insertions, 1673 deletions
diff --git a/.gitreview b/.gitreview new file mode 100644 index 0000000..261a6f0 --- /dev/null +++ b/.gitreview @@ -0,0 +1,4 @@ +[gerrit] +host=review.openstack.org +port=29418 +project=openstack/glance.store.git diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100644 index 0000000..125e241 --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- + +import os +import sys + +sys.path.insert(0, os.path.abspath('../..')) +# -- General configuration ---------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'oslosphinx'] + +# autodoc generation is a bit aggressive and a nuisance when doing heavy +# text edit cycles. +# execute "export SPHINX_DEBUG=1" in your terminal to disable + +# Add any paths that contain templates here, relative to this directory. +# templates_path = [] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'glance.store' +copyright = u'2014, OpenStack Foundation' + +# If true, '()' will be appended to :func: etc. cross-reference text. +add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +add_module_names = True + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# -- Options for HTML output -------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +# html_theme_path = ["."] +# html_theme = '_theme' +html_static_path = ['static'] + +# Output file base name for HTML help builder. +htmlhelp_basename = '%sdoc' % project + +git_cmd = "git log --pretty=format:'%ad, commit %h' --date=local -n1" +html_last_updated_fmt = os.popen(git_cmd).read() + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass +# [howto/manual]). +latex_documents = [ + ('index', + '%s.tex' % project, + '%s Documentation' % project, + 'OpenStack Foundation', 'manual'), +] + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {'http://docs.python.org/': None} diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 0000000..11a5044 --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,29 @@ +glance.store +============ + +The glance.store library supports the creation, deletion and gather of data +assets from/to a set of several, different, storage technologies + +Contents +======== + +.. toctree:: + :maxdepth: 1 + +Release Notes +============= + +0.0.1a1 +------- + +* Initial release of glance.store_. + +.. _glance.store: https://wiki.openstack.org/wiki/Glance/Store + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/glance/store/_drivers/swift.py b/glance/store/_drivers/swift.py deleted file mode 100644 index 8e79d00..0000000 --- a/glance/store/_drivers/swift.py +++ /dev/null @@ -1,687 +0,0 @@ -# Copyright 2010-2011 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Storage backend for SWIFT""" - -from __future__ import absolute_import - -import hashlib -import httplib -import logging -import math -import urllib -import urlparse - -from oslo.config import cfg - -from glance.common import auth -from glance.common import exception -from glance.openstack.common import excutils -import glance.store -import glance.store.driver -import glance.store.location - -try: - import swiftclient -except ImportError: - pass - -LOG = logging.getLogger(__name__) - -DEFAULT_CONTAINER = 'glance' -DEFAULT_LARGE_OBJECT_SIZE = 5 * 1024 # 5GB -DEFAULT_LARGE_OBJECT_CHUNK_SIZE = 200 # 200M -ONE_MB = 1000 * 1024 - -swift_opts = [ - cfg.BoolOpt('swift_enable_snet', default=False, - help=_('Whether to use ServiceNET to communicate with the ' - 'Swift storage servers.')), - cfg.StrOpt('swift_store_auth_address', - help=_('The address where the Swift authentication service ' - 'is listening.')), - cfg.StrOpt('swift_store_user', secret=True, - help=_('The user to authenticate against the Swift ' - 'authentication service')), - cfg.StrOpt('swift_store_key', secret=True, - help=_('Auth key for the user authenticating against the ' - 'Swift authentication service.')), - cfg.StrOpt('swift_store_auth_version', default='2', - help=_('Version of the authentication service to use. ' - 'Valid versions are 2 for keystone and 1 for swauth ' - 'and rackspace')), - cfg.BoolOpt('swift_store_auth_insecure', default=False, - help=_('If True, swiftclient won\'t check for a valid SSL ' - 'certificate when authenticating.')), - cfg.StrOpt('swift_store_region', - help=_('The region of the swift endpoint to be used for ' - 'single tenant. This setting is only necessary if the ' - 'tenant has multiple swift endpoints.')), - cfg.StrOpt('swift_store_endpoint_type', default='publicURL', - help=_('A string giving the endpoint type of the swift ' - 'service to use (publicURL, adminURL or internalURL). ' - 'This setting is only used if swift_store_auth_version ' - 'is 2.')), - cfg.StrOpt('swift_store_service_type', default='object-store', - help=_('A string giving the service type of the swift service ' - 'to use. This setting is only used if ' - 'swift_store_auth_version is 2.')), - cfg.StrOpt('swift_store_container', - default=DEFAULT_CONTAINER, - help=_('Container within the account that the account should ' - 'use for storing images in Swift.')), - cfg.IntOpt('swift_store_large_object_size', - default=DEFAULT_LARGE_OBJECT_SIZE, - help=_('The size, in MB, that Glance will start chunking image ' - 'files and do a large object manifest in Swift')), - cfg.IntOpt('swift_store_large_object_chunk_size', - default=DEFAULT_LARGE_OBJECT_CHUNK_SIZE, - help=_('The amount of data written to a temporary disk buffer ' - 'during the process of chunking the image file.')), - cfg.BoolOpt('swift_store_create_container_on_put', default=False, - help=_('A boolean value that determines if we create the ' - 'container if it does not exist.')), - cfg.BoolOpt('swift_store_multi_tenant', default=False, - help=_('If set to True, enables multi-tenant storage ' - 'mode which causes Glance images to be stored in ' - 'tenant specific Swift accounts.')), - cfg.ListOpt('swift_store_admin_tenants', default=[], - help=_('A list of tenants that will be granted read/write ' - 'access on all Swift containers created by Glance in ' - 'multi-tenant mode.')), - cfg.BoolOpt('swift_store_ssl_compression', default=True, - help=_('If set to False, disables SSL layer compression of ' - 'https swift requests. Setting to False may improve ' - 'performance for images which are already in a ' - 'compressed format, eg qcow2.')), -] - -CONF = cfg.CONF -CONF.register_opts(swift_opts) - - -class StoreLocation(glance.store.location.StoreLocation): - - """ - Class describing a Swift URI. A Swift URI can look like any of - the following: - - swift://user:pass@authurl.com/container/obj-id - swift://account:user:pass@authurl.com/container/obj-id - swift+http://user:pass@authurl.com/container/obj-id - swift+https://user:pass@authurl.com/container/obj-id - - When using multi-tenant a URI might look like this (a storage URL): - - swift+https://example.com/container/obj-id - - The swift+http:// URIs indicate there is an HTTP authentication URL. - The default for Swift is an HTTPS authentication URL, so swift:// and - swift+https:// are the same... - """ - - def process_specs(self): - self.scheme = self.specs.get('scheme', 'swift+https') - self.user = self.specs.get('user') - self.key = self.specs.get('key') - self.auth_or_store_url = self.specs.get('auth_or_store_url') - self.container = self.specs.get('container') - self.obj = self.specs.get('obj') - - def _get_credstring(self): - if self.user and self.key: - return '%s:%s@' % (urllib.quote(self.user), urllib.quote(self.key)) - return '' - - def get_uri(self): - auth_or_store_url = self.auth_or_store_url - if auth_or_store_url.startswith('http://'): - auth_or_store_url = auth_or_store_url[len('http://'):] - elif auth_or_store_url.startswith('https://'): - auth_or_store_url = auth_or_store_url[len('https://'):] - - credstring = self._get_credstring() - auth_or_store_url = auth_or_store_url.strip('/') - container = self.container.strip('/') - obj = self.obj.strip('/') - - return '%s://%s%s/%s/%s' % (self.scheme, credstring, auth_or_store_url, - container, obj) - - def parse_uri(self, uri): - """ - Parse URLs. This method fixes an issue where credentials specified - in the URL are interpreted differently in Python 2.6.1+ than prior - versions of Python. It also deals with the peculiarity that new-style - Swift URIs have where a username can contain a ':', like so: - - swift://account:user:pass@authurl.com/container/obj - """ - # Make sure that URIs that contain multiple schemes, such as: - # swift://user:pass@http://authurl.com/v1/container/obj - # are immediately rejected. - if uri.count('://') != 1: - reason = _("URI cannot contain more than one occurrence " - "of a scheme. If you have specified a URI like " - "swift://user:pass@http://authurl.com/v1/container/obj" - ", you need to change it to use the " - "swift+http:// scheme, like so: " - "swift+http://user:pass@authurl.com/v1/container/obj") - LOG.debug(_("Invalid store URI: %(reason)s"), {'reason': reason}) - raise exceptions.BadStoreUri(message=reason) - - pieces = urlparse.urlparse(uri) - assert pieces.scheme in ('swift', 'swift+http', 'swift+https') - self.scheme = pieces.scheme - netloc = pieces.netloc - path = pieces.path.lstrip('/') - if netloc != '': - # > Python 2.6.1 - if '@' in netloc: - creds, netloc = netloc.split('@') - else: - creds = None - else: - # Python 2.6.1 compat - # see lp659445 and Python issue7904 - if '@' in path: - creds, path = path.split('@') - else: - creds = None - netloc = path[0:path.find('/')].strip('/') - path = path[path.find('/'):].strip('/') - if creds: - cred_parts = creds.split(':') - if len(cred_parts) != 2: - reason = (_("Badly formed credentials in Swift URI.")) - LOG.debug(reason) - raise exceptions.BadStoreUri() - user, key = cred_parts - self.user = urllib.unquote(user) - self.key = urllib.unquote(key) - else: - self.user = None - self.key = None - path_parts = path.split('/') - try: - self.obj = path_parts.pop() - self.container = path_parts.pop() - if not netloc.startswith('http'): - # push hostname back into the remaining to build full authurl - path_parts.insert(0, netloc) - self.auth_or_store_url = '/'.join(path_parts) - except IndexError: - reason = _("Badly formed Swift URI.") - LOG.debug(reason) - raise exceptions.BadStoreUri() - - @property - def swift_url(self): - """ - Creates a fully-qualified auth url that the Swift client library can - use. The scheme for the auth_url is determined using the scheme - included in the `location` field. - - HTTPS is assumed, unless 'swift+http' is specified. - """ - if self.auth_or_store_url.startswith('http'): - return self.auth_or_store_url - else: - if self.scheme in ('swift+https', 'swift'): - auth_scheme = 'https://' - else: - auth_scheme = 'http://' - - return ''.join([auth_scheme, self.auth_or_store_url]) - - -def Store(context=None, loc=None): - if (CONF.swift_store_multi_tenant and - (loc is None or loc.store_location.user is None)): - return MultiTenantStore(context, loc) - return SingleTenantStore(context, loc) - - -class BaseStore(glance.store.driver.Store): - CHUNKSIZE = 65536 - - def get_schemes(self): - return ('swift+https', 'swift', 'swift+http') - - def configure(self): - _obj_size = self._option_get('swift_store_large_object_size') - self.large_object_size = _obj_size * ONE_MB - _chunk_size = self._option_get('swift_store_large_object_chunk_size') - self.large_object_chunk_size = _chunk_size * ONE_MB - self.admin_tenants = CONF.swift_store_admin_tenants - self.region = CONF.swift_store_region - self.service_type = CONF.swift_store_service_type - self.endpoint_type = CONF.swift_store_endpoint_type - self.snet = CONF.swift_enable_snet - self.insecure = CONF.swift_store_auth_insecure - self.ssl_compression = CONF.swift_store_ssl_compression - - def get(self, location, offset=0, chunk_size=None, context=None): - location = location.store_location - if not connection: - connection = self.get_connection(location) - - try: - resp_headers, resp_body = connection.get_object( - container=location.container, obj=location.obj, - resp_chunk_size=self.CHUNKSIZE) - except swiftclient.ClientException as e: - if e.http_status == httplib.NOT_FOUND: - msg = _("Swift could not find object %s.") % location.obj - LOG.warn(msg) - raise exceptions.NotFound(msg) - else: - raise - - class ResponseIndexable(glance.store.Indexable): - def another(self): - try: - return self.wrapped.next() - except StopIteration: - return '' - - length = int(resp_headers.get('content-length', 0)) - return (ResponseIndexable(resp_body, length), length) - - def get_size(self, location, connection=None): - location = location.store_location - if not connection: - connection = self.get_connection(location) - try: - resp_headers = connection.head_object( - container=location.container, obj=location.obj) - return int(resp_headers.get('content-length', 0)) - except Exception: - return 0 - - def _option_get(self, param): - result = getattr(CONF, param) - if not result: - reason = (_("Could not find %(param)s in configuration " - "options.") % {'param': param}) - LOG.error(reason) - raise exceptions.BadStoreConfiguration(store_name="swift", - reason=reason) - return result - - def _delete_stale_chunks(self, connection, container, chunk_list): - for chunk in chunk_list: - LOG.debug(_("Deleting chunk %s") % chunk) - try: - connection.delete_object(container, chunk) - except Exception: - msg = _("Failed to delete orphaned chunk %s/%s") - LOG.exception(msg, container, chunk) - - def add(self, image_id, image_file, image_size, connection=None): - location = self.create_location(image_id) - if not connection: - connection = self.get_connection(location) - - self._create_container_if_missing(location.container, connection) - - LOG.debug(_("Adding image object '%(obj_name)s' " - "to Swift") % dict(obj_name=location.obj)) - try: - if image_size > 0 and image_size < self.large_object_size: - # Image size is known, and is less than large_object_size. - # Send to Swift with regular PUT. - obj_etag = connection.put_object(location.container, - location.obj, image_file, - content_length=image_size) - else: - # Write the image into Swift in chunks. - chunk_id = 1 - if image_size > 0: - total_chunks = str(int( - math.ceil(float(image_size) / - float(self.large_object_chunk_size)))) - else: - # image_size == 0 is when we don't know the size - # of the image. This can occur with older clients - # that don't inspect the payload size. - LOG.debug(_("Cannot determine image size. Adding as a " - "segmented object to Swift.")) - total_chunks = '?' - - checksum = hashlib.md5() - written_chunks = [] - combined_chunks_size = 0 - while True: - chunk_size = self.large_object_chunk_size - if image_size == 0: - content_length = None - else: - left = image_size - combined_chunks_size - if left == 0: - break - if chunk_size > left: - chunk_size = left - content_length = chunk_size - - chunk_name = "%s-%05d" % (location.obj, chunk_id) - reader = ChunkReader(image_file, checksum, chunk_size) - try: - chunk_etag = connection.put_object( - location.container, chunk_name, reader, - content_length=content_length) - written_chunks.append(chunk_name) - except Exception: - # Delete orphaned segments from swift backend - with excutils.save_and_reraise_exception(): - LOG.exception(_("Error during chunked upload to " - "backend, deleting stale chunks")) - self._delete_stale_chunks(connection, - location.container, - written_chunks) - - bytes_read = reader.bytes_read - msg = (_("Wrote chunk %(chunk_name)s (%(chunk_id)d/" - "%(total_chunks)s) of length %(bytes_read)d " - "to Swift returning MD5 of content: " - "%(chunk_etag)s") % - {'chunk_name': chunk_name, - 'chunk_id': chunk_id, - 'total_chunks': total_chunks, - 'bytes_read': bytes_read, - 'chunk_etag': chunk_etag}) - LOG.debug(msg) - - if bytes_read == 0: - # Delete the last chunk, because it's of zero size. - # This will happen if size == 0. - LOG.debug(_("Deleting final zero-length chunk")) - connection.delete_object(location.container, - chunk_name) - break - - chunk_id += 1 - combined_chunks_size += bytes_read - - # In the case we have been given an unknown image size, - # set the size to the total size of the combined chunks. - if image_size == 0: - image_size = combined_chunks_size - - # Now we write the object manifest and return the - # manifest's etag... - manifest = "%s/%s-" % (location.container, location.obj) - headers = {'ETag': hashlib.md5("").hexdigest(), - 'X-Object-Manifest': manifest} - - # The ETag returned for the manifest is actually the - # MD5 hash of the concatenated checksums of the strings - # of each chunk...so we ignore this result in favour of - # the MD5 of the entire image file contents, so that - # users can verify the image file contents accordingly - connection.put_object(location.container, location.obj, - None, headers=headers) - obj_etag = checksum.hexdigest() - - # NOTE: We return the user and key here! Have to because - # location is used by the API server to return the actual - # image data. We *really* should consider NOT returning - # the location attribute from GET /images/<ID> and - # GET /images/details - - return (location.get_uri(), image_size, obj_etag, {}) - except swiftclient.ClientException as e: - if e.http_status == httplib.CONFLICT: - raise exceptions.Duplicate(_("Swift already has an image at " - "this location")) - msg = (_("Failed to add object to Swift.\n" - "Got error from Swift: %(e)s") % {'e': e}) - LOG.error(msg) - raise glance.store.BackendException(msg) - - def delete(self, location, connection=None): - location = location.store_location - if not connection: - connection = self.get_connection(location) - - try: - # We request the manifest for the object. If one exists, - # that means the object was uploaded in chunks/segments, - # and we need to delete all the chunks as well as the - # manifest. - manifest = None - try: - headers = connection.head_object( - location.container, location.obj) - manifest = headers.get('x-object-manifest') - except swiftclient.ClientException as e: - if e.http_status != httplib.NOT_FOUND: - raise - if manifest: - # Delete all the chunks before the object manifest itself - obj_container, obj_prefix = manifest.split('/', 1) - segments = connection.get_container( - obj_container, prefix=obj_prefix)[1] - for segment in segments: - # TODO(jaypipes): This would be an easy area to parallelize - # since we're simply sending off parallelizable requests - # to Swift to delete stuff. It's not like we're going to - # be hogging up network or file I/O here... - connection.delete_object(obj_container, - segment['name']) - - # Delete object (or, in segmented case, the manifest) - connection.delete_object(location.container, location.obj) - - except swiftclient.ClientException as e: - if e.http_status == httplib.NOT_FOUND: - msg = _("Swift could not find image at URI.") - raise exceptions.NotFound(msg) - else: - raise - - def _create_container_if_missing(self, container, connection): - """ - Creates a missing container in Swift if the - ``swift_store_create_container_on_put`` option is set. - - :param container: Name of container to create - :param connection: Connection to swift service - """ - try: - connection.head_container(container) - except swiftclient.ClientException as e: - if e.http_status == httplib.NOT_FOUND: - if CONF.swift_store_create_container_on_put: - try: - connection.put_container(container) - except swiftclient.ClientException as e: - msg = (_("Failed to add container to Swift.\n" - "Got error from Swift: %(e)s") % {'e': e}) - raise glance.store.BackendException(msg) - else: - msg = (_("The container %(container)s does not exist in " - "Swift. Please set the " - "swift_store_create_container_on_put option" - "to add container to Swift automatically.") % - {'container': container}) - raise glance.store.BackendException(msg) - else: - raise - - def get_connection(self): - raise NotImplemented() - - def create_location(self): - raise NotImplemented() - - -class SingleTenantStore(BaseStore): - EXAMPLE_URL = "swift://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<FILE>" - - def configure(self): - super(SingleTenantStore, self).configure() - self.auth_version = self._option_get('swift_store_auth_version') - - def configure_add(self): - self.auth_address = self._option_get('swift_store_auth_address') - if self.auth_address.startswith('http://'): - self.scheme = 'swift+http' - else: - self.scheme = 'swift+https' - self.container = CONF.swift_store_container - self.user = self._option_get('swift_store_user') - self.key = self._option_get('swift_store_key') - - def create_location(self, image_id): - specs = {'scheme': self.scheme, - 'container': self.container, - 'obj': str(image_id), - 'auth_or_store_url': self.auth_address, - 'user': self.user, - 'key': self.key} - return StoreLocation(specs) - - def get_connection(self, location): - if not location.user: - reason = (_("Location is missing user:password information.")) - LOG.debug(reason) - raise exceptions.BadStoreUri(message=reason) - - auth_url = location.swift_url - if not auth_url.endswith('/'): - auth_url += '/' - - if self.auth_version == '2': - try: - tenant_name, user = location.user.split(':') - except ValueError: - reason = (_("Badly formed tenant:user '%(user)s' in " - "Swift URI") % {'user': location.user}) - LOG.debug(reason) - raise exceptions.BadStoreUri() - else: - tenant_name = None - user = location.user - - os_options = {} - if self.region: - os_options['region_name'] = self.region - os_options['endpoint_type'] = self.endpoint_type - os_options['service_type'] = self.service_type - - return swiftclient.Connection( - auth_url, user, location.key, insecure=self.insecure, - tenant_name=tenant_name, snet=self.snet, - auth_version=self.auth_version, os_options=os_options, - ssl_compression=self.ssl_compression) - - -class MultiTenantStore(BaseStore): - EXAMPLE_URL = "swift://<SWIFT_URL>/<CONTAINER>/<FILE>" - - def configure_add(self): - self.container = CONF.swift_store_container - if self.context is None: - reason = _("Multi-tenant Swift storage requires a context.") - raise exceptions.BadStoreConfiguration(store_name="swift", - reason=reason) - if self.context.service_catalog is None: - reason = _("Multi-tenant Swift storage requires " - "a service catalog.") - raise exceptions.BadStoreConfiguration(store_name="swift", - reason=reason) - self.storage_url = auth.get_endpoint( - self.context.service_catalog, service_type=self.service_type, - endpoint_region=self.region, endpoint_type=self.endpoint_type) - if self.storage_url.startswith('http://'): - self.scheme = 'swift+http' - else: - self.scheme = 'swift+https' - - def delete(self, location, connection=None): - if not connection: - connection = self.get_connection(location.store_location) - super(MultiTenantStore, self).delete(location, connection) - connection.delete_container(location.store_location.container) - - def set_acls(self, location, public=False, read_tenants=None, - write_tenants=None, connection=None): - location = location.store_location - if not connection: - connection = self.get_connection(location) - - if read_tenants is None: - read_tenants = [] - if write_tenants is None: - write_tenants = [] - - headers = {} - if public: - headers['X-Container-Read'] = ".r:*,.rlistings" - elif read_tenants: - headers['X-Container-Read'] = ','.join('%s:*' % i - for i in read_tenants) - else: - headers['X-Container-Read'] = '' - - write_tenants.extend(self.admin_tenants) - if write_tenants: - headers['X-Container-Write'] = ','.join('%s:*' % i - for i in write_tenants) - else: - headers['X-Container-Write'] = '' - - try: - connection.post_container(location.container, headers=headers) - except swiftclient.ClientException as e: - if e.http_status == httplib.NOT_FOUND: - msg = _("Swift could not find image at URI.") - raise exceptions.NotFound(msg) - else: - raise - - def create_location(self, image_id): - specs = {'scheme': self.scheme, - 'container': self.container + '_' + str(image_id), - 'obj': str(image_id), - 'auth_or_store_url': self.storage_url} - return StoreLocation(specs) - - def get_connection(self, location): - return swiftclient.Connection( - None, self.context.user, None, - preauthurl=location.swift_url, - preauthtoken=self.context.auth_tok, - tenant_name=self.context.tenant, - auth_version='2', snet=self.snet, insecure=self.insecure, - ssl_compression=self.ssl_compression) - - -class ChunkReader(object): - def __init__(self, fd, checksum, total): - self.fd = fd - self.checksum = checksum - self.total = total - self.bytes_read = 0 - - def read(self, i): - left = self.total - self.bytes_read - if i > left: - i = left - result = self.fd.read(i) - self.bytes_read += len(result) - self.checksum.update(result) - return result diff --git a/glance/store/_drivers/vmware_datastore.py b/glance/store/_drivers/vmware_datastore.py index b81be3f..57fad7f 100644 --- a/glance/store/_drivers/vmware_datastore.py +++ b/glance/store/_drivers/vmware_datastore.py @@ -176,9 +176,12 @@ class StoreLocation(location.StoreLocation): return '%s?%s' % (base_url, self.query) - def _is_valid_path(self, path): - sdir = self.conf.glance_store.vmware_store_image_dir.strip('/') - return path.startswith(os.path.join(DS_URL_PREFIX, sdir)) + # NOTE(flaper87): Commenting out for now, it's probably better to do + # it during image add/get. This validation relies on a config param + # which doesn't make sense to have in the StoreLocation instance. + #def _is_valid_path(self, path): + # sdir = self.conf.glance_store.vmware_store_image_dir.strip('/') + # return path.startswith(os.path.join(DS_URL_PREFIX, sdir)) def parse_uri(self, uri): if not uri.startswith('%s://' % STORE_SCHEME): @@ -189,19 +192,14 @@ class StoreLocation(location.StoreLocation): (self.scheme, self.server_host, path, params, query, fragment) = urlparse.urlparse(uri) if not query: - path = path.split('?') - if self._is_valid_path(path[0]): - self.path = path[0] - self.query = path[1] - return - #elif self._is_valid_path(path): - else: - self.path = path - self.query = query - return - reason = 'Badly formed VMware datastore URI %(uri)s.' % {'uri': uri} - LOG.debug(reason) - raise exceptions.BadStoreUri(reason) + path, query = path.split('?') + + self.path = path + self.query = query + # NOTE(flaper87): Read comment on `_is_valid_path` + #reason = 'Badly formed VMware datastore URI %(uri)s.' % {'uri': uri} + #LOG.debug(reason) + #raise exceptions.BadStoreUri(reason) class Store(glance.store.Store): diff --git a/requirements.txt b/requirements.txt index 0dfb490..ac5e692 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,3 @@ -pbr>=0.5.21,<1.0 - oslo.config>=1.2.0 oslo.i18n>=0.1.0 stevedore>=0.12 diff --git a/test-requirements.txt b/test-requirements.txt index a975a91..0154dd5 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -6,6 +6,20 @@ mock>=1.0 # Unit testing fixtures>=0.3.14 -python-subunit -testrepository>=0.0.17 -testtools>=0.9.32 +python-subunit>=0.0.18 +testrepository>=0.0.18 +testscenarios>=0.4 +testtools>=0.9.34 +oslotest + +# this is required for the docs build jobs +sphinx>=1.1.2,!=1.2.0,<1.3 +oslosphinx + +### Store specific packages + +# For S3 storage backend +boto>=2.12.0,!=2.13.0 + +# For VMware storage backend. +oslo.vmware>=0.4 # Apache-2.0 diff --git a/tests/unit/test_swift_store.py b/tests/unit/test_swift_store.py deleted file mode 100644 index eaa2a14..0000000 --- a/tests/unit/test_swift_store.py +++ /dev/null @@ -1,965 +0,0 @@ -# Copyright 2011 OpenStack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Tests the Swift backend store""" - -import hashlib -import httplib -import mock -import StringIO -import tempfile -import urllib -import uuid - -from oslo.config import cfg -import swiftclient - -from glance.store import exceptions -from glance.store.location import get_location_from_uri -from glance.store._drivers import swift -from glance.tests.unit import base - -CONF = cfg.CONF - -FAKE_UUID = lambda: str(uuid.uuid4()) - -Store = glance.store.swift.Store -FIVE_KB = 5 * 1024 -FIVE_GB = 5 * 1024 * 3 -MAX_SWIFT_OBJECT_SIZE = FIVE_GB -SWIFT_PUT_OBJECT_CALLS = 0 -SWIFT_CONF = {'verbose': True, - 'debug': True, - 'known_stores': ['glance.store.swift.Store'], - 'default_store': 'swift', - 'swift_store_user': 'user', - 'swift_store_key': 'key', - 'swift_store_auth_address': 'localhost:8080', - 'swift_store_container': 'glance'} - - -# We stub out as little as possible to ensure that the code paths -# between glance.store.swift and swiftclient are tested -# thoroughly -def stub_out_swiftclient(test, swift_store_auth_version): - fixture_containers = ['glance'] - fixture_container_headers = {} - fixture_headers = { - 'glance/%s' % FAKE_UUID: { - 'content-length': FIVE_KB, - 'etag': 'c2e5db72bd7fd153f53ede5da5a06de3' - } - } - fixture_objects = {'glance/%s' % FAKE_UUID: - StringIO.StringIO("*" * FIVE_KB)} - - def fake_head_container(url, token, container, **kwargs): - if container not in fixture_containers: - msg = "No container %s found" % container - raise swiftclient.ClientException(msg, - http_status=httplib.NOT_FOUND) - return fixture_container_headers - - def fake_put_container(url, token, container, **kwargs): - fixture_containers.append(container) - - def fake_post_container(url, token, container, headers, http_conn=None): - for key, value in headers.iteritems(): - fixture_container_headers[key] = value - - def fake_put_object(url, token, container, name, contents, **kwargs): - # PUT returns the ETag header for the newly-added object - # Large object manifest... - global SWIFT_PUT_OBJECT_CALLS - SWIFT_PUT_OBJECT_CALLS += 1 - CHUNKSIZE = 64 * units.Ki - fixture_key = "%s/%s" % (container, name) - if fixture_key not in fixture_headers: - if kwargs.get('headers'): - etag = kwargs['headers']['ETag'] - fixture_headers[fixture_key] = {'manifest': True, - 'etag': etag} - return etag - if hasattr(contents, 'read'): - fixture_object = StringIO.StringIO() - chunk = contents.read(CHUNKSIZE) - checksum = hashlib.md5() - while chunk: - fixture_object.write(chunk) - checksum.update(chunk) - chunk = contents.read(CHUNKSIZE) - etag = checksum.hexdigest() - else: - fixture_object = StringIO.StringIO(contents) - etag = hashlib.md5(fixture_object.getvalue()).hexdigest() - read_len = fixture_object.len - if read_len > MAX_SWIFT_OBJECT_SIZE: - msg = ('Image size:%d exceeds Swift max:%d' % - (read_len, MAX_SWIFT_OBJECT_SIZE)) - raise swiftclient.ClientException( - msg, http_status=httplib.REQUEST_ENTITY_TOO_LARGE) - fixture_objects[fixture_key] = fixture_object - fixture_headers[fixture_key] = { - 'content-length': read_len, - 'etag': etag} - return etag - else: - msg = ("Object PUT failed - Object with key %s already exists" - % fixture_key) - raise swiftclient.ClientException(msg, - http_status=httplib.CONFLICT) - - def fake_get_object(url, token, container, name, **kwargs): - # GET returns the tuple (list of headers, file object) - fixture_key = "%s/%s" % (container, name) - if fixture_key not in fixture_headers: - msg = "Object GET failed" - raise swiftclient.ClientException(msg, - http_status=httplib.NOT_FOUND) - - fixture = fixture_headers[fixture_key] - if 'manifest' in fixture: - # Large object manifest... we return a file containing - # all objects with prefix of this fixture key - chunk_keys = sorted([k for k in fixture_headers.keys() - if k.startswith(fixture_key) and - k != fixture_key]) - result = StringIO.StringIO() - for key in chunk_keys: - result.write(fixture_objects[key].getvalue()) - return fixture_headers[fixture_key], result - - else: - return fixture_headers[fixture_key], fixture_objects[fixture_key] - - def fake_head_object(url, token, container, name, **kwargs): - # HEAD returns the list of headers for an object - try: - fixture_key = "%s/%s" % (container, name) - return fixture_headers[fixture_key] - except KeyError: - msg = "Object HEAD failed - Object does not exist" - raise swiftclient.ClientException(msg, - http_status=httplib.NOT_FOUND) - - def fake_delete_object(url, token, container, name, **kwargs): - # DELETE returns nothing - fixture_key = "%s/%s" % (container, name) - if fixture_key not in fixture_headers: - msg = "Object DELETE failed - Object does not exist" - raise swiftclient.ClientException(msg, - http_status=httplib.NOT_FOUND) - else: - del fixture_headers[fixture_key] - del fixture_objects[fixture_key] - - def fake_http_connection(*args, **kwargs): - return None - - def fake_get_auth(url, user, key, snet, auth_version, **kwargs): - if url is None: - return None, None - if 'http' in url and '://' not in url: - raise ValueError('Invalid url %s' % url) - # Check the auth version against the configured value - if swift_store_auth_version != auth_version: - msg = 'AUTHENTICATION failed (version mismatch)' - raise swiftclient.ClientException(msg) - return None, None - - to_mock = [('head_container', fake_head_container), - ('put_container', fake_put_container), - ('post_container', fake_post_container), - ('put_object', fake_put_object), - ('delete_object', fake_delete_object), - ('head_object', fake_head_object), - ('get_object', fake_get_object), - ('get_auth', fake_get_auth), - ('http_connection', fake_http_connection)] - - for (meth, fake_meth) in to_mock: - mocked = mock.patch.object(swiftclient.client, meth).start() - mocked.side_effect = fake_meth - test.add_cleanUp(mocked.stop) - - -class SwiftTests(object): - - @property - def swift_store_user(self): - return urllib.quote(CONF.swift_store_user) - - def test_get_size(self): - """ - Test that we can get the size of an object in the swift store - """ - uri = "swift://%s:key@auth_address/glance/%s" % ( - self.swift_store_user, FAKE_UUID) - loc = get_location_from_uri(uri) - image_size = self.store.get_size(loc) - self.assertEqual(image_size, 5120) - - def test_get_size_with_multi_tenant_on(self): - """Test that single tenant uris work with multi tenant on.""" - uri = ("swift://%s:key@auth_address/glance/%s" % - (self.swift_store_user, FAKE_UUID)) - self.config(swift_store_multi_tenant=True) - #NOTE(markwash): ensure the image is found - context = glance.context.RequestContext() - size = glance.store.get_size_from_backend(context, uri) - self.assertEqual(size, 5120) - - def test_get(self): - """Test a "normal" retrieval of an image in chunks""" - uri = "swift://%s:key@auth_address/glance/%s" % ( - self.swift_store_user, FAKE_UUID) - loc = get_location_from_uri(uri) - (image_swift, image_size) = self.store.get(loc) - self.assertEqual(image_size, 5120) - - expected_data = "*" * FIVE_KB - data = "" - - for chunk in image_swift: - data += chunk - self.assertEqual(expected_data, data) - - def test_get_with_http_auth(self): - """ - Test a retrieval from Swift with an HTTP authurl. This is - specified either via a Location header with swift+http:// or using - http:// in the swift_store_auth_address config value - """ - loc = get_location_from_uri("swift+http://%s:key@auth_address/" - "glance/%s" % - (self.swift_store_user, FAKE_UUID)) - (image_swift, image_size) = self.store.get(loc) - self.assertEqual(image_size, 5120) - - expected_data = "*" * FIVE_KB - data = "" - - for chunk in image_swift: - data += chunk - self.assertEqual(expected_data, data) - - def test_get_non_existing(self): - """ - Test that trying to retrieve a swift that doesn't exist - raises an error - """ - loc = get_location_from_uri("swift://%s:key@authurl/glance/noexist" % ( - self.swift_store_user)) - self.assertRaises(exceptions.NotFound, - self.store.get, - loc) - - def test_add(self): - """Test that we can add an image via the swift backend""" - expected_swift_size = FIVE_KB - expected_swift_contents = "*" * expected_swift_size - expected_checksum = hashlib.md5(expected_swift_contents).hexdigest() - expected_image_id = str(uuid.uuid4()) - loc = 'swift+https://%s:key@localhost:8080/glance/%s' - expected_location = loc % (self.swift_store_user, - expected_image_id) - image_swift = StringIO.StringIO(expected_swift_contents) - - global SWIFT_PUT_OBJECT_CALLS - SWIFT_PUT_OBJECT_CALLS = 0 - - location, size, checksum, _ = self.store.add(expected_image_id, - image_swift, - expected_swift_size) - - self.assertEqual(expected_location, location) - self.assertEqual(expected_swift_size, size) - self.assertEqual(expected_checksum, checksum) - # Expecting a single object to be created on Swift i.e. no chunking. - self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 1) - - loc = get_location_from_uri(expected_location) - (new_image_swift, new_image_size) = self.store.get(loc) - new_image_contents = new_image_swift.getvalue() - new_image_swift_size = len(new_image_swift) - - self.assertEqual(expected_swift_contents, new_image_contents) - self.assertEqual(expected_swift_size, new_image_swift_size) - - def test_add_auth_url_variations(self): - """ - Test that we can add an image via the swift backend with - a variety of different auth_address values - """ - variations = { - 'http://localhost:80': 'swift+http://%s:key@localhost:80' - '/glance/%s', - 'http://localhost': 'swift+http://%s:key@localhost/glance/%s', - 'http://localhost/v1': 'swift+http://%s:key@localhost' - '/v1/glance/%s', - 'http://localhost/v1/': 'swift+http://%s:key@localhost' - '/v1/glance/%s', - 'https://localhost': 'swift+https://%s:key@localhost/glance/%s', - 'https://localhost:8080': 'swift+https://%s:key@localhost:8080' - '/glance/%s', - 'https://localhost/v1': 'swift+https://%s:key@localhost' - '/v1/glance/%s', - 'https://localhost/v1/': 'swift+https://%s:key@localhost' - '/v1/glance/%s', - 'localhost': 'swift+https://%s:key@localhost/glance/%s', - 'localhost:8080/v1': 'swift+https://%s:key@localhost:8080' - '/v1/glance/%s', - } - - for variation, expected_location in variations.items(): - image_id = str(uuid.uuid4()) - expected_location = expected_location % ( - self.swift_store_user, image_id) - expected_swift_size = FIVE_KB - expected_swift_contents = "*" * expected_swift_size - expected_checksum = \ - hashlib.md5(expected_swift_contents).hexdigest() - - image_swift = StringIO.StringIO(expected_swift_contents) - - global SWIFT_PUT_OBJECT_CALLS - SWIFT_PUT_OBJECT_CALLS = 0 - - self.config(swift_store_auth_address=variation) - self.store = Store() - location, size, checksum, _ = self.store.add(image_id, image_swift, - expected_swift_size) - - self.assertEqual(expected_location, location) - self.assertEqual(expected_swift_size, size) - self.assertEqual(expected_checksum, checksum) - self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 1) - - loc = get_location_from_uri(expected_location) - (new_image_swift, new_image_size) = self.store.get(loc) - new_image_contents = new_image_swift.getvalue() - new_image_swift_size = len(new_image_swift) - - self.assertEqual(expected_swift_contents, new_image_contents) - self.assertEqual(expected_swift_size, new_image_swift_size) - - def test_add_no_container_no_create(self): - """ - Tests that adding an image with a non-existing container - raises an appropriate exception - """ - self.config(swift_store_create_container_on_put=False, - swift_store_container='noexist') - self.store = Store() - - image_swift = StringIO.StringIO("nevergonnamakeit") - - global SWIFT_PUT_OBJECT_CALLS - SWIFT_PUT_OBJECT_CALLS = 0 - - # We check the exception text to ensure the container - # missing text is found in it, otherwise, we would have - # simply used self.assertRaises here - exception_caught = False - try: - self.store.add(str(uuid.uuid4()), image_swift, 0) - except backend.BackendException as e: - exception_caught = True - self.assertTrue("container noexist does not exist " - "in Swift" in str(e)) - self.assertTrue(exception_caught) - self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 0) - - def test_add_no_container_and_create(self): - """ - Tests that adding an image with a non-existing container - creates the container automatically if flag is set - """ - expected_swift_size = FIVE_KB - expected_swift_contents = "*" * expected_swift_size - expected_checksum = hashlib.md5(expected_swift_contents).hexdigest() - expected_image_id = str(uuid.uuid4()) - loc = 'swift+https://%s:key@localhost:8080/noexist/%s' - expected_location = loc % (self.swift_store_user, - expected_image_id) - image_swift = StringIO.StringIO(expected_swift_contents) - - global SWIFT_PUT_OBJECT_CALLS - SWIFT_PUT_OBJECT_CALLS = 0 - - self.config(swift_store_create_container_on_put=True, - swift_store_container='noexist') - self.store = Store() - location, size, checksum, _ = self.store.add(expected_image_id, - image_swift, - expected_swift_size) - - self.assertEqual(expected_location, location) - self.assertEqual(expected_swift_size, size) - self.assertEqual(expected_checksum, checksum) - self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 1) - - loc = get_location_from_uri(expected_location) - (new_image_swift, new_image_size) = self.store.get(loc) - new_image_contents = new_image_swift.getvalue() - new_image_swift_size = len(new_image_swift) - - self.assertEqual(expected_swift_contents, new_image_contents) - self.assertEqual(expected_swift_size, new_image_swift_size) - - def test_add_large_object(self): - """ - Tests that adding a very large image. We simulate the large - object by setting store.large_object_size to a small number - and then verify that there have been a number of calls to - put_object()... - """ - expected_swift_size = FIVE_KB - expected_swift_contents = "*" * expected_swift_size - expected_checksum = hashlib.md5(expected_swift_contents).hexdigest() - expected_image_id = str(uuid.uuid4()) - loc = 'swift+https://%s:key@localhost:8080/glance/%s' - expected_location = loc % (self.swift_store_user, - expected_image_id) - image_swift = StringIO.StringIO(expected_swift_contents) - - global SWIFT_PUT_OBJECT_CALLS - SWIFT_PUT_OBJECT_CALLS = 0 - - self.config(swift_store_container='glance') - self.store = Store() - orig_max_size = self.store.large_object_size - orig_temp_size = self.store.large_object_chunk_size - try: - self.store.large_object_size = 1024 - self.store.large_object_chunk_size = 1024 - location, size, checksum, _ = self.store.add(expected_image_id, - image_swift, - expected_swift_size) - finally: - self.store.large_object_chunk_size = orig_temp_size - self.store.large_object_size = orig_max_size - - self.assertEqual(expected_location, location) - self.assertEqual(expected_swift_size, size) - self.assertEqual(expected_checksum, checksum) - # Expecting 6 objects to be created on Swift -- 5 chunks and 1 - # manifest. - self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 6) - - loc = get_location_from_uri(expected_location) - (new_image_swift, new_image_size) = self.store.get(loc) - new_image_contents = new_image_swift.getvalue() - new_image_swift_size = len(new_image_swift) - - self.assertEqual(expected_swift_contents, new_image_contents) - self.assertEqual(expected_swift_size, new_image_swift_size) - - def test_add_large_object_zero_size(self): - """ - Tests that adding an image to Swift which has both an unknown size and - exceeds Swift's maximum limit of 5GB is correctly uploaded. - - We avoid the overhead of creating a 5GB object for this test by - temporarily setting MAX_SWIFT_OBJECT_SIZE to 1KB, and then adding - an object of 5KB. - - Bug lp:891738 - """ - # Set up a 'large' image of 5KB - expected_swift_size = FIVE_KB - expected_swift_contents = "*" * expected_swift_size - expected_checksum = hashlib.md5(expected_swift_contents).hexdigest() - expected_image_id = str(uuid.uuid4()) - loc = 'swift+https://%s:key@localhost:8080/glance/%s' - expected_location = loc % (self.swift_store_user, - expected_image_id) - image_swift = StringIO.StringIO(expected_swift_contents) - - global SWIFT_PUT_OBJECT_CALLS - SWIFT_PUT_OBJECT_CALLS = 0 - - # Temporarily set Swift MAX_SWIFT_OBJECT_SIZE to 1KB and add our image, - # explicitly setting the image_length to 0 - self.config(swift_store_container='glance') - self.store = Store() - orig_max_size = self.store.large_object_size - orig_temp_size = self.store.large_object_chunk_size - global MAX_SWIFT_OBJECT_SIZE - orig_max_swift_object_size = MAX_SWIFT_OBJECT_SIZE - try: - MAX_SWIFT_OBJECT_SIZE = 1024 - self.store.large_object_size = 1024 - self.store.large_object_chunk_size = 1024 - location, size, checksum, _ = self.store.add(expected_image_id, - image_swift, 0) - finally: - self.store.large_object_chunk_size = orig_temp_size - self.store.large_object_size = orig_max_size - MAX_SWIFT_OBJECT_SIZE = orig_max_swift_object_size - - self.assertEqual(expected_location, location) - self.assertEqual(expected_swift_size, size) - self.assertEqual(expected_checksum, checksum) - # Expecting 7 calls to put_object -- 5 chunks, a zero chunk which is - # then deleted, and the manifest. Note the difference with above - # where the image_size is specified in advance (there's no zero chunk - # in that case). - self.assertEqual(SWIFT_PUT_OBJECT_CALLS, 7) - - loc = get_location_from_uri(expected_location) - (new_image_swift, new_image_size) = self.store.get(loc) - new_image_contents = new_image_swift.getvalue() - new_image_swift_size = len(new_image_swift) - - self.assertEqual(expected_swift_contents, new_image_contents) - self.assertEqual(expected_swift_size, new_image_swift_size) - - def test_add_already_existing(self): - """ - Tests that adding an image with an existing identifier - raises an appropriate exception - """ - image_swift = StringIO.StringIO("nevergonnamakeit") - self.assertRaises(exceptions.Duplicate, - self.store.add, - FAKE_UUID, image_swift, 0) - - def test_add_saves_and_reraises_and_not_uses_wildcard_raise(self): - image_id = str(uuid.uuid4()) - swift_size = self.store.large_object_size = 1024 - loc = 'swift+https://%s:key@localhost:8080/glance/%s' - swift_contents = "*" * swift_size - connection = mock.Mock() - - def fake_delete_chunk(connection, - container, - chunks): - try: - raise Exception() - except Exception: - pass - - image_swift = StringIO.StringIO(swift_contents) - connection.put_object.side_effect = exceptions.ClientConnectionError - self.store._delete_stale_chunks = fake_delete_chunk - - self.assertRaises(exceptions.ClientConnectionError, - self.store.add, - image_id, - image_swift, - swift_size, - connection) - - def _option_required(self, key): - conf = self.getConfig() - conf[key] = None - - try: - self.config(**conf) - self.store = Store() - return self.store.add == self.store.add_disabled - except Exception: - return False - return False - - def test_no_user(self): - """ - Tests that options without user disables the add method - """ - self.assertTrue(self._option_required('swift_store_user')) - - def test_no_key(self): - """ - Tests that options without key disables the add method - """ - self.assertTrue(self._option_required('swift_store_key')) - - def test_no_auth_address(self): - """ - Tests that options without auth address disables the add method - """ - self.assertTrue(self._option_required('swift_store_auth_address')) - - def test_delete(self): - """ - Test we can delete an existing image in the swift store - """ - uri = "swift://%s:key@authurl/glance/%s" % ( - self.swift_store_user, FAKE_UUID) - loc = get_location_from_uri(uri) - self.store.delete(loc) - - self.assertRaises(exceptions.NotFound, self.store.get, loc) - - def test_delete_non_existing(self): - """ - Test that trying to delete a swift that doesn't exist - raises an error - """ - loc = get_location_from_uri("swift://%s:key@authurl/glance/noexist" % ( - self.swift_store_user)) - self.assertRaises(exceptions.NotFound, self.store.delete, loc) - - def test_read_acl_public(self): - """ - Test that we can set a public read acl. - """ - self.config(swift_store_multi_tenant=True) - context = glance.context.RequestContext() - store = Store(context) - uri = "swift+http://storeurl/glance/%s" % FAKE_UUID - loc = get_location_from_uri(uri) - store.set_acls(loc, public=True) - container_headers = swiftclient.client.head_container('x', 'y', - 'glance') - self.assertEqual(container_headers['X-Container-Read'], - ".r:*,.rlistings") - - def test_read_acl_tenants(self): - """ - Test that we can set read acl for tenants. - """ - self.config(swift_store_multi_tenant=True) - context = glance.context.RequestContext() - store = Store(context) - uri = "swift+http://storeurl/glance/%s" % FAKE_UUID - loc = get_location_from_uri(uri) - read_tenants = ['matt', 'mark'] - store.set_acls(loc, read_tenants=read_tenants) - container_headers = swiftclient.client.head_container('x', 'y', - 'glance') - self.assertEqual(container_headers['X-Container-Read'], - 'matt:*,mark:*') - - def test_write_acls(self): - """ - Test that we can set write acl for tenants. - """ - self.config(swift_store_multi_tenant=True) - context = glance.context.RequestContext() - store = Store(context) - uri = "swift+http://storeurl/glance/%s" % FAKE_UUID - loc = get_location_from_uri(uri) - read_tenants = ['frank', 'jim'] - store.set_acls(loc, write_tenants=read_tenants) - container_headers = swiftclient.client.head_container('x', 'y', - 'glance') - self.assertEqual(container_headers['X-Container-Write'], - 'frank:*,jim:*') - - -class TestStoreAuthV1(base.StoreClearingUnitTest, SwiftTests): - - def getConfig(self): - conf = SWIFT_CONF.copy() - conf['swift_store_auth_version'] = '1' - conf['swift_store_user'] = 'user' - return conf - - def setUp(self): - """Establish a clean test environment""" - conf = self.getConfig() - self.config(**conf) - super(TestStoreAuthV1, self).setUp() - stub_out_swiftclient(self, conf['swift_store_auth_version']) - self.store = Store() - -class TestStoreAuthV2(TestStoreAuthV1): - - def getConfig(self): - conf = super(TestStoreAuthV2, self).getConfig() - conf['swift_store_user'] = 'tenant:user' - conf['swift_store_auth_version'] = '2' - return conf - - def test_v2_with_no_tenant(self): - conf = self.getConfig() - conf['swift_store_user'] = 'failme' - uri = "swift://%s:key@auth_address/glance/%s" % ( - conf['swift_store_user'], FAKE_UUID) - loc = get_location_from_uri(uri) - self.assertRaises(exceptions.BadStoreUri, - self.store.get, - loc) - - def test_v2_multi_tenant_location(self): - conf = self.getConfig() - conf['swift_store_multi_tenant'] = True - uri = "swift://auth_address/glance/%s" % (FAKE_UUID) - loc = get_location_from_uri(uri) - self.assertEqual('swift', loc.store_name) - - -class FakeConnection(object): - def __init__(self, authurl, user, key, retries=5, preauthurl=None, - preauthtoken=None, snet=False, starting_backoff=1, - tenant_name=None, os_options={}, auth_version="1", - insecure=False, ssl_compression=True): - self.authurl = authurl - self.user = user - self.key = key - self.preauthurl = preauthurl - self.preauthtoken = preauthtoken - self.snet = snet - self.tenant_name = tenant_name - self.os_options = os_options - self.auth_version = auth_version - self.insecure = insecure - - -class TestSingleTenantStoreConnections(base.IsolatedUnitTest): - def setUp(self): - super(TestSingleTenantStoreConnections, self).setUp() - self.stubs.Set(swiftclient, 'Connection', FakeConnection) - self.store = glance.store.swift.SingleTenantStore() - specs = {'scheme': 'swift', - 'auth_or_store_url': 'example.com/v2/', - 'user': 'tenant:user', - 'key': 'abcdefg', - 'container': 'cont', - 'obj': 'object'} - self.location = glance.store.swift.StoreLocation(specs) - - def test_basic_connection(self): - connection = self.store.get_connection(self.location) - self.assertEqual(connection.authurl, 'https://example.com/v2/') - self.assertEqual(connection.auth_version, '2') - self.assertEqual(connection.user, 'user') - self.assertEqual(connection.tenant_name, 'tenant') - self.assertEqual(connection.key, 'abcdefg') - self.assertFalse(connection.snet) - self.assertEqual(connection.preauthurl, None) - self.assertEqual(connection.preauthtoken, None) - self.assertFalse(connection.insecure) - self.assertEqual(connection.os_options, - {'service_type': 'object-store', - 'endpoint_type': 'publicURL'}) - - def test_connection_with_no_trailing_slash(self): - self.location.auth_or_store_url = 'example.com/v2' - connection = self.store.get_connection(self.location) - self.assertEqual(connection.authurl, 'https://example.com/v2/') - - def test_connection_insecure(self): - self.config(swift_store_auth_insecure=True) - self.store.configure() - connection = self.store.get_connection(self.location) - self.assertTrue(connection.insecure) - - def test_connection_with_auth_v1(self): - self.config(swift_store_auth_version='1') - self.store.configure() - self.location.user = 'auth_v1_user' - connection = self.store.get_connection(self.location) - self.assertEqual(connection.auth_version, '1') - self.assertEqual(connection.user, 'auth_v1_user') - self.assertEqual(connection.tenant_name, None) - - def test_connection_invalid_user(self): - self.store.configure() - self.location.user = 'invalid:format:user' - self.assertRaises(exceptions.BadStoreUri, - self.store.get_connection, self.location) - - def test_connection_missing_user(self): - self.store.configure() - self.location.user = None - self.assertRaises(exceptions.BadStoreUri, - self.store.get_connection, self.location) - - def test_connection_with_region(self): - self.config(swift_store_region='Sahara') - self.store.configure() - connection = self.store.get_connection(self.location) - self.assertEqual(connection.os_options, - {'region_name': 'Sahara', - 'service_type': 'object-store', - 'endpoint_type': 'publicURL'}) - - def test_connection_with_service_type(self): - self.config(swift_store_service_type='shoe-store') - self.store.configure() - connection = self.store.get_connection(self.location) - self.assertEqual(connection.os_options, - {'service_type': 'shoe-store', - 'endpoint_type': 'publicURL'}) - - def test_connection_with_endpoint_type(self): - self.config(swift_store_endpoint_type='internalURL') - self.store.configure() - connection = self.store.get_connection(self.location) - self.assertEqual(connection.os_options, - {'service_type': 'object-store', - 'endpoint_type': 'internalURL'}) - - def test_connection_with_snet(self): - self.config(swift_enable_snet=True) - self.store.configure() - connection = self.store.get_connection(self.location) - self.assertTrue(connection.snet) - - -class TestMultiTenantStoreConnections(base.IsolatedUnitTest): - def setUp(self): - super(TestMultiTenantStoreConnections, self).setUp() - self.stubs.Set(swiftclient, 'Connection', FakeConnection) - self.context = glance.context.RequestContext( - user='user', tenant='tenant', auth_tok='0123') - self.store = glance.store.swift.MultiTenantStore(self.context) - specs = {'scheme': 'swift', - 'auth_or_store_url': 'example.com', - 'container': 'cont', - 'obj': 'object'} - self.location = glance.store.swift.StoreLocation(specs) - - def test_basic_connection(self): - self.store.configure() - connection = self.store.get_connection(self.location) - self.assertEqual(connection.authurl, None) - self.assertEqual(connection.auth_version, '2') - self.assertEqual(connection.user, 'user') - self.assertEqual(connection.tenant_name, 'tenant') - self.assertEqual(connection.key, None) - self.assertFalse(connection.snet) - self.assertEqual(connection.preauthurl, 'https://example.com') - self.assertEqual(connection.preauthtoken, '0123') - self.assertEqual(connection.os_options, {}) - - def test_connection_with_snet(self): - self.config(swift_enable_snet=True) - self.store.configure() - connection = self.store.get_connection(self.location) - self.assertTrue(connection.snet) - - -class FakeGetEndpoint(object): - def __init__(self, response): - self.response = response - - def __call__(self, service_catalog, service_type=None, - endpoint_region=None, endpoint_type=None): - self.service_type = service_type - self.endpoint_region = endpoint_region - self.endpoint_type = endpoint_type - return self.response - - -class TestCreatingLocations(base.IsolatedUnitTest): - def test_single_tenant_location(self): - self.config(swift_store_auth_address='example.com/v2', - swift_store_container='container', - swift_store_user='tenant:user', - swift_store_key='auth_key') - store = glance.store.swift.SingleTenantStore() - location = store.create_location('image-id') - self.assertEqual(location.scheme, 'swift+https') - self.assertEqual(location.swift_url, 'https://example.com/v2') - self.assertEqual(location.container, 'container') - self.assertEqual(location.obj, 'image-id') - self.assertEqual(location.user, 'tenant:user') - self.assertEqual(location.key, 'auth_key') - - def test_single_tenant_location_http(self): - self.config(swift_store_auth_address='http://example.com/v2', - swift_store_container='container', - swift_store_user='tenant:user', - swift_store_key='auth_key') - store = glance.store.swift.SingleTenantStore() - location = store.create_location('image-id') - self.assertEqual(location.scheme, 'swift+http') - self.assertEqual(location.swift_url, 'http://example.com/v2') - - def test_multi_tenant_location(self): - self.config(swift_store_container='container') - fake_get_endpoint = FakeGetEndpoint('https://some_endpoint') - self.stubs.Set(glance.store.common.auth, 'get_endpoint', fake_get_endpoint) - context = glance.context.RequestContext( - user='user', tenant='tenant', auth_tok='123', - service_catalog={}) - store = glance.store.swift.MultiTenantStore(context) - location = store.create_location('image-id') - self.assertEqual(location.scheme, 'swift+https') - self.assertEqual(location.swift_url, 'https://some_endpoint') - self.assertEqual(location.container, 'container_image-id') - self.assertEqual(location.obj, 'image-id') - self.assertEqual(location.user, None) - self.assertEqual(location.key, None) - self.assertEqual(fake_get_endpoint.service_type, 'object-store') - - def test_multi_tenant_location_http(self): - fake_get_endpoint = FakeGetEndpoint('http://some_endpoint') - self.stubs.Set(glance.store.common.auth, 'get_endpoint', fake_get_endpoint) - context = glance.context.RequestContext( - user='user', tenant='tenant', auth_tok='123', - service_catalog={}) - store = glance.store.swift.MultiTenantStore(context) - location = store.create_location('image-id') - self.assertEqual(location.scheme, 'swift+http') - self.assertEqual(location.swift_url, 'http://some_endpoint') - - def test_multi_tenant_location_with_region(self): - self.config(swift_store_region='WestCarolina') - fake_get_endpoint = FakeGetEndpoint('https://some_endpoint') - self.stubs.Set(glance.store.common.auth, 'get_endpoint', fake_get_endpoint) - context = glance.context.RequestContext( - user='user', tenant='tenant', auth_tok='123', - service_catalog={}) - store = glance.store.swift.MultiTenantStore(context) - self.assertEqual(fake_get_endpoint.endpoint_region, 'WestCarolina') - - def test_multi_tenant_location_custom_service_type(self): - self.config(swift_store_service_type='toy-store') - fake_get_endpoint = FakeGetEndpoint('https://some_endpoint') - self.stubs.Set(glance.store.common.auth, 'get_endpoint', fake_get_endpoint) - context = glance.context.RequestContext( - user='user', tenant='tenant', auth_tok='123', - service_catalog={}) - store = glance.store.swift.MultiTenantStore(context) - self.assertEqual(fake_get_endpoint.service_type, 'toy-store') - - def test_multi_tenant_location_custom_endpoint_type(self): - self.config(swift_store_endpoint_type='InternalURL') - fake_get_endpoint = FakeGetEndpoint('https://some_endpoint') - self.stubs.Set(glance.store.common.auth, 'get_endpoint', fake_get_endpoint) - context = glance.context.RequestContext( - user='user', tenant='tenant', auth_tok='123', - service_catalog={}) - store = glance.store.swift.MultiTenantStore(context) - self.assertEqual(fake_get_endpoint.endpoint_type, 'InternalURL') - - -class TestChunkReader(base.StoreClearingUnitTest): - - def test_read_all_data(self): - """ - Replicate what goes on in the Swift driver with the - repeated creation of the ChunkReader object - """ - CHUNKSIZE = 100 - checksum = hashlib.md5() - data_file = tempfile.NamedTemporaryFile() - data_file.write('*' * units.Ki) - data_file.flush() - infile = open(data_file.name, 'rb') - bytes_read = 0 - while True: - cr = glance.store.swift.ChunkReader(infile, checksum, CHUNKSIZE) - chunk = cr.read(CHUNKSIZE) - bytes_read += len(chunk) - if not chunk: - break - self.assertEqual(1024, bytes_read) - data_file.close() |