diff options
| -rw-r--r-- | etc/rootwrap.conf | 27 | ||||
| -rw-r--r-- | etc/rootwrap.d/glance_cinder_store.filters | 29 | ||||
| -rw-r--r-- | glance_store/_drivers/cinder.py | 485 | ||||
| -rw-r--r-- | glance_store/_drivers/http.py | 158 | ||||
| -rw-r--r-- | glance_store/_drivers/swift/store.py | 10 | ||||
| -rw-r--r-- | glance_store/_drivers/vmware_datastore.py | 220 | ||||
| -rw-r--r-- | glance_store/tests/unit/test_cinder_store.py | 285 | ||||
| -rw-r--r-- | glance_store/tests/unit/test_http_store.py | 73 | ||||
| -rw-r--r-- | glance_store/tests/unit/test_opts.py | 11 | ||||
| -rw-r--r-- | glance_store/tests/unit/test_swift_store.py | 55 | ||||
| -rw-r--r-- | glance_store/tests/unit/test_vmware_store.py | 195 | ||||
| -rw-r--r-- | glance_store/tests/utils.py | 16 | ||||
| -rw-r--r-- | releasenotes/notes/support-cinder-upload-c85849d9c88bbd7e.yaml | 8 | ||||
| -rw-r--r-- | releasenotes/notes/vmware-store-requests-369485d2cfdb6175.yaml | 6 | ||||
| -rw-r--r-- | requirements.txt | 9 | ||||
| -rw-r--r-- | setup.cfg | 5 |
16 files changed, 1189 insertions, 403 deletions
diff --git a/etc/rootwrap.conf b/etc/rootwrap.conf new file mode 100644 index 0000000..c376050 --- /dev/null +++ b/etc/rootwrap.conf @@ -0,0 +1,27 @@ +# Configuration for glance-rootwrap +# This file should be owned by (and only-writeable by) the root user + +[DEFAULT] +# List of directories to load filter definitions from (separated by ','). +# These directories MUST all be only writeable by root ! +filters_path=/etc/glance/rootwrap.d,/usr/share/glance/rootwrap + +# List of directories to search executables in, in case filters do not +# explicitely specify a full path (separated by ',') +# If not specified, defaults to system PATH environment variable. +# These directories MUST all be only writeable by root ! +exec_dirs=/sbin,/usr/sbin,/bin,/usr/bin,/usr/local/bin,/usr/local/sbin + +# Enable logging to syslog +# Default value is False +use_syslog=False + +# Which syslog facility to use. +# Valid values include auth, authpriv, syslog, local0, local1... +# Default value is 'syslog' +syslog_log_facility=syslog + +# Which messages to log. +# INFO means log all usage +# ERROR means only log unsuccessful attempts +syslog_log_level=ERROR diff --git a/etc/rootwrap.d/glance_cinder_store.filters b/etc/rootwrap.d/glance_cinder_store.filters new file mode 100644 index 0000000..2e3c92f --- /dev/null +++ b/etc/rootwrap.d/glance_cinder_store.filters @@ -0,0 +1,29 @@ +# glance-rootwrap command filters for glance cinder store +# This file should be owned by (and only-writeable by) the root user + +[Filters] +# cinder store driver +disk_chown: RegExpFilter, chown, root, chown, \d+, /dev/(?!.*/\.\.).* + +# os-brick +mount: CommandFilter, mount, root +blockdev: RegExpFilter, blockdev, root, blockdev, (--getsize64|--flushbufs), /dev/.* +tee: CommandFilter, tee, root +mkdir: CommandFilter, mkdir, root +chown: RegExpFilter, chown, root, chown root:root /etc/pstorage/clusters/(?!.*/\.\.).* +ip: CommandFilter, ip, root +dd: CommandFilter, dd, root +iscsiadm: CommandFilter, iscsiadm, root +aoe-revalidate: CommandFilter, aoe-revalidate, root +aoe-discover: CommandFilter, aoe-discover, root +aoe-flush: CommandFilter, aoe-flush, root +read_initiator: ReadFileFilter, /etc/iscsi/initiatorname.iscsi +multipath: CommandFilter, multipath, root +multipathd: CommandFilter, multipathd, root +systool: CommandFilter, systool, root +sg_scan: CommandFilter, sg_scan, root +cp: CommandFilter, cp, root +drv_cfg: CommandFilter, /opt/emc/scaleio/sdc/bin/drv_cfg, root, /opt/emc/scaleio/sdc/bin/drv_cfg, --query_guid +sds_cli: CommandFilter, /usr/local/bin/sds/sds_cli, root +vgc-cluster: CommandFilter, vgc-cluster, root +scsi_id: CommandFilter, /lib/udev/scsi_id, root diff --git a/glance_store/_drivers/cinder.py b/glance_store/_drivers/cinder.py index bac8ccd..d7f4a9b 100644 --- a/glance_store/_drivers/cinder.py +++ b/glance_store/_drivers/cinder.py @@ -12,8 +12,15 @@ """Storage backend for Cinder""" +import contextlib +import errno +import hashlib import logging +import os +import socket +import time +from oslo_concurrency import processutils from oslo_config import cfg from oslo_utils import units @@ -21,86 +28,129 @@ from glance_store import capabilities from glance_store.common import utils import glance_store.driver from glance_store import exceptions -from glance_store.i18n import _ +from glance_store.i18n import _, _LE, _LW, _LI import glance_store.location +from keystoneclient import exceptions as keystone_exc +from keystoneclient import service_catalog as keystone_sc try: from cinderclient import exceptions as cinder_exception - from cinderclient import service_catalog from cinderclient.v2 import client as cinderclient + from os_brick.initiator import connector except ImportError: cinder_exception = None - service_catalog = None cinderclient = None + connector = None + +CONF = cfg.CONF LOG = logging.getLogger(__name__) _CINDER_OPTS = [ cfg.StrOpt('cinder_catalog_info', - default='volume:cinder:publicURL', - help='Info to match when looking for cinder in the service ' - 'catalog. Format is : separated values of the form: ' - '<service_type>:<service_name>:<endpoint_type>'), + default='volumev2::publicURL', + help=_('Info to match when looking for cinder in the service ' + 'catalog. Format is : separated values of the form: ' + '<service_type>:<service_name>:<endpoint_type>')), cfg.StrOpt('cinder_endpoint_template', - help='Override service catalog lookup with template for cinder ' - 'endpoint e.g. http://localhost:8776/v1/%(project_id)s'), - cfg.StrOpt('os_region_name', - help='Region name of this node'), + help=_('Override service catalog lookup with template for ' + 'cinder endpoint e.g. ' + 'http://localhost:8776/v2/%(tenant)s')), + cfg.StrOpt('cinder_os_region_name', deprecated_name='os_region_name', + help=_('Region name of this node. If specified, it will be ' + 'used to locate OpenStack services for stores.')), cfg.StrOpt('cinder_ca_certificates_file', - help='Location of ca certicates file to use for cinder client ' - 'requests.'), + help=_('Location of ca certicates file to use for cinder ' + 'client requests.')), cfg.IntOpt('cinder_http_retries', default=3, - help='Number of cinderclient retries on failed http calls'), + help=_('Number of cinderclient retries on failed http calls')), + cfg.IntOpt('cinder_state_transition_timeout', + default=300, + help=_('Time period of time in seconds to wait for a cinder ' + 'volume transition to complete.')), cfg.BoolOpt('cinder_api_insecure', default=False, - help='Allow to perform insecure SSL requests to cinder'), + help=_('Allow to perform insecure SSL requests to cinder')), + cfg.StrOpt('cinder_store_auth_address', + default=None, + help=_('The address where the Cinder authentication service ' + 'is listening. If <None>, the cinder endpoint in the ' + 'service catalog is used.')), + cfg.StrOpt('cinder_store_user_name', + default=None, + help=_('User name to authenticate against Cinder. If <None>, ' + 'the user of current context is used.')), + cfg.StrOpt('cinder_store_password', secret=True, + default=None, + help=_('Password for the user authenticating against Cinder. ' + 'If <None>, the current context auth token is used.')), + cfg.StrOpt('cinder_store_project_name', + default=None, + help=_('Project name where the image is stored in Cinder. ' + 'If <None>, the project in current context is used.')), + cfg.StrOpt('rootwrap_config', + default='/etc/glance/rootwrap.conf', + help=_('Path to the rootwrap configuration file to use for ' + 'running commands as root.')), ] -def get_cinderclient(conf, context): - if conf.glance_store.cinder_endpoint_template: - url = conf.glance_store.cinder_endpoint_template % context.to_dict() - else: - info = conf.glance_store.cinder_catalog_info - service_type, service_name, endpoint_type = info.split(':') - - # extract the region if set in configuration - if conf.glance_store.os_region_name: - attr = 'region' - filter_value = conf.glance_store.os_region_name - else: - attr = None - filter_value = None +def get_root_helper(): + return 'sudo glance-rootwrap %s' % CONF.glance_store.rootwrap_config - # FIXME: the cinderclient ServiceCatalog object is mis-named. - # It actually contains the entire access blob. - # Only needed parts of the service catalog are passed in, see - # nova/context.py. - compat_catalog = { - 'access': {'serviceCatalog': context.service_catalog or []}} - sc = service_catalog.ServiceCatalog(compat_catalog) - url = sc.url_for(attr=attr, - filter_value=filter_value, - service_type=service_type, - service_name=service_name, - endpoint_type=endpoint_type) +def is_user_overriden(conf): + return all([conf.glance_store.get('cinder_store_' + key) + for key in ['user_name', 'password', + 'project_name', 'auth_address']]) - LOG.debug(_('Cinderclient connection created using URL: %s') % url) +def get_cinderclient(conf, context=None): glance_store = conf.glance_store - c = cinderclient.Client(context.user, - context.auth_token, - project_id=context.tenant, + user_overriden = is_user_overriden(conf) + if user_overriden: + username = glance_store.cinder_store_user_name + password = glance_store.cinder_store_password + project = glance_store.cinder_store_project_name + url = glance_store.cinder_store_auth_address + else: + username = context.user + password = context.auth_token + project = context.tenant + + if glance_store.cinder_endpoint_template: + url = glance_store.cinder_endpoint_template % context.to_dict() + else: + info = glance_store.cinder_catalog_info + service_type, service_name, endpoint_type = info.split(':') + sc = {'serviceCatalog': context.service_catalog} + try: + url = keystone_sc.ServiceCatalogV2(sc).url_for( + region_name=glance_store.cinder_os_region_name, + service_type=service_type, + service_name=service_name, + endpoint_type=endpoint_type) + except keystone_exc.EndpointNotFound: + reason = _("Failed to find Cinder from a service catalog.") + raise exceptions.BadStoreConfiguration(store_name="cinder", + reason=reason) + + c = cinderclient.Client(username, + password, + project, auth_url=url, insecure=glance_store.cinder_api_insecure, retries=glance_store.cinder_http_retries, cacert=glance_store.cinder_ca_certificates_file) + LOG.debug('Cinderclient connection created for user %(user)s using URL: ' + '%(url)s.', {'user': username, 'url': url}) + # noauth extracts user_id:project_id from auth_token - c.client.auth_token = context.auth_token or '%s:%s' % (context.user, - context.tenant) + if not user_overriden: + c.client.auth_token = context.auth_token or '%s:%s' % (username, + project) c.client.management_url = url return c @@ -131,34 +181,206 @@ class StoreLocation(glance_store.location.StoreLocation): raise exceptions.BadStoreUri(message=reason) +@contextlib.contextmanager +def temporary_chown(path): + owner_uid = os.getuid() + orig_uid = os.stat(path).st_uid + + if orig_uid != owner_uid: + processutils.execute('chown', owner_uid, path, + run_as_root=True, + root_helper=get_root_helper()) + try: + yield + finally: + if orig_uid != owner_uid: + processutils.execute('chown', orig_uid, path, + run_as_root=True, + root_helper=get_root_helper()) + + class Store(glance_store.driver.Store): """Cinder backend store adapter.""" - _CAPABILITIES = capabilities.BitMasks.DRIVER_REUSABLE + _CAPABILITIES = (capabilities.BitMasks.READ_RANDOM | + capabilities.BitMasks.WRITE_ACCESS | + capabilities.BitMasks.DRIVER_REUSABLE) OPTIONS = _CINDER_OPTS EXAMPLE_URL = "cinder://<VOLUME_ID>" + def __init__(self, *args, **kargs): + super(Store, self).__init__(*args, **kargs) + LOG.warning(_LW("Cinder store is considered experimental. " + "Current deployers should be aware that the use " + "of it in production right now may be risky.")) + def get_schemes(self): return ('cinder',) - def _check_context(self, context): - """ - Configure the Store to use the stored configuration options - Any store that needs special configuration should implement - this method. If the store was not able to successfully configure - itself, it should raise `exceptions.BadStoreConfiguration` - """ - + def _check_context(self, context, require_tenant=False): + user_overriden = is_user_overriden(self.conf) + if user_overriden and not require_tenant: + return if context is None: reason = _("Cinder storage requires a context.") raise exceptions.BadStoreConfiguration(store_name="cinder", reason=reason) - if context.service_catalog is None: + if not user_overriden and context.service_catalog is None: reason = _("Cinder storage requires a service catalog.") raise exceptions.BadStoreConfiguration(store_name="cinder", reason=reason) + def _wait_volume_status(self, volume, status_transition, status_expected): + max_recheck_wait = 15 + timeout = self.conf.glance_store.cinder_state_transition_timeout + volume = volume.manager.get(volume.id) + tries = 0 + elapsed = 0 + while volume.status == status_transition: + if elapsed >= timeout: + msg = (_('Timeout while waiting while volume %(volume_id)s ' + 'status is %(status)s.') + % {'volume_id': volume.id, 'status': status_transition}) + LOG.error(msg) + raise exceptions.BackendException(msg) + + wait = min(0.5 * 2 ** tries, max_recheck_wait) + time.sleep(wait) + tries += 1 + elapsed += wait + volume = volume.manager.get(volume.id) + if volume.status != status_expected: + msg = (_('The status of volume %(volume_id)s is unexpected: ' + 'status = %(status)s, expected = %(expected)s.') + % {'volume_id': volume.id, 'status': volume.status, + 'expected': status_expected}) + LOG.error(msg) + raise exceptions.BackendException(msg) + return volume + + @contextlib.contextmanager + def _open_cinder_volume(self, client, volume, mode): + attach_mode = 'rw' if mode == 'wb' else 'ro' + device = None + root_helper = get_root_helper() + host = socket.gethostname() + properties = connector.get_connector_properties(root_helper, host, + False, False) + + try: + volume.reserve(volume) + except cinder_exception.ClientException as e: + msg = (_('Failed to reserve volume %(volume_id)s: %(error)s') + % {'volume_id': volume.id, 'error': e}) + LOG.error(msg) + raise exceptions.BackendException(msg) + + try: + connection_info = volume.initialize_connection(volume, properties) + conn = connector.InitiatorConnector.factory( + connection_info['driver_volume_type'], root_helper) + device = conn.connect_volume(connection_info['data']) + volume.attach(None, None, attach_mode, host_name=host) + volume = self._wait_volume_status(volume, 'attaching', 'in-use') + LOG.debug('Opening host device "%s"', device['path']) + with temporary_chown(device['path']), \ + open(device['path'], mode) as f: + yield f + except Exception: + LOG.exception(_LE('Exception while accessing to cinder volume ' + '%(volume_id)s.'), {'volume_id': volume.id}) + raise + finally: + if volume.status == 'in-use': + volume.begin_detaching(volume) + elif volume.status == 'attaching': + volume.unreserve(volume) + + if device: + try: + conn.disconnect_volume(connection_info['data'], device) + except Exception: + LOG.exception(_LE('Failed to disconnect volume ' + '%(volume_id)s.'), + {'volume_id': volume.id}) + + try: + volume.terminate_connection(volume, properties) + except Exception: + LOG.exception(_LE('Failed to terminate connection of volume ' + '%(volume_id)s.'), {'volume_id': volume.id}) + + try: + client.volumes.detach(volume) + except Exception: + LOG.exception(_LE('Failed to detach volume %(volume_id)s.'), + {'volume_id': volume.id}) + + def _cinder_volume_data_iterator(self, client, volume, max_size, offset=0, + chunk_size=None, partial_length=None): + chunk_size = chunk_size if chunk_size else self.READ_CHUNKSIZE + partial = partial_length is not None + with self._open_cinder_volume(client, volume, 'rb') as fp: + if offset: + fp.seek(offset) + max_size -= offset + while True: + if partial: + size = min(chunk_size, partial_length, max_size) + else: + size = min(chunk_size, max_size) + + chunk = fp.read(size) + if chunk: + yield chunk + max_size -= len(chunk) + if max_size <= 0: + break + if partial: + partial_length -= len(chunk) + if partial_length <= 0: + break + else: + break + + @capabilities.check + def get(self, location, offset=0, chunk_size=None, context=None): + """ + Takes a `glance_store.location.Location` object that indicates + where to find the image file, and returns a tuple of generator + (for reading the image file) and image_size + + :param location `glance_store.location.Location` object, supplied + from glance_store.location.get_location_from_uri() + :param offset: offset to start reading + :param chunk_size: size to read, or None to get all the image + :param context: Request context + :raises `glance_store.exceptions.NotFound` if image does not exist + """ + + loc = location.store_location + self._check_context(context) + try: + client = get_cinderclient(self.conf, context) + volume = client.volumes.get(loc.volume_id) + size = int(volume.metadata.get('image_size', + volume.size * units.Gi)) + iterator = self._cinder_volume_data_iterator( + client, volume, size, offset=offset, + chunk_size=self.READ_CHUNKSIZE, partial_length=chunk_size) + return (iterator, chunk_size or size) + except cinder_exception.NotFound: + reason = _("Failed to get image size due to " + "volume can not be found: %s") % volume.id + LOG.error(reason) + raise exceptions.NotFound(reason) + except cinder_exception.ClientException as e: + msg = (_('Failed to get image volume %(volume_id): %(error)s') + % {'volume_id': loc.volume_id, 'error': e}) + LOG.error(msg) + raise exceptions.BackendException(msg) + def get_size(self, location, context=None): """ Takes a `glance_store.location.Location` object that indicates @@ -178,12 +400,145 @@ class Store(glance_store.driver.Store): context).volumes.get(loc.volume_id) # GB unit convert to byte return volume.size * units.Gi - except cinder_exception.NotFound as e: - reason = _("Failed to get image size due to " - "volume can not be found: %s") % self.volume_id - LOG.error(reason) - raise exceptions.NotFound(reason) - except Exception as e: - LOG.exception(_("Failed to get image size due to " - "internal error: %s") % e) + except cinder_exception.NotFound: + raise exceptions.NotFound(image=loc.volume_id) + except Exception: + LOG.exception(_LE("Failed to get image size due to " + "internal error.")) return 0 + + @capabilities.check + def add(self, image_id, image_file, image_size, context=None, + verifier=None): + """ + Stores an image file with supplied identifier to the backend + storage system and returns a tuple containing information + about the stored image. + + :param image_id: The opaque image identifier + :param image_file: The image data to write, as a file-like object + :param image_size: The size of the image data to write, in bytes + :param context: The request context + :param verifier: An object used to verify signatures for images + + :retval tuple of URL in backing store, bytes written, checksum + and a dictionary with storage system specific information + :raises `glance_store.exceptions.Duplicate` if the image already + existed + """ + + self._check_context(context, require_tenant=True) + client = get_cinderclient(self.conf, context) + + checksum = hashlib.md5() + bytes_written = 0 + size_gb = int((image_size + units.Gi - 1) / units.Gi) + if size_gb == 0: + size_gb = 1 + name = "image-%s" % image_id + owner = context.tenant + metadata = {'glance_image_id': image_id, + 'image_size': str(image_size), + 'image_owner': owner} + LOG.debug('Creating a new volume: image_size=%d size_gb=%d', + image_size, size_gb) + if image_size == 0: + LOG.info(_LI("Since image size is zero, we will be doing " + "resize-before-write for each GB which " + "will be considerably slower than normal.")) + volume = client.volumes.create(size_gb, name=name, metadata=metadata) + volume = self._wait_volume_status(volume, 'creating', 'available') + + failed = True + need_extend = True + buf = None + try: + while need_extend: + with self._open_cinder_volume(client, volume, 'wb') as f: + f.seek(bytes_written) + if buf: + f.write(buf) + bytes_written += len(buf) + while True: + buf = image_file.read(self.WRITE_CHUNKSIZE) + if not buf: + need_extend = False + break + checksum.update(buf) + if verifier: + verifier.update(buf) + if (bytes_written + len(buf) > size_gb * units.Gi and + image_size == 0): + break + f.write(buf) + bytes_written += len(buf) + + if need_extend: + size_gb += 1 + LOG.debug("Extending volume %(volume_id)s to %(size)s GB.", + {'volume_id': volume.id, 'size': size_gb}) + volume.extend(volume, size_gb) + try: + volume = self._wait_volume_status(volume, + 'extending', + 'available') + except exceptions.BackendException: + raise exceptions.StorageFull() + + failed = False + except IOError as e: + # Convert IOError reasons to Glance Store exceptions + errors = {errno.EFBIG: exceptions.StorageFull(), + errno.ENOSPC: exceptions.StorageFull(), + errno.EACCES: exceptions.StorageWriteDenied()} + raise errors.get(e.errno, e) + finally: + if failed: + LOG.error(_LE("Failed to write to volume %(volume_id)s."), + {'volume_id': volume.id}) + try: + volume.delete() + except Exception: + LOG.exception(_LE('Failed to delete of volume ' + '%(volume_id)s.'), + {'volume_id': volume.id}) + + if image_size == 0: + metadata.update({'image_size': str(bytes_written)}) + volume.update_all_metadata(metadata) + volume.update_readonly_flag(volume, True) + + checksum_hex = checksum.hexdigest() + + LOG.debug("Wrote %(bytes_written)d bytes to volume %(volume_id)s " + "with checksum %(checksum_hex)s.", + {'bytes_written': bytes_written, + 'volume_id': volume.id, + 'checksum_hex': checksum_hex}) + + return ('cinder://%s' % volume.id, bytes_written, checksum_hex, {}) + + @capabilities.check + def delete(self, location, context=None): + """ + Takes a `glance_store.location.Location` object that indicates + where to find the image file to delete + + :location `glance_store.location.Location` object, supplied + from glance_store.location.get_location_from_uri() + + :raises NotFound if image does not exist + :raises Forbidden if cannot delete because of permissions + """ + loc = location.store_location + self._check_context(context) + try: + volume = get_cinderclient(self.conf, + context).volumes.get(loc.volume_id) + volume.delete() + except cinder_exception.NotFound: + raise exceptions.NotFound(image=loc.volume_id) + except cinder_exception.ClientException as e: + msg = (_('Failed to delete volume %(volume_id)s: %(error)s') % + {'volume_id': loc.volume_id, 'error': e}) + raise exceptions.BackendException(msg) diff --git a/glance_store/_drivers/http.py b/glance_store/_drivers/http.py index 6aab258..3e858bc 100644 --- a/glance_store/_drivers/http.py +++ b/glance_store/_drivers/http.py @@ -14,17 +14,18 @@ # under the License. import logging -import socket +from oslo_config import cfg from oslo_utils import encodeutils -from six.moves import http_client + from six.moves import urllib +import requests + from glance_store import capabilities import glance_store.driver from glance_store import exceptions from glance_store.i18n import _ -from glance_store.i18n import _LE import glance_store.location LOG = logging.getLogger(__name__) @@ -32,6 +33,26 @@ LOG = logging.getLogger(__name__) MAX_REDIRECTS = 5 +_HTTP_OPTS = [ + cfg.StrOpt('https_ca_certificates_file', + help=_('Specify the path to the CA bundle file to use in ' + 'verifying the remote server certificate.')), + cfg.BoolOpt('https_insecure', + default=True, + help=_('If true, the remote server certificate is not ' + 'verified. If false, then the default CA truststore is ' + 'used for verification. This option is ignored if ' + '"https_ca_certificates_file" is set.')), + cfg.DictOpt('http_proxy_information', + default={}, + help=_('Specify the http/https proxy information that should ' + 'be used to connect to the remote server. The proxy ' + 'information should be a key value pair of the ' + 'scheme and proxy. e.g. http:10.0.0.1:3128. You can ' + 'specify proxies for multiple schemes by seperating ' + 'the key value pairs with a comma.' + 'e.g. http:10.0.0.1:3128, https:10.0.0.1:1080.'))] + class StoreLocation(glance_store.location.StoreLocation): @@ -109,14 +130,16 @@ def http_response_iterator(conn, response, size): Return an iterator for a file-like object. :param conn: HTTP(S) Connection - :param response: http_client.HTTPResponse object + :param response: urllib3.HTTPResponse object :param size: Chunk size to iterate with """ - chunk = response.read(size) - while chunk: - yield chunk + try: chunk = response.read(size) - conn.close() + while chunk: + yield chunk + chunk = response.read(size) + finally: + conn.close() class Store(glance_store.driver.Store): @@ -125,6 +148,7 @@ class Store(glance_store.driver.Store): _CAPABILITIES = (capabilities.BitMasks.READ_ACCESS | capabilities.BitMasks.DRIVER_REUSABLE) + OPTIONS = _HTTP_OPTS @capabilities.check def get(self, location, offset=0, chunk_size=None, context=None): @@ -138,11 +162,11 @@ class Store(glance_store.driver.Store): """ try: conn, resp, content_length = self._query(location, 'GET') - except socket.error: - reason = _LE("Remote server where the image is present " - "is unavailable.") - LOG.error(reason) - raise exceptions.RemoteServiceUnavailable() + except requests.exceptions.ConnectionError: + reason = _("Remote server where the image is present " + "is unavailable.") + LOG.exception(reason) + raise exceptions.RemoteServiceUnavailable(message=reason) iterator = http_response_iterator(conn, resp, self.READ_CHUNKSIZE) @@ -166,63 +190,95 @@ class Store(glance_store.driver.Store): :param location: `glance_store.location.Location` object, supplied from glance_store.location.get_location_from_uri() """ + conn = None try: - size = self._query(location, 'HEAD')[2] - except (socket.error, http_client.HTTPException) as exc: + conn, resp, size = self._query(location, 'HEAD') + except requests.exceptions.ConnectionError as exc: err_msg = encodeutils.exception_to_unicode(exc) reason = _("The HTTP URL is invalid: %s") % err_msg LOG.info(reason) raise exceptions.BadStoreUri(message=reason) + finally: + # NOTE(sabari): Close the connection as the request was made with + # stream=True + if conn is not None: + conn.close() return size - def _query(self, location, verb, depth=0): - if depth > MAX_REDIRECTS: + def _query(self, location, verb): + redirects_followed = 0 + + while redirects_followed < MAX_REDIRECTS: + loc = location.store_location + + conn = self._get_response(loc, verb) + + # NOTE(sigmavirus24): If it was generally successful, break early + if conn.status_code < 300: + break + + self._check_store_uri(conn, loc) + + redirects_followed += 1 + + # NOTE(sigmavirus24): Close the response so we don't leak sockets + conn.close() + + location = self._new_location(location, conn.headers['location']) + else: reason = (_("The HTTP URL exceeded %s maximum " "redirects.") % MAX_REDIRECTS) LOG.debug(reason) raise exceptions.MaxRedirectsExceeded(message=reason) - loc = location.store_location - conn_class = self._get_conn_class(loc) - conn = conn_class(loc.netloc) - conn.request(verb, loc.path, "", {}) - resp = conn.getresponse() + resp = conn.raw + + content_length = int(resp.getheader('content-length', 0)) + return (conn, resp, content_length) + + def _new_location(self, old_location, url): + store_name = old_location.store_name + store_class = old_location.store_location.__class__ + image_id = old_location.image_id + store_specs = old_location.store_specs + return glance_store.location.Location(store_name, + store_class, + self.conf, + uri=url, + image_id=image_id, + store_specs=store_specs) + + @staticmethod + def _check_store_uri(conn, loc): + # TODO(sigmavirus24): Make this a staticmethod # Check for bad status codes - if resp.status >= 400: - if resp.status == http_client.NOT_FOUND: + if conn.status_code >= 400: + if conn.status_code == requests.codes.not_found: reason = _("HTTP datastore could not find image at URI.") LOG.debug(reason) raise exceptions.NotFound(message=reason) reason = (_("HTTP URL %(url)s returned a " - "%(status)s status code.") % - dict(url=loc.path, status=resp.status)) + "%(status)s status code. \nThe response body:\n" + "%(body)s") % + {'url': loc.path, 'status': conn.status_code, + 'body': conn.text}) LOG.debug(reason) raise exceptions.BadStoreUri(message=reason) - location_header = resp.getheader("location") - if location_header: - if resp.status not in (301, 302): - reason = (_("The HTTP URL %(url)s attempted to redirect " - "with an invalid %(status)s status code.") % - dict(url=loc.path, status=resp.status)) - LOG.info(reason) - raise exceptions.BadStoreUri(message=reason) - location_class = glance_store.location.Location - new_loc = location_class(location.store_name, - location.store_location.__class__, - self.conf, - uri=location_header, - image_id=location.image_id, - store_specs=location.store_specs) - return self._query(new_loc, verb, depth + 1) - content_length = int(resp.getheader('content-length', 0)) - return (conn, resp, content_length) + if conn.is_redirect and conn.status_code not in (301, 302): + reason = (_("The HTTP URL %(url)s attempted to redirect " + "with an invalid %(status)s status code.") % + {'url': loc.path, 'status': conn.status_code}) + LOG.info(reason) + raise exceptions.BadStoreUri(message=reason) - def _get_conn_class(self, loc): - """ - Returns connection class for accessing the resource. Useful - for dependency injection and stubouts in testing... - """ - return {'http': http_client.HTTPConnection, - 'https': http_client.HTTPSConnection}[loc.scheme] + def _get_response(self, location, verb): + if not hasattr(self, 'session'): + self.session = requests.Session() + ca_bundle = self.conf.glance_store.https_ca_certificates_file + disable_https = self.conf.glance_store.https_insecure + self.session.verify = ca_bundle if ca_bundle else not disable_https + self.session.proxies = self.conf.glance_store.http_proxy_information + return self.session.request(verb, location.get_uri(), stream=True, + allow_redirects=False) diff --git a/glance_store/_drivers/swift/store.py b/glance_store/_drivers/swift/store.py index c32d621..bf42d05 100644 --- a/glance_store/_drivers/swift/store.py +++ b/glance_store/_drivers/swift/store.py @@ -19,6 +19,7 @@ import hashlib import logging import math +from keystoneclient import exceptions as keystone_exc from keystoneclient import service_catalog as keystone_sc from oslo_config import cfg from oslo_utils import encodeutils @@ -1036,8 +1037,15 @@ class MultiTenantStore(BaseStore): return StoreLocation(specs, self.conf) def get_connection(self, location, context=None): + try: + storage_url = self._get_endpoint(context) + except (exceptions.BadStoreConfiguration, + keystone_exc.EndpointNotFound) as e: + LOG.debug("Cannot obtain swift endpoint url from Service Catalog: " + "%s. Use url stored in database.", e) + storage_url = location.swift_url return swiftclient.Connection( - preauthurl=location.swift_url, + preauthurl=storage_url, preauthtoken=context.auth_token, insecure=self.insecure, ssl_compression=self.ssl_compression, diff --git a/glance_store/_drivers/vmware_datastore.py b/glance_store/_drivers/vmware_datastore.py index 98bd0c8..819fc56 100644 --- a/glance_store/_drivers/vmware_datastore.py +++ b/glance_store/_drivers/vmware_datastore.py @@ -31,15 +31,20 @@ try: from oslo_vmware import vim_util except ImportError: api = None -from six.moves import http_client + from six.moves import urllib +import six.moves.urllib.parse as urlparse +import requests +from requests import adapters +from requests.packages.urllib3.util import retry import six # NOTE(jokke): simplified transition to py3, behaves like py2 xrange from six.moves import range import glance_store from glance_store import capabilities +from glance_store.common import utils from glance_store import exceptions from glance_store.i18n import _ from glance_store.i18n import _LE @@ -48,6 +53,7 @@ from glance_store import location LOG = logging.getLogger(__name__) +CHUNKSIZE = 1024 * 64 # 64kB MAX_REDIRECTS = 5 DEFAULT_STORE_IMAGE_DIR = '/openstack_glance' DS_URL_PREFIX = '/folder' @@ -138,49 +144,6 @@ class _Reader(object): return self._size -class _ChunkReader(_Reader): - - def __init__(self, data, verifier=None, blocksize=8192): - self.blocksize = blocksize - self.current_chunk = b"" - self.closed = False - super(_ChunkReader, self).__init__(data, verifier) - - def read(self, size=None): - ret = b"" - while size is None or size >= len(self.current_chunk): - ret += self.current_chunk - if size is not None: - size -= len(self.current_chunk) - if self.closed: - self.current_chunk = b"" - break - self._get_chunk() - else: - ret += self.current_chunk[:size] - self.current_chunk = self.current_chunk[size:] - return ret - - def _get_chunk(self): - if not self.closed: - chunk = self.data.read(self.blocksize) - chunk_len = len(chunk) - self._size += chunk_len - self.checksum.update(chunk) - if self.verifier: - self.verifier.update(chunk) - if chunk: - if six.PY3: - size_header = ('%x\r\n' % chunk_len).encode('ascii') - self.current_chunk = b''.join((size_header, chunk, - b'\r\n')) - else: - self.current_chunk = b'%x\r\n%s\r\n' % (chunk_len, chunk) - else: - self.current_chunk = b'0\r\n\r\n' - self.closed = True - - class StoreLocation(location.StoreLocation): """Class describing an VMware URI. @@ -248,6 +211,16 @@ class StoreLocation(location.StoreLocation): if ds_name: self.datastore_name = ds_name[0] + @property + def https_url(self): + """ + Creates a https url that can be used to upload/download data from a + vmware store. + """ + parsed_url = urlparse.urlparse(self.get_uri()) + new_url = parsed_url._replace(scheme='https') + return urlparse.urlunparse(new_url) + class Store(glance_store.Store): """An implementation of the VMware datastore adapter.""" @@ -445,15 +418,13 @@ class Store(glance_store.Store): are 201 Created and 200 OK. """ ds = self.select_datastore(image_size) + image_file = _Reader(image_file, verifier) + headers = {} if image_size > 0: - headers = {'Content-Length': image_size} - image_file = _Reader(image_file, verifier) + headers.update({'Content-Length': image_size}) + data = image_file else: - # NOTE (arnaud): use chunk encoding when the image is still being - # generated by the server (ex: stream optimized disks generated by - # Nova). - headers = {'Transfer-Encoding': 'chunked'} - image_file = _ChunkReader(image_file, verifier) + data = utils.chunkiter(image_file, CHUNKSIZE) loc = StoreLocation({'scheme': self.scheme, 'server_host': self.server_host, 'image_dir': self.store_image_dir, @@ -463,13 +434,15 @@ class Store(glance_store.Store): # NOTE(arnaud): use a decorator when the config is not tied to self cookie = self._build_vim_cookie_header(True) headers = dict(headers) - headers['Cookie'] = cookie - conn_class = self._get_http_conn_class() - conn = conn_class(loc.server_host) - url = urllib.parse.quote('%s?%s' % (loc.path, loc.query)) + headers.update({'Cookie': cookie}) + session = new_session(self.api_insecure) + + url = loc.https_url try: - conn.request('PUT', url, image_file, headers) + response = session.put(url, data=data, headers=headers) except IOError as e: + # TODO(sigmavirus24): Figure out what the new exception type would + # be in requests. # When a session is not authenticated, the socket is closed by # the server after sending the response. http_client has an open # issue with https that raises Broken Pipe @@ -482,17 +455,19 @@ class Store(glance_store.Store): 'url': url, 'e': e} LOG.error(msg) + raise exceptions.BackendException(msg) except Exception: with excutils.save_and_reraise_exception(): LOG.exception(_LE('Failed to upload content of image ' '%(image)s'), {'image': image_id}) - res = conn.getresponse() - if res.status == http_client.CONFLICT: + + res = response.raw + if res.status == requests.codes.conflict: raise exceptions.Duplicate(_("Image file %(image_id)s already " "exists!") % {'image_id': image_id}) - if res.status not in (http_client.CREATED, http_client.OK): + if res.status not in (requests.codes.created, requests.codes.ok): msg = (_LE('Failed to upload content of image %(image)s. ' 'The request returned an unexpected status: %(status)s.' '\nThe response body:\n%(body)s') % @@ -534,7 +509,15 @@ class Store(glance_store.Store): :param location: `glance_store.location.Location` object, supplied from glance_store.location.get_location_from_uri() """ - return self._query(location, 'HEAD')[2] + conn = None + try: + conn, resp, size = self._query(location, 'HEAD') + return size + finally: + # NOTE(sabari): Close the connection as the request was made with + # stream=True. + if conn is not None: + conn.close() @capabilities.check def delete(self, location, context=None): @@ -566,30 +549,59 @@ class Store(glance_store.Store): LOG.exception(_LE('Failed to delete image %(image)s ' 'content.') % {'image': location.image_id}) - def _query(self, location, method, depth=0): - if depth > MAX_REDIRECTS: + def _query(self, location, method): + session = new_session(self.api_insecure) + loc = location.store_location + redirects_followed = 0 + # TODO(sabari): The redirect logic was added to handle cases when the + # backend redirects http url's to https. But the store never makes a + # http request and hence this can be safely removed. + while redirects_followed < MAX_REDIRECTS: + conn, resp = self._retry_request(session, method, location) + + # NOTE(sigmavirus24): _retry_request handles 4xx and 5xx errors so + # if the response is not a redirect, we can return early. + if not conn.is_redirect: + break + + redirects_followed += 1 + + location_header = conn.headers.get('location') + if location_header: + if resp.status not in (301, 302): + reason = (_("The HTTP URL %(path)s attempted to redirect " + "with an invalid %(status)s status code.") + % {'path': loc.path, 'status': resp.status}) + LOG.info(reason) + raise exceptions.BadStoreUri(message=reason) + conn.close() + location = self._new_location(location, location_header) + else: + # NOTE(sigmavirus24): We exceeded the maximum number of redirects msg = ("The HTTP URL exceeded %(max_redirects)s maximum " "redirects.", {'max_redirects': MAX_REDIRECTS}) LOG.debug(msg) raise exceptions.MaxRedirectsExceeded(redirects=MAX_REDIRECTS) + + content_length = int(resp.getheader('content-length', 0)) + + return (conn, resp, content_length) + + def _retry_request(self, session, method, location): loc = location.store_location # NOTE(arnaud): use a decorator when the config is not tied to self for i in range(self.api_retry_count + 1): cookie = self._build_vim_cookie_header() headers = {'Cookie': cookie} - try: - conn = self._get_http_conn(method, loc, headers) - resp = conn.getresponse() - except Exception: - with excutils.save_and_reraise_exception(): - LOG.exception(_LE('Failed to access image %(image)s ' - 'content.') % {'image': - location.image_id}) + conn = session.request(method, loc.https_url, headers=headers, + stream=True) + resp = conn.raw + if resp.status >= 400: - if resp.status == http_client.UNAUTHORIZED: + if resp.status == requests.codes.unauthorized: self.reset_session() continue - if resp.status == http_client.NOT_FOUND: + if resp.status == requests.codes.not_found: reason = _('VMware datastore could not find image at URI.') LOG.info(reason) raise exceptions.NotFound(message=reason) @@ -598,34 +610,36 @@ class Store(glance_store.Store): LOG.debug(msg) raise exceptions.BadStoreUri(msg) break - location_header = resp.getheader('location') - if location_header: - if resp.status not in (301, 302): - reason = (_("The HTTP URL %(path)s attempted to redirect " - "with an invalid %(status)s status code.") - % {'path': loc.path, 'status': resp.status}) - LOG.info(reason) - raise exceptions.BadStoreUri(message=reason) - location_class = glance_store.location.Location - new_loc = location_class(location.store_name, - location.store_location.__class__, - uri=location_header, - image_id=location.image_id, - store_specs=location.store_specs) - return self._query(new_loc, method, depth + 1) - content_length = int(resp.getheader('content-length', 0)) - - return (conn, resp, content_length) - - def _get_http_conn(self, method, loc, headers, content=None): - conn_class = self._get_http_conn_class() - conn = conn_class(loc.server_host) - url = urllib.parse.quote('%s?%s' % (loc.path, loc.query)) - conn.request(method, url, content, headers) - - return conn - - def _get_http_conn_class(self): - if self.api_insecure: - return http_client.HTTPConnection - return http_client.HTTPSConnection + return conn, resp + + def _new_location(self, old_location, url): + store_name = old_location.store_name + store_class = old_location.store_location.__class__ + image_id = old_location.image_id + store_specs = old_location.store_specs + # Note(sabari): The redirect url will have a scheme 'http(s)', but the + # store only accepts url with scheme 'vsphere'. Thus, replacing with + # store's scheme. + parsed_url = urlparse.urlparse(url) + new_url = parsed_url._replace(scheme='vsphere') + vsphere_url = urlparse.urlunparse(new_url) + return glance_store.location.Location(store_name, + store_class, + self.conf, + uri=vsphere_url, + image_id=image_id, + store_specs=store_specs) + + +def new_session(insecure=False, total_retries=None): + session = requests.Session() + if total_retries is not None: + http_adapter = adapters.HTTPAdapter( + max_retries=retry.Retry(total=total_retries)) + https_adapter = adapters.HTTPAdapter( + max_retries=retry.Retry(total=total_retries)) + session.mount('http://', http_adapter) + session.mount('https://', https_adapter) + if insecure: + session.verify = False + return session diff --git a/glance_store/tests/unit/test_cinder_store.py b/glance_store/tests/unit/test_cinder_store.py index 069c716..582dbde 100644 --- a/glance_store/tests/unit/test_cinder_store.py +++ b/glance_store/tests/unit/test_cinder_store.py @@ -13,11 +13,21 @@ # License for the specific language governing permissions and limitations # under the License. +import contextlib +import errno +import hashlib import mock -from oslo_utils import units +import os import six +import socket +import tempfile +import time +import uuid + +from os_brick.initiator import connector +from oslo_concurrency import processutils +from oslo_utils import units -import glance_store from glance_store._drivers import cinder from glance_store import exceptions from glance_store import location @@ -39,6 +49,140 @@ class TestCinderStore(base.StoreBaseTest, self.store = cinder.Store(self.conf) self.store.configure() self.register_store_schemes(self.store, 'cinder') + self.store.READ_CHUNKSIZE = 4096 + self.store.WRITE_CHUNKSIZE = 4096 + + fake_sc = [{u'endpoints': [{u'publicURL': u'http://foo/public_url'}], + u'endpoints_links': [], + u'name': u'cinder', + u'type': u'volumev2'}] + self.context = FakeObject(service_catalog=fake_sc, + user='fake_user', + auth_token='fake_token', + tenant='fake_tenant') + + def test_get_cinderclient(self): + cc = cinder.get_cinderclient(self.conf, self.context) + self.assertEqual('fake_token', cc.client.auth_token) + self.assertEqual('http://foo/public_url', cc.client.management_url) + + def test_get_cinderclient_with_user_overriden(self): + self.config(cinder_store_user_name='test_user') + self.config(cinder_store_password='test_password') + self.config(cinder_store_project_name='test_project') + self.config(cinder_store_auth_address='test_address') + cc = cinder.get_cinderclient(self.conf, self.context) + self.assertIsNone(cc.client.auth_token) + self.assertEqual('test_address', cc.client.management_url) + + def test_temporary_chown(self): + class fake_stat(object): + st_uid = 1 + + with mock.patch.object(os, 'stat', return_value=fake_stat()), \ + mock.patch.object(os, 'getuid', return_value=2), \ + mock.patch.object(processutils, 'execute') as mock_execute, \ + mock.patch.object(cinder, 'get_root_helper', + return_value='sudo'): + with cinder.temporary_chown('test'): + pass + expected_calls = [mock.call('chown', 2, 'test', run_as_root=True, + root_helper='sudo'), + mock.call('chown', 1, 'test', run_as_root=True, + root_helper='sudo')] + self.assertEqual(expected_calls, mock_execute.call_args_list) + + @mock.patch.object(time, 'sleep') + def test_wait_volume_status(self, mock_sleep): + fake_manager = FakeObject(get=mock.Mock()) + volume_available = FakeObject(manager=fake_manager, + id='fake-id', + status='available') + volume_in_use = FakeObject(manager=fake_manager, + id='fake-id', + status='in-use') + fake_manager.get.side_effect = [volume_available, volume_in_use] + self.assertEqual(volume_in_use, + self.store._wait_volume_status( + volume_available, 'available', 'in-use')) + fake_manager.get.assert_called_with('fake-id') + mock_sleep.assert_called_once_with(0.5) + + @mock.patch.object(time, 'sleep') + def test_wait_volume_status_unexpected(self, mock_sleep): + fake_manager = FakeObject(get=mock.Mock()) + volume_available = FakeObject(manager=fake_manager, + id='fake-id', + status='error') + fake_manager.get.return_value = volume_available + self.assertRaises(exceptions.BackendException, + self.store._wait_volume_status, + volume_available, 'available', 'in-use') + fake_manager.get.assert_called_with('fake-id') + + @mock.patch.object(time, 'sleep') + def test_wait_volume_status_timeout(self, mock_sleep): + fake_manager = FakeObject(get=mock.Mock()) + volume_available = FakeObject(manager=fake_manager, + id='fake-id', + status='available') + fake_manager.get.return_value = volume_available + self.assertRaises(exceptions.BackendException, + self.store._wait_volume_status, + volume_available, 'available', 'in-use') + fake_manager.get.assert_called_with('fake-id') + + def _test_open_cinder_volume(self, open_mode, attach_mode, error): + fake_volume = mock.MagicMock(id=str(uuid.uuid4()), status='available') + fake_volumes = FakeObject(get=lambda id: fake_volume, + detach=mock.Mock()) + fake_client = FakeObject(volumes=fake_volumes) + _, fake_dev_path = tempfile.mkstemp(dir=self.test_dir) + fake_devinfo = {'path': fake_dev_path} + fake_connector = FakeObject( + connect_volume=mock.Mock(return_value=fake_devinfo), + disconnect_volume=mock.Mock()) + + @contextlib.contextmanager + def fake_chown(path): + yield + + def do_open(): + with self.store._open_cinder_volume( + fake_client, fake_volume, open_mode): + if error: + raise error + + with mock.patch.object(cinder.Store, + '_wait_volume_status', + return_value=fake_volume), \ + mock.patch.object(cinder, 'temporary_chown', + side_effect=fake_chown), \ + mock.patch.object(cinder, 'get_root_helper'), \ + mock.patch.object(connector, 'get_connector_properties'), \ + mock.patch.object(connector.InitiatorConnector, 'factory', + return_value=fake_connector): + + if error: + self.assertRaises(error, do_open) + else: + do_open() + + fake_connector.connect_volume.assert_called_once_with(mock.ANY) + fake_connector.disconnect_volume.assert_called_once_with( + mock.ANY, fake_devinfo) + fake_volume.attach.assert_called_once_with( + None, None, attach_mode, host_name=socket.gethostname()) + fake_volumes.detach.assert_called_once_with(fake_volume) + + def test_open_cinder_volume_rw(self): + self._test_open_cinder_volume('wb', 'rw', None) + + def test_open_cinder_volume_ro(self): + self._test_open_cinder_volume('rb', 'ro', None) + + def test_open_cinder_volume_error(self): + self._test_open_cinder_volume('wb', 'rw', IOError) def test_cinder_configure_add(self): self.assertRaises(exceptions.BadStoreConfiguration, @@ -50,9 +194,46 @@ class TestCinderStore(base.StoreBaseTest, self.store._check_context(FakeObject(service_catalog='fake')) + def test_cinder_get(self): + expected_size = 5 * units.Ki + expected_file_contents = b"*" * expected_size + volume_file = six.BytesIO(expected_file_contents) + fake_client = FakeObject(auth_token=None, management_url=None) + fake_volume_uuid = str(uuid.uuid4()) + fake_volume = mock.MagicMock(id=fake_volume_uuid, + metadata={'image_size': expected_size}, + status='available') + fake_volume.manager.get.return_value = fake_volume + fake_volumes = FakeObject(get=lambda id: fake_volume) + + @contextlib.contextmanager + def fake_open(client, volume, mode): + self.assertEqual(mode, 'rb') + yield volume_file + + with mock.patch.object(cinder, 'get_cinderclient') as mock_cc, \ + mock.patch.object(self.store, '_open_cinder_volume', + side_effect=fake_open): + mock_cc.return_value = FakeObject(client=fake_client, + volumes=fake_volumes) + uri = "cinder://%s" % fake_volume_uuid + loc = location.get_location_from_uri(uri, conf=self.conf) + (image_file, image_size) = self.store.get(loc, + context=self.context) + + expected_num_chunks = 2 + data = b"" + num_chunks = 0 + + for chunk in image_file: + num_chunks += 1 + data += chunk + self.assertEqual(expected_num_chunks, num_chunks) + self.assertEqual(expected_file_contents, data) + def test_cinder_get_size(self): fake_client = FakeObject(auth_token=None, management_url=None) - fake_volume_uuid = '12345678-9012-3455-6789-012345678901' + fake_volume_uuid = str(uuid.uuid4()) fake_volume = FakeObject(size=5) fake_volumes = {fake_volume_uuid: fake_volume} @@ -60,31 +241,81 @@ class TestCinderStore(base.StoreBaseTest, mocked_cc.return_value = FakeObject(client=fake_client, volumes=fake_volumes) - fake_sc = [{u'endpoints': [{u'publicURL': u'foo_public_url'}], - u'endpoints_links': [], - u'name': u'cinder', - u'type': u'volume'}] - fake_context = FakeObject(service_catalog=fake_sc, - user='fake_uer', - auth_tok='fake_token', - tenant='fake_tenant') - uri = 'cinder://%s' % fake_volume_uuid loc = location.get_location_from_uri(uri, conf=self.conf) - image_size = self.store.get_size(loc, context=fake_context) + image_size = self.store.get_size(loc, context=self.context) self.assertEqual(image_size, fake_volume.size * units.Gi) - def test_cinder_delete_raise_error(self): - uri = 'cinder://12345678-9012-3455-6789-012345678901' - loc = location.get_location_from_uri(uri, conf=self.conf) - self.assertRaises(exceptions.StoreDeleteNotSupported, - self.store.delete, loc) - self.assertRaises(exceptions.StoreDeleteNotSupported, - glance_store.delete_from_backend, uri, {}) - - def test_cinder_add_raise_error(self): - self.assertRaises(exceptions.StoreAddDisabled, - self.store.add, None, None, None, None) - self.assertRaises(exceptions.StoreAddDisabled, - glance_store.add_to_backend, None, None, - None, None, 'cinder') + def _test_cinder_add(self, fake_volume, volume_file, size_kb=5, + verifier=None): + expected_image_id = str(uuid.uuid4()) + expected_size = size_kb * units.Ki + expected_file_contents = b"*" * expected_size + image_file = six.BytesIO(expected_file_contents) + expected_checksum = hashlib.md5(expected_file_contents).hexdigest() + expected_location = 'cinder://%s' % fake_volume.id + fake_client = FakeObject(auth_token=None, management_url=None) + fake_volume.manager.get.return_value = fake_volume + fake_volumes = FakeObject(create=mock.Mock(return_value=fake_volume)) + + @contextlib.contextmanager + def fake_open(client, volume, mode): + self.assertEqual(mode, 'wb') + yield volume_file + + with mock.patch.object(cinder, 'get_cinderclient') as mock_cc, \ + mock.patch.object(self.store, '_open_cinder_volume', + side_effect=fake_open): + mock_cc.return_value = FakeObject(client=fake_client, + volumes=fake_volumes) + loc, size, checksum, _ = self.store.add(expected_image_id, + image_file, + expected_size, + self.context, + verifier) + self.assertEqual(expected_location, loc) + self.assertEqual(expected_size, size) + self.assertEqual(expected_checksum, checksum) + fake_volumes.create.assert_called_once_with( + 1, + name='image-%s' % expected_image_id, + metadata={'image_owner': self.context.tenant, + 'glance_image_id': expected_image_id, + 'image_size': str(expected_size)}) + + def test_cinder_add(self): + fake_volume = mock.MagicMock(id=str(uuid.uuid4()), status='available') + volume_file = six.BytesIO() + self._test_cinder_add(fake_volume, volume_file) + + def test_cinder_add_with_verifier(self): + fake_volume = mock.MagicMock(id=str(uuid.uuid4()), status='available') + volume_file = six.BytesIO() + verifier = mock.MagicMock() + self._test_cinder_add(fake_volume, volume_file, 1, verifier) + verifier.update.assert_called_with(b"*" * units.Ki) + + def test_cinder_add_volume_full(self): + e = IOError() + volume_file = six.BytesIO() + e.errno = errno.ENOSPC + fake_volume = mock.MagicMock(id=str(uuid.uuid4()), status='available') + with mock.patch.object(volume_file, 'write', side_effect=e): + self.assertRaises(exceptions.StorageFull, + self._test_cinder_add, fake_volume, volume_file) + fake_volume.delete.assert_called_once_with() + + def test_cinder_delete(self): + fake_client = FakeObject(auth_token=None, management_url=None) + fake_volume_uuid = str(uuid.uuid4()) + fake_volume = FakeObject(delete=mock.Mock()) + fake_volumes = {fake_volume_uuid: fake_volume} + + with mock.patch.object(cinder, 'get_cinderclient') as mocked_cc: + mocked_cc.return_value = FakeObject(client=fake_client, + volumes=fake_volumes) + + uri = 'cinder://%s' % fake_volume_uuid + loc = location.get_location_from_uri(uri, conf=self.conf) + self.store.delete(loc, context=self.context) + fake_volume.delete.assert_called_once_with() diff --git a/glance_store/tests/unit/test_http_store.py b/glance_store/tests/unit/test_http_store.py index 3617d1f..967a9fe 100644 --- a/glance_store/tests/unit/test_http_store.py +++ b/glance_store/tests/unit/test_http_store.py @@ -15,7 +15,7 @@ import mock -from six.moves import http_client +import requests import glance_store from glance_store._drivers import http @@ -36,25 +36,19 @@ class TestHttpStore(base.StoreBaseTest, self.store = http.Store(self.conf) self.register_store_schemes(self.store, 'http') - def _mock_httplib(self): - """Mock httplib connection object. + def _mock_requests(self): + """Mock requests session object. - Should be called when need to mock httplib response and request - objects. + Should be called when we need to mock request/response objects. """ - response = mock.patch('six.moves.http_client' - '.HTTPConnection.getresponse') - self.response = response.start() - self.response.return_value = utils.FakeHTTPResponse() - self.addCleanup(response.stop) - - request = mock.patch('six.moves.http_client.HTTPConnection.request') + request = mock.patch('requests.Session.request') self.request = request.start() - self.request.side_effect = lambda w, x, y, z: None self.addCleanup(request.stop) def test_http_get(self): - self._mock_httplib() + self._mock_requests() + self.request.return_value = utils.fake_response() + uri = "http://netloc/path/to/file.tar.gz" expected_returns = ['I ', 'am', ' a', ' t', 'ea', 'po', 't,', ' s', 'ho', 'rt', ' a', 'nd', ' s', 'to', 'ut', '\n'] @@ -74,16 +68,16 @@ class TestHttpStore(base.StoreBaseTest, # Add two layers of redirects to the response stack, which will # return the default 200 OK with the expected data after resolving # both redirects. - self._mock_httplib() + self._mock_requests() redirect1 = {"location": "http://example.com/teapot.img"} redirect2 = {"location": "http://example.com/teapot_real.img"} - responses = [utils.FakeHTTPResponse(status=302, headers=redirect1), - utils.FakeHTTPResponse(status=301, headers=redirect2), - utils.FakeHTTPResponse()] + responses = [utils.fake_response(), + utils.fake_response(status_code=301, headers=redirect2), + utils.fake_response(status_code=302, headers=redirect1)] - def getresponse(): + def getresponse(*args, **kwargs): return responses.pop() - self.response.side_effect = getresponse + self.request.side_effect = getresponse uri = "http://netloc/path/to/file.tar.gz" expected_returns = ['I ', 'am', ' a', ' t', 'ea', 'po', 't,', ' s', @@ -91,46 +85,49 @@ class TestHttpStore(base.StoreBaseTest, loc = location.get_location_from_uri(uri, conf=self.conf) (image_file, image_size) = self.store.get(loc) + self.assertEqual(0, len(responses)) self.assertEqual(image_size, 31) chunks = [c for c in image_file] self.assertEqual(chunks, expected_returns) def test_http_get_max_redirects(self): - self._mock_httplib() + self._mock_requests() redirect = {"location": "http://example.com/teapot.img"} - responses = ([utils.FakeHTTPResponse(status=302, headers=redirect)] + responses = ([utils.fake_response(status_code=302, headers=redirect)] * (http.MAX_REDIRECTS + 2)) - def getresponse(): + def getresponse(*args, **kwargs): return responses.pop() - self.response.side_effect = getresponse + self.request.side_effect = getresponse uri = "http://netloc/path/to/file.tar.gz" loc = location.get_location_from_uri(uri, conf=self.conf) self.assertRaises(exceptions.MaxRedirectsExceeded, self.store.get, loc) def test_http_get_redirect_invalid(self): - self._mock_httplib() + self._mock_requests() redirect = {"location": "http://example.com/teapot.img"} - redirect_resp = utils.FakeHTTPResponse(status=307, headers=redirect) - self.response.return_value = redirect_resp + redirect_resp = utils.fake_response(status_code=307, headers=redirect) + self.request.return_value = redirect_resp uri = "http://netloc/path/to/file.tar.gz" loc = location.get_location_from_uri(uri, conf=self.conf) self.assertRaises(exceptions.BadStoreUri, self.store.get, loc) def test_http_get_not_found(self): - self._mock_httplib() - fake = utils.FakeHTTPResponse(status=404, data="404 Not Found") - self.response.return_value = fake + self._mock_requests() + fake = utils.fake_response(status_code=404, content="404 Not Found") + self.request.return_value = fake uri = "http://netloc/path/to/file.tar.gz" loc = location.get_location_from_uri(uri, conf=self.conf) self.assertRaises(exceptions.NotFound, self.store.get, loc) def test_http_delete_raise_error(self): - self._mock_httplib() + self._mock_requests() + self.request.return_value = utils.fake_response() + uri = "https://netloc/path/to/file.tar.gz" loc = location.get_location_from_uri(uri, conf=self.conf) self.assertRaises(exceptions.StoreDeleteNotSupported, @@ -146,17 +143,21 @@ class TestHttpStore(base.StoreBaseTest, None, None, 'http') def test_http_get_size_with_non_existent_image_raises_Not_Found(self): - self._mock_httplib() - fake = utils.FakeHTTPResponse(status=404, data="404 Not Found") - self.response.return_value = fake + self._mock_requests() + self.request.return_value = utils.fake_response( + status_code=404, content='404 Not Found') uri = "http://netloc/path/to/file.tar.gz" loc = location.get_location_from_uri(uri, conf=self.conf) self.assertRaises(exceptions.NotFound, self.store.get_size, loc) + self.request.assert_called_once_with('HEAD', uri, stream=True, + allow_redirects=False) def test_http_get_size_bad_status_line(self): - self._mock_httplib() - self.response.side_effect = http_client.BadStatusLine(line='') + self._mock_requests() + # Note(sabari): Low-level httplib.BadStatusLine will be raised as + # ConnectionErorr after migrating to requests. + self.request.side_effect = requests.exceptions.ConnectionError uri = "http://netloc/path/to/file.tar.gz" loc = location.get_location_from_uri(uri, conf=self.conf) diff --git a/glance_store/tests/unit/test_opts.py b/glance_store/tests/unit/test_opts.py index 658fd07..a8eab08 100644 --- a/glance_store/tests/unit/test_opts.py +++ b/glance_store/tests/unit/test_opts.py @@ -64,17 +64,26 @@ class OptsTestCase(base.StoreBaseTest): 'cinder_catalog_info', 'cinder_endpoint_template', 'cinder_http_retries', + 'cinder_os_region_name', + 'cinder_state_transition_timeout', + 'cinder_store_auth_address', + 'cinder_store_user_name', + 'cinder_store_password', + 'cinder_store_project_name', 'default_swift_reference', + 'https_insecure', 'filesystem_store_datadir', 'filesystem_store_datadirs', 'filesystem_store_file_perm', 'filesystem_store_metadata_file', - 'os_region_name', + 'http_proxy_information', + 'https_ca_certificates_file', 'rbd_store_ceph_conf', 'rbd_store_chunk_size', 'rbd_store_pool', 'rbd_store_user', 'rados_connect_timeout', + 'rootwrap_config', 's3_store_access_key', 's3_store_bucket', 's3_store_bucket_url_format', diff --git a/glance_store/tests/unit/test_swift_store.py b/glance_store/tests/unit/test_swift_store.py index 33abe0a..cf1e3d3 100644 --- a/glance_store/tests/unit/test_swift_store.py +++ b/glance_store/tests/unit/test_swift_store.py @@ -1520,8 +1520,61 @@ class TestMultiTenantStoreConnections(base.StoreBaseTest): self.location = swift.StoreLocation(specs, self.conf) self.addCleanup(self.conf.reset) - def test_basic_connection(self): + def test_basic_connection_no_catalog(self): + self.store.configure() + connection = self.store.get_connection(self.location, + context=self.context) + self.assertIsNone(connection.authurl) + self.assertEqual(connection.auth_version, '1') + self.assertIsNone(connection.user) + self.assertIsNone(connection.tenant_name) + self.assertIsNone(connection.key) + self.assertEqual(connection.preauthurl, 'https://example.com') + self.assertEqual(connection.preauthtoken, '0123') + self.assertEqual(connection.os_options, {}) + + def test_connection_with_endpoint_from_catalog(self): + self.store.configure() + self.context.service_catalog = [ + { + 'endpoint_links': [], + 'endpoints': [ + { + 'region': 'RegionOne', + 'publicURL': 'https://scexample.com', + }, + ], + 'type': 'object-store', + 'name': 'Object Storage Service', + } + ] + connection = self.store.get_connection(self.location, + context=self.context) + self.assertIsNone(connection.authurl) + self.assertEqual(connection.auth_version, '1') + self.assertIsNone(connection.user) + self.assertIsNone(connection.tenant_name) + self.assertIsNone(connection.key) + self.assertEqual(connection.preauthurl, 'https://scexample.com') + self.assertEqual(connection.preauthtoken, '0123') + self.assertEqual(connection.os_options, {}) + + def test_connection_with_no_endpoint_found(self): self.store.configure() + self.context.service_catalog = [ + { + 'endpoint_links': [], + 'endpoints': [ + { + 'region': 'RegionOne', + 'publicURL': 'https://scexample.com', + }, + ], + 'type': 'object-store', + 'name': 'Object Storage Service', + } + ] + self.store.service_type = 'incorrect-store' connection = self.store.get_connection(self.location, context=self.context) self.assertIsNone(connection.authurl) diff --git a/glance_store/tests/unit/test_vmware_store.py b/glance_store/tests/unit/test_vmware_store.py index 35507a1..3f95adb 100644 --- a/glance_store/tests/unit/test_vmware_store.py +++ b/glance_store/tests/unit/test_vmware_store.py @@ -65,24 +65,6 @@ def format_location(host_ip, folder_name, image_id, datastores): image_id, datacenter_path, datastore_name)) -class FakeHTTPConnection(object): - - def __init__(self, status=200, *args, **kwargs): - self.status = status - self.no_response_body = kwargs.get('no_response_body', False) - pass - - def getresponse(self): - return utils.FakeHTTPResponse(status=self.status, - no_response_body=self.no_response_body) - - def request(self, *_args, **_kwargs): - pass - - def close(self): - pass - - def fake_datastore_obj(*args, **kwargs): dc_obj = oslo_datacenter.Datacenter(ref='fake-ref', name='fake-name') @@ -130,8 +112,8 @@ class TestStore(base.StoreBaseTest, loc = location.get_location_from_uri( "vsphere://127.0.0.1/folder/openstack_glance/%s" "?dsName=ds1&dcPath=dc1" % FAKE_UUID, conf=self.conf) - with self._mock_http_connection() as HttpConn: - HttpConn.return_value = FakeHTTPConnection() + with mock.patch('requests.Session.request') as HttpConn: + HttpConn.return_value = utils.fake_response() (image_file, image_size) = self.store.get(loc) self.assertEqual(image_size, expected_image_size) chunks = [c for c in image_file] @@ -146,8 +128,8 @@ class TestStore(base.StoreBaseTest, loc = location.get_location_from_uri( "vsphere://127.0.0.1/folder/openstack_glan" "ce/%s?dsName=ds1&dcPath=dc1" % FAKE_UUID, conf=self.conf) - with self._mock_http_connection() as HttpConn: - HttpConn.return_value = FakeHTTPConnection(status=404) + with mock.patch('requests.Session.request') as HttpConn: + HttpConn.return_value = utils.fake_response(status_code=404) self.assertRaises(exceptions.NotFound, self.store.get, loc) @mock.patch.object(vm_store.Store, 'select_datastore') @@ -170,8 +152,8 @@ class TestStore(base.StoreBaseTest, expected_image_id, VMWARE_DS['vmware_datastores']) image = six.BytesIO(expected_contents) - with self._mock_http_connection() as HttpConn: - HttpConn.return_value = FakeHTTPConnection() + with mock.patch('requests.Session.request') as HttpConn: + HttpConn.return_value = utils.fake_response() location, size, checksum, _ = self.store.add(expected_image_id, image, expected_size) @@ -204,8 +186,8 @@ class TestStore(base.StoreBaseTest, expected_image_id, VMWARE_DS['vmware_datastores']) image = six.BytesIO(expected_contents) - with self._mock_http_connection() as HttpConn: - HttpConn.return_value = FakeHTTPConnection() + with mock.patch('requests.Session.request') as HttpConn: + HttpConn.return_value = utils.fake_response() location, size, checksum, _ = self.store.add(expected_image_id, image, 0) self.assertEqual(utils.sort_url_by_qs_keys(expected_location), @@ -222,14 +204,14 @@ class TestStore(base.StoreBaseTest, size = FIVE_KB contents = b"*" * size image = six.BytesIO(contents) - with self._mock_http_connection() as HttpConn: - HttpConn.return_value = FakeHTTPConnection() + with mock.patch('requests.Session.request') as HttpConn: + HttpConn.return_value = utils.fake_response() self.store.add(image_id, image, size, verifier=verifier) fake_reader.assert_called_with(image, verifier) @mock.patch.object(vm_store.Store, 'select_datastore') - @mock.patch('glance_store._drivers.vmware_datastore._ChunkReader') + @mock.patch('glance_store._drivers.vmware_datastore._Reader') def test_add_with_verifier_size_zero(self, fake_reader, fake_select_ds): """Test that the verifier is passed to the _ChunkReader during add.""" verifier = mock.MagicMock(name='mock_verifier') @@ -237,8 +219,8 @@ class TestStore(base.StoreBaseTest, size = FIVE_KB contents = b"*" * size image = six.BytesIO(contents) - with self._mock_http_connection() as HttpConn: - HttpConn.return_value = FakeHTTPConnection() + with mock.patch('requests.Session.request') as HttpConn: + HttpConn.return_value = utils.fake_response() self.store.add(image_id, image, 0, verifier=verifier) fake_reader.assert_called_with(image, verifier) @@ -249,12 +231,12 @@ class TestStore(base.StoreBaseTest, loc = location.get_location_from_uri( "vsphere://127.0.0.1/folder/openstack_glance/%s?" "dsName=ds1&dcPath=dc1" % FAKE_UUID, conf=self.conf) - with self._mock_http_connection() as HttpConn: - HttpConn.return_value = FakeHTTPConnection() + with mock.patch('requests.Session.request') as HttpConn: + HttpConn.return_value = utils.fake_response() vm_store.Store._service_content = mock.Mock() self.store.delete(loc) - with self._mock_http_connection() as HttpConn: - HttpConn.return_value = FakeHTTPConnection(status=404) + with mock.patch('requests.Session.request') as HttpConn: + HttpConn.return_value = utils.fake_response(status_code=404) self.assertRaises(exceptions.NotFound, self.store.get, loc) @mock.patch('oslo_vmware.api.VMwareAPISession') @@ -278,8 +260,8 @@ class TestStore(base.StoreBaseTest, loc = location.get_location_from_uri( "vsphere://127.0.0.1/folder/openstack_glance/%s" "?dsName=ds1&dcPath=dc1" % FAKE_UUID, conf=self.conf) - with self._mock_http_connection() as HttpConn: - HttpConn.return_value = FakeHTTPConnection() + with mock.patch('requests.Session.request') as HttpConn: + HttpConn.return_value = utils.fake_response() image_size = self.store.get_size(loc) self.assertEqual(image_size, 31) @@ -292,8 +274,8 @@ class TestStore(base.StoreBaseTest, loc = location.get_location_from_uri( "vsphere://127.0.0.1/folder/openstack_glan" "ce/%s?dsName=ds1&dcPath=dc1" % FAKE_UUID, conf=self.conf) - with self._mock_http_connection() as HttpConn: - HttpConn.return_value = FakeHTTPConnection(status=404) + with mock.patch('requests.Session.request') as HttpConn: + HttpConn.return_value = utils.fake_response(status_code=404) self.assertRaises(exceptions.NotFound, self.store.get_size, loc) def test_reader_full(self): @@ -324,76 +306,6 @@ class TestStore(base.StoreBaseTest, reader.read() verifier.update.assert_called_with(content) - def test_chunkreader_image_fits_in_blocksize(self): - """ - Test that the image file reader returns the expected chunk of data - when the block size is larger than the image. - """ - content = b'XXX' - image = six.BytesIO(content) - expected_checksum = hashlib.md5(content).hexdigest() - reader = vm_store._ChunkReader(image) - ret = reader.read() - if six.PY3: - expected_chunk = ('%x\r\n%s\r\n' - % (len(content), content.decode('ascii'))) - expected_chunk = expected_chunk.encode('ascii') - else: - expected_chunk = b'%x\r\n%s\r\n' % (len(content), content) - last_chunk = b'0\r\n\r\n' - self.assertEqual(expected_chunk + last_chunk, ret) - self.assertEqual(len(content), reader.size) - self.assertEqual(expected_checksum, reader.checksum.hexdigest()) - self.assertTrue(reader.closed) - ret = reader.read() - self.assertEqual(len(content), reader.size) - self.assertEqual(expected_checksum, reader.checksum.hexdigest()) - self.assertTrue(reader.closed) - self.assertEqual(b'', ret) - - def test_chunkreader_image_larger_blocksize(self): - """ - Test that the image file reader returns the expected chunks when - the block size specified is smaller than the image. - """ - content = b'XXX' - image = six.BytesIO(content) - expected_checksum = hashlib.md5(content).hexdigest() - last_chunk = b'0\r\n\r\n' - reader = vm_store._ChunkReader(image, blocksize=1) - ret = reader.read() - expected_chunk = b'1\r\nX\r\n' - expected = (expected_chunk + expected_chunk + expected_chunk - + last_chunk) - self.assertEqual(expected, - ret) - self.assertEqual(expected_checksum, reader.checksum.hexdigest()) - self.assertEqual(len(content), reader.size) - self.assertTrue(reader.closed) - - def test_chunkreader_size(self): - """Test that the image reader takes into account the specified size.""" - content = b'XXX' - image = six.BytesIO(content) - expected_checksum = hashlib.md5(content).hexdigest() - reader = vm_store._ChunkReader(image, blocksize=1) - ret = reader.read(size=3) - self.assertEqual(b'1\r\n', ret) - ret = reader.read(size=1) - self.assertEqual(b'X', ret) - ret = reader.read() - self.assertEqual(expected_checksum, reader.checksum.hexdigest()) - self.assertEqual(len(content), reader.size) - self.assertTrue(reader.closed) - - def test_chunkreader_with_verifier(self): - content = b'XXX' - image = six.BytesIO(content) - verifier = mock.MagicMock(name='mock_verifier') - reader = vm_store._ChunkReader(image, verifier) - reader.read(size=3) - verifier.update.assert_called_with(content) - def test_sanity_check_api_retry_count(self): """Test that sanity check raises if api_retry_count is <= 0.""" self.store.conf.glance_store.vmware_api_retry_count = -1 @@ -475,8 +387,8 @@ class TestStore(base.StoreBaseTest, expected_contents = b"*" * expected_size image = six.BytesIO(expected_contents) self.session = mock.Mock() - with self._mock_http_connection() as HttpConn: - HttpConn.return_value = FakeHTTPConnection(status=401) + with mock.patch('requests.Session.request') as HttpConn: + HttpConn.return_value = utils.fake_response(status_code=401) self.assertRaises(exceptions.BackendException, self.store.add, expected_image_id, image, expected_size) @@ -491,8 +403,8 @@ class TestStore(base.StoreBaseTest, image = six.BytesIO(expected_contents) self.session = mock.Mock() with self._mock_http_connection() as HttpConn: - HttpConn.return_value = FakeHTTPConnection(status=500, - no_response_body=True) + HttpConn.return_value = utils.fake_response(status_code=500, + no_response_body=True) self.assertRaises(exceptions.BackendException, self.store.add, expected_image_id, image, expected_size) @@ -532,7 +444,7 @@ class TestStore(base.StoreBaseTest, expected_contents = b"*" * expected_size image = six.BytesIO(expected_contents) self.session = mock.Mock() - with self._mock_http_connection() as HttpConn: + with mock.patch('requests.Session.request') as HttpConn: HttpConn.request.side_effect = IOError self.assertRaises(exceptions.BackendException, self.store.add, @@ -660,3 +572,58 @@ class TestStore(base.StoreBaseTest, 'FindByInventoryPath', self.store.session.vim.service_content.searchIndex, inventoryPath=datacenter_path) + + @mock.patch('oslo_vmware.api.VMwareAPISession') + def test_http_get_redirect(self, mock_api_session): + # Add two layers of redirects to the response stack, which will + # return the default 200 OK with the expected data after resolving + # both redirects. + redirect1 = {"location": "https://example.com?dsName=ds1&dcPath=dc1"} + redirect2 = {"location": "https://example.com?dsName=ds2&dcPath=dc2"} + responses = [utils.fake_response(), + utils.fake_response(status_code=302, headers=redirect1), + utils.fake_response(status_code=301, headers=redirect2)] + + def getresponse(*args, **kwargs): + return responses.pop() + + expected_image_size = 31 + expected_returns = ['I am a teapot, short and stout\n'] + loc = location.get_location_from_uri( + "vsphere://127.0.0.1/folder/openstack_glance/%s" + "?dsName=ds1&dcPath=dc1" % FAKE_UUID, conf=self.conf) + with mock.patch('requests.Session.request') as HttpConn: + HttpConn.side_effect = getresponse + (image_file, image_size) = self.store.get(loc) + self.assertEqual(image_size, expected_image_size) + chunks = [c for c in image_file] + self.assertEqual(expected_returns, chunks) + + @mock.patch('oslo_vmware.api.VMwareAPISession') + def test_http_get_max_redirects(self, mock_api_session): + redirect = {"location": "https://example.com?dsName=ds1&dcPath=dc1"} + responses = ([utils.fake_response(status_code=302, headers=redirect)] + * (vm_store.MAX_REDIRECTS + 1)) + + def getresponse(*args, **kwargs): + return responses.pop() + + loc = location.get_location_from_uri( + "vsphere://127.0.0.1/folder/openstack_glance/%s" + "?dsName=ds1&dcPath=dc1" % FAKE_UUID, conf=self.conf) + with mock.patch('requests.Session.request') as HttpConn: + HttpConn.side_effect = getresponse + self.assertRaises(exceptions.MaxRedirectsExceeded, self.store.get, + loc) + + @mock.patch('oslo_vmware.api.VMwareAPISession') + def test_http_get_redirect_invalid(self, mock_api_session): + redirect = {"location": "https://example.com?dsName=ds1&dcPath=dc1"} + + loc = location.get_location_from_uri( + "vsphere://127.0.0.1/folder/openstack_glance/%s" + "?dsName=ds1&dcPath=dc1" % FAKE_UUID, conf=self.conf) + with mock.patch('requests.Session.request') as HttpConn: + HttpConn.return_value = utils.fake_response(status_code=307, + headers=redirect) + self.assertRaises(exceptions.BadStoreUri, self.store.get, loc) diff --git a/glance_store/tests/utils.py b/glance_store/tests/utils.py index 0d9bef2..2f3a90f 100644 --- a/glance_store/tests/utils.py +++ b/glance_store/tests/utils.py @@ -16,6 +16,8 @@ import six from six.moves import urllib +import requests + def sort_url_by_qs_keys(url): # NOTE(kragniz): this only sorts the keys of the query string of a url. @@ -57,3 +59,17 @@ class FakeHTTPResponse(object): def read(self, amt): self.data.read(amt) + + def release_conn(self): + pass + + def close(self): + self.data.close() + + +def fake_response(status_code=200, headers=None, content=None, **kwargs): + r = requests.models.Response() + r.status_code = status_code + r.headers = headers or {} + r.raw = FakeHTTPResponse(status_code, headers, content, kwargs) + return r diff --git a/releasenotes/notes/support-cinder-upload-c85849d9c88bbd7e.yaml b/releasenotes/notes/support-cinder-upload-c85849d9c88bbd7e.yaml new file mode 100644 index 0000000..06327ff --- /dev/null +++ b/releasenotes/notes/support-cinder-upload-c85849d9c88bbd7e.yaml @@ -0,0 +1,8 @@ +--- +features: + - Implemented image uploading, downloading and deletion for cinder store. + It also supports new settings to put image volumes into a specific project + to hide them from users and to control them based on ACL of the images. + Note that cinder store is currently considered experimental, so + current deployers should be aware that the use of it in production right + now may be risky. diff --git a/releasenotes/notes/vmware-store-requests-369485d2cfdb6175.yaml b/releasenotes/notes/vmware-store-requests-369485d2cfdb6175.yaml new file mode 100644 index 0000000..060f3e5 --- /dev/null +++ b/releasenotes/notes/vmware-store-requests-369485d2cfdb6175.yaml @@ -0,0 +1,6 @@ +--- +security: + - Previously the VMWare Datastore was using HTTPS Connections from httplib + which do not verify the connection. By switching to using requests library + the VMware storage backend now verifies HTTPS connection to vCenter server + and thus addresses the vulnerabilities described in OSSN-0033. diff --git a/requirements.txt b/requirements.txt index ba89d45..f102881 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,17 +1,18 @@ # The order of packages is significant, because pip processes them in the order # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -oslo.config>=3.4.0 # Apache-2.0 +oslo.config>=3.7.0 # Apache-2.0 oslo.i18n>=2.1.0 # Apache-2.0 oslo.serialization>=1.10.0 # Apache-2.0 -oslo.utils>=3.4.0 # Apache-2.0 -oslo.concurrency>=2.3.0 # Apache-2.0 +oslo.utils>=3.5.0 # Apache-2.0 +oslo.concurrency>=3.5.0 # Apache-2.0 stevedore>=1.5.0 # Apache-2.0 enum34;python_version=='2.7' or python_version=='2.6' or python_version=='3.3' # BSD -eventlet>=0.18.2 # MIT +eventlet!=0.18.3,>=0.18.2 # MIT six>=1.9.0 # MIT debtcollector>=1.2.0 # Apache-2.0 jsonschema!=2.5.0,<3.0.0,>=2.0.0 # MIT python-keystoneclient!=1.8.0,!=2.1.0,>=1.6.0 # Apache-2.0 +requests!=2.9.0,>=2.8.1 # Apache-2.0 @@ -50,6 +50,9 @@ glance_store.drivers = oslo.config.opts = glance.store = glance_store.backend:_list_opts +console_scripts = + glance-rootwrap = oslo_rootwrap.cmd:main + [extras] # Dependencies for each of the optional stores s3 = @@ -61,6 +64,8 @@ swift = python-swiftclient>=2.2.0 # Apache-2.0 cinder = python-cinderclient>=1.3.1 # Apache-2.0 + os-brick>=1.0.0 # Apache-2.0 + oslo.rootwrap>=2.0.0 # Apache-2.0 [build_sphinx] source-dir = doc/source |