diff options
author | Zuul <zuul@review.opendev.org> | 2019-06-06 16:33:22 +0000 |
---|---|---|
committer | Gerrit Code Review <review@openstack.org> | 2019-06-06 16:33:22 +0000 |
commit | 6266ddf45f4d006cd7a8304224ec0ba0774489c1 (patch) | |
tree | 92eb1902a8a8097739962c59774b73d27dd65162 | |
parent | 0c73abc6614898eb75663337c05e78fa9cd5b659 (diff) | |
parent | d5dc27e7d389cd00ece53e62f2b1e643ea115fd3 (diff) | |
download | glance_store-6266ddf45f4d006cd7a8304224ec0ba0774489c1.tar.gz |
Merge "Add location prefix url to store instance"0.29.0
-rw-r--r-- | glance_store/_drivers/cinder.py | 5 | ||||
-rw-r--r-- | glance_store/_drivers/filesystem.py | 6 | ||||
-rw-r--r-- | glance_store/_drivers/http.py | 10 | ||||
-rw-r--r-- | glance_store/_drivers/rbd.py | 17 | ||||
-rw-r--r-- | glance_store/_drivers/sheepdog.py | 7 | ||||
-rw-r--r-- | glance_store/_drivers/swift/store.py | 55 | ||||
-rw-r--r-- | glance_store/_drivers/vmware_datastore.py | 13 | ||||
-rw-r--r-- | glance_store/driver.py | 5 | ||||
-rw-r--r-- | glance_store/tests/unit/test_multistore_cinder.py | 5 | ||||
-rw-r--r-- | glance_store/tests/unit/test_multistore_filesystem.py | 10 | ||||
-rw-r--r-- | glance_store/tests/unit/test_multistore_rbd.py | 8 | ||||
-rw-r--r-- | glance_store/tests/unit/test_multistore_sheepdog.py | 6 | ||||
-rw-r--r-- | glance_store/tests/unit/test_multistore_vmware.py | 6 | ||||
-rw-r--r-- | glance_store/tests/unit/test_swift_store_multibackend.py | 18 |
14 files changed, 159 insertions, 12 deletions
diff --git a/glance_store/_drivers/cinder.py b/glance_store/_drivers/cinder.py index e0a9586..b6bd2e1 100644 --- a/glance_store/_drivers/cinder.py +++ b/glance_store/_drivers/cinder.py @@ -435,10 +435,15 @@ class Store(glance_store.driver.Store): def __init__(self, *args, **kargs): super(Store, self).__init__(*args, **kargs) + if self.backend_group: + self._set_url_prefix() def get_schemes(self): return ('cinder',) + def _set_url_prefix(self): + self._url_prefix = "cinder://" + def _check_context(self, context, require_tenant=False): user_overriden = is_user_overriden(self.conf, backend=self.backend_group) diff --git a/glance_store/_drivers/filesystem.py b/glance_store/_drivers/filesystem.py index 0825706..57204de 100644 --- a/glance_store/_drivers/filesystem.py +++ b/glance_store/_drivers/filesystem.py @@ -460,10 +460,16 @@ class Store(glance_store.driver.Store): reverse=True) self._create_image_directories(directory_paths) + if self.backend_group: + self._set_url_prefix() if meta_file: self._validate_metadata(meta_file) + def _set_url_prefix(self): + path = self._find_best_datadir(0) + self._url_prefix = "%s://%s" % ('file', path) + def _check_directory_paths(self, datadir_path, directory_paths, priority_paths): """ diff --git a/glance_store/_drivers/http.py b/glance_store/_drivers/http.py index d2a821a..b0b70f5 100644 --- a/glance_store/_drivers/http.py +++ b/glance_store/_drivers/http.py @@ -191,6 +191,16 @@ class Store(glance_store.driver.Store): capabilities.BitMasks.DRIVER_REUSABLE) OPTIONS = _HTTP_OPTS + def __init__(self, *args, **kargs): + super(Store, self).__init__(*args, **kargs) + if self.backend_group: + self._set_url_prefix() + + def _set_url_prefix(self): + # NOTE(abhishekk): HTTP store url either starts with http + # or https, so default _url_prefix is set to http. + self._url_prefix = "http" + @capabilities.check def get(self, location, offset=0, chunk_size=None, context=None): """ diff --git a/glance_store/_drivers/rbd.py b/glance_store/_drivers/rbd.py index a931f54..0e3e487 100644 --- a/glance_store/_drivers/rbd.py +++ b/glance_store/_drivers/rbd.py @@ -319,6 +319,23 @@ class Store(driver.Store): LOG.error(reason) raise exceptions.BadStoreConfiguration(store_name='rbd', reason=reason) + if self.backend_group: + self._set_url_prefix() + + def _set_url_prefix(self): + fsid = None + with self.get_connection(conffile=self.conf_file, + rados_id=self.user) as conn: + if hasattr(conn, 'get_fsid'): + fsid = encodeutils.safe_decode(conn.get_fsid()) + + if fsid and self.pool: + # ensure nothing contains / or any other url-unsafe character + safe_fsid = urllib.parse.quote(fsid, '') + safe_pool = urllib.parse.quote(self.pool, '') + self._url_prefix = "rbd://%s/%s/" % (safe_fsid, safe_pool) + else: + self._url_prefix = "rbd://" @capabilities.check def get(self, location, offset=0, chunk_size=None, context=None): diff --git a/glance_store/_drivers/sheepdog.py b/glance_store/_drivers/sheepdog.py index 591efe8..2b1f451 100644 --- a/glance_store/_drivers/sheepdog.py +++ b/glance_store/_drivers/sheepdog.py @@ -270,6 +270,10 @@ class Store(glance_store.driver.Store): def get_schemes(self): return ('sheepdog',) + def _set_url_prefix(self): + self._url_prefix = "%s://%s:%s:" % ( + 'sheepdog', self.addr, self.port) + def configure_add(self): """ Configure the Store to use the stored configuration options @@ -304,6 +308,9 @@ class Store(glance_store.driver.Store): raise exceptions.BadStoreConfiguration(store_name='sheepdog', reason=reason) + if self.backend_group: + self._set_url_prefix() + @capabilities.check def get(self, location, offset=0, chunk_size=None, context=None): """ diff --git a/glance_store/_drivers/swift/store.py b/glance_store/_drivers/swift/store.py index 933ba42..d7a324e 100644 --- a/glance_store/_drivers/swift/store.py +++ b/glance_store/_drivers/swift/store.py @@ -1222,6 +1222,9 @@ class BaseStore(driver.Store): msg = _("There is no Connection Manager implemented for %s class.") raise NotImplementedError(msg % self.__class__.__name__) + def _set_url_prefix(self, context=None): + raise NotImplementedError() + class SingleTenantStore(BaseStore): EXAMPLE_URL = "swift://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<FILE>" @@ -1280,6 +1283,50 @@ class SingleTenantStore(BaseStore): LOG.error(reason) raise exceptions.BadStoreConfiguration(store_name="swift", reason=reason) + if self.backend_group: + self._set_url_prefix() + + def _get_credstring(self): + if self.user and self.key: + return '%s:%s' % (urllib.parse.quote(self.user), + urllib.parse.quote(self.key)) + return '' + + def _set_url_prefix(self, context=None): + auth_or_store_url = self.auth_address + if auth_or_store_url.startswith('http://'): + auth_or_store_url = auth_or_store_url[len('http://'):] + elif auth_or_store_url.startswith('https://'): + auth_or_store_url = auth_or_store_url[len('https://'):] + + credstring = self._get_credstring() + auth_or_store_url = auth_or_store_url.strip('/') + container = self.container.strip('/') + + if sutils.is_multiple_swift_store_accounts_enabled( + self.conf, backend=self.backend_group): + include_creds = False + else: + include_creds = True + + if not include_creds: + store = getattr(self.conf, + self.backend_group).default_swift_reference + + self._url_prefix = '%s://%s/%s/' % ( + 'swift+config', store, container) + return + + if self.scheme == 'swift+config': + if self.ssl_enabled: + self.scheme = 'swift+https' + else: + self.scheme = 'swift+http' + if credstring != '': + credstring = "%s@" % credstring + + self._url_prefix = '%s://%s%s/%s/' % ( + self.scheme, credstring, auth_or_store_url, container) def create_location(self, image_id, context=None): container_name = self.get_container_name(image_id, self.container) @@ -1502,6 +1549,11 @@ class MultiTenantStore(BaseStore): return StoreLocation(specs, self.conf, backend_group=self.backend_group) + def _set_url_prefix(self, context=None): + ep = self._get_endpoint(context) + self._url_prefix = "%s://%s:%s_" % ( + self.scheme, ep, self.container) + def get_connection(self, location, context=None): return swiftclient.Connection( preauthurl=location.swift_url, @@ -1536,6 +1588,9 @@ class MultiTenantStore(BaseStore): project_domain_name = default_swift_reference.get( 'project_domain_name') + if self.backend_group: + self._set_url_prefix(context=context) + # create client for multitenant user(trustor) trustor_auth = ks_identity.V3Token(auth_url=auth_address, token=context.auth_token, diff --git a/glance_store/_drivers/vmware_datastore.py b/glance_store/_drivers/vmware_datastore.py index 2b6c4f8..07a3abb 100644 --- a/glance_store/_drivers/vmware_datastore.py +++ b/glance_store/_drivers/vmware_datastore.py @@ -515,6 +515,19 @@ class Store(glance_store.Store): store_conf = self.conf.glance_store self.store_image_dir = store_conf.vmware_store_image_dir + if self.backend_group: + self._set_url_prefix() + + def _set_url_prefix(self): + path = os.path.join(DS_URL_PREFIX, + self.store_image_dir) + + if netutils.is_valid_ipv6(self.server_host): + self._url_prefix = '%s://[%s]%s' % (self.scheme, + self.server_host, path) + else: + self._url_prefix = '%s://%s%s' % (self.scheme, + self.server_host, path) def select_datastore(self, image_size): """Select a datastore with free space larger than image size.""" diff --git a/glance_store/driver.py b/glance_store/driver.py index 0881a47..b11012a 100644 --- a/glance_store/driver.py +++ b/glance_store/driver.py @@ -61,6 +61,7 @@ class Store(capabilities.StoreCapability): self.conf = conf self.backend_group = backend self.store_location_class = None + self._url_prefix = None try: if self.OPTIONS is not None: @@ -75,6 +76,10 @@ class Store(capabilities.StoreCapability): except cfg.DuplicateOptError: pass + @property + def url_prefix(self): + return self._url_prefix + def configure(self, re_raise_bsc=False): """ Configure the store to use the stored configuration options diff --git a/glance_store/tests/unit/test_multistore_cinder.py b/glance_store/tests/unit/test_multistore_cinder.py index 9f12310..acb9a29 100644 --- a/glance_store/tests/unit/test_multistore_cinder.py +++ b/glance_store/tests/unit/test_multistore_cinder.py @@ -67,8 +67,8 @@ class TestMultiCinderStore(base.MultiStoreBaseTest, # Ensure stores + locations cleared location.SCHEME_TO_CLS_BACKEND_MAP = {} - store.create_multi_stores(self.conf) + self.addCleanup(setattr, location, 'SCHEME_TO_CLS_BACKEND_MAP', dict()) self.test_dir = self.useFixture(fixtures.TempDir()).path @@ -89,6 +89,9 @@ class TestMultiCinderStore(base.MultiStoreBaseTest, auth_token='fake_token', tenant='fake_tenant') + def test_location_url_prefix_is_set(self): + self.assertEqual("cinder://", self.store.url_prefix) + def test_get_cinderclient(self): cc = cinder.get_cinderclient(self.conf, self.context, backend="cinder1") diff --git a/glance_store/tests/unit/test_multistore_filesystem.py b/glance_store/tests/unit/test_multistore_filesystem.py index 7c16fab..08130a9 100644 --- a/glance_store/tests/unit/test_multistore_filesystem.py +++ b/glance_store/tests/unit/test_multistore_filesystem.py @@ -50,21 +50,21 @@ class TestMultiStore(base.MultiStoreBaseTest, def setUp(self): """Establish a clean test environment.""" super(TestMultiStore, self).setUp() - enabled_backends = { + self.enabled_backends = { "file1": "file", "file2": "file", } self.conf = self._CONF self.conf(args=[]) self.conf.register_opt(cfg.DictOpt('enabled_backends')) - self.config(enabled_backends=enabled_backends) + self.config(enabled_backends=self.enabled_backends) store.register_store_opts(self.conf) self.config(default_backend='file1', group='glance_store') # Ensure stores + locations cleared location.SCHEME_TO_CLS_BACKEND_MAP = {} - store.create_multi_stores(self.conf) + self.addCleanup(setattr, location, 'SCHEME_TO_CLS_BACKEND_MAP', dict()) self.test_dir = self.useFixture(fixtures.TempDir()).path @@ -96,6 +96,10 @@ class TestMultiStore(base.MultiStoreBaseTest, return self.store.add(expected_image_id, image_file, expected_file_size) + def test_location_url_prefix_is_set(self): + expected_url_prefix = "file://%s" % self.test_dir + self.assertEqual(expected_url_prefix, self.store.url_prefix) + def test_get(self): """Test a "normal" retrieval of an image in chunks.""" # First add an image... diff --git a/glance_store/tests/unit/test_multistore_rbd.py b/glance_store/tests/unit/test_multistore_rbd.py index ccc5259..4d4d74b 100644 --- a/glance_store/tests/unit/test_multistore_rbd.py +++ b/glance_store/tests/unit/test_multistore_rbd.py @@ -190,7 +190,9 @@ class TestMultiStore(base.MultiStoreBaseTest, # Ensure stores + locations cleared g_location.SCHEME_TO_CLS_BACKEND_MAP = {} - store.create_multi_stores(self.conf) + with mock.patch.object(rbd_store.Store, '_set_url_prefix'): + store.create_multi_stores(self.conf) + self.addCleanup(setattr, g_location, 'SCHEME_TO_CLS_BACKEND_MAP', dict()) self.addCleanup(self.conf.reset) @@ -212,6 +214,10 @@ class TestMultiStore(base.MultiStoreBaseTest, self.data_len = 3 * units.Ki self.data_iter = six.BytesIO(b'*' * self.data_len) + def test_location_url_prefix_is_set(self): + expected_url_prefix = "rbd://" + self.assertEqual(expected_url_prefix, self.store.url_prefix) + def test_add_w_image_size_zero(self): """Assert that correct size is returned even though 0 was provided.""" self.store.chunk_size = units.Ki diff --git a/glance_store/tests/unit/test_multistore_sheepdog.py b/glance_store/tests/unit/test_multistore_sheepdog.py index 999eba1..5d2cbc8 100644 --- a/glance_store/tests/unit/test_multistore_sheepdog.py +++ b/glance_store/tests/unit/test_multistore_sheepdog.py @@ -59,8 +59,8 @@ class TestSheepdogMultiStore(base.MultiStoreBaseTest, # Ensure stores + locations cleared location.SCHEME_TO_CLS_BACKEND_MAP = {} - store.create_multi_stores(self.conf) + self.addCleanup(setattr, location, 'SCHEME_TO_CLS_BACKEND_MAP', dict()) self.addCleanup(self.conf.reset) @@ -71,6 +71,10 @@ class TestSheepdogMultiStore(base.MultiStoreBaseTest, 'addr': '127.0.0.1', 'port': 7000} + def test_location_url_prefix_is_set(self): + expected_url_prefix = "sheepdog://127.0.0.1:7000:" + self.assertEqual(expected_url_prefix, self.store.url_prefix) + @mock.patch.object(sheepdog.SheepdogImage, 'write') @mock.patch.object(sheepdog.SheepdogImage, 'create') @mock.patch.object(sheepdog.SheepdogImage, 'exist') diff --git a/glance_store/tests/unit/test_multistore_vmware.py b/glance_store/tests/unit/test_multistore_vmware.py index 04c63ca..7ca8a7a 100644 --- a/glance_store/tests/unit/test_multistore_vmware.py +++ b/glance_store/tests/unit/test_multistore_vmware.py @@ -115,8 +115,8 @@ class TestMultiStore(base.MultiStoreBaseTest, vmware_store_image_dir='/openstack_glance_1') # Ensure stores + locations cleared location.SCHEME_TO_CLS_BACKEND_MAP = {} - store.create_multi_stores(self.conf) + self.addCleanup(setattr, location, 'SCHEME_TO_CLS_BACKEND_MAP', dict()) self.addCleanup(self.conf.reset) @@ -131,6 +131,10 @@ class TestMultiStore(base.MultiStoreBaseTest, def _mock_http_connection(self): return mock.patch('six.moves.http_client.HTTPConnection') + def test_location_url_prefix_is_set(self): + expected_url_prefix = "vsphere://127.0.0.1/openstack_glance" + self.assertEqual(expected_url_prefix, self.store.url_prefix) + @mock.patch('oslo_vmware.api.VMwareAPISession') def test_get(self, mock_api_session): """Test a "normal" retrieval of an image in chunks.""" diff --git a/glance_store/tests/unit/test_swift_store_multibackend.py b/glance_store/tests/unit/test_swift_store_multibackend.py index 20eb409..9c6931f 100644 --- a/glance_store/tests/unit/test_swift_store_multibackend.py +++ b/glance_store/tests/unit/test_swift_store_multibackend.py @@ -964,6 +964,12 @@ class SwiftTests(object): self.assertEqual(expected_swift_contents, new_image_contents) self.assertEqual(expected_swift_size, new_image_swift_size) + def test_location_url_prefix_is_set(self): + self.store = Store(self.conf, backend="swift1") + self.store.configure() + expected_url_prefix = "swift+config://ref1/glance/" + self.assertEqual(expected_url_prefix, self.store.url_prefix) + def test_add_already_existing(self): """ Tests that adding an image with an existing identifier @@ -1245,17 +1251,19 @@ class SwiftTests(object): def test_init_client_multi_tenant(self): """Test that keystone client was initialized correctly""" - self._init_client(verify=True, swift_store_multi_tenant=True, - swift_store_config_file=None) + with mock.patch.object(swift.MultiTenantStore, '_set_url_prefix'): + self._init_client(verify=True, swift_store_multi_tenant=True, + swift_store_config_file=None) def test_init_client_multi_tenant_insecure(self): """ Test that keystone client was initialized correctly with no certificate verification. """ - self._init_client(verify=False, swift_store_multi_tenant=True, - swift_store_auth_insecure=True, - swift_store_config_file=None) + with mock.patch.object(swift.MultiTenantStore, '_set_url_prefix'): + self._init_client(verify=False, swift_store_multi_tenant=True, + swift_store_auth_insecure=True, + swift_store_config_file=None) @mock.patch("glance_store._drivers.swift.store.ks_identity") @mock.patch("glance_store._drivers.swift.store.ks_session") |