Merge "Multi store support for cinder driver"

3 files changed, 463 insertions, 23 deletions

diff --git a/glance_store/_drivers/cinder.py b/glance_store/_drivers/cinder.py
index 10917be..fb787dc 100644
--- a/glance_store/_drivers/cinder.py
+++ b/glance_store/_drivers/cinder.py
@@ -308,19 +308,33 @@ Related options:
 ]
 
 
-def get_root_helper():
-    return 'sudo glance-rootwrap %s' % CONF.glance_store.rootwrap_config
+def get_root_helper(backend=None):
+    if backend:
+        rootwrap = getattr(CONF, backend).rootwrap_config
+    else:
+        rootwrap = CONF.glance_store.rootwrap_config
+
+    return 'sudo glance-rootwrap %s' % rootwrap
+
 
+def is_user_overriden(conf, backend=None):
+    if backend:
+        store_conf = getattr(conf, backend)
+    else:
+        store_conf = conf.glance_store
 
-def is_user_overriden(conf):
-    return all([conf.glance_store.get('cinder_store_' + key)
+    return all([store_conf.get('cinder_store_' + key)
                 for key in ['user_name', 'password',
                             'project_name', 'auth_address']])
 
 
-def get_cinderclient(conf, context=None):
-    glance_store = conf.glance_store
-    user_overriden = is_user_overriden(conf)
+def get_cinderclient(conf, context=None, backend=None):
+    if backend:
+        glance_store = getattr(conf, backend)
+    else:
+        glance_store = conf.glance_store
+
+    user_overriden = is_user_overriden(conf, backend=backend)
     if user_overriden:
         username = glance_store.cinder_store_user_name
         password = glance_store.cinder_store_password
@@ -392,21 +406,21 @@ class StoreLocation(glance_store.location.StoreLocation):
 
 
 @contextlib.contextmanager
-def temporary_chown(path):
+def temporary_chown(path, backend=None):
     owner_uid = os.getuid()
     orig_uid = os.stat(path).st_uid
 
     if orig_uid != owner_uid:
         processutils.execute('chown', owner_uid, path,
                              run_as_root=True,
-                             root_helper=get_root_helper())
+                             root_helper=get_root_helper(backend=backend))
     try:
         yield
     finally:
         if orig_uid != owner_uid:
             processutils.execute('chown', orig_uid, path,
                                  run_as_root=True,
-                                 root_helper=get_root_helper())
+                                 root_helper=get_root_helper(backend=backend))
 
 
 class Store(glance_store.driver.Store):
@@ -429,7 +443,8 @@ class Store(glance_store.driver.Store):
         return ('cinder',)
 
     def _check_context(self, context, require_tenant=False):
-        user_overriden = is_user_overriden(self.conf)
+        user_overriden = is_user_overriden(self.conf,
+                                           backend=self.backend_group)
         if user_overriden and not require_tenant:
             return
         if context is None:
@@ -443,7 +458,12 @@ class Store(glance_store.driver.Store):
 
     def _wait_volume_status(self, volume, status_transition, status_expected):
         max_recheck_wait = 15
-        timeout = self.conf.glance_store.cinder_state_transition_timeout
+        if self.backend_group:
+            timeout = getattr(
+                self.conf, self.backend_group).cinder_state_transition_timeout
+        else:
+            timeout = self.conf.glance_store.cinder_state_transition_timeout
+
         volume = volume.manager.get(volume.id)
         tries = 0
         elapsed = 0
@@ -473,7 +493,7 @@ class Store(glance_store.driver.Store):
     def _open_cinder_volume(self, client, volume, mode):
         attach_mode = 'rw' if mode == 'wb' else 'ro'
         device = None
-        root_helper = get_root_helper()
+        root_helper = get_root_helper(backend=self.backend_group)
         priv_context.init(root_helper=shlex.split(root_helper))
         host = socket.gethostname()
         properties = connector.get_connector_properties(root_helper, host,
@@ -499,7 +519,8 @@ class Store(glance_store.driver.Store):
                not conn.do_local_attach):
                 yield device['path']
             else:
-                with temporary_chown(device['path']), \
+                with temporary_chown(device['path'],
+                                     backend=self.backend_group), \
                         open(device['path'], mode) as f:
                     yield f
         except Exception:
@@ -577,7 +598,8 @@ class Store(glance_store.driver.Store):
         loc = location.store_location
         self._check_context(context)
         try:
-            client = get_cinderclient(self.conf, context)
+            client = get_cinderclient(self.conf, context,
+                                      backend=self.backend_group)
             volume = client.volumes.get(loc.volume_id)
             size = int(volume.metadata.get('image_size',
                                            volume.size * units.Gi))
@@ -611,8 +633,9 @@ class Store(glance_store.driver.Store):
 
         try:
             self._check_context(context)
-            volume = get_cinderclient(self.conf,
-                                      context).volumes.get(loc.volume_id)
+            volume = get_cinderclient(
+                self.conf, context,
+                backend=self.backend_group).volumes.get(loc.volume_id)
             return int(volume.metadata.get('image_size',
                                            volume.size * units.Gi))
         except cinder_exception.NotFound:
@@ -643,7 +666,8 @@ class Store(glance_store.driver.Store):
         """
 
         self._check_context(context, require_tenant=True)
-        client = get_cinderclient(self.conf, context)
+        client = get_cinderclient(self.conf, context,
+                                  backend=self.backend_group)
 
         checksum = hashlib.md5()
         bytes_written = 0
@@ -655,7 +679,13 @@ class Store(glance_store.driver.Store):
         metadata = {'glance_image_id': image_id,
                     'image_size': str(image_size),
                     'image_owner': owner}
-        volume_type = self.conf.glance_store.cinder_volume_type
+
+        if self.backend_group:
+            volume_type = getattr(self.conf,
+                                  self.backend_group).cinder_volume_type
+        else:
+            volume_type = self.conf.glance_store.cinder_volume_type
+
         LOG.debug('Creating a new volume: image_size=%d size_gb=%d type=%s',
                   image_size, size_gb, volume_type or 'None')
         if image_size == 0:
@@ -735,7 +765,12 @@ class Store(glance_store.driver.Store):
                    'volume_id': volume.id,
                    'checksum_hex': checksum_hex})
 
-        return ('cinder://%s' % volume.id, bytes_written, checksum_hex, {})
+        image_metadata = {}
+        if self.backend_group:
+            image_metadata['backend'] = u"%s" % self.backend_group
+
+        return ('cinder://%s' % volume.id, bytes_written,
+                checksum_hex, image_metadata)
 
     @capabilities.check
     def delete(self, location, context=None):
@@ -752,8 +787,9 @@ class Store(glance_store.driver.Store):
         loc = location.store_location
         self._check_context(context)
         try:
-            volume = get_cinderclient(self.conf,
-                                      context).volumes.get(loc.volume_id)
+            volume = get_cinderclient(
+                self.conf, context,
+                backend=self.backend_group).volumes.get(loc.volume_id)
             volume.delete()
         except cinder_exception.NotFound:
             raise exceptions.NotFound(image=loc.volume_id)
diff --git a/glance_store/tests/unit/test_cinder_store.py b/glance_store/tests/unit/test_cinder_store.py
index 27103f4..60890ae 100644
--- a/glance_store/tests/unit/test_cinder_store.py
+++ b/glance_store/tests/unit/test_cinder_store.py
@@ -156,7 +156,7 @@ class TestCinderStore(base.StoreBaseTest,
             disconnect_volume=mock.Mock())
 
         @contextlib.contextmanager
-        def fake_chown(path):
+        def fake_chown(path, backend=None):
             yield
 
         def do_open():
diff --git a/glance_store/tests/unit/test_multistore_cinder.py b/glance_store/tests/unit/test_multistore_cinder.py
new file mode 100644
index 0000000..57c89d3
--- /dev/null
+++ b/glance_store/tests/unit/test_multistore_cinder.py
@@ -0,0 +1,404 @@
+# Copyright 2018-2019 RedHat Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import contextlib
+import errno
+import hashlib
+import mock
+import os
+import six
+import socket
+import tempfile
+import time
+import uuid
+
+import fixtures
+from os_brick.initiator import connector
+from oslo_concurrency import processutils
+from oslo_config import cfg
+from oslo_utils import units
+
+import glance_store as store
+from glance_store._drivers import cinder
+from glance_store import exceptions
+from glance_store import location
+from glance_store.tests import base
+from glance_store.tests.unit import test_store_capabilities as test_cap
+
+
+class FakeObject(object):
+    def __init__(self, **kwargs):
+        for name, value in kwargs.items():
+            setattr(self, name, value)
+
+
+class TestMultiCinderStore(base.MultiStoreBaseTest,
+                           test_cap.TestStoreCapabilitiesChecking):
+
+    # NOTE(flaper87): temporary until we
+    # can move to a fully-local lib.
+    # (Swift store's fault)
+    _CONF = cfg.ConfigOpts()
+
+    def setUp(self):
+        super(TestMultiCinderStore, self).setUp()
+        enabled_backends = {
+            "cinder1": "cinder",
+            "cinder2": "cinder"
+        }
+        self.conf = self._CONF
+        self.conf(args=[])
+        self.conf.register_opt(cfg.DictOpt('enabled_backends'))
+        self.config(enabled_backends=enabled_backends)
+        store.register_store_opts(self.conf)
+        self.config(default_backend='cinder1', group='glance_store')
+
+        # Ensure stores + locations cleared
+        location.SCHEME_TO_CLS_BACKEND_MAP = {}
+
+        store.create_multi_stores(self.conf)
+        self.addCleanup(setattr, location, 'SCHEME_TO_CLS_BACKEND_MAP',
+                        dict())
+        self.test_dir = self.useFixture(fixtures.TempDir()).path
+        self.addCleanup(self.conf.reset)
+
+        self.store = cinder.Store(self.conf, backend="cinder1")
+        self.store.configure()
+        self.register_store_backend_schemes(self.store, 'cinder', 'cinder1')
+        self.store.READ_CHUNKSIZE = 4096
+        self.store.WRITE_CHUNKSIZE = 4096
+
+        fake_sc = [{u'endpoints': [{u'publicURL': u'http://foo/public_url'}],
+                    u'endpoints_links': [],
+                    u'name': u'cinder',
+                    u'type': u'volumev2'}]
+        self.context = FakeObject(service_catalog=fake_sc,
+                                  user='fake_user',
+                                  auth_token='fake_token',
+                                  tenant='fake_tenant')
+
+    def test_get_cinderclient(self):
+        cc = cinder.get_cinderclient(self.conf, self.context,
+                                     backend="cinder1")
+        self.assertEqual('fake_token', cc.client.auth_token)
+        self.assertEqual('http://foo/public_url', cc.client.management_url)
+
+    def test_get_cinderclient_with_user_overriden(self):
+        self.config(cinder_store_user_name='test_user', group="cinder1")
+        self.config(cinder_store_password='test_password', group="cinder1")
+        self.config(cinder_store_project_name='test_project', group="cinder1")
+        self.config(cinder_store_auth_address='test_address', group="cinder1")
+        cc = cinder.get_cinderclient(self.conf, self.context,
+                                     backend="cinder1")
+        self.assertIsNone(cc.client.auth_token)
+        self.assertEqual('test_address', cc.client.management_url)
+
+    def test_temporary_chown(self):
+        class fake_stat(object):
+            st_uid = 1
+
+        with mock.patch.object(os, 'stat', return_value=fake_stat()), \
+                mock.patch.object(os, 'getuid', return_value=2), \
+                mock.patch.object(processutils, 'execute') as mock_execute, \
+                mock.patch.object(cinder, 'get_root_helper',
+                                  return_value='sudo'):
+            with cinder.temporary_chown('test'):
+                pass
+            expected_calls = [mock.call('chown', 2, 'test', run_as_root=True,
+                                        root_helper='sudo'),
+                              mock.call('chown', 1, 'test', run_as_root=True,
+                                        root_helper='sudo')]
+            self.assertEqual(expected_calls, mock_execute.call_args_list)
+
+    @mock.patch.object(time, 'sleep')
+    def test_wait_volume_status(self, mock_sleep):
+        fake_manager = FakeObject(get=mock.Mock())
+        volume_available = FakeObject(manager=fake_manager,
+                                      id='fake-id',
+                                      status='available')
+        volume_in_use = FakeObject(manager=fake_manager,
+                                   id='fake-id',
+                                   status='in-use')
+        fake_manager.get.side_effect = [volume_available, volume_in_use]
+        self.assertEqual(volume_in_use,
+                         self.store._wait_volume_status(
+                             volume_available, 'available', 'in-use'))
+        fake_manager.get.assert_called_with('fake-id')
+        mock_sleep.assert_called_once_with(0.5)
+
+    @mock.patch.object(time, 'sleep')
+    def test_wait_volume_status_unexpected(self, mock_sleep):
+        fake_manager = FakeObject(get=mock.Mock())
+        volume_available = FakeObject(manager=fake_manager,
+                                      id='fake-id',
+                                      status='error')
+        fake_manager.get.return_value = volume_available
+        self.assertRaises(exceptions.BackendException,
+                          self.store._wait_volume_status,
+                          volume_available, 'available', 'in-use')
+        fake_manager.get.assert_called_with('fake-id')
+
+    @mock.patch.object(time, 'sleep')
+    def test_wait_volume_status_timeout(self, mock_sleep):
+        fake_manager = FakeObject(get=mock.Mock())
+        volume_available = FakeObject(manager=fake_manager,
+                                      id='fake-id',
+                                      status='available')
+        fake_manager.get.return_value = volume_available
+        self.assertRaises(exceptions.BackendException,
+                          self.store._wait_volume_status,
+                          volume_available, 'available', 'in-use')
+        fake_manager.get.assert_called_with('fake-id')
+
+    def _test_open_cinder_volume(self, open_mode, attach_mode, error):
+        fake_volume = mock.MagicMock(id=str(uuid.uuid4()), status='available')
+        fake_volumes = FakeObject(get=lambda id: fake_volume,
+                                  detach=mock.Mock())
+        fake_client = FakeObject(volumes=fake_volumes)
+        _, fake_dev_path = tempfile.mkstemp(dir=self.test_dir)
+        fake_devinfo = {'path': fake_dev_path}
+        fake_connector = FakeObject(
+            connect_volume=mock.Mock(return_value=fake_devinfo),
+            disconnect_volume=mock.Mock())
+
+        @contextlib.contextmanager
+        def fake_chown(path, backend=None):
+            yield
+
+        def do_open():
+            with self.store._open_cinder_volume(
+                    fake_client, fake_volume, open_mode):
+                if error:
+                    raise error
+
+        def fake_factory(protocol, root_helper, **kwargs):
+            self.assertEqual(fake_volume.initialize_connection.return_value,
+                             kwargs['conn'])
+            return fake_connector
+
+        root_helper = "sudo glance-rootwrap /etc/glance/rootwrap.conf"
+        with mock.patch.object(cinder.Store,
+                               '_wait_volume_status',
+                               return_value=fake_volume), \
+                mock.patch.object(cinder, 'temporary_chown',
+                                  side_effect=fake_chown), \
+                mock.patch.object(cinder, 'get_root_helper',
+                                  return_value=root_helper), \
+                mock.patch.object(connector, 'get_connector_properties'), \
+                mock.patch.object(connector.InitiatorConnector, 'factory',
+                                  side_effect=fake_factory):
+
+            if error:
+                self.assertRaises(error, do_open)
+            else:
+                do_open()
+
+            fake_connector.connect_volume.assert_called_once_with(mock.ANY)
+            fake_connector.disconnect_volume.assert_called_once_with(
+                mock.ANY, fake_devinfo)
+            fake_volume.attach.assert_called_once_with(
+                None, 'glance_store', attach_mode,
+                host_name=socket.gethostname())
+            fake_volumes.detach.assert_called_once_with(fake_volume)
+
+    def test_open_cinder_volume_rw(self):
+        self._test_open_cinder_volume('wb', 'rw', None)
+
+    def test_open_cinder_volume_ro(self):
+        self._test_open_cinder_volume('rb', 'ro', None)
+
+    def test_open_cinder_volume_error(self):
+        self._test_open_cinder_volume('wb', 'rw', IOError)
+
+    def test_cinder_configure_add(self):
+        self.assertRaises(exceptions.BadStoreConfiguration,
+                          self.store._check_context, None)
+
+        self.assertRaises(exceptions.BadStoreConfiguration,
+                          self.store._check_context,
+                          FakeObject(service_catalog=None))
+
+        self.store._check_context(FakeObject(service_catalog='fake'))
+
+    def test_cinder_get(self):
+        expected_size = 5 * units.Ki
+        expected_file_contents = b"*" * expected_size
+        volume_file = six.BytesIO(expected_file_contents)
+        fake_client = FakeObject(auth_token=None, management_url=None)
+        fake_volume_uuid = str(uuid.uuid4())
+        fake_volume = mock.MagicMock(id=fake_volume_uuid,
+                                     metadata={'image_size': expected_size},
+                                     status='available')
+        fake_volume.manager.get.return_value = fake_volume
+        fake_volumes = FakeObject(get=lambda id: fake_volume)
+
+        @contextlib.contextmanager
+        def fake_open(client, volume, mode):
+            self.assertEqual('rb', mode)
+            yield volume_file
+
+        with mock.patch.object(cinder, 'get_cinderclient') as mock_cc, \
+                mock.patch.object(self.store, '_open_cinder_volume',
+                                  side_effect=fake_open):
+            mock_cc.return_value = FakeObject(client=fake_client,
+                                              volumes=fake_volumes)
+            uri = "cinder://%s" % fake_volume_uuid
+            loc = location.get_location_from_uri_and_backend(uri,
+                                                             "cinder1",
+                                                             conf=self.conf)
+            (image_file, image_size) = self.store.get(loc,
+                                                      context=self.context)
+
+            expected_num_chunks = 2
+            data = b""
+            num_chunks = 0
+
+            for chunk in image_file:
+                num_chunks += 1
+                data += chunk
+            self.assertEqual(expected_num_chunks, num_chunks)
+            self.assertEqual(expected_file_contents, data)
+
+    def test_cinder_get_size(self):
+        fake_client = FakeObject(auth_token=None, management_url=None)
+        fake_volume_uuid = str(uuid.uuid4())
+        fake_volume = FakeObject(size=5, metadata={})
+        fake_volumes = {fake_volume_uuid: fake_volume}
+
+        with mock.patch.object(cinder, 'get_cinderclient') as mocked_cc:
+            mocked_cc.return_value = FakeObject(client=fake_client,
+                                                volumes=fake_volumes)
+
+            uri = 'cinder://%s' % fake_volume_uuid
+            loc = location.get_location_from_uri_and_backend(uri,
+                                                             "cinder1",
+                                                             conf=self.conf)
+            image_size = self.store.get_size(loc, context=self.context)
+            self.assertEqual(fake_volume.size * units.Gi, image_size)
+
+    def test_cinder_get_size_with_metadata(self):
+        fake_client = FakeObject(auth_token=None, management_url=None)
+        fake_volume_uuid = str(uuid.uuid4())
+        expected_image_size = 4500 * units.Mi
+        fake_volume = FakeObject(size=5,
+                                 metadata={'image_size': expected_image_size})
+        fake_volumes = {fake_volume_uuid: fake_volume}
+
+        with mock.patch.object(cinder, 'get_cinderclient') as mocked_cc:
+            mocked_cc.return_value = FakeObject(client=fake_client,
+                                                volumes=fake_volumes)
+
+            uri = 'cinder://%s' % fake_volume_uuid
+            loc = location.get_location_from_uri_and_backend(uri,
+                                                             "cinder1",
+                                                             conf=self.conf)
+            image_size = self.store.get_size(loc, context=self.context)
+            self.assertEqual(expected_image_size, image_size)
+
+    def _test_cinder_add(self, fake_volume, volume_file, size_kb=5,
+                         verifier=None, backend="cinder1"):
+        expected_image_id = str(uuid.uuid4())
+        expected_size = size_kb * units.Ki
+        expected_file_contents = b"*" * expected_size
+        image_file = six.BytesIO(expected_file_contents)
+        expected_checksum = hashlib.md5(expected_file_contents).hexdigest()
+        expected_location = 'cinder://%s' % fake_volume.id
+        fake_client = FakeObject(auth_token=None, management_url=None)
+        fake_volume.manager.get.return_value = fake_volume
+        fake_volumes = FakeObject(create=mock.Mock(return_value=fake_volume))
+        self.config(cinder_volume_type='some_type', group=backend)
+
+        @contextlib.contextmanager
+        def fake_open(client, volume, mode):
+            self.assertEqual('wb', mode)
+            yield volume_file
+
+        with mock.patch.object(cinder, 'get_cinderclient') as mock_cc, \
+                mock.patch.object(self.store, '_open_cinder_volume',
+                                  side_effect=fake_open):
+            mock_cc.return_value = FakeObject(client=fake_client,
+                                              volumes=fake_volumes)
+            loc, size, checksum, metadata = self.store.add(expected_image_id,
+                                                           image_file,
+                                                           expected_size,
+                                                           self.context,
+                                                           verifier)
+            self.assertEqual(expected_location, loc)
+            self.assertEqual(expected_size, size)
+            self.assertEqual(expected_checksum, checksum)
+            fake_volumes.create.assert_called_once_with(
+                1,
+                name='image-%s' % expected_image_id,
+                metadata={'image_owner': self.context.tenant,
+                          'glance_image_id': expected_image_id,
+                          'image_size': str(expected_size)},
+                volume_type='some_type')
+            self.assertEqual(backend, metadata["backend"])
+
+    def test_cinder_add(self):
+        fake_volume = mock.MagicMock(id=str(uuid.uuid4()),
+                                     status='available',
+                                     size=1)
+        volume_file = six.BytesIO()
+        self._test_cinder_add(fake_volume, volume_file)
+
+    def test_cinder_add_with_verifier(self):
+        fake_volume = mock.MagicMock(id=str(uuid.uuid4()),
+                                     status='available',
+                                     size=1)
+        volume_file = six.BytesIO()
+        verifier = mock.MagicMock()
+        self._test_cinder_add(fake_volume, volume_file, 1, verifier)
+        verifier.update.assert_called_with(b"*" * units.Ki)
+
+    def test_cinder_add_volume_full(self):
+        e = IOError()
+        volume_file = six.BytesIO()
+        e.errno = errno.ENOSPC
+        fake_volume = mock.MagicMock(id=str(uuid.uuid4()),
+                                     status='available',
+                                     size=1)
+        with mock.patch.object(volume_file, 'write', side_effect=e):
+            self.assertRaises(exceptions.StorageFull,
+                              self._test_cinder_add, fake_volume, volume_file)
+        fake_volume.delete.assert_called_once_with()
+
+    def test_cinder_delete(self):
+        fake_client = FakeObject(auth_token=None, management_url=None)
+        fake_volume_uuid = str(uuid.uuid4())
+        fake_volume = FakeObject(delete=mock.Mock())
+        fake_volumes = {fake_volume_uuid: fake_volume}
+
+        with mock.patch.object(cinder, 'get_cinderclient') as mocked_cc:
+            mocked_cc.return_value = FakeObject(client=fake_client,
+                                                volumes=fake_volumes)
+
+            uri = 'cinder://%s' % fake_volume_uuid
+            loc = location.get_location_from_uri_and_backend(uri,
+                                                             "cinder1",
+                                                             conf=self.conf)
+            self.store.delete(loc, context=self.context)
+            fake_volume.delete.assert_called_once_with()
+
+    def test_cinder_add_different_backend(self):
+        self.store = cinder.Store(self.conf, backend="cinder2")
+        self.store.configure()
+        self.register_store_backend_schemes(self.store, 'cinder', 'cinder2')
+
+        fake_volume = mock.MagicMock(id=str(uuid.uuid4()),
+                                     status='available',
+                                     size=1)
+        volume_file = six.BytesIO()
+        self._test_cinder_add(fake_volume, volume_file, backend="cinder2")