diff options
author | Alexandre Arents <alexandre.arents@corp.ovh.com> | 2020-06-09 14:17:37 +0000 |
---|---|---|
committer | Alexandre Arents <alexandre.arents@corp.ovh.com> | 2020-06-30 07:21:02 +0000 |
commit | be9b7358473ca5276cff3c6491f88cd512d118b8 (patch) | |
tree | 717563f8a8daa332c996b570aa2df7765fc8fa40 /nova | |
parent | 6bb0c4fdabb98f168c530617b7c7a8f9396075fc (diff) | |
download | nova-be9b7358473ca5276cff3c6491f88cd512d118b8.tar.gz |
Snapshot: offload glance upload in a native thread
Execute glance upload in a native thread as it may block the current
coroutine until it completes.
Despite the fact we use eventlet monkey_patching [1] to achieve cooperative
yielding for network IO, file IO on busy file system may still get
nova-compute hanging.
Stick those IO in a native thread using eventlet tpool.execute() [2]
avoid this issue.
[1] https://eventlet.net/doc/patching.html
[2] https://eventlet.net/doc/threading.html
Closes-Bug: #1874032
Change-Id: I8dbc579e0037969aab4f2bb500fccfbde4190726
Diffstat (limited to 'nova')
-rw-r--r-- | nova/image/glance.py | 7 | ||||
-rw-r--r-- | nova/tests/unit/image/test_glance.py | 8 | ||||
-rw-r--r-- | nova/tests/unit/test_utils.py | 6 | ||||
-rw-r--r-- | nova/utils.py | 5 |
4 files changed, 24 insertions, 2 deletions
diff --git a/nova/image/glance.py b/nova/image/glance.py index daa397b2e6..6ebf578386 100644 --- a/nova/image/glance.py +++ b/nova/image/glance.py @@ -492,7 +492,12 @@ class GlanceImageServiceV2(object): _reraise_translated_exception() def _upload_data(self, context, image_id, data): - self._client.call(context, 2, 'upload', args=(image_id, data)) + # NOTE(aarents) offload upload in a native thread as it can block + # coroutine in busy environment. + utils.tpool_execute(self._client.call, + context, 2, 'upload', + args=(image_id, data)) + return self._client.call(context, 2, 'get', args=(image_id,)) def _get_image_create_disk_format_default(self, context): diff --git a/nova/tests/unit/image/test_glance.py b/nova/tests/unit/image/test_glance.py index e035fcca77..8dcb2e241f 100644 --- a/nova/tests/unit/image/test_glance.py +++ b/nova/tests/unit/image/test_glance.py @@ -1724,11 +1724,13 @@ class TestCreate(test.NoDBTestCase): class TestUpdate(test.NoDBTestCase): """Tests the update method of the GlanceImageServiceV2.""" + @mock.patch('nova.utils.tpool_execute', + side_effect=nova.utils.tpool_execute) @mock.patch('nova.image.glance.GlanceImageServiceV2.show') @mock.patch('nova.image.glance._translate_from_glance') @mock.patch('nova.image.glance._translate_to_glance') def test_update_success_v2( - self, trans_to_mock, trans_from_mock, show_mock): + self, trans_to_mock, trans_from_mock, show_mock, texec_mock): image = { 'id': mock.sentinel.image_id, 'name': mock.sentinel.name, @@ -1777,6 +1779,10 @@ class TestUpdate(test.NoDBTestCase): data=mock.sentinel.data) self.assertEqual(3, client.call.call_count) + texec_mock.assert_called_once_with( + client.call, ctx, 2, 'upload', + args=(mock.sentinel.image_id, + mock.sentinel.data)) @mock.patch('nova.image.glance.GlanceImageServiceV2.show') @mock.patch('nova.image.glance._translate_from_glance') diff --git a/nova/tests/unit/test_utils.py b/nova/tests/unit/test_utils.py index 23a080f6a2..d6cb92b206 100644 --- a/nova/tests/unit/test_utils.py +++ b/nova/tests/unit/test_utils.py @@ -225,6 +225,12 @@ class GenericUtilsTestCase(test.NoDBTestCase): utils.ssh_execute('remotehost', 'ls', '-l') mock_execute.assert_called_once_with(*expected_args) + @mock.patch('nova.utils.generate_uid') + def test_tpool_execute(self, mock_generate): + expected_kargs = {'size': 12} + utils.tpool_execute(utils.generate_uid, 'mytopic', size=12) + mock_generate.assert_called_once_with('mytopic', **expected_kargs) + def test_generate_hostid(self): host = 'host' project_id = '9b9e3c847e904b0686e8ffb20e4c6381' diff --git a/nova/utils.py b/nova/utils.py index ed26d006f2..e2d9d5e657 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -698,6 +698,11 @@ def spawn_n(func, *args, **kwargs): eventlet.spawn_n(context_wrapper, *args, **kwargs) +def tpool_execute(func, *args, **kwargs): + """Run func in a native thread""" + eventlet.tpool.execute(func, *args, **kwargs) + + def is_none_string(val): """Check if a string represents a None value. """ |