diff options
author | Zuul <zuul@review.opendev.org> | 2023-03-22 16:00:44 +0000 |
---|---|---|
committer | Gerrit Code Review <review@openstack.org> | 2023-03-22 16:00:44 +0000 |
commit | 31c279ff0ea0d1ede1a118d7189dcd009c14e23a (patch) | |
tree | 7db9558ffb740364a787ca34c0619b4c458eb2b0 /test | |
parent | cd82d15506307326afb2497e704df89ef04fb6f6 (diff) | |
parent | 6a8675e897e634abd26d993181ac4a45b9cf16f7 (diff) | |
download | python-swiftclient-31c279ff0ea0d1ede1a118d7189dcd009c14e23a.tar.gz |
Merge "Use SLO by default for segmented uploads if the cluster supports it"
Diffstat (limited to 'test')
-rw-r--r-- | test/unit/test_service.py | 80 | ||||
-rw-r--r-- | test/unit/test_shell.py | 65 |
2 files changed, 141 insertions, 4 deletions
diff --git a/test/unit/test_service.py b/test/unit/test_service.py index cdab61f..9935125 100644 --- a/test/unit/test_service.py +++ b/test/unit/test_service.py @@ -1410,7 +1410,7 @@ class TestServiceUpload(_TestServiceBase): self.assertTrue(fp.closed, 'Failed to close open(%s)' % formatted_args) - def test_upload_object_job_file_with_unicode_path(self): + def test_upload_dlo_job_file_with_unicode_path(self): # Uploading a file results in the file object being wrapped in a # LengthWrapper. This test sets the options in such a way that much # of _upload_object_job is skipped bringing the critical path down @@ -1471,6 +1471,84 @@ class TestServiceUpload(_TestServiceBase): headers={}, response_dict={}) + def test_upload_slo_job_file_with_unicode_path(self): + # Uploading a file results in the file object being wrapped in a + # LengthWrapper. This test sets the options in such a way that much + # of _upload_object_job is skipped bringing the critical path down + # to around 60 lines to ease testing. + with tempfile.NamedTemporaryFile() as f: + f.write(b'a' * 30) + f.flush() + expected_r = { + 'action': 'upload_object', + 'attempts': 2, + 'container': 'test_c', + 'headers': {}, + 'large_object': True, + 'object': 'テスト/dummy.dat', + 'manifest_response_dict': {}, + 'segment_results': [ + {'action': 'upload_segment', + 'success': True, + 'segment_index': i, + 'segment_location': 'test_c+segments/slo/%d' % i, + 'segment_etag': 'part-etag', + 'segment_size': 1000 + i} + for i in range(3)], + 'status': 'uploaded', + 'success': True, + } + expected_mtime = '%f' % os.path.getmtime(f.name) + + mock_conn = mock.Mock() + mock_conn.put_object.return_value = '' + type(mock_conn).attempts = mock.PropertyMock(return_value=2) + + s = SwiftService() + with mock.patch.object(s, '_upload_segment_job') as mock_job: + mock_job.side_effect = [ + {'action': 'upload_segment', + 'success': True, + 'segment_index': i, + 'segment_location': 'test_c+segments/slo/%d' % i, + 'segment_etag': 'part-etag', + 'segment_size': 1000 + i} + for i in range(3)] + + r = s._upload_object_job(conn=mock_conn, + container='test_c', + source=f.name, + obj='テスト/dummy.dat', + options=dict(s._options, + segment_size=10, + leave_segments=True, + use_slo=True)) + + mtime = r['headers']['x-object-meta-mtime'] + self.assertEqual(expected_mtime, mtime) + del r['headers']['x-object-meta-mtime'] + + self.assertEqual(r['path'], f.name) + del r['path'] + + self.assertEqual(r, expected_r) + self.assertEqual(mock_conn.put_object.call_count, 1) + mock_conn.put_object.assert_called_with( + 'test_c', + 'テスト/dummy.dat', + ' '.join([ + '[{"path": "test_c+segments/slo/0",', + '"etag": "part-etag", "size_bytes": 1000},', + '{"path": "test_c+segments/slo/1",', + '"etag": "part-etag", "size_bytes": 1001},', + '{"path": "test_c+segments/slo/2",', + '"etag": "part-etag", "size_bytes": 1002}]', + ]), + query_string='multipart-manifest=put', + headers={}, + response_dict={}, + ) + def test_upload_segment_job(self): with tempfile.NamedTemporaryFile() as f: f.write(b'a' * 10) diff --git a/test/unit/test_shell.py b/test/unit/test_shell.py index f103246..5e69f4a 100644 --- a/test/unit/test_shell.py +++ b/test/unit/test_shell.py @@ -861,7 +861,8 @@ class TestShell(unittest.TestCase): # Upload in segments connection.return_value.head_container.return_value = { 'x-storage-policy': 'one'} - argv = ["", "upload", "container", self.tmpfile, "-S", "10"] + argv = ["", "upload", "container", self.tmpfile, + "-S", "10", "--use-dlo"] with open(self.tmpfile, "wb") as fh: fh.write(b'12345678901234567890') swiftclient.shell.main(argv) @@ -907,6 +908,62 @@ class TestShell(unittest.TestCase): query_string='multipart-manifest=put', response_dict=mock.ANY) + # detect no SLO + connection.reset_mock() + connection.return_value.head_container.return_value = { + 'x-storage-policy': 'one'} + argv = ["", "upload", "container/pseudo-folder/nested", + self.tmpfile, "-S", "10"] + with open(self.tmpfile, "wb") as fh: + fh.write(b'12345678901234567890') + connection.return_value.get_capabilities.return_value = {} + swiftclient.shell.main(argv) + expected_calls = [mock.call('container', + {}, + response_dict={}), + mock.call('container_segments', + {'X-Storage-Policy': 'one'}, + response_dict={})] + connection.return_value.put_container.assert_has_calls(expected_calls) + connection.return_value.put_object.assert_called_with( + 'container', + 'pseudo-folder/nested' + self.tmpfile, + '', + content_length=0, + headers={'x-object-manifest': mock.ANY, + 'x-object-meta-mtime': mock.ANY}, + response_dict={}) + + # detect SLO + connection.reset_mock() + connection.return_value.head_container.return_value = { + 'x-storage-policy': 'one'} + argv = ["", "upload", "container/pseudo-folder/nested", + self.tmpfile, "-S", "10"] + with open(self.tmpfile, "wb") as fh: + fh.write(b'12345678901234567890') + connection.return_value.get_capabilities.return_value = { + 'slo': {}, + } + swiftclient.shell.main(argv) + expected_calls = [mock.call('container', + {}, + response_dict={}), + mock.call('container_segments', + {'X-Storage-Policy': 'one'}, + response_dict={})] + print(connection.return_value.mock_calls) + connection.return_value.put_container.assert_has_calls(expected_calls) + connection.return_value.put_object.assert_called_with( + 'container', + 'pseudo-folder/nested' + self.tmpfile, + mock.ANY, + headers={ + 'x-object-meta-mtime': mock.ANY, + }, + query_string='multipart-manifest=put', + response_dict=mock.ANY) + @mock.patch('swiftclient.shell.walk') @mock.patch('swiftclient.service.Connection') def test_upload_skip_container_put(self, connection, walk): @@ -934,7 +991,7 @@ class TestShell(unittest.TestCase): connection.return_value.head_container.return_value = { 'x-storage-policy': 'one'} argv = ["", "upload", "container", "--skip-container-put", - self.tmpfile, "-S", "10"] + self.tmpfile, "-S", "10", "--use-dlo"] with open(self.tmpfile, "wb") as fh: fh.write(b'12345678901234567890') swiftclient.shell.main(argv) @@ -1187,7 +1244,7 @@ class TestShell(unittest.TestCase): connection.return_value.attempts = 0 connection.return_value.put_object.return_value = EMPTY_ETAG argv = ["", "upload", "container", self.tmpfile, "-S", "10", - "-C", "container"] + "-C", "container", "--use-dlo"] with open(self.tmpfile, "wb") as fh: fh.write(b'12345678901234567890') swiftclient.shell.main(argv) @@ -3581,6 +3638,7 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): args, env = self._make_cmd('upload', cmd_args=[self.cont, self.obj, + '--use-dlo', '--leave-segments', '--segment-size=10', '--segment-container=%s' @@ -3621,6 +3679,7 @@ class TestCrossAccountObjectAccess(TestBase, MockHttpTest): args, env = self._make_cmd('upload', cmd_args=[self.cont, self.obj, + '--use-dlo', '--leave-segments', '--segment-size=10']) with mock.patch('swiftclient.client.ksclient_v3', self.fake_ks): |