diff options
author | Victor Stinner <vstinner@redhat.com> | 2015-05-20 12:31:59 -0700 |
---|---|---|
committer | Victor Stinner <vstinner@redhat.com> | 2015-06-18 13:30:03 +0200 |
commit | 70abda6ab996df8fc8ef0fded0c4ac8664ccdfce (patch) | |
tree | 48286211b6c1c2c0816ac41b1cf7efa1aef716c1 | |
parent | 0ae8f4102e7aeebb75fe5f65c02ed9bfa76cebb5 (diff) | |
download | glance_store-70abda6ab996df8fc8ef0fded0c4ac8664ccdfce.tar.gz |
Port swift driver to Python 3
* On Python 3, swift_retry_iter() now uses seek()+tell() to get the
length of a BytesIO object instead of getting the len attribute,
because the len attribute has been removed in Python 3.
* Replace six.StringIO with six.BytesIO for image content
* Use bytes to handle image content
* Replace int / 2 with int // 2 to use integer division
Change-Id: I0972df318002a6f91c3ef7ddc91330c635ba7e41
-rw-r--r-- | glance_store/_drivers/swift/store.py | 14 | ||||
-rw-r--r-- | glance_store/tests/unit/test_swift_store.py | 100 |
2 files changed, 66 insertions, 48 deletions
diff --git a/glance_store/_drivers/swift/store.py b/glance_store/_drivers/swift/store.py index f3fd7ff..84f813c 100644 --- a/glance_store/_drivers/swift/store.py +++ b/glance_store/_drivers/swift/store.py @@ -126,6 +126,18 @@ _SWIFT_OPTS = [ def swift_retry_iter(resp_iter, length, store, location, context): + if not length and isinstance(resp_iter, six.BytesIO): + if six.PY3: + # On Python 3, io.BytesIO does not have a len attribute, instead + # go the end using seek to get the size of the file + pos = resp_iter.tell() + resp_iter.seek(0, 2) + length = resp_iter.tell() + resp_iter.seek(pos) + else: + # On Python 2, StringIO has a len attribute + length = resp_iter.len + length = length if length else (resp_iter.len if hasattr(resp_iter, 'len') else 0) retries = 0 @@ -577,7 +589,7 @@ class BaseStore(driver.Store): # Now we write the object manifest and return the # manifest's etag... manifest = "%s/%s-" % (location.container, location.obj) - headers = {'ETag': hashlib.md5("").hexdigest(), + headers = {'ETag': hashlib.md5(b"").hexdigest(), 'X-Object-Manifest': manifest} # The ETag returned for the manifest is actually the diff --git a/glance_store/tests/unit/test_swift_store.py b/glance_store/tests/unit/test_swift_store.py index d4547c8..b217453 100644 --- a/glance_store/tests/unit/test_swift_store.py +++ b/glance_store/tests/unit/test_swift_store.py @@ -79,8 +79,8 @@ def stub_out_swiftclient(stubs, swift_store_auth_version): }, 'glance/%s' % FAKE_UUID2: {'x-static-large-object': 'true', }, } - fixture_objects = {'glance/%s' % FAKE_UUID: six.StringIO("*" * FIVE_KB), - 'glance/%s' % FAKE_UUID2: six.StringIO("*" * FIVE_KB), } + fixture_objects = {'glance/%s' % FAKE_UUID: six.BytesIO(b"*" * FIVE_KB), + 'glance/%s' % FAKE_UUID2: six.BytesIO(b"*" * FIVE_KB), } def fake_head_container(url, token, container, **kwargs): if container not in fixture_containers: @@ -113,18 +113,20 @@ def stub_out_swiftclient(stubs, swift_store_auth_version): fixture_objects[fixture_key] = None return etag if hasattr(contents, 'read'): - fixture_object = six.StringIO() + fixture_object = six.BytesIO() + read_len = 0 chunk = contents.read(CHUNKSIZE) checksum = hashlib.md5() while chunk: fixture_object.write(chunk) + read_len += len(chunk) checksum.update(chunk) chunk = contents.read(CHUNKSIZE) etag = checksum.hexdigest() else: - fixture_object = six.StringIO(contents) + fixture_object = six.BytesIO(contents) + read_len = len(contents) etag = hashlib.md5(fixture_object.getvalue()).hexdigest() - read_len = fixture_object.len if read_len > MAX_SWIFT_OBJECT_SIZE: msg = ('Image size:%d exceeds Swift max:%d' % (read_len, MAX_SWIFT_OBJECT_SIZE)) @@ -163,7 +165,7 @@ def stub_out_swiftclient(stubs, swift_store_auth_version): chunk_keys = sorted([k for k in fixture_headers.keys() if k.startswith(fixture_key) and k != fixture_key]) - result = six.StringIO() + result = six.BytesIO() for key in chunk_keys: result.write(fixture_objects[key].getvalue()) else: @@ -171,7 +173,7 @@ def stub_out_swiftclient(stubs, swift_store_auth_version): if byte_range is not None: start = int(byte_range.split('=')[1].strip('-')) - result = six.StringIO(result.getvalue()[start:]) + result = six.BytesIO(result.getvalue()[start:]) fixture_headers[fixture_key]['content-length'] = len( result.getvalue()) @@ -266,8 +268,8 @@ class SwiftTests(object): (image_swift, image_size) = self.store.get(loc) self.assertEqual(image_size, 5120) - expected_data = "*" * FIVE_KB - data = "" + expected_data = b"*" * FIVE_KB + data = b"" for chunk in image_swift: data += chunk @@ -283,16 +285,17 @@ class SwiftTests(object): loc = location.get_location_from_uri(uri, conf=self.conf) ctxt = mock.MagicMock() (image_swift, image_size) = self.store.get(loc, context=ctxt) - resp_full = ''.join([chunk for chunk in image_swift.wrapped]) - resp_half = resp_full[:len(resp_full) / 2] + resp_full = b''.join([chunk for chunk in image_swift.wrapped]) + resp_half = resp_full[:len(resp_full) // 2] + resp_half = six.BytesIO(resp_half) image_swift.wrapped = swift.swift_retry_iter(resp_half, image_size, self.store, loc.store_location, ctxt) self.assertEqual(image_size, 5120) - expected_data = "*" * FIVE_KB - data = "" + expected_data = b"*" * FIVE_KB + data = b"" for chunk in image_swift: data += chunk @@ -312,8 +315,8 @@ class SwiftTests(object): (image_swift, image_size) = self.store.get(loc, context=ctxt) self.assertEqual(image_size, 5120) - expected_data = "*" * FIVE_KB - data = "" + expected_data = b"*" * FIVE_KB + data = b"" for chunk in image_swift: data += chunk @@ -336,16 +339,16 @@ class SwiftTests(object): mock.Mock(return_value=False)) def test_add(self): """Test that we can add an image via the swift backend.""" - reload(swift) + moves.reload_module(swift) self.store = Store(self.conf) self.store.configure() expected_swift_size = FIVE_KB - expected_swift_contents = "*" * expected_swift_size + expected_swift_contents = b"*" * expected_swift_size expected_checksum = hashlib.md5(expected_swift_contents).hexdigest() expected_image_id = str(uuid.uuid4()) loc = "swift+https://tenant%%3Auser1:key@localhost:8080/glance/%s" expected_location = loc % (expected_image_id) - image_swift = six.StringIO(expected_swift_contents) + image_swift = six.BytesIO(expected_swift_contents) global SWIFT_PUT_OBJECT_CALLS SWIFT_PUT_OBJECT_CALLS = 0 @@ -362,7 +365,7 @@ class SwiftTests(object): loc = location.get_location_from_uri(expected_location, conf=self.conf) (new_image_swift, new_image_size) = self.store.get(loc) - new_image_contents = ''.join([chunk for chunk in new_image_swift]) + new_image_contents = b''.join([chunk for chunk in new_image_swift]) new_image_swift_size = len(new_image_swift) self.assertEqual(expected_swift_contents, new_image_contents) @@ -378,9 +381,9 @@ class SwiftTests(object): self.store.configure() expected_swift_size = FIVE_KB - expected_swift_contents = "*" * expected_swift_size + expected_swift_contents = b"*" * expected_swift_size expected_image_id = str(uuid.uuid4()) - image_swift = six.StringIO(expected_swift_contents) + image_swift = six.BytesIO(expected_swift_contents) global SWIFT_PUT_OBJECT_CALLS SWIFT_PUT_OBJECT_CALLS = 0 loc = 'swift+config://store_2/glance/%s' @@ -413,11 +416,11 @@ class SwiftTests(object): image_id = str(uuid.uuid4()) expected_location = expected_location % image_id expected_swift_size = FIVE_KB - expected_swift_contents = "*" * expected_swift_size + expected_swift_contents = b"*" * expected_swift_size expected_checksum = \ hashlib.md5(expected_swift_contents).hexdigest() - image_swift = six.StringIO(expected_swift_contents) + image_swift = six.BytesIO(expected_swift_contents) global SWIFT_PUT_OBJECT_CALLS SWIFT_PUT_OBJECT_CALLS = 0 @@ -437,7 +440,7 @@ class SwiftTests(object): loc = location.get_location_from_uri(expected_location, conf=self.conf) (new_image_swift, new_image_size) = self.store.get(loc) - new_image_contents = ''.join([chunk for chunk in new_image_swift]) + new_image_contents = b''.join([chunk for chunk in new_image_swift]) new_image_swift_size = len(new_image_swift) self.assertEqual(expected_swift_contents, new_image_contents) @@ -458,7 +461,7 @@ class SwiftTests(object): self.store = Store(self.conf) self.store.configure() - image_swift = six.StringIO("nevergonnamakeit") + image_swift = six.BytesIO(b"nevergonnamakeit") global SWIFT_PUT_OBJECT_CALLS SWIFT_PUT_OBJECT_CALLS = 0 @@ -485,12 +488,12 @@ class SwiftTests(object): creates the container automatically if flag is set """ expected_swift_size = FIVE_KB - expected_swift_contents = "*" * expected_swift_size + expected_swift_contents = b"*" * expected_swift_size expected_checksum = hashlib.md5(expected_swift_contents).hexdigest() expected_image_id = str(uuid.uuid4()) loc = 'swift+config://ref1/noexist/%s' expected_location = loc % (expected_image_id) - image_swift = six.StringIO(expected_swift_contents) + image_swift = six.BytesIO(expected_swift_contents) global SWIFT_PUT_OBJECT_CALLS SWIFT_PUT_OBJECT_CALLS = 0 @@ -513,7 +516,7 @@ class SwiftTests(object): loc = location.get_location_from_uri(expected_location, conf=self.conf) (new_image_swift, new_image_size) = self.store.get(loc) - new_image_contents = ''.join([chunk for chunk in new_image_swift]) + new_image_contents = b''.join([chunk for chunk in new_image_swift]) new_image_swift_size = len(new_image_swift) self.assertEqual(expected_swift_contents, new_image_contents) @@ -528,13 +531,13 @@ class SwiftTests(object): multi containers will create the container automatically if flag is set """ expected_swift_size = FIVE_KB - expected_swift_contents = "*" * expected_swift_size + expected_swift_contents = b"*" * expected_swift_size expected_checksum = hashlib.md5(expected_swift_contents).hexdigest() expected_image_id = str(uuid.uuid4()) container = 'randomname_' + expected_image_id[:2] loc = 'swift+config://ref1/%s/%s' expected_location = loc % (container, expected_image_id) - image_swift = six.StringIO(expected_swift_contents) + image_swift = six.BytesIO(expected_swift_contents) global SWIFT_PUT_OBJECT_CALLS SWIFT_PUT_OBJECT_CALLS = 0 @@ -558,7 +561,7 @@ class SwiftTests(object): loc = location.get_location_from_uri(expected_location, conf=self.conf) (new_image_swift, new_image_size) = self.store.get(loc) - new_image_contents = ''.join([chunk for chunk in new_image_swift]) + new_image_contents = b''.join([chunk for chunk in new_image_swift]) new_image_swift_size = len(new_image_swift) self.assertEqual(expected_swift_contents, new_image_contents) @@ -586,7 +589,7 @@ class SwiftTests(object): self.store = Store(self.conf) self.store.configure() - image_swift = six.StringIO("nevergonnamakeit") + image_swift = six.BytesIO(b"nevergonnamakeit") global SWIFT_PUT_OBJECT_CALLS SWIFT_PUT_OBJECT_CALLS = 0 @@ -610,12 +613,12 @@ class SwiftTests(object): mock.Mock(return_value=False)) def test_multi_container_doesnt_impact_multi_tenant_add(self): expected_swift_size = FIVE_KB - expected_swift_contents = "*" * expected_swift_size + expected_swift_contents = b"*" * expected_swift_size expected_image_id = str(uuid.uuid4()) expected_container = 'container_' + expected_image_id loc = 'swift+https://some_endpoint/%s/%s' expected_location = loc % (expected_container, expected_image_id) - image_swift = six.StringIO(expected_swift_contents) + image_swift = six.BytesIO(expected_swift_contents) global SWIFT_PUT_OBJECT_CALLS SWIFT_PUT_OBJECT_CALLS = 0 @@ -646,12 +649,12 @@ class SwiftTests(object): put_object()... """ expected_swift_size = FIVE_KB - expected_swift_contents = "*" * expected_swift_size + expected_swift_contents = b"*" * expected_swift_size expected_checksum = hashlib.md5(expected_swift_contents).hexdigest() expected_image_id = str(uuid.uuid4()) loc = 'swift+config://ref1/glance/%s' expected_location = loc % (expected_image_id) - image_swift = six.StringIO(expected_swift_contents) + image_swift = six.BytesIO(expected_swift_contents) global SWIFT_PUT_OBJECT_CALLS SWIFT_PUT_OBJECT_CALLS = 0 @@ -679,7 +682,7 @@ class SwiftTests(object): loc = location.get_location_from_uri(expected_location, conf=self.conf) (new_image_swift, new_image_size) = self.store.get(loc) - new_image_contents = ''.join([chunk for chunk in new_image_swift]) + new_image_contents = b''.join([chunk for chunk in new_image_swift]) new_image_swift_size = len(new_image_contents) self.assertEqual(expected_swift_contents, new_image_contents) @@ -698,12 +701,12 @@ class SwiftTests(object): """ # Set up a 'large' image of 5KB expected_swift_size = FIVE_KB - expected_swift_contents = "*" * expected_swift_size + expected_swift_contents = b"*" * expected_swift_size expected_checksum = hashlib.md5(expected_swift_contents).hexdigest() expected_image_id = str(uuid.uuid4()) loc = 'swift+config://ref1/glance/%s' expected_location = loc % (expected_image_id) - image_swift = six.StringIO(expected_swift_contents) + image_swift = six.BytesIO(expected_swift_contents) global SWIFT_PUT_OBJECT_CALLS SWIFT_PUT_OBJECT_CALLS = 0 @@ -739,7 +742,7 @@ class SwiftTests(object): loc = location.get_location_from_uri(expected_location, conf=self.conf) (new_image_swift, new_image_size) = self.store.get(loc) - new_image_contents = ''.join([chunk for chunk in new_image_swift]) + new_image_contents = b''.join([chunk for chunk in new_image_swift]) new_image_swift_size = len(new_image_contents) self.assertEqual(expected_swift_contents, new_image_contents) @@ -752,7 +755,7 @@ class SwiftTests(object): """ self.store = Store(self.conf) self.store.configure() - image_swift = six.StringIO("nevergonnamakeit") + image_swift = six.BytesIO(b"nevergonnamakeit") self.assertRaises(exceptions.Duplicate, self.store.add, FAKE_UUID, image_swift, 0) @@ -1268,7 +1271,7 @@ class TestMultiTenantStoreContext(base.StoreBaseTest): m.get("http://127.0.0.1/glance_123/123") store.get(loc, context=ctx) - self.assertEqual('0123', m.last_request.headers['X-Auth-Token']) + self.assertEqual(b'0123', m.last_request.headers['X-Auth-Token']) @requests_mock.mock() def test_upload_context(self, m): @@ -1281,15 +1284,18 @@ class TestMultiTenantStoreContext(base.StoreBaseTest): self.config(swift_store_multi_tenant=True) store = Store(self.conf) store.configure() - pseudo_file = six.StringIO('Some data') + content = b'Some data' + pseudo_file = six.BytesIO(content) ctx = mock.MagicMock( service_catalog=self.service_catalog, user='tenant:user1', tenant='tenant', auth_token='0123') - store.add('123', pseudo_file, pseudo_file.len, + store.add('123', pseudo_file, len(content), context=ctx) - self.assertEqual('0123', head_req.last_request.headers['X-Auth-Token']) - self.assertEqual('0123', put_req.last_request.headers['X-Auth-Token']) + self.assertEqual(b'0123', + head_req.last_request.headers['X-Auth-Token']) + self.assertEqual(b'0123', + put_req.last_request.headers['X-Auth-Token']) class FakeGetEndpoint(object): @@ -1435,7 +1441,7 @@ class TestChunkReader(base.StoreBaseTest): CHUNKSIZE = 100 checksum = hashlib.md5() data_file = tempfile.NamedTemporaryFile() - data_file.write('*' * units.Ki) + data_file.write(b'*' * units.Ki) data_file.flush() infile = open(data_file.name, 'rb') bytes_read = 0 |