Merge "Port swift driver to Python 3"

2 files changed, 66 insertions, 48 deletions

diff --git a/glance_store/_drivers/swift/store.py b/glance_store/_drivers/swift/store.py
index 9e20616..4a4bc31 100644
--- a/glance_store/_drivers/swift/store.py
+++ b/glance_store/_drivers/swift/store.py
@@ -126,6 +126,18 @@ _SWIFT_OPTS = [
 
 
 def swift_retry_iter(resp_iter, length, store, location, context):
+    if not length and isinstance(resp_iter, six.BytesIO):
+        if six.PY3:
+            # On Python 3, io.BytesIO does not have a len attribute, instead
+            # go the end using seek to get the size of the file
+            pos = resp_iter.tell()
+            resp_iter.seek(0, 2)
+            length = resp_iter.tell()
+            resp_iter.seek(pos)
+        else:
+            # On Python 2, StringIO has a len attribute
+            length = resp_iter.len
+
     length = length if length else (resp_iter.len
                                     if hasattr(resp_iter, 'len') else 0)
     retries = 0
@@ -577,7 +589,7 @@ class BaseStore(driver.Store):
                 # Now we write the object manifest and return the
                 # manifest's etag...
                 manifest = "%s/%s-" % (location.container, location.obj)
-                headers = {'ETag': hashlib.md5("").hexdigest(),
+                headers = {'ETag': hashlib.md5(b"").hexdigest(),
                            'X-Object-Manifest': manifest}
 
                 # The ETag returned for the manifest is actually the
diff --git a/glance_store/tests/unit/test_swift_store.py b/glance_store/tests/unit/test_swift_store.py
index c83381e..f4b8781 100644
--- a/glance_store/tests/unit/test_swift_store.py
+++ b/glance_store/tests/unit/test_swift_store.py
@@ -79,8 +79,8 @@ def stub_out_swiftclient(stubs, swift_store_auth_version):
         },
         'glance/%s' % FAKE_UUID2: {'x-static-large-object': 'true', },
     }
-    fixture_objects = {'glance/%s' % FAKE_UUID: six.StringIO("*" * FIVE_KB),
-                       'glance/%s' % FAKE_UUID2: six.StringIO("*" * FIVE_KB), }
+    fixture_objects = {'glance/%s' % FAKE_UUID: six.BytesIO(b"*" * FIVE_KB),
+                       'glance/%s' % FAKE_UUID2: six.BytesIO(b"*" * FIVE_KB), }
 
     def fake_head_container(url, token, container, **kwargs):
         if container not in fixture_containers:
@@ -113,18 +113,20 @@ def stub_out_swiftclient(stubs, swift_store_auth_version):
                 fixture_objects[fixture_key] = None
                 return etag
             if hasattr(contents, 'read'):
-                fixture_object = six.StringIO()
+                fixture_object = six.BytesIO()
+                read_len = 0
                 chunk = contents.read(CHUNKSIZE)
                 checksum = hashlib.md5()
                 while chunk:
                     fixture_object.write(chunk)
+                    read_len += len(chunk)
                     checksum.update(chunk)
                     chunk = contents.read(CHUNKSIZE)
                 etag = checksum.hexdigest()
             else:
-                fixture_object = six.StringIO(contents)
+                fixture_object = six.BytesIO(contents)
+                read_len = len(contents)
                 etag = hashlib.md5(fixture_object.getvalue()).hexdigest()
-            read_len = fixture_object.len
             if read_len > MAX_SWIFT_OBJECT_SIZE:
                 msg = ('Image size:%d exceeds Swift max:%d' %
                        (read_len, MAX_SWIFT_OBJECT_SIZE))
@@ -163,7 +165,7 @@ def stub_out_swiftclient(stubs, swift_store_auth_version):
             chunk_keys = sorted([k for k in fixture_headers.keys()
                                  if k.startswith(fixture_key) and
                                  k != fixture_key])
-            result = six.StringIO()
+            result = six.BytesIO()
             for key in chunk_keys:
                 result.write(fixture_objects[key].getvalue())
         else:
@@ -171,7 +173,7 @@ def stub_out_swiftclient(stubs, swift_store_auth_version):
 
         if byte_range is not None:
             start = int(byte_range.split('=')[1].strip('-'))
-            result = six.StringIO(result.getvalue()[start:])
+            result = six.BytesIO(result.getvalue()[start:])
             fixture_headers[fixture_key]['content-length'] = len(
                 result.getvalue())
 
@@ -266,8 +268,8 @@ class SwiftTests(object):
         (image_swift, image_size) = self.store.get(loc)
         self.assertEqual(image_size, 5120)
 
-        expected_data = "*" * FIVE_KB
-        data = ""
+        expected_data = b"*" * FIVE_KB
+        data = b""
 
         for chunk in image_swift:
             data += chunk
@@ -283,16 +285,17 @@ class SwiftTests(object):
         loc = location.get_location_from_uri(uri, conf=self.conf)
         ctxt = mock.MagicMock()
         (image_swift, image_size) = self.store.get(loc, context=ctxt)
-        resp_full = ''.join([chunk for chunk in image_swift.wrapped])
-        resp_half = resp_full[:len(resp_full) / 2]
+        resp_full = b''.join([chunk for chunk in image_swift.wrapped])
+        resp_half = resp_full[:len(resp_full) // 2]
+        resp_half = six.BytesIO(resp_half)
         image_swift.wrapped = swift.swift_retry_iter(resp_half, image_size,
                                                      self.store,
                                                      loc.store_location,
                                                      ctxt)
         self.assertEqual(image_size, 5120)
 
-        expected_data = "*" * FIVE_KB
-        data = ""
+        expected_data = b"*" * FIVE_KB
+        data = b""
 
         for chunk in image_swift:
             data += chunk
@@ -312,8 +315,8 @@ class SwiftTests(object):
         (image_swift, image_size) = self.store.get(loc, context=ctxt)
         self.assertEqual(image_size, 5120)
 
-        expected_data = "*" * FIVE_KB
-        data = ""
+        expected_data = b"*" * FIVE_KB
+        data = b""
 
         for chunk in image_swift:
             data += chunk
@@ -336,16 +339,16 @@ class SwiftTests(object):
                 mock.Mock(return_value=False))
     def test_add(self):
         """Test that we can add an image via the swift backend."""
-        reload(swift)
+        moves.reload_module(swift)
         self.store = Store(self.conf)
         self.store.configure()
         expected_swift_size = FIVE_KB
-        expected_swift_contents = "*" * expected_swift_size
+        expected_swift_contents = b"*" * expected_swift_size
         expected_checksum = hashlib.md5(expected_swift_contents).hexdigest()
         expected_image_id = str(uuid.uuid4())
         loc = "swift+https://tenant%%3Auser1:key@localhost:8080/glance/%s"
         expected_location = loc % (expected_image_id)
-        image_swift = six.StringIO(expected_swift_contents)
+        image_swift = six.BytesIO(expected_swift_contents)
 
         global SWIFT_PUT_OBJECT_CALLS
         SWIFT_PUT_OBJECT_CALLS = 0
@@ -362,7 +365,7 @@ class SwiftTests(object):
 
         loc = location.get_location_from_uri(expected_location, conf=self.conf)
         (new_image_swift, new_image_size) = self.store.get(loc)
-        new_image_contents = ''.join([chunk for chunk in new_image_swift])
+        new_image_contents = b''.join([chunk for chunk in new_image_swift])
         new_image_swift_size = len(new_image_swift)
 
         self.assertEqual(expected_swift_contents, new_image_contents)
@@ -378,9 +381,9 @@ class SwiftTests(object):
         self.store.configure()
 
         expected_swift_size = FIVE_KB
-        expected_swift_contents = "*" * expected_swift_size
+        expected_swift_contents = b"*" * expected_swift_size
         expected_image_id = str(uuid.uuid4())
-        image_swift = six.StringIO(expected_swift_contents)
+        image_swift = six.BytesIO(expected_swift_contents)
         global SWIFT_PUT_OBJECT_CALLS
         SWIFT_PUT_OBJECT_CALLS = 0
         loc = 'swift+config://store_2/glance/%s'
@@ -413,11 +416,11 @@ class SwiftTests(object):
             image_id = str(uuid.uuid4())
             expected_location = expected_location % image_id
             expected_swift_size = FIVE_KB
-            expected_swift_contents = "*" * expected_swift_size
+            expected_swift_contents = b"*" * expected_swift_size
             expected_checksum = \
                 hashlib.md5(expected_swift_contents).hexdigest()
 
-            image_swift = six.StringIO(expected_swift_contents)
+            image_swift = six.BytesIO(expected_swift_contents)
 
             global SWIFT_PUT_OBJECT_CALLS
             SWIFT_PUT_OBJECT_CALLS = 0
@@ -437,7 +440,7 @@ class SwiftTests(object):
             loc = location.get_location_from_uri(expected_location,
                                                  conf=self.conf)
             (new_image_swift, new_image_size) = self.store.get(loc)
-            new_image_contents = ''.join([chunk for chunk in new_image_swift])
+            new_image_contents = b''.join([chunk for chunk in new_image_swift])
             new_image_swift_size = len(new_image_swift)
 
             self.assertEqual(expected_swift_contents, new_image_contents)
@@ -458,7 +461,7 @@ class SwiftTests(object):
         self.store = Store(self.conf)
         self.store.configure()
 
-        image_swift = six.StringIO("nevergonnamakeit")
+        image_swift = six.BytesIO(b"nevergonnamakeit")
 
         global SWIFT_PUT_OBJECT_CALLS
         SWIFT_PUT_OBJECT_CALLS = 0
@@ -485,12 +488,12 @@ class SwiftTests(object):
         creates the container automatically if flag is set
         """
         expected_swift_size = FIVE_KB
-        expected_swift_contents = "*" * expected_swift_size
+        expected_swift_contents = b"*" * expected_swift_size
         expected_checksum = hashlib.md5(expected_swift_contents).hexdigest()
         expected_image_id = str(uuid.uuid4())
         loc = 'swift+config://ref1/noexist/%s'
         expected_location = loc % (expected_image_id)
-        image_swift = six.StringIO(expected_swift_contents)
+        image_swift = six.BytesIO(expected_swift_contents)
 
         global SWIFT_PUT_OBJECT_CALLS
         SWIFT_PUT_OBJECT_CALLS = 0
@@ -513,7 +516,7 @@ class SwiftTests(object):
 
         loc = location.get_location_from_uri(expected_location, conf=self.conf)
         (new_image_swift, new_image_size) = self.store.get(loc)
-        new_image_contents = ''.join([chunk for chunk in new_image_swift])
+        new_image_contents = b''.join([chunk for chunk in new_image_swift])
         new_image_swift_size = len(new_image_swift)
 
         self.assertEqual(expected_swift_contents, new_image_contents)
@@ -528,13 +531,13 @@ class SwiftTests(object):
         multi containers will create the container automatically if flag is set
         """
         expected_swift_size = FIVE_KB
-        expected_swift_contents = "*" * expected_swift_size
+        expected_swift_contents = b"*" * expected_swift_size
         expected_checksum = hashlib.md5(expected_swift_contents).hexdigest()
         expected_image_id = str(uuid.uuid4())
         container = 'randomname_' + expected_image_id[:2]
         loc = 'swift+config://ref1/%s/%s'
         expected_location = loc % (container, expected_image_id)
-        image_swift = six.StringIO(expected_swift_contents)
+        image_swift = six.BytesIO(expected_swift_contents)
 
         global SWIFT_PUT_OBJECT_CALLS
         SWIFT_PUT_OBJECT_CALLS = 0
@@ -558,7 +561,7 @@ class SwiftTests(object):
 
         loc = location.get_location_from_uri(expected_location, conf=self.conf)
         (new_image_swift, new_image_size) = self.store.get(loc)
-        new_image_contents = ''.join([chunk for chunk in new_image_swift])
+        new_image_contents = b''.join([chunk for chunk in new_image_swift])
         new_image_swift_size = len(new_image_swift)
 
         self.assertEqual(expected_swift_contents, new_image_contents)
@@ -586,7 +589,7 @@ class SwiftTests(object):
         self.store = Store(self.conf)
         self.store.configure()
 
-        image_swift = six.StringIO("nevergonnamakeit")
+        image_swift = six.BytesIO(b"nevergonnamakeit")
 
         global SWIFT_PUT_OBJECT_CALLS
         SWIFT_PUT_OBJECT_CALLS = 0
@@ -610,12 +613,12 @@ class SwiftTests(object):
                 mock.Mock(return_value=False))
     def test_multi_container_doesnt_impact_multi_tenant_add(self):
         expected_swift_size = FIVE_KB
-        expected_swift_contents = "*" * expected_swift_size
+        expected_swift_contents = b"*" * expected_swift_size
         expected_image_id = str(uuid.uuid4())
         expected_container = 'container_' + expected_image_id
         loc = 'swift+https://some_endpoint/%s/%s'
         expected_location = loc % (expected_container, expected_image_id)
-        image_swift = six.StringIO(expected_swift_contents)
+        image_swift = six.BytesIO(expected_swift_contents)
 
         global SWIFT_PUT_OBJECT_CALLS
         SWIFT_PUT_OBJECT_CALLS = 0
@@ -646,12 +649,12 @@ class SwiftTests(object):
         put_object()...
         """
         expected_swift_size = FIVE_KB
-        expected_swift_contents = "*" * expected_swift_size
+        expected_swift_contents = b"*" * expected_swift_size
         expected_checksum = hashlib.md5(expected_swift_contents).hexdigest()
         expected_image_id = str(uuid.uuid4())
         loc = 'swift+config://ref1/glance/%s'
         expected_location = loc % (expected_image_id)
-        image_swift = six.StringIO(expected_swift_contents)
+        image_swift = six.BytesIO(expected_swift_contents)
 
         global SWIFT_PUT_OBJECT_CALLS
         SWIFT_PUT_OBJECT_CALLS = 0
@@ -679,7 +682,7 @@ class SwiftTests(object):
 
         loc = location.get_location_from_uri(expected_location, conf=self.conf)
         (new_image_swift, new_image_size) = self.store.get(loc)
-        new_image_contents = ''.join([chunk for chunk in new_image_swift])
+        new_image_contents = b''.join([chunk for chunk in new_image_swift])
         new_image_swift_size = len(new_image_contents)
 
         self.assertEqual(expected_swift_contents, new_image_contents)
@@ -698,12 +701,12 @@ class SwiftTests(object):
         """
         # Set up a 'large' image of 5KB
         expected_swift_size = FIVE_KB
-        expected_swift_contents = "*" * expected_swift_size
+        expected_swift_contents = b"*" * expected_swift_size
         expected_checksum = hashlib.md5(expected_swift_contents).hexdigest()
         expected_image_id = str(uuid.uuid4())
         loc = 'swift+config://ref1/glance/%s'
         expected_location = loc % (expected_image_id)
-        image_swift = six.StringIO(expected_swift_contents)
+        image_swift = six.BytesIO(expected_swift_contents)
 
         global SWIFT_PUT_OBJECT_CALLS
         SWIFT_PUT_OBJECT_CALLS = 0
@@ -739,7 +742,7 @@ class SwiftTests(object):
 
         loc = location.get_location_from_uri(expected_location, conf=self.conf)
         (new_image_swift, new_image_size) = self.store.get(loc)
-        new_image_contents = ''.join([chunk for chunk in new_image_swift])
+        new_image_contents = b''.join([chunk for chunk in new_image_swift])
         new_image_swift_size = len(new_image_contents)
 
         self.assertEqual(expected_swift_contents, new_image_contents)
@@ -752,7 +755,7 @@ class SwiftTests(object):
         """
         self.store = Store(self.conf)
         self.store.configure()
-        image_swift = six.StringIO("nevergonnamakeit")
+        image_swift = six.BytesIO(b"nevergonnamakeit")
         self.assertRaises(exceptions.Duplicate,
                           self.store.add,
                           FAKE_UUID, image_swift, 0)
@@ -1268,7 +1271,7 @@ class TestMultiTenantStoreContext(base.StoreBaseTest):
 
         m.get("http://127.0.0.1/glance_123/123")
         store.get(loc, context=ctx)
-        self.assertEqual('0123', m.last_request.headers['X-Auth-Token'])
+        self.assertEqual(b'0123', m.last_request.headers['X-Auth-Token'])
 
     @requests_mock.mock()
     def test_upload_context(self, m):
@@ -1281,15 +1284,18 @@ class TestMultiTenantStoreContext(base.StoreBaseTest):
         self.config(swift_store_multi_tenant=True)
         store = Store(self.conf)
         store.configure()
-        pseudo_file = six.StringIO('Some data')
+        content = b'Some data'
+        pseudo_file = six.BytesIO(content)
         ctx = mock.MagicMock(
             service_catalog=self.service_catalog, user='tenant:user1',
             tenant='tenant', auth_token='0123')
-        store.add('123', pseudo_file, pseudo_file.len,
+        store.add('123', pseudo_file, len(content),
                   context=ctx)
 
-        self.assertEqual('0123', head_req.last_request.headers['X-Auth-Token'])
-        self.assertEqual('0123', put_req.last_request.headers['X-Auth-Token'])
+        self.assertEqual(b'0123',
+                         head_req.last_request.headers['X-Auth-Token'])
+        self.assertEqual(b'0123',
+                         put_req.last_request.headers['X-Auth-Token'])
 
 
 class FakeGetEndpoint(object):
@@ -1435,7 +1441,7 @@ class TestChunkReader(base.StoreBaseTest):
         CHUNKSIZE = 100
         checksum = hashlib.md5()
         data_file = tempfile.NamedTemporaryFile()
-        data_file.write('*' * units.Ki)
+        data_file.write(b'*' * units.Ki)
         data_file.flush()
         infile = open(data_file.name, 'rb')
         bytes_read = 0