Merge pull request #2357 from felixonmars/glacier-py3

glacier module: add backward-compatible support for Python 3.3+. Fixes #2357.

14 files changed, 72 insertions, 68 deletions

diff --git a/boto/compat.py b/boto/compat.py
index c91e808d..c6a9828d 100644
--- a/boto/compat.py
+++ b/boto/compat.py
@@ -50,6 +50,7 @@ from boto.vendored import six
 from boto.vendored.six import BytesIO, StringIO
 from boto.vendored.six.moves import filter, http_client, map, _thread, \
                                     urllib, zip
+from boto.vendored.six.moves.queue import Queue
 from boto.vendored.six.moves.configparser import SafeConfigParser
 from boto.vendored.six.moves.urllib.parse import parse_qs, quote, unquote, \
                                                  urlparse, urlsplit
diff --git a/boto/glacier/concurrent.py b/boto/glacier/concurrent.py
index 6c038eab..93a12d4f 100644
--- a/boto/glacier/concurrent.py
+++ b/boto/glacier/concurrent.py
@@ -27,7 +27,7 @@ import threading
 import hashlib
 import time
 import logging
-from Queue import Queue, Empty
+from boto.compat import Queue
 import binascii
 
 from boto.glacier.utils import DEFAULT_PART_SIZE, minimum_part_size, \
diff --git a/boto/glacier/response.py b/boto/glacier/response.py
index 78d9f5f9..a67ec61d 100644
--- a/boto/glacier/response.py
+++ b/boto/glacier/response.py
@@ -37,7 +37,7 @@ class GlacierResponse(dict):
             for header_name, item_name in response_headers:
                 self[item_name] = http_response.getheader(header_name)
         if http_response.getheader('Content-Type') == 'application/json':
-            body = json.loads(http_response.read())
+            body = json.loads(http_response.read().decode('utf-8'))
             self.update(body)
         size = http_response.getheader('Content-Length', None)
         if size is not None:
diff --git a/boto/glacier/utils.py b/boto/glacier/utils.py
index fe53bb8f..21da67ba 100644
--- a/boto/glacier/utils.py
+++ b/boto/glacier/utils.py
@@ -21,6 +21,7 @@
 #
 import hashlib
 import math
+import binascii
 
 
 _MEGABYTE = 1024 * 1024
@@ -76,7 +77,7 @@ def chunk_hashes(bytestring, chunk_size=_MEGABYTE):
         end = (i + 1) * chunk_size
         hashes.append(hashlib.sha256(bytestring[start:end]).digest())
     if not hashes:
-        return [hashlib.sha256('').digest()]
+        return [hashlib.sha256(b'').digest()]
     return hashes
 
 
@@ -123,18 +124,18 @@ def compute_hashes_from_fileobj(fileobj, chunk_size=1024 * 1024):
     """
     linear_hash = hashlib.sha256()
     chunks = []
-    chunk = fileobj.read(chunk_size)
+    chunk = fileobj.read(chunk_size).encode('utf-8')
     while chunk:
         linear_hash.update(chunk)
         chunks.append(hashlib.sha256(chunk).digest())
         chunk = fileobj.read(chunk_size)
     if not chunks:
-        chunks = [hashlib.sha256('').digest()]
+        chunks = [hashlib.sha256(b'').digest()]
     return linear_hash.hexdigest(), bytes_to_hex(tree_hash(chunks))
 
 
 def bytes_to_hex(str_as_bytes):
-    return ''.join(["%02x" % ord(x) for x in str_as_bytes]).strip()
+    return binascii.hexlify(str_as_bytes)
 
 
 def tree_hash_from_str(str_as_bytes):
diff --git a/boto/glacier/vault.py b/boto/glacier/vault.py
index 6070ffb8..e447c188 100644
--- a/boto/glacier/vault.py
+++ b/boto/glacier/vault.py
@@ -22,6 +22,8 @@
 # IN THE SOFTWARE.
 #
 from __future__ import with_statement
+import codecs
+from boto.compat import six
 from boto.glacier.exceptions import UploadArchiveError
 from boto.glacier.job import Job
 from boto.glacier.writer import compute_hashes_from_fileobj, \
@@ -55,8 +57,6 @@ class Vault(object):
         if response_data:
             for response_name, attr_name, default in self.ResponseDataElements:
                 value = response_data[response_name]
-                if isinstance(value, unicode):
-                    value = value.encode('utf8')
                 setattr(self, attr_name, value)
         else:
             for response_name, attr_name, default in self.ResponseDataElements:
@@ -228,7 +228,7 @@ class Vault(object):
         for part_desc in part_list_response['Parts']:
             part_index = self._range_string_to_part_index(
                 part_desc['RangeInBytes'], part_size)
-            part_tree_hash = part_desc['SHA256TreeHash'].decode('hex')
+            part_tree_hash = codecs.decode(part_desc['SHA256TreeHash'], 'hex_codec')
             part_hash_map[part_index] = part_tree_hash
 
         if not file_obj:
diff --git a/boto/glacier/writer.py b/boto/glacier/writer.py
index ad0ab265..fa3161ab 100644
--- a/boto/glacier/writer.py
+++ b/boto/glacier/writer.py
@@ -53,7 +53,7 @@ class _Partitioner(object):
         self._buffer_size = 0
 
     def write(self, data):
-        if data == '':
+        if data == b'':
             return
         self._buffer.append(data)
         self._buffer_size += len(data)
@@ -61,7 +61,7 @@ class _Partitioner(object):
             self._send_part()
 
     def _send_part(self):
-        data = ''.join(self._buffer)
+        data = b''.join(self._buffer)
         # Put back any data remaining over the part size into the
         # buffer
         if len(data) > self.part_size:
@@ -164,7 +164,7 @@ class _Uploader(object):
 def generate_parts_from_fobj(fobj, part_size):
     data = fobj.read(part_size)
     while data:
-        yield data
+        yield data.encode('utf-8')
         data = fobj.read(part_size)
 
 
diff --git a/tests/test.py b/tests/test.py
index 618bf636..9ed4536e 100755
--- a/tests/test.py
+++ b/tests/test.py
@@ -44,6 +44,7 @@ PY3_WHITELIST = (
     'tests/unit/directconnect',
     'tests/unit/ecs',
     'tests/unit/elasticache',
+    'tests/unit/glacier',
     'tests/unit/iam',
     'tests/unit/ec2/elb',
     'tests/unit/manage',
diff --git a/tests/unit/glacier/test_concurrent.py b/tests/unit/glacier/test_concurrent.py
index 08d7c560..37d8f180 100644
--- a/tests/unit/glacier/test_concurrent.py
+++ b/tests/unit/glacier/test_concurrent.py
@@ -21,7 +21,7 @@
 # IN THE SOFTWARE.
 #
 import tempfile
-from Queue import Queue
+from boto.compat import Queue
 
 import mock
 from tests.unit import unittest
@@ -41,7 +41,7 @@ class FakeThreadedConcurrentUploader(ConcurrentUploader):
 
     def _wait_for_upload_threads(self, hash_chunks, result_queue, total_parts):
         for i in range(total_parts):
-            hash_chunks[i] = 'foo'
+            hash_chunks[i] = b'foo'
 
 
 class FakeThreadedConcurrentDownloader(ConcurrentDownloader):
diff --git a/tests/unit/glacier/test_job.py b/tests/unit/glacier/test_job.py
index 5f77beb6..2948f8d2 100644
--- a/tests/unit/glacier/test_job.py
+++ b/tests/unit/glacier/test_job.py
@@ -19,7 +19,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 # IN THE SOFTWARE.
 #
-from StringIO import StringIO
+from boto.compat import StringIO
 from tests.unit import unittest
 import mock
 
diff --git a/tests/unit/glacier/test_layer1.py b/tests/unit/glacier/test_layer1.py
index 4cabbeae..1e6490bf 100644
--- a/tests/unit/glacier/test_layer1.py
+++ b/tests/unit/glacier/test_layer1.py
@@ -34,7 +34,7 @@ class GlacierVaultsOperations(GlacierLayer1ConnectionBase):
                    u'RequestId': None,
                    u'VaultList': [self.vault_info]}
         self.set_http_response(status_code=200, header=self.json_header,
-                               body=json.dumps(content))
+                               body=json.dumps(content).encode('utf-8'))
         api_response = self.service_connection.list_vaults()
         self.assertDictEqual(content, api_response)
 
@@ -42,7 +42,7 @@ class GlacierVaultsOperations(GlacierLayer1ConnectionBase):
         content = copy.copy(self.vault_info)
         content[u'RequestId'] = None
         self.set_http_response(status_code=200, header=self.json_header,
-                               body=json.dumps(content))
+                               body=json.dumps(content).encode('utf-8'))
         api_response = self.service_connection.describe_vault(self.vault_name)
         self.assertDictEqual(content, api_response)
 
@@ -66,7 +66,7 @@ class GlacierJobOperations(GlacierLayer1ConnectionBase):
                    u'Location': None,
                    u'RequestId': None}
         self.set_http_response(status_code=202, header=self.json_header,
-                               body=json.dumps(content))
+                               body=json.dumps(content).encode('utf-8'))
         api_response = self.service_connection.initiate_job(self.vault_name,
                                                             self.job_content)
         self.assertDictEqual(content, api_response)
@@ -83,7 +83,7 @@ class GlacierJobOperations(GlacierLayer1ConnectionBase):
 class GlacierUploadArchiveResets(GlacierLayer1ConnectionBase):
     def test_upload_archive(self):
         fake_data = tempfile.NamedTemporaryFile()
-        fake_data.write('foobarbaz')
+        fake_data.write(b'foobarbaz')
         # First seek to a non zero offset.
         fake_data.seek(2)
         self.set_http_response(status_code=201)
diff --git a/tests/unit/glacier/test_layer2.py b/tests/unit/glacier/test_layer2.py
index d454e588..eec175d3 100644
--- a/tests/unit/glacier/test_layer2.py
+++ b/tests/unit/glacier/test_layer2.py
@@ -25,13 +25,14 @@ from tests.unit import unittest
 
 from mock import call, Mock, patch, sentinel
 
+import codecs
 from boto.glacier.layer1 import Layer1
 from boto.glacier.layer2 import Layer2
 import boto.glacier.vault
 from boto.glacier.vault import Vault
 from boto.glacier.vault import Job
 
-from StringIO import StringIO
+from boto.compat import StringIO
 
 from datetime import datetime, tzinfo, timedelta
 
@@ -297,7 +298,7 @@ class TestVault(GlacierLayer2Base):
             sentinel.upload_id, file_obj=sentinel.file_obj)
         mock_resume_file_upload.assert_called_once_with(
             self.vault, sentinel.upload_id, part_size, sentinel.file_obj,
-            {0: '12'.decode('hex'), 1: '34'.decode('hex')})
+            {0: codecs.decode('12', 'hex_codec'), 1: codecs.decode('34', 'hex_codec')})
 
 
 class TestJob(GlacierLayer2Base):
@@ -308,11 +309,11 @@ class TestJob(GlacierLayer2Base):
 
     def test_get_job_output(self):
         self.mock_layer1.get_job_output.return_value = "TEST_OUTPUT"
-        self.job.get_output((0,100))
+        self.job.get_output((0, 100))
         self.mock_layer1.get_job_output.assert_called_with(
             "examplevault",
             "HkF9p6o7yjhFx-K3CGl6fuSm6VzW9T7esGQfco8nUXVYwS0jlb5gq1JZ55yHgt5vP"
-            "54ZShjoQzQVVh7vEXAMPLEjobID", (0,100))
+            "54ZShjoQzQVVh7vEXAMPLEjobID", (0, 100))
 
 class TestRangeStringParsing(unittest.TestCase):
     def test_simple_range(self):
diff --git a/tests/unit/glacier/test_utils.py b/tests/unit/glacier/test_utils.py
index ee621889..a051b59e 100644
--- a/tests/unit/glacier/test_utils.py
+++ b/tests/unit/glacier/test_utils.py
@@ -61,22 +61,22 @@ class TestPartSizeCalculations(unittest.TestCase):
 
 class TestChunking(unittest.TestCase):
     def test_chunk_hashes_exact(self):
-        chunks = chunk_hashes('a' * (2 * 1024 * 1024))
+        chunks = chunk_hashes(b'a' * (2 * 1024 * 1024))
         self.assertEqual(len(chunks), 2)
-        self.assertEqual(chunks[0], sha256('a' * 1024 * 1024).digest())
+        self.assertEqual(chunks[0], sha256(b'a' * 1024 * 1024).digest())
 
     def test_chunks_with_leftovers(self):
-        bytestring = 'a' * (2 * 1024 * 1024 + 20)
+        bytestring = b'a' * (2 * 1024 * 1024 + 20)
         chunks = chunk_hashes(bytestring)
         self.assertEqual(len(chunks), 3)
-        self.assertEqual(chunks[0], sha256('a' * 1024 * 1024).digest())
-        self.assertEqual(chunks[1], sha256('a' * 1024 * 1024).digest())
-        self.assertEqual(chunks[2], sha256('a' * 20).digest())
+        self.assertEqual(chunks[0], sha256(b'a' * 1024 * 1024).digest())
+        self.assertEqual(chunks[1], sha256(b'a' * 1024 * 1024).digest())
+        self.assertEqual(chunks[2], sha256(b'a' * 20).digest())
 
     def test_less_than_one_chunk(self):
-        chunks = chunk_hashes('aaaa')
+        chunks = chunk_hashes(b'aaaa')
         self.assertEqual(len(chunks), 1)
-        self.assertEqual(chunks[0], sha256('aaaa').digest())
+        self.assertEqual(chunks[0], sha256(b'aaaa').digest())
 
 
 class TestTreeHash(unittest.TestCase):
@@ -92,25 +92,25 @@ class TestTreeHash(unittest.TestCase):
         return calculated
 
     def test_tree_hash_calculations(self):
-        one_meg_bytestring = 'a' * (1 * 1024 * 1024)
-        two_meg_bytestring = 'a' * (2 * 1024 * 1024)
-        four_meg_bytestring = 'a' * (4 * 1024 * 1024)
-        bigger_bytestring = four_meg_bytestring + 'a' * 20
+        one_meg_bytestring = b'a' * (1 * 1024 * 1024)
+        two_meg_bytestring = b'a' * (2 * 1024 * 1024)
+        four_meg_bytestring = b'a' * (4 * 1024 * 1024)
+        bigger_bytestring = four_meg_bytestring + b'a' * 20
 
         self.assertEqual(
             self.calculate_tree_hash(one_meg_bytestring),
-            '9bc1b2a288b26af7257a36277ae3816a7d4f16e89c1e7e77d0a5c48bad62b360')
+            b'9bc1b2a288b26af7257a36277ae3816a7d4f16e89c1e7e77d0a5c48bad62b360')
         self.assertEqual(
             self.calculate_tree_hash(two_meg_bytestring),
-            '560c2c9333c719cb00cfdffee3ba293db17f58743cdd1f7e4055373ae6300afa')
+            b'560c2c9333c719cb00cfdffee3ba293db17f58743cdd1f7e4055373ae6300afa')
         self.assertEqual(
             self.calculate_tree_hash(four_meg_bytestring),
-            '9491cb2ed1d4e7cd53215f4017c23ec4ad21d7050a1e6bb636c4f67e8cddb844')
+            b'9491cb2ed1d4e7cd53215f4017c23ec4ad21d7050a1e6bb636c4f67e8cddb844')
         self.assertEqual(
             self.calculate_tree_hash(bigger_bytestring),
-            '12f3cbd6101b981cde074039f6f728071da8879d6f632de8afc7cdf00661b08f')
+            b'12f3cbd6101b981cde074039f6f728071da8879d6f632de8afc7cdf00661b08f')
 
     def test_empty_tree_hash(self):
         self.assertEqual(
             self.calculate_tree_hash(''),
-            'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855')
+            b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855')
diff --git a/tests/unit/glacier/test_vault.py b/tests/unit/glacier/test_vault.py
index 7861b63a..a01e7454 100644
--- a/tests/unit/glacier/test_vault.py
+++ b/tests/unit/glacier/test_vault.py
@@ -21,7 +21,7 @@
 # IN THE SOFTWARE.
 #
 import unittest
-from cStringIO import StringIO
+from boto.compat import StringIO
 
 import mock
 from mock import ANY
@@ -111,7 +111,7 @@ class TestVault(unittest.TestCase):
                 return 'something'
 
             def read(self, amt=None):
-                return """{
+                return b"""{
   "Action": "ArchiveRetrieval",
   "ArchiveId": "NkbByEejwEggmBz2fTHgJrg0XBoDfjP4q6iu87-EXAMPLEArchiveId",
   "ArchiveSizeInBytes": 16777216,
diff --git a/tests/unit/glacier/test_writer.py b/tests/unit/glacier/test_writer.py
index 43757ebb..c7066b97 100644
--- a/tests/unit/glacier/test_writer.py
+++ b/tests/unit/glacier/test_writer.py
@@ -21,7 +21,7 @@
 #
 from hashlib import sha256
 import itertools
-from StringIO import StringIO
+from boto.compat import StringIO
 
 from tests.unit import unittest
 from mock import (
@@ -104,70 +104,70 @@ class TestWriter(unittest.TestCase):
             self.writer.write(write_data)
         self.writer.close()
 
-        data = ''.join(write_list)
+        data = b''.join(write_list)
         upload_part_calls, data_tree_hashes = calculate_mock_vault_calls(
             data, self.part_size, self.chunk_size)
         check_mock_vault_calls(
             self.vault, upload_part_calls, data_tree_hashes, len(data))
 
     def test_single_byte_write(self):
-        self.check_write(['1'])
+        self.check_write([b'1'])
 
     def test_one_part_write(self):
-        self.check_write(['1234'])
+        self.check_write([b'1234'])
 
     def test_split_write_1(self):
-        self.check_write(['1', '234'])
+        self.check_write([b'1', b'234'])
 
     def test_split_write_2(self):
-        self.check_write(['12', '34'])
+        self.check_write([b'12', b'34'])
 
     def test_split_write_3(self):
-        self.check_write(['123', '4'])
+        self.check_write([b'123', b'4'])
 
     def test_one_part_plus_one_write(self):
-        self.check_write(['12345'])
+        self.check_write([b'12345'])
 
     def test_returns_archive_id(self):
-        self.writer.write('1')
+        self.writer.write(b'1')
         self.writer.close()
         self.assertEquals(sentinel.archive_id, self.writer.get_archive_id())
 
     def test_current_tree_hash(self):
-        self.writer.write('1234')
-        self.writer.write('567')
+        self.writer.write(b'1234')
+        self.writer.write(b'567')
         hash_1 = self.writer.current_tree_hash
         self.assertEqual(hash_1,
-            '\x0e\xb0\x11Z\x1d\x1f\n\x10|\xf76\xa6\xf5' +
-            '\x83\xd1\xd5"bU\x0c\x95\xa8<\xf5\x81\xef\x0e\x0f\x95\n\xb7k'
+            b'\x0e\xb0\x11Z\x1d\x1f\n\x10|\xf76\xa6\xf5' +
+            b'\x83\xd1\xd5"bU\x0c\x95\xa8<\xf5\x81\xef\x0e\x0f\x95\n\xb7k'
         )
 
         # This hash will be different, since the content has changed.
-        self.writer.write('22i3uy')
+        self.writer.write(b'22i3uy')
         hash_2 = self.writer.current_tree_hash
         self.assertEqual(hash_2,
-            '\x7f\xf4\x97\x82U]\x81R\x05#^\xe8\x1c\xd19' +
-            '\xe8\x1f\x9e\xe0\x1aO\xaad\xe5\x06"\xa5\xc0\xa8AdL'
+            b'\x7f\xf4\x97\x82U]\x81R\x05#^\xe8\x1c\xd19' +
+            b'\xe8\x1f\x9e\xe0\x1aO\xaad\xe5\x06"\xa5\xc0\xa8AdL'
         )
         self.writer.close()
 
         # Check the final tree hash, post-close.
         final_hash = self.writer.current_tree_hash
         self.assertEqual(final_hash,
-            ';\x1a\xb8!=\xf0\x14#\x83\x11\xd5\x0b\x0f' +
-            '\xc7D\xe4\x8e\xd1W\x99z\x14\x06\xb9D\xd0\xf0*\x93\xa2\x8e\xf9'
+            b';\x1a\xb8!=\xf0\x14#\x83\x11\xd5\x0b\x0f' +
+            b'\xc7D\xe4\x8e\xd1W\x99z\x14\x06\xb9D\xd0\xf0*\x93\xa2\x8e\xf9'
         )
         # Then assert we don't get a different one on a subsequent call.
         self.assertEqual(final_hash, self.writer.current_tree_hash)
 
     def test_current_uploaded_size(self):
-        self.writer.write('1234')
-        self.writer.write('567')
+        self.writer.write(b'1234')
+        self.writer.write(b'567')
         size_1 = self.writer.current_uploaded_size
         self.assertEqual(size_1, 4)
 
         # This hash will be different, since the content has changed.
-        self.writer.write('22i3uy')
+        self.writer.write(b'22i3uy')
         size_2 = self.writer.current_uploaded_size
         self.assertEqual(size_2, 12)
         self.writer.close()
@@ -190,7 +190,7 @@ class TestResume(unittest.TestCase):
         self.part_size = 4 # power of 2
 
     def check_no_resume(self, data, resume_set=set()):
-        fobj = StringIO(data)
+        fobj = StringIO(data.decode('utf-8'))
         part_hash_map = {}
         for part_index in resume_set:
             start = self.part_size * part_index
@@ -212,16 +212,16 @@ class TestResume(unittest.TestCase):
             self.vault, resume_upload_part_calls, data_tree_hashes, len(data))
 
     def test_one_part_no_resume(self):
-        self.check_no_resume('1234')
+        self.check_no_resume(b'1234')
 
     def test_two_parts_no_resume(self):
-        self.check_no_resume('12345678')
+        self.check_no_resume(b'12345678')
 
     def test_one_part_resume(self):
-        self.check_no_resume('1234', resume_set=set([0]))
+        self.check_no_resume(b'1234', resume_set=set([0]))
 
     def test_two_parts_one_resume(self):
-        self.check_no_resume('12345678', resume_set=set([1]))
+        self.check_no_resume(b'12345678', resume_set=set([1]))
 
     def test_returns_archive_id(self):
         archive_id = resume_file_upload(