diff options
author | Cat Lee Ball 🎷🐛 <cball@google.com> | 2019-03-27 17:21:19 -0700 |
---|---|---|
committer | Cat Lee Ball 🎷🐛 <cball@google.com> | 2019-03-27 17:21:19 -0700 |
commit | 931c74757207b8de69229bd4187415da14784edd (patch) | |
tree | 20f537a65b17a31af3c41f08e1c83185aba1bfbf /boto | |
parent | 9e1cd3bd76e738d80630f1bd9160fd87c8eab865 (diff) | |
download | boto-931c74757207b8de69229bd4187415da14784edd.tar.gz |
Use get_utf8able_str over get_utf8_value
Diffstat (limited to 'boto')
-rw-r--r-- | boto/auth.py | 15 | ||||
-rw-r--r-- | boto/connection.py | 2 | ||||
-rw-r--r-- | boto/gs/bucket.py | 6 | ||||
-rwxr-xr-x | boto/gs/connection.py | 7 | ||||
-rw-r--r-- | boto/gs/key.py | 7 | ||||
-rw-r--r-- | boto/s3/bucket.py | 8 | ||||
-rw-r--r-- | boto/s3/connection.py | 6 | ||||
-rw-r--r-- | boto/utils.py | 41 |
8 files changed, 66 insertions, 26 deletions
diff --git a/boto/auth.py b/boto/auth.py index d062349f..77c2c2bc 100644 --- a/boto/auth.py +++ b/boto/auth.py @@ -42,6 +42,7 @@ import posixpath from boto.compat import urllib, encodebytes, parse_qs_safe, urlparse, six from boto.auth_handler import AuthHandler from boto.exception import BotoClientError +from boto.utils import get_utf8able_str try: from hashlib import sha1 as sha @@ -383,7 +384,7 @@ class HmacAuthV4Handler(AuthHandler, HmacKeys): parameter_names = sorted(http_request.params.keys()) pairs = [] for pname in parameter_names: - pval = boto.utils.get_utf8_value(http_request.params[pname]) + pval = get_utf8able_str(http_request.params[pname]) pairs.append(urllib.parse.quote(pname, safe=''.encode('ascii')) + '=' + urllib.parse.quote(pval, safe='-_~'.encode('ascii'))) @@ -396,7 +397,7 @@ class HmacAuthV4Handler(AuthHandler, HmacKeys): return "" l = [] for param in sorted(http_request.params): - value = boto.utils.get_utf8_value(http_request.params[param]) + value = get_utf8able_str(http_request.params[param]) l.append('%s=%s' % (urllib.parse.quote(param, safe='-_.~'), urllib.parse.quote(value, safe='-_.~'))) return '&'.join(l) @@ -623,7 +624,7 @@ class S3HmacAuthV4Handler(HmacAuthV4Handler, AuthHandler): # query string. l = [] for param in sorted(http_request.params): - value = boto.utils.get_utf8_value(http_request.params[param]) + value = get_utf8able_str(http_request.params[param]) l.append('%s=%s' % (urllib.parse.quote(param, safe='-_.~'), urllib.parse.quote(value, safe='-_.~'))) return '&'.join(l) @@ -836,7 +837,7 @@ class STSAnonHandler(AuthHandler): keys.sort(key=lambda x: x.lower()) pairs = [] for key in keys: - val = boto.utils.get_utf8_value(params[key]) + val = get_utf8able_str(params[key]) pairs.append(key + '=' + self._escape_value(six.ensure_str(val))) return '&'.join(pairs) @@ -897,7 +898,7 @@ class QuerySignatureV0AuthHandler(QuerySignatureHelper, AuthHandler): keys.sort(cmp=lambda x, y: cmp(x.lower(), y.lower())) pairs = [] for key in keys: - val = boto.utils.get_utf8_value(params[key]) + val = get_utf8able_str(params[key]) pairs.append(key + '=' + urllib.parse.quote(val)) qs = '&'.join(pairs) return (qs, base64.b64encode(hmac.digest())) @@ -924,7 +925,7 @@ class QuerySignatureV1AuthHandler(QuerySignatureHelper, AuthHandler): pairs = [] for key in keys: hmac.update(key.encode('utf-8')) - val = boto.utils.get_utf8_value(params[key]) + val = six.ensure_str(params[key]) hmac.update(val) pairs.append(key + '=' + urllib.parse.quote(val)) qs = '&'.join(pairs) @@ -948,7 +949,7 @@ class QuerySignatureV2AuthHandler(QuerySignatureHelper, AuthHandler): keys = sorted(params.keys()) pairs = [] for key in keys: - val = boto.utils.get_utf8_value(params[key]) + val = get_utf8able_str(params[key]) pairs.append(urllib.parse.quote(key, safe='') + '=' + urllib.parse.quote(val, safe='-_~')) qs = '&'.join(pairs) diff --git a/boto/connection.py b/boto/connection.py index 85d12d3f..a0d89a51 100644 --- a/boto/connection.py +++ b/boto/connection.py @@ -1105,9 +1105,11 @@ class AWSQueryConnection(AWSAuthConnection): def _required_auth_capability(self): return [] + def get_utf8_value(self, value): return boto.utils.get_utf8_value(value) + def make_request(self, action, params=None, path='/', verb='GET'): http_request = self.build_base_http_request(verb, path, None, params, {}, '', diff --git a/boto/gs/bucket.py b/boto/gs/bucket.py index 01ac0839..2777e829 100644 --- a/boto/gs/bucket.py +++ b/boto/gs/bucket.py @@ -40,7 +40,7 @@ from boto.gs.lifecycle import LifecycleConfig from boto.gs.key import Key as GSKey from boto.s3.acl import Policy from boto.s3.bucket import Bucket as S3Bucket -from boto.utils import get_utf8_value +from boto.utils import get_utf8able_str from boto.compat import quote from boto.compat import six @@ -644,7 +644,7 @@ class Bucket(S3Bucket): :param str storage_class: A string containing the storage class. :param dict headers: Additional headers to send with the request. """ - req_body = self.StorageClassBody % (get_utf8_value(storage_class)) + req_body = self.StorageClassBody % (get_utf8able_str(storage_class)) self.set_subresource(STORAGE_CLASS_ARG, req_body, headers=headers) # Method with same signature as boto.s3.bucket.Bucket.add_email_grant(), @@ -883,7 +883,7 @@ class Bucket(S3Bucket): body = self.WebsiteBody % (main_page_frag, error_frag) response = self.connection.make_request( - 'PUT', get_utf8_value(self.name), data=get_utf8_value(body), + 'PUT', get_utf8able_str(self.name), data=get_utf8able_str(body), query_args='websiteConfig', headers=headers) body = response.read() if response.status == 200: diff --git a/boto/gs/connection.py b/boto/gs/connection.py index 9a2e4a2b..70ce6ec6 100755 --- a/boto/gs/connection.py +++ b/boto/gs/connection.py @@ -23,7 +23,8 @@ from boto.gs.bucket import Bucket from boto.s3.connection import S3Connection from boto.s3.connection import SubdomainCallingFormat from boto.s3.connection import check_lowercase_bucketname -from boto.utils import get_utf8_value +from boto.compat import six +from boto.utils import get_utf8able_str class Location(object): DEFAULT = 'US' @@ -91,8 +92,8 @@ class GSConnection(S3Connection): data = ('<CreateBucketConfiguration>%s%s</CreateBucketConfiguration>' % (location_elem, storage_class_elem)) response = self.make_request( - 'PUT', get_utf8_value(bucket_name), headers=headers, - data=get_utf8_value(data)) + 'PUT', get_utf8able_str(bucket_name), headers=headers, + data=get_utf8able_str(data)) body = response.read() if response.status == 409: raise self.provider.storage_create_error( diff --git a/boto/gs/key.py b/boto/gs/key.py index b1d4189f..0f720bb5 100644 --- a/boto/gs/key.py +++ b/boto/gs/key.py @@ -24,12 +24,11 @@ import binascii import os import re -from boto.compat import StringIO +from boto.compat import StringIO, six from boto.exception import BotoClientError from boto.s3.key import Key as S3Key from boto.s3.keyfile import KeyFile -from boto.utils import compute_hash -from boto.utils import get_utf8_value +from boto.utils import compute_hash, get_utf8able_str class Key(S3Key): """ @@ -707,7 +706,7 @@ class Key(S3Key): self.md5 = None self.base64md5 = None - fp = StringIO(get_utf8_value(s)) + fp = StringIO(get_utf8able_str(s)) r = self.set_contents_from_file(fp, headers, replace, cb, num_cb, policy, md5, if_generation=if_generation) diff --git a/boto/s3/bucket.py b/boto/s3/bucket.py index fdd040c8..6e806141 100644 --- a/boto/s3/bucket.py +++ b/boto/s3/bucket.py @@ -51,6 +51,7 @@ import re import base64 from collections import defaultdict from boto.compat import BytesIO, six, StringIO, urllib +from boto.utils import get_utf8able_str # as per http://goo.gl/BDuud (02/19/2011) @@ -848,11 +849,8 @@ class Bucket(object): """ headers = headers or {} provider = self.connection.provider - if six.PY3: - if isinstance(src_key_name, bytes): - src_key_name = src_key_name.decode('utf-8') - else: - src_key_name = boto.utils.get_utf8_value(src_key_name) + src_key_name = get_utf8able_str(src_key_name) + if preserve_acl: if self.name == src_bucket_name: src_bucket = self diff --git a/boto/s3/connection.py b/boto/s3/connection.py index 224bf882..6893c309 100644 --- a/boto/s3/connection.py +++ b/boto/s3/connection.py @@ -88,7 +88,7 @@ class _CallingFormat(object): return self.get_bucket_server(server, bucket) def build_auth_path(self, bucket, key=''): - key = boto.utils.get_utf8_value(key) + key = get_utf8able_str(key) if isinstance(bucket, bytes): bucket = bucket.decode('utf-8') path = '' @@ -97,7 +97,7 @@ class _CallingFormat(object): return path + '/%s' % urllib.parse.quote(key) def build_path_base(self, bucket, key=''): - key = boto.utils.get_utf8_value(key) + key = get_utf8able_str(key) return '/%s' % urllib.parse.quote(key) @@ -121,7 +121,7 @@ class OrdinaryCallingFormat(_CallingFormat): return server def build_path_base(self, bucket, key=''): - key = boto.utils.get_utf8_value(key) + key = get_utf8able_str(key) path_base = '/' if bucket: path_base += "%s/" % bucket diff --git a/boto/utils.py b/boto/utils.py index 35637772..ed39c253 100644 --- a/boto/utils.py +++ b/boto/utils.py @@ -1101,6 +1101,46 @@ def parse_host(hostname): else: return hostname.split(':', 1)[0] + +def get_utf8able_str(s, errors='strict'): + """Returns a UTF8-encodable string in PY3, UTF8 bytes in PY2. + + This method is similar to six's `ensure_str()`, except it also + makes sure that any bytes passed in can be decoded using the + utf-8 codec (and raises a UnicodeDecodeError if not). + """ + if six.PY2: + # We want to return utf-8 encoded bytes. + if isinstance(s, six.text_type): + return s.encode('utf-8', errors) + if isinstance(s, six.binary_type): + # Verify the bytes can be represented in utf-8 + s.decode('utf-8') + return s + else: + # We want to return a unicode/str object. + if isinstance(s, six.text_type): + return s + if isinstance(s, six.binary_type): + s = s.decode('utf-8') + return s + raise TypeError('not expecting type "%s"' % type(s)) + + +def get_utf8_value(value): + if isinstance(value, bytes): + value.decode('utf-8') + return value + + if not isinstance(value, six.string_types): + value = six.text_type(value) + + if isinstance(value, six.text_type): + value = value.encode('utf-8') + + return value + + def print_to_fd(*objects, **kwargs): """A Python 2/3 compatible analogue to the print function. @@ -1179,4 +1219,3 @@ def write_to_fd(fd, data): fd.write(six.ensure_binary(data)) else: fd.write(data) - |