diff options
author | Mike Schwartz <mfschwartz@google.com> | 2010-09-14 17:55:10 -0700 |
---|---|---|
committer | Mike Schwartz <mfschwartz@google.com> | 2010-09-14 17:55:10 -0700 |
commit | a273c8695d2ac816f0b47c4a3d6adbba653f9b91 (patch) | |
tree | d0ce30fbd940a3bfe724136774583dbea75a7f9d | |
parent | c9d1be26e17881abd000ec8ac08a770eb1b723ec (diff) | |
download | boto-a273c8695d2ac816f0b47c4a3d6adbba653f9b91.tar.gz |
Introduced class hierarchy of storage service exceptions, and hooked into code through Provider class.
-rw-r--r-- | boto/exception.py | 76 | ||||
-rw-r--r--[-rwxr-xr-x] | boto/file/bucket.py | 21 | ||||
-rw-r--r--[-rwxr-xr-x] | boto/gs/bucket.py | 12 | ||||
-rw-r--r-- | boto/provider.py | 36 | ||||
-rw-r--r-- | boto/s3/bucket.py | 60 | ||||
-rw-r--r-- | boto/s3/connection.py | 15 | ||||
-rw-r--r-- | boto/s3/key.py | 37 | ||||
-rw-r--r-- | boto/tests/test_gsconnection.py | 1 |
8 files changed, 180 insertions, 78 deletions
diff --git a/boto/exception.py b/boto/exception.py index 595eac64..31ad206e 100644 --- a/boto/exception.py +++ b/boto/exception.py @@ -49,12 +49,24 @@ class SDBPersistenceError(StandardError): pass -class S3PermissionsError(BotoClientError): +class StoragePermissionsError(BotoClientError): + """ + Permissions error when accessing a bucket or key on a storage service. + """ + pass + +class S3PermissionsError(StoragePermissionsError): """ Permissions error when accessing a bucket or key on S3. """ pass +class GSPermissionsError(StoragePermissionsError): + """ + Permissions error when accessing a bucket or key on GS. + """ + pass + class BotoServerError(StandardError): def __init__(self, status, reason, body=None): @@ -135,9 +147,9 @@ class ConsoleOutput: else: setattr(self, name, value) -class S3CreateError(BotoServerError): +class StorageCreateError(BotoServerError): """ - Error creating a bucket or key on S3. + Error creating a bucket or key on a storage service. """ def __init__(self, status, reason, body=None): self.bucket = None @@ -149,12 +161,36 @@ class S3CreateError(BotoServerError): else: return BotoServerError.endElement(self, name, value, connection) -class S3CopyError(BotoServerError): +class S3CreateError(StorageCreateError): + """ + Error creating a bucket or key on S3. + """ + pass + +class GSCreateError(StorageCreateError): + """ + Error creating a bucket or key on GS. + """ + pass + +class StorageCopyError(BotoServerError): + """ + Error copying a key on a storage service. + """ + pass + +class S3CopyError(StorageCopyError): """ Error copying a key on S3. """ pass +class GSCopyError(StorageCopyError): + """ + Error copying a key on GS. + """ + pass + class SQSError(BotoServerError): """ General Error on Simple Queue Service. @@ -193,10 +229,10 @@ class SQSDecodeError(BotoClientError): def __str__(self): return 'SQSDecodeError: %s' % self.reason - -class S3ResponseError(BotoServerError): + +class StorageResponseError(BotoServerError): """ - Error in response from S3. + Error in response from a storage service. """ def __init__(self, status, reason, body=None): self.resource = None @@ -216,6 +252,18 @@ class S3ResponseError(BotoServerError): for p in ('resource'): setattr(self, p, None) +class S3ResponseError(StorageResponseError): + """ + Error in response from S3. + """ + pass + +class GSResponseError(StorageResponseError): + """ + Error in response from GS. + """ + pass + class EC2ResponseError(BotoServerError): """ Error in response from EC2. @@ -285,12 +333,24 @@ class AWSConnectionError(BotoClientError): """ pass -class S3DataError(BotoClientError): +class StorageDataError(BotoClientError): + """ + Error receiving data from a storage service. + """ + pass + +class S3DataError(StorageDataError): """ Error receiving data from S3. """ pass +class GSDataError(StorageDataError): + """ + Error receiving data from GS. + """ + pass + class FPSResponseError(BotoServerError): pass diff --git a/boto/file/bucket.py b/boto/file/bucket.py index f1055884..be01cff6 100755..100644 --- a/boto/file/bucket.py +++ b/boto/file/bucket.py @@ -24,7 +24,7 @@ import os from key import Key from boto.file.simpleresultset import SimpleResultSet -from boto.exception import S3ResponseError +from boto.provider import Provider from boto.s3.bucketlistresultset import BucketListResultSet class Bucket(object): @@ -54,10 +54,7 @@ class Bucket(object): :type mfa_token: tuple or list of strings :param mfa_token: Unused in this subclass. """ - try: - os.remove(key_name) - except OSError, e: - raise S3ResponseError(409, e.strerror) + os.remove(key_name) def get_all_keys(self, headers=None, **params): """ @@ -85,11 +82,8 @@ class Bucket(object): :rtype: :class:`boto.file.key.Key` :returns: A Key object from this bucket. """ - try: - fp = open(key_name, 'rb') - return Key(self.name, key_name, fp) - except OSError, e: - raise S3ResponseError(409, e.strerror) + fp = open(key_name, 'rb') + return Key(self.name, key_name, fp) def new_key(self, key_name=None): """ @@ -104,8 +98,5 @@ class Bucket(object): dir_name = os.path.dirname(key_name) if dir_name and not os.path.exists(dir_name): os.makedirs(dir_name) - try: - fp = open(key_name, 'wb') - return Key(self.name, key_name, fp) - except OSError, e: - raise S3ResponseError(409, e.strerror) + fp = open(key_name, 'wb') + return Key(self.name, key_name, fp) diff --git a/boto/gs/bucket.py b/boto/gs/bucket.py index df9165f2..fd3aa1ff 100755..100644 --- a/boto/gs/bucket.py +++ b/boto/gs/bucket.py @@ -21,7 +21,8 @@ import boto from boto import handler -from boto.exception import InvalidAclError, S3ResponseError, S3PermissionsError +from boto.exception import InvalidAclError +from boto.provider import Provider from boto.gs.acl import ACL from boto.gs.acl import SupportedPermissions as GSPermissions from boto.gs.key import Key as GSKey @@ -52,7 +53,8 @@ class Bucket(S3Bucket): xml.sax.parseString(body, h) return acl else: - raise S3ResponseError(response.status, response.reason, body) + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) # Method with same signature as boto.s3.bucket.Bucket.add_email_grant(), # to allow polymorphic treatment at application layer. @@ -82,7 +84,8 @@ class Bucket(S3Bucket): a long time! """ if permission not in GSPermissions: - raise S3PermissionsError('Unknown Permission: %s' % permission) + raise self.connection.provider.storage_permissions_error( + 'Unknown Permission: %s' % permission) acl = self.get_acl(headers=headers) acl.add_email_grant(permission, email_address) self.set_acl(acl, headers=headers) @@ -116,7 +119,8 @@ class Bucket(S3Bucket): a long time! """ if permission not in GSPermissions: - raise S3PermissionsError('Unknown Permission: %s' % permission) + raise self.connection.provider.storage_permissions_error( + 'Unknown Permission: %s' % permission) acl = self.get_acl(headers=headers) acl.add_user_grant(permission, user_id) self.set_acl(acl, headers=headers) diff --git a/boto/provider.py b/boto/provider.py index 026cb6df..84893bd9 100644 --- a/boto/provider.py +++ b/boto/provider.py @@ -51,6 +51,13 @@ STORAGE_CLASS_HEADER_KEY = 'storage-class' MFA_HEADER_KEY = 'mfa-header' VERSION_ID_HEADER_KEY = 'version-id-header' +STORAGE_COPY_ERROR = 'StorageCopyError' +STORAGE_CREATE_ERROR = 'StorageCreateError' +STORAGE_DATA_ERROR = 'StorageDataError' +STORAGE_PERMISSIONS_ERROR = 'StoragePermissionsError' +STORAGE_RESPONSE_ERROR = 'StorageResponseError' + + class Provider(object): CredentialMap = { @@ -109,6 +116,23 @@ class Provider(object): MFA_HEADER_KEY : None, } } + + ErrorMap = { + 'aws' : { + STORAGE_COPY_ERROR : boto.exception.S3CopyError, + STORAGE_CREATE_ERROR : boto.exception.S3CreateError, + STORAGE_DATA_ERROR : boto.exception.S3DataError, + STORAGE_PERMISSIONS_ERROR : boto.exception.S3PermissionsError, + STORAGE_RESPONSE_ERROR : boto.exception.S3ResponseError, + }, + 'google' : { + STORAGE_COPY_ERROR : boto.exception.GSCopyError, + STORAGE_CREATE_ERROR : boto.exception.GSCreateError, + STORAGE_DATA_ERROR : boto.exception.GSDataError, + STORAGE_PERMISSIONS_ERROR : boto.exception.GSPermissionsError, + STORAGE_RESPONSE_ERROR : boto.exception.GSResponseError, + } + } def __init__(self, name, access_key=None, secret_key=None): self.host = None @@ -119,6 +143,7 @@ class Provider(object): self.canned_acls = self.CannedAclsMap[self.name] self.get_credentials(access_key, secret_key) self.configure_headers() + self.configure_errors() # allow config file to override default host host_opt_name = '%s_host' % self.HostKeyMap[self.name] if (config.has_option('Credentials', host_opt_name)): @@ -147,7 +172,8 @@ class Provider(object): self.acl_header = header_info_map[ACL_HEADER_KEY] self.auth_header = header_info_map[AUTH_HEADER_KEY] self.copy_source_header = header_info_map[COPY_SOURCE_HEADER_KEY] - self.copy_source_version_id = header_info_map[COPY_SOURCE_VERSION_ID_HEADER_KEY] + self.copy_source_version_id = header_info_map[ + COPY_SOURCE_VERSION_ID_HEADER_KEY] self.date_header = header_info_map[DATE_HEADER_KEY] self.delete_marker = header_info_map[DELETE_MARKER_HEADER_KEY] self.metadata_directive_header = ( @@ -157,6 +183,14 @@ class Provider(object): self.version_id = header_info_map[VERSION_ID_HEADER_KEY] self.mfa_header = header_info_map[MFA_HEADER_KEY] + def configure_errors(self): + error_map = self.ErrorMap[self.name] + self.storage_copy_error = error_map[STORAGE_COPY_ERROR] + self.storage_create_error = error_map[STORAGE_CREATE_ERROR] + self.storage_data_error = error_map[STORAGE_DATA_ERROR] + self.storage_permissions_error = error_map[STORAGE_PERMISSIONS_ERROR] + self.storage_response_error = error_map[STORAGE_RESPONSE_ERROR] + # Static utility method for getting default Provider. def get_default(): return Provider('aws') diff --git a/boto/s3/bucket.py b/boto/s3/bucket.py index fc374de3..0acfdbb3 100644 --- a/boto/s3/bucket.py +++ b/boto/s3/bucket.py @@ -23,13 +23,13 @@ import boto from boto import handler +from boto.provider import Provider from boto.resultset import ResultSet from boto.s3.acl import ACL, Policy, CannedACLStrings, Grant from boto.s3.key import Key from boto.s3.prefix import Prefix from boto.s3.deletemarker import DeleteMarker from boto.s3.user import User -from boto.exception import S3ResponseError, S3PermissionsError, S3CopyError from boto.s3.bucketlistresultset import BucketListResultSet from boto.s3.bucketlistresultset import VersionedBucketListResultSet import boto.utils @@ -150,7 +150,6 @@ class Bucket(object): k.content_type = response.getheader('content-type') k.content_encoding = response.getheader('content-encoding') k.last_modified = response.getheader('last-modified') - k.cache_control = response.getheader('cache-control') k.size = int(response.getheader('content-length')) k.name = key_name k.handle_version_headers(response) @@ -160,7 +159,8 @@ class Bucket(object): response.read() return None else: - raise S3ResponseError(response.status, response.reason, '') + raise self.connection.provider.storage_response_error( + response.status, response.reason, '') def list(self, prefix='', delimiter='', marker='', headers=None): """ @@ -249,7 +249,8 @@ class Bucket(object): xml.sax.parseString(body, h) return rs else: - raise S3ResponseError(response.status, response.reason, body) + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) def get_all_keys(self, headers=None, **params): """ @@ -361,6 +362,7 @@ class Bucket(object): deleting versioned objects from a bucket that has the MFADelete option on the bucket. """ + provider = self.connection.provider if version_id: query_args = 'versionId=%s' % version_id else: @@ -368,14 +370,14 @@ class Bucket(object): if mfa_token: if not headers: headers = {} - provider = self.connection.provider headers[provider.mfa_header] = ' '.join(mfa_token) response = self.connection.make_request('DELETE', self.name, key_name, headers=headers, query_args=query_args) body = response.read() if response.status != 204: - raise S3ResponseError(response.status, response.reason, body) + raise provider.storage_response_error(response.status, + response.reason, body) def copy_key(self, new_key_name, src_bucket_name, src_key_name, metadata=None, src_version_id=None, @@ -449,13 +451,13 @@ class Bucket(object): h = handler.XmlHandler(key, self) xml.sax.parseString(body, h) if hasattr(key, 'Error'): - raise S3CopyError(key.Code, key.Message, body) + raise provider.storage_copy_error(key.Code, key.Message, body) key.handle_version_headers(response) if preserve_acl: self.set_xml_acl(acl, new_key_name) return key else: - raise S3ResponseError(response.status, response.reason, body) + raise provider.storage_response_error(response.status, response.reason, body) def set_canned_acl(self, acl_str, key_name='', headers=None, version_id=None): @@ -473,7 +475,8 @@ class Bucket(object): headers=headers, query_args=query_args) body = response.read() if response.status != 200: - raise S3ResponseError(response.status, response.reason, body) + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) def get_xml_acl(self, key_name='', headers=None, version_id=None): query_args = 'acl' @@ -484,7 +487,8 @@ class Bucket(object): headers=headers) body = response.read() if response.status != 200: - raise S3ResponseError(response.status, response.reason, body) + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) return body def set_xml_acl(self, acl_str, key_name='', headers=None, version_id=None): @@ -497,7 +501,8 @@ class Bucket(object): headers=headers) body = response.read() if response.status != 200: - raise S3ResponseError(response.status, response.reason, body) + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) def set_acl(self, acl_or_str, key_name='', headers=None, version_id=None): if isinstance(acl_or_str, Policy): @@ -521,7 +526,8 @@ class Bucket(object): xml.sax.parseString(body, h) return policy else: - raise S3ResponseError(response.status, response.reason, body) + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) def make_public(self, recursive=False, headers=None): self.set_canned_acl('public-read', headers=headers) @@ -555,7 +561,8 @@ class Bucket(object): a long time! """ if permission not in S3Permissions: - raise S3PermissionsError('Unknown Permission: %s' % permission) + raise self.connection.provider.storage_permissions_error( + 'Unknown Permission: %s' % permission) policy = self.get_acl(headers=headers) policy.acl.add_email_grant(permission, email_address) self.set_acl(policy, headers=headers) @@ -587,7 +594,8 @@ class Bucket(object): a long time! """ if permission not in S3Permissions: - raise S3PermissionsError('Unknown Permission: %s' % permission) + raise self.connection.provider.storage_permissions_error( + 'Unknown Permission: %s' % permission) policy = self.get_acl(headers=headers) policy.acl.add_user_grant(permission, user_id) self.set_acl(policy, headers=headers) @@ -616,7 +624,8 @@ class Bucket(object): xml.sax.parseString(body, h) return rs.LocationConstraint else: - raise S3ResponseError(response.status, response.reason, body) + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) def enable_logging(self, target_bucket, target_prefix='', headers=None): if isinstance(target_bucket, Bucket): @@ -628,7 +637,8 @@ class Bucket(object): if response.status == 200: return True else: - raise S3ResponseError(response.status, response.reason, body) + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) def disable_logging(self, headers=None): body = self.EmptyBucketLoggingBody @@ -638,7 +648,8 @@ class Bucket(object): if response.status == 200: return True else: - raise S3ResponseError(response.status, response.reason, body) + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) def get_logging_status(self, headers=None): response = self.connection.make_request('GET', self.name, @@ -647,7 +658,8 @@ class Bucket(object): if response.status == 200: return body else: - raise S3ResponseError(response.status, response.reason, body) + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) def set_as_logging_target(self, headers=None): policy = self.get_acl(headers=headers) @@ -664,7 +676,8 @@ class Bucket(object): if response.status == 200: return body else: - raise S3ResponseError(response.status, response.reason, body) + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) def set_request_payment(self, payer='BucketOwner', headers=None): body = self.BucketPaymentBody % payer @@ -674,7 +687,8 @@ class Bucket(object): if response.status == 200: return True else: - raise S3ResponseError(response.status, response.reason, body) + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) def configure_versioning(self, versioning, mfa_delete=False, mfa_token=None, headers=None): @@ -723,7 +737,8 @@ class Bucket(object): if response.status == 200: return True else: - raise S3ResponseError(response.status, response.reason, body) + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) def get_versioning_status(self, headers=None): """ @@ -751,7 +766,8 @@ class Bucket(object): d['MfaDelete'] = mfa.group(1) return d else: - raise S3ResponseError(response.status, response.reason, body) + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) def delete(self, headers=None): return self.connection.delete_bucket(self.name, headers=headers) diff --git a/boto/s3/connection.py b/boto/s3/connection.py index 5fa11f0b..33117b72 100644 --- a/boto/s3/connection.py +++ b/boto/s3/connection.py @@ -27,10 +27,11 @@ import time import boto.utils from boto.connection import AWSAuthConnection from boto import handler +from boto.provider import Provider from boto.s3.bucket import Bucket from boto.s3.key import Key from boto.resultset import ResultSet -from boto.exception import S3ResponseError, S3CreateError, BotoClientError +from boto.exception import BotoClientError def check_lowercase_bucketname(n): """ @@ -297,7 +298,8 @@ class S3Connection(AWSAuthConnection): response = self.make_request('GET') body = response.read() if response.status > 300: - raise S3ResponseError(response.status, response.reason, body) + raise self.provider.storage_response_error( + response.status, response.reason, body) rs = ResultSet([('Bucket', self.bucket_class)]) h = handler.XmlHandler(rs, self) xml.sax.parseString(body, h) @@ -365,17 +367,20 @@ class S3Connection(AWSAuthConnection): data=data) body = response.read() if response.status == 409: - raise S3CreateError(response.status, response.reason, body) + raise self.provider.storage_create_error( + response.status, response.reason, body) if response.status == 200: return self.bucket_class(self, bucket_name) else: - raise S3ResponseError(response.status, response.reason, body) + raise self.provider.storage_response_error( + response.status, response.reason, body) def delete_bucket(self, bucket, headers=None): response = self.make_request('DELETE', bucket, headers=headers) body = response.read() if response.status != 204: - raise S3ResponseError(response.status, response.reason, body) + raise self.connection.storage_response_error( + response.status, response.reason, body) def make_request(self, method, bucket='', key='', headers=None, data='', query_args=None, sender=None): diff --git a/boto/s3/key.py b/boto/s3/key.py index fd6af570..f80ff67f 100644 --- a/boto/s3/key.py +++ b/boto/s3/key.py @@ -25,7 +25,8 @@ import rfc822 import StringIO import base64 import boto.utils -from boto.exception import S3ResponseError, S3DataError, BotoClientError +from boto.exception import BotoClientError +from boto.provider import Provider from boto.s3.user import User from boto import UserAgent try: @@ -117,15 +118,16 @@ class Key(object): if self.resp == None: self.mode = 'r' + provider = self.bucket.connection.provider self.resp = self.bucket.connection.make_request('GET', self.bucket.name, self.name, headers, query_args=query_args) if self.resp.status < 199 or self.resp.status > 299: body = self.resp.read() - raise S3ResponseError(self.resp.status, self.resp.reason, body) + raise provider.storage_response_error(self.resp.status, + self.resp.reason, body) response_headers = self.resp.msg - provider = self.bucket.connection.provider self.metadata = boto.utils.get_aws_metadata(response_headers, provider) for name,value in response_headers.items(): @@ -404,6 +406,8 @@ class Key(object): your callback to be called with each buffer read. """ + provider = self.bucket.connection.provider + def sender(http_conn, method, path, data, headers): http_conn.putrequest(method, path) for key in headers: @@ -444,10 +448,12 @@ class Key(object): elif response.status >= 200 and response.status <= 299: self.etag = response.getheader('etag') if self.etag != '"%s"' % self.md5: - raise S3DataError('ETag from S3 did not match computed MD5') + raise provider.storage_data_error( + 'ETag from S3 did not match computed MD5') return response else: - raise S3ResponseError(response.status, response.reason, body) + raise provider.storage_response_error( + response.status, response.reason, body) if not headers: headers = {} @@ -456,21 +462,9 @@ class Key(object): headers['User-Agent'] = UserAgent headers['Content-MD5'] = self.base64md5 if self.storage_class != 'STANDARD': - provider = self.bucket.connection.provider headers[provider.storage_class_header] = self.storage_class - # - # If values are passed in the headers param, they take precedence - # and will override the object attributes. If not, the value of - # the attributes will be used unless they are still set to None - # - if 'Content-Encoding' in headers: + if headers.has_key('Content-Encoding'): self.content_encoding = headers['Content-Encoding'] - elif self.content_encoding: - headers['Content-Encoding'] = self.content_encoding - if 'Cache-Control' in headers: - self.cache_control = headers['Cache-Control'] - elif self.cache_control: - headers['Cache-Control'] = self.cache_control if headers.has_key('Content-Type'): self.content_type = headers['Content-Type'] elif self.path: @@ -482,8 +476,7 @@ class Key(object): headers['Content-Type'] = self.content_type headers['Content-Length'] = str(self.size) headers['Expect'] = '100-Continue' - headers = boto.utils.merge_meta(headers, self.metadata, - self.bucket.connection.provider) + headers = boto.utils.merge_meta(headers, self.metadata, provider) resp = self.bucket.connection.make_request('PUT', self.bucket.name, self.name, headers, sender=sender) @@ -921,7 +914,7 @@ class Key(object): policy.acl.add_email_grant(permission, email_address) self.set_acl(policy, headers=headers) - def add_user_grant(self, permission, user_id, headers=None): + def add_user_grant(self, permission, user_id): """ Convenience method that provides a quick way to add a canonical user grant to a key. This method retrieves the current ACL, creates a new grant based on the parameters @@ -939,4 +932,4 @@ class Key(object): """ policy = self.get_acl() policy.acl.add_user_grant(permission, user_id) - self.set_acl(policy, headers=headers) + self.set_acl(policy) diff --git a/boto/tests/test_gsconnection.py b/boto/tests/test_gsconnection.py index 34679a83..c6c90716 100644 --- a/boto/tests/test_gsconnection.py +++ b/boto/tests/test_gsconnection.py @@ -32,7 +32,6 @@ import time import os import urllib from boto.gs.connection import GSConnection -from boto.exception import S3PermissionsError class GSConnectionTest (unittest.TestCase): |