From eab0aaef969bf658d7586d5437bfe081d89ea236 Mon Sep 17 00:00:00 2001 From: Samuel Sutch Date: Tue, 7 Feb 2012 11:21:50 -0800 Subject: Unbound local error in get_dynamodb_type when using set types. --- boto/dynamodb/layer2.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/boto/dynamodb/layer2.py b/boto/dynamodb/layer2.py index 858c3998..b972e841 100644 --- a/boto/dynamodb/layer2.py +++ b/boto/dynamodb/layer2.py @@ -183,6 +183,7 @@ class Layer2(object): the corresponding Amazon DynamoDB type. If the value passed in is not a supported type, raise a TypeError. """ + dynamodb_type = None if is_num(val): dynamodb_type = 'N' elif is_str(val): @@ -192,7 +193,7 @@ class Layer2(object): dynamodb_type = 'NS' elif False not in map(is_str, val): dynamodb_type = 'SS' - else: + if dynamodb_type is None: raise TypeError('Unsupported type "%s" for value "%s"' % (type(val), val)) return dynamodb_type -- cgit v1.2.1 From c464bf041ceae48d9003f8132450a74e3483944b Mon Sep 17 00:00:00 2001 From: Mitch Garnaat Date: Wed, 8 Feb 2012 18:49:04 -0800 Subject: Fixing an obscure bug that occurs when people use the DynamoDB type strings as attribute names. Closes #567 --- boto/dynamodb/layer2.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/boto/dynamodb/layer2.py b/boto/dynamodb/layer2.py index 858c3998..072200ea 100644 --- a/boto/dynamodb/layer2.py +++ b/boto/dynamodb/layer2.py @@ -51,6 +51,8 @@ def item_object_hook(dct): This hook will transform Amazon DynamoDB JSON responses to something that maps directly to native Python types. """ + if len(dct.keys()) > 1: + return dct if 'S' in dct: return dct['S'] if 'N' in dct: -- cgit v1.2.1 From 0b266d419189fd8fdaef7abd17aac8683a789d5e Mon Sep 17 00:00:00 2001 From: Mitch Garnaat Date: Thu, 9 Feb 2012 11:43:13 -0800 Subject: Fixing a couple of issues in the DynamoDB tutorial. Fixes #569. --- docs/source/dynamodb_tut.rst | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/source/dynamodb_tut.rst b/docs/source/dynamodb_tut.rst index 40ff3fa2..39405908 100644 --- a/docs/source/dynamodb_tut.rst +++ b/docs/source/dynamodb_tut.rst @@ -23,10 +23,9 @@ To do so, the most straight forward way is the following:: Bear in mind that if you have your credentials in boto config in your home -directory, the two keyword arguments in the call above are not needed. Also -important to note is that just as any other AWS service, DynamoDB is -region-specific and as such you might want to specify which region to connect -to, by default, it'll connect to the US-EAST-1 region. +directory, the two keyword arguments in the call above are not needed. At this +time, Amazon DynamoDB is available only in the US-EAST-1 region and the +above command will automatically connect to that region. The :py:func:`boto.connect_dynamodb` functions returns a :py:class:`boto.dynamodb.layer2.Layer2` instance, which is a high-level API @@ -225,7 +224,7 @@ Deleting Tables There are two easy ways to delete a table. Through your top-level :py:class:`Layer2 ` object:: - >>> conn.delete_table('messages') + >>> conn.delete_table(table) Or by getting the table, then using :py:meth:`Table.delete `:: -- cgit v1.2.1 From 96fa697ed0411266a288e00f1081bf0ddc6810e6 Mon Sep 17 00:00:00 2001 From: Thomas O'Dowd Date: Mon, 6 Feb 2012 12:16:35 +0900 Subject: Various improvements around file pointer behavior and get_file/send_file. - When setting contents, the file pointer now starts reading content where you positioned it and returns the file pointer up to where you finished reading. This is useful for multipart uploads of large files where it is desirable to split them into different parts. - Rewind the file pointer to original start point when requests are retried if the file object supports it. - Computing md5 for a file pointer now returns the file pointer to the initial position instead of at the start of the file. It also takes a size of bytes to read from the file pointer which enables it to work well with multipart uploads. - If the provider supports chunking, delay the md5 calculation to when the file is being sent on the wire and send the Content-MD5 in the trailer. AWS doesn't support chunking but Google Storage does. Sending in the trailer is disabled currently though as GS doesn't seem to support it right now. - Improve the callback so that it doesn't call too many times and also doesn't call twice or more for the final chunk. - Fix callback for get_file such that the Key size is available when figuring out cb_count. This fixes an exception also. - get_file() now calculates the md5 as the stream is being read. It is useful to have key.md5 available after reading contents in case you want to verify what you read without re-calculating the md5 of a paricularly large piece of data. key.md5 is set. - after send_file() key.size will be accurate for the data that you sent. - Added unit tests for most of the work. --- boto/s3/key.py | 249 +++++++++++++++++++++++++---------- boto/s3/multipart.py | 5 +- boto/utils.py | 36 +++-- tests/s3/test_key.py | 321 +++++++++++++++++++++++++++++++++++++++++++++ tests/s3/test_multipart.py | 32 +++++ tests/test.py | 11 +- 6 files changed, 570 insertions(+), 84 deletions(-) create mode 100644 tests/s3/test_key.py diff --git a/boto/s3/key.py b/boto/s3/key.py index 6afe77a1..5ed1a8d2 100644 --- a/boto/s3/key.py +++ b/boto/s3/key.py @@ -26,6 +26,7 @@ import re import rfc822 import StringIO import base64 +import math import urllib import boto.utils from boto.exception import BotoClientError @@ -454,12 +455,17 @@ class Key(object): expires_in_absolute) def send_file(self, fp, headers=None, cb=None, num_cb=10, - query_args=None, chunked_transfer=False): + query_args=None, chunked_transfer=False, size=None): """ Upload a file to a key into a bucket on S3. :type fp: file - :param fp: The file pointer to upload + :param fp: The file pointer to upload. The file pointer must point + point at the offset from which you wish to upload. + ie. if uploading the full file, it should point at the + start of the file. Normally when a file is opened for + reading, the fp will point at the first byte. See the + bytes parameter below for more info. :type headers: dict :param headers: The headers to pass along with the PUT request @@ -480,20 +486,43 @@ class Key(object): transfer. Providing a negative integer will cause your callback to be called with each buffer read. + :type size: int + :param size: (optional) The Maximum number of bytes to read from + the file pointer (fp). This is useful when uploading + a file in multiple parts where you are splitting the + file up into different ranges to be uploaded. If not + specified, the default behaviour is to read all bytes + from the file pointer. Less bytes may be available. """ provider = self.bucket.connection.provider + try: + spos = fp.tell() + except IOError: + spos = None + self.read_from_stream = False def sender(http_conn, method, path, data, headers): + # This function is called repeatedly for temporary retries + # so we must be sure the file pointer is pointing at the + # start of the data. + if spos is not None and spos != fp.tell(): + fp.seek(spos) + elif spos is None and self.read_from_stream: + # if seek is not supported, and we've read from this + # stream already, then we need to abort retries to + # avoid setting bad data. + raise provider.storage_data_error( + 'Cannot retry failed request. fp does not support seeking.') + http_conn.putrequest(method, path) for key in headers: http_conn.putheader(key, headers[key]) http_conn.endheaders() - if chunked_transfer: - # MD5 for the stream has to be calculated on the fly, as - # we don't know the size of the stream before hand. + if chunked_transfer and not self.base64md5: + # MD5 for the stream has to be calculated on the fly. m = md5() else: - fp.seek(0) + m = None save_debug = self.bucket.connection.debug self.bucket.connection.debug = 0 @@ -503,48 +532,74 @@ class Key(object): # Use the getattr approach to allow this to work in AppEngine. if getattr(http_conn, 'debuglevel', 0) < 3: http_conn.set_debuglevel(0) + + data_len = 0 if cb: - if chunked_transfer: + if size: + cb_size = size + elif self.size: + cb_size = self.size + else: + cb_size = 0 + if chunked_transfer and cb_size == 0: # For chunked Transfer, we call the cb for every 1MB - # of data transferred. + # of data transferred, except when we know size. cb_count = (1024 * 1024)/self.BufferSize - self.size = 0 - elif num_cb > 2: - cb_count = self.size / self.BufferSize / (num_cb-2) + elif num_cb > 1: + cb_count = int(math.ceil(cb_size/self.BufferSize/(num_cb-1.0))) elif num_cb < 0: cb_count = -1 else: cb_count = 0 - i = total_bytes = 0 - cb(total_bytes, self.size) - l = fp.read(self.BufferSize) - while len(l) > 0: + i = 0 + cb(data_len, cb_size) + + bytes_togo = size + if bytes_togo and bytes_togo < self.BufferSize: + chunk = fp.read(bytes_togo) + else: + chunk = fp.read(self.BufferSize) + if spos is None: + # read at least something from a non-seekable fp. + self.read_from_stream = True + while chunk: + chunk_len = len(chunk) + data_len += chunk_len if chunked_transfer: - http_conn.send('%x;\r\n' % len(l)) - http_conn.send(l) + http_conn.send('%x;\r\n' % chunk_len) + http_conn.send(chunk) http_conn.send('\r\n') else: - http_conn.send(l) + http_conn.send(chunk) + if m: + m.update(chunk) + if bytes_togo: + bytes_togo -= chunk_len + if bytes_togo <= 0: + break if cb: - total_bytes += len(l) i += 1 if i == cb_count or cb_count == -1: - cb(total_bytes, self.size) + cb(data_len, cb_size) i = 0 - if chunked_transfer: - m.update(l) - l = fp.read(self.BufferSize) + if bytes_togo and bytes_togo < self.BufferSize: + chunk = fp.read(bytes_togo) + else: + chunk = fp.read(self.BufferSize) + + self.size = data_len if chunked_transfer: http_conn.send('0\r\n') + if m: + # Use the chunked trailer for the digest + hd = m.hexdigest() + self.md5, self.base64md5 = self.get_md5_from_hexdigest(hd) + # http_conn.send("Content-MD5: %s\r\n" % self.base64md5) http_conn.send('\r\n') - if cb: - self.size = total_bytes - # Get the md5 which is calculated on the fly. - self.md5 = m.hexdigest() - else: - fp.seek(0) - if cb: - cb(total_bytes, self.size) + + if cb and (cb_count <= 1 or i > 0) and data_len > 0: + cb(data_len, cb_size) + response = http_conn.getresponse() body = response.read() http_conn.set_debuglevel(save_debug) @@ -568,8 +623,6 @@ class Key(object): else: headers = headers.copy() headers['User-Agent'] = UserAgent - if self.base64md5: - headers['Content-MD5'] = self.base64md5 if self.storage_class != 'STANDARD': headers[provider.storage_class_header] = self.storage_class if headers.has_key('Content-Encoding'): @@ -591,7 +644,13 @@ class Key(object): headers['Content-Type'] = self.content_type else: headers['Content-Type'] = self.content_type - if not chunked_transfer: + if self.base64md5: + headers['Content-MD5'] = self.base64md5 + if chunked_transfer: + headers['Transfer-Encoding'] = 'chunked' + #if not self.base64md5: + # headers['Trailer'] = "Content-MD5" + else: headers['Content-Length'] = str(self.size) headers['Expect'] = '100-Continue' headers = boto.utils.merge_meta(headers, self.metadata, provider) @@ -601,22 +660,29 @@ class Key(object): query_args=query_args) self.handle_version_headers(resp, force=True) - def compute_md5(self, fp): + def compute_md5(self, fp, size=None): """ :type fp: file :param fp: File pointer to the file to MD5 hash. The file pointer - will be reset to the beginning of the file before the + will be reset to the same position before the method returns. + :type size: int + :param size: (optional) The Maximum number of bytes to read from + the file pointer (fp). This is useful when uploading + a file in multiple parts where the file is being + split inplace into different parts. Less bytes may + be available. + :rtype: tuple :return: A tuple containing the hex digest version of the MD5 hash as the first element and the base64 encoded version of the plain digest as the second element. """ - tup = compute_md5(fp) - # Returned values are MD5 hash, base64 encoded MD5 hash, and file size. + tup = compute_md5(fp, size=size) + # Returned values are MD5 hash, base64 encoded MD5 hash, and data size. # The internal implementation of compute_md5() needs to return the - # file size but we don't want to return that value to the external + # data size but we don't want to return that value to the external # caller because it changes the class interface (i.e. it might # break some code) so we consume the third tuple value here and # return the remainder of the tuple to the caller, thereby preserving @@ -689,9 +755,6 @@ class Key(object): if policy: headers[provider.acl_header] = policy - # Set the Transfer Encoding for Streams. - headers['Transfer-Encoding'] = 'chunked' - if reduced_redundancy: self.storage_class = 'REDUCED_REDUNDANCY' if provider.storage_class_header: @@ -699,8 +762,7 @@ class Key(object): if self.bucket != None: if not replace: - k = self.bucket.lookup(self.name) - if k: + if self.bucket.lookup(self.name): return self.send_file(fp, headers, cb, num_cb, query_args, chunked_transfer=True) @@ -708,7 +770,7 @@ class Key(object): def set_contents_from_file(self, fp, headers=None, replace=True, cb=None, num_cb=10, policy=None, md5=None, reduced_redundancy=False, query_args=None, - encrypt_key=False): + encrypt_key=False, size=None): """ Store an object in S3 using the name of the Key object as the key in S3 and the contents of the file pointed to by 'fp' as the @@ -769,6 +831,14 @@ class Key(object): be encrypted on the server-side by S3 and will be stored in an encrypted form while at rest in S3. + + :type size: int + :param size: (optional) The Maximum number of bytes to read from + the file pointer (fp). This is useful when uploading + a file in multiple parts where you are splitting the + file up into different ranges to be uploaded. If not + specified, the default behaviour is to read all bytes + from the file pointer. Less bytes may be available. """ provider = self.bucket.connection.provider if headers is None: @@ -786,23 +856,43 @@ class Key(object): # What if different providers provide different classes? if hasattr(fp, 'name'): self.path = fp.name + if self.bucket != None: - if not md5: - md5 = self.compute_md5(fp) + if not md5 and provider.supports_chunked_transfer(): + # defer md5 calculation to on the fly and + # we don't know anything about size yet. + chunked_transfer = True + self.size = None else: - # even if md5 is provided, still need to set size of content - fp.seek(0, 2) - self.size = fp.tell() - fp.seek(0) - self.md5 = md5[0] - self.base64md5 = md5[1] + chunked_transfer = False + if not md5: + # compute_md5() and also set self.size to actual + # size of the bytes read computing the md5. + md5 = self.compute_md5(fp, size) + # adjust size if required + size = self.size + elif size: + self.size = size + else: + # If md5 is provided, still need to size so + # calculate based on bytes to end of content + spos = fp.tell() + fp.seek(0, os.SEEK_END) + self.size = fp.tell() - spos + fp.seek(spos) + size = self.size + self.md5 = md5[0] + self.base64md5 = md5[1] + if self.name == None: self.name = self.md5 if not replace: - k = self.bucket.lookup(self.name) - if k: + if self.bucket.lookup(self.name): return - self.send_file(fp, headers, cb, num_cb, query_args) + + self.send_file(fp, headers=headers, cb=cb, num_cb=num_cb, + query_args=query_args, chunked_transfer=chunked_transfer, + size=size) def set_contents_from_filename(self, filename, headers=None, replace=True, cb=None, num_cb=10, policy=None, md5=None, @@ -980,15 +1070,6 @@ class Key(object): the stored object in the response. See http://goo.gl/EWOPb for details. """ - if cb: - if num_cb > 2: - cb_count = self.size / self.BufferSize / (num_cb-2) - elif num_cb < 0: - cb_count = -1 - else: - cb_count = 0 - i = total_bytes = 0 - cb(total_bytes, self.size) save_debug = self.bucket.connection.debug if self.bucket.connection.debug == 1: self.bucket.connection.debug = 0 @@ -996,6 +1077,9 @@ class Key(object): query_args = [] if torrent: query_args.append('torrent') + m = None + else: + m = md5() # If a version_id is passed in, use that. If not, check to see # if the Key object has an explicit version_id and, if so, use that. # Otherwise, don't pass a version_id query param. @@ -1009,16 +1093,43 @@ class Key(object): query_args = '&'.join(query_args) self.open('r', headers, query_args=query_args, override_num_retries=override_num_retries) + + data_len = 0 + if cb: + if self.size is None: + cb_size = 0 + else: + cb_size = self.size + if self.size is None and num_cb != -1: + # If size is not available due to chunked transfer for example, + # we'll call the cb for every 1MB of data transferred. + cb_count = (1024 * 1024)/self.BufferSize + elif num_cb > 1: + cb_count = int(math.ceil(cb_size/self.BufferSize/(num_cb-1.0))) + elif num_cb < 0: + cb_count = -1 + else: + cb_count = 0 + i = 0 + cb(data_len, cb_size) for bytes in self: fp.write(bytes) + data_len += len(bytes) + if m: + m.update(bytes) if cb: - total_bytes += len(bytes) + if cb_size > 0 and data_len >= cb_size: + break i += 1 if i == cb_count or cb_count == -1: - cb(total_bytes, self.size) + cb(data_len, cb_size) i = 0 - if cb: - cb(total_bytes, self.size) + if cb and (cb_count <= 1 or i > 0) and data_len > 0: + cb(data_len, cb_size) + if m: + self.md5 = m.hexdigest() + if self.size is None and not torrent and not headers.has_key("Range"): + self.size = data_len self.close() self.bucket.connection.debug = save_debug diff --git a/boto/s3/multipart.py b/boto/s3/multipart.py index 5a2d2784..4eddc397 100644 --- a/boto/s3/multipart.py +++ b/boto/s3/multipart.py @@ -211,7 +211,8 @@ class MultiPartUpload(object): return self._parts def upload_part_from_file(self, fp, part_num, headers=None, replace=True, - cb=None, num_cb=10, policy=None, md5=None): + cb=None, num_cb=10, policy=None, md5=None, + size=None): """ Upload another part of this MultiPart Upload. @@ -230,7 +231,7 @@ class MultiPartUpload(object): key = self.bucket.new_key(self.key_name) key.set_contents_from_file(fp, headers, replace, cb, num_cb, policy, md5, reduced_redundancy=False, - query_args=query_args) + query_args=query_args, size=size) def copy_part_from_key(self, src_bucket_name, src_key_name, part_num, start=None, end=None): diff --git a/boto/utils.py b/boto/utils.py index 197fe12a..2eeb3d32 100644 --- a/boto/utils.py +++ b/boto/utils.py @@ -702,34 +702,52 @@ def guess_mime_type(content, deftype): break return(rtype) -def compute_md5(fp, buf_size=8192): +def compute_md5(fp, buf_size=8192, size=None): """ Compute MD5 hash on passed file and return results in a tuple of values. :type fp: file :param fp: File pointer to the file to MD5 hash. The file pointer - will be reset to the beginning of the file before the + will be reset to its current location before the method returns. :type buf_size: integer :param buf_size: Number of bytes per read request. + :type size: int + :param size: (optional) The Maximum number of bytes to read from + the file pointer (fp). This is useful when uploading + a file in multiple parts where the file is being + split inplace into different parts. Less bytes may + be available. + :rtype: tuple :return: A tuple containing the hex digest version of the MD5 hash as the first element, the base64 encoded version of the - plain digest as the second element and the file size as + plain digest as the second element and the data size as the third element. """ m = md5() - fp.seek(0) - s = fp.read(buf_size) + spos = fp.tell() + if size and size < buf_size: + s = fp.read(size) + else: + s = fp.read(buf_size) while s: m.update(s) - s = fp.read(buf_size) + if size: + size -= len(s) + if size <= 0: + break + if size and size < buf_size: + s = fp.read(size) + else: + s = fp.read(buf_size) hex_md5 = m.hexdigest() base64md5 = base64.encodestring(m.digest()) if base64md5[-1] == '\n': base64md5 = base64md5[0:-1] - file_size = fp.tell() - fp.seek(0) - return (hex_md5, base64md5, file_size) + # data_size based on bytes read. + data_size = fp.tell() - spos + fp.seek(spos) + return (hex_md5, base64md5, data_size) diff --git a/tests/s3/test_key.py b/tests/s3/test_key.py new file mode 100644 index 00000000..c961b317 --- /dev/null +++ b/tests/s3/test_key.py @@ -0,0 +1,321 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ +# All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Some unit tests for S3 Key +""" + +import unittest +import time +import StringIO +from boto.s3.connection import S3Connection +from boto.s3.key import Key +from boto.exception import S3ResponseError + +class S3KeyTest (unittest.TestCase): + + def setUp(self): + self.conn = S3Connection() + self.bucket_name = 'keytest-%d' % int(time.time()) + self.bucket = self.conn.create_bucket(self.bucket_name) + + def tearDown(self): + for key in self.bucket: + key.delete() + self.bucket.delete() + + def test_set_contents_as_file(self): + content="01234567890123456789" + sfp = StringIO.StringIO(content) + + # fp is set at 0 for just opened (for read) files. + # set_contents should write full content to key. + k = self.bucket.new_key("k") + k.set_contents_from_file(sfp) + self.assertEqual(k.size, 20) + kn = self.bucket.new_key("k") + ks = kn.get_contents_as_string() + self.assertEqual(ks, content) + + # set fp to 5 and set contents. this should + # set "567890123456789" to the key + sfp.seek(5) + k = self.bucket.new_key("k") + k.set_contents_from_file(sfp) + self.assertEqual(k.size, 15) + kn = self.bucket.new_key("k") + ks = kn.get_contents_as_string() + self.assertEqual(ks, content[5:]) + + # set fp to 5 and only set 5 bytes. this should + # write the value "56789" to the key. + sfp.seek(5) + k = self.bucket.new_key("k") + k.set_contents_from_file(sfp, size=5) + self.assertEqual(k.size, 5) + self.assertEqual(sfp.tell(), 10) + kn = self.bucket.new_key("k") + ks = kn.get_contents_as_string() + self.assertEqual(ks, content[5:10]) + + def test_set_contents_with_md5(self): + content="01234567890123456789" + sfp = StringIO.StringIO(content) + + # fp is set at 0 for just opened (for read) files. + # set_contents should write full content to key. + k = self.bucket.new_key("k") + good_md5 = k.compute_md5(sfp) + k.set_contents_from_file(sfp, md5=good_md5) + kn = self.bucket.new_key("k") + ks = kn.get_contents_as_string() + self.assertEqual(ks, content) + + # set fp to 5 and only set 5 bytes. this should + # write the value "56789" to the key. + sfp.seek(5) + k = self.bucket.new_key("k") + good_md5 = k.compute_md5(sfp, size=5) + k.set_contents_from_file(sfp, size=5, md5=good_md5) + self.assertEqual(sfp.tell(), 10) + kn = self.bucket.new_key("k") + ks = kn.get_contents_as_string() + self.assertEqual(ks, content[5:10]) + + # let's try a wrong md5 by just altering it. + k = self.bucket.new_key("k") + sfp.seek(0) + hexdig,base64 = k.compute_md5(sfp) + bad_md5 = (hexdig, base64[3:]) + try: + k.set_contents_from_file(sfp, md5=bad_md5) + self.fail("should fail with bad md5") + except S3ResponseError: + pass + + def test_get_contents_with_md5(self): + content="01234567890123456789" + sfp = StringIO.StringIO(content) + + k = self.bucket.new_key("k") + k.set_contents_from_file(sfp) + kn = self.bucket.new_key("k") + s = kn.get_contents_as_string() + self.assertEqual(kn.md5, k.md5) + self.assertEqual(s, content) + + def test_file_callback(self): + def callback(wrote, total): + self.my_cb_cnt += 1 + self.assertNotEqual(wrote, self.my_cb_last, "called twice with same value") + self.my_cb_last = wrote + + # Zero bytes written => 1 call + self.my_cb_cnt = 0 + self.my_cb_last = None + k = self.bucket.new_key("k") + k.BufferSize = 2 + sfp = StringIO.StringIO("") + k.set_contents_from_file(sfp, cb=callback, num_cb=10) + self.assertEqual(self.my_cb_cnt, 1) + self.assertEqual(self.my_cb_last, 0) + sfp.close() + + # Read back zero bytes => 1 call + self.my_cb_cnt = 0 + self.my_cb_last = None + s = k.get_contents_as_string(cb=callback) + self.assertEqual(self.my_cb_cnt, 1) + self.assertEqual(self.my_cb_last, 0) + + content="01234567890123456789" + sfp = StringIO.StringIO(content) + + # expect 2 calls due start/finish + self.my_cb_cnt = 0 + self.my_cb_last = None + k = self.bucket.new_key("k") + k.set_contents_from_file(sfp, cb=callback, num_cb=10) + self.assertEqual(self.my_cb_cnt, 2) + self.assertEqual(self.my_cb_last, 20) + + # Read back all bytes => 2 calls + self.my_cb_cnt = 0 + self.my_cb_last = None + s = k.get_contents_as_string(cb=callback) + self.assertEqual(self.my_cb_cnt, 2) + self.assertEqual(self.my_cb_last, 20) + self.assertEqual(s, content) + + # rewind sfp and try upload again. -1 should call + # for every read/write so that should make 11 when bs=2 + sfp.seek(0) + self.my_cb_cnt = 0 + self.my_cb_last = None + k = self.bucket.new_key("k") + k.BufferSize = 2 + k.set_contents_from_file(sfp, cb=callback, num_cb=-1) + self.assertEqual(self.my_cb_cnt, 11) + self.assertEqual(self.my_cb_last, 20) + + # Read back all bytes => 11 calls + self.my_cb_cnt = 0 + self.my_cb_last = None + s = k.get_contents_as_string(cb=callback, num_cb=-1) + self.assertEqual(self.my_cb_cnt, 11) + self.assertEqual(self.my_cb_last, 20) + self.assertEqual(s, content) + + # no more than 1 times => 2 times + # last time always 20 bytes + sfp.seek(0) + self.my_cb_cnt = 0 + self.my_cb_last = None + k = self.bucket.new_key("k") + k.BufferSize = 2 + k.set_contents_from_file(sfp, cb=callback, num_cb=1) + self.assertTrue(self.my_cb_cnt <= 2) + self.assertEqual(self.my_cb_last, 20) + + # no more than 1 times => 2 times + self.my_cb_cnt = 0 + self.my_cb_last = None + s = k.get_contents_as_string(cb=callback, num_cb=1) + self.assertTrue(self.my_cb_cnt <= 2) + self.assertEqual(self.my_cb_last, 20) + self.assertEqual(s, content) + + # no more than 2 times + # last time always 20 bytes + sfp.seek(0) + self.my_cb_cnt = 0 + self.my_cb_last = None + k = self.bucket.new_key("k") + k.BufferSize = 2 + k.set_contents_from_file(sfp, cb=callback, num_cb=2) + self.assertTrue(self.my_cb_cnt <= 2) + self.assertEqual(self.my_cb_last, 20) + + # no more than 2 times + self.my_cb_cnt = 0 + self.my_cb_last = None + s = k.get_contents_as_string(cb=callback, num_cb=2) + self.assertTrue(self.my_cb_cnt <= 2) + self.assertEqual(self.my_cb_last, 20) + self.assertEqual(s, content) + + # no more than 3 times + # last time always 20 bytes + sfp.seek(0) + self.my_cb_cnt = 0 + self.my_cb_last = None + k = self.bucket.new_key("k") + k.BufferSize = 2 + k.set_contents_from_file(sfp, cb=callback, num_cb=3) + self.assertTrue(self.my_cb_cnt <= 3) + self.assertEqual(self.my_cb_last, 20) + + # no more than 3 times + self.my_cb_cnt = 0 + self.my_cb_last = None + s = k.get_contents_as_string(cb=callback, num_cb=3) + self.assertTrue(self.my_cb_cnt <= 3) + self.assertEqual(self.my_cb_last, 20) + self.assertEqual(s, content) + + # no more than 4 times + # last time always 20 bytes + sfp.seek(0) + self.my_cb_cnt = 0 + self.my_cb_last = None + k = self.bucket.new_key("k") + k.BufferSize = 2 + k.set_contents_from_file(sfp, cb=callback, num_cb=4) + self.assertTrue(self.my_cb_cnt <= 4) + self.assertEqual(self.my_cb_last, 20) + + # no more than 4 times + self.my_cb_cnt = 0 + self.my_cb_last = None + s = k.get_contents_as_string(cb=callback, num_cb=4) + self.assertTrue(self.my_cb_cnt <= 4) + self.assertEqual(self.my_cb_last, 20) + self.assertEqual(s, content) + + # no more than 6 times + # last time always 20 bytes + sfp.seek(0) + self.my_cb_cnt = 0 + self.my_cb_last = None + k = self.bucket.new_key("k") + k.BufferSize = 2 + k.set_contents_from_file(sfp, cb=callback, num_cb=6) + self.assertTrue(self.my_cb_cnt <= 6) + self.assertEqual(self.my_cb_last, 20) + + # no more than 6 times + self.my_cb_cnt = 0 + self.my_cb_last = None + s = k.get_contents_as_string(cb=callback, num_cb=6) + self.assertTrue(self.my_cb_cnt <= 6) + self.assertEqual(self.my_cb_last, 20) + self.assertEqual(s, content) + + # no more than 10 times + # last time always 20 bytes + sfp.seek(0) + self.my_cb_cnt = 0 + self.my_cb_last = None + k = self.bucket.new_key("k") + k.BufferSize = 2 + k.set_contents_from_file(sfp, cb=callback, num_cb=10) + self.assertTrue(self.my_cb_cnt <= 10) + self.assertEqual(self.my_cb_last, 20) + + # no more than 10 times + self.my_cb_cnt = 0 + self.my_cb_last = None + s = k.get_contents_as_string(cb=callback, num_cb=10) + self.assertTrue(self.my_cb_cnt <= 10) + self.assertEqual(self.my_cb_last, 20) + self.assertEqual(s, content) + + # no more than 1000 times + # last time always 20 bytes + sfp.seek(0) + self.my_cb_cnt = 0 + self.my_cb_last = None + k = self.bucket.new_key("k") + k.BufferSize = 2 + k.set_contents_from_file(sfp, cb=callback, num_cb=1000) + self.assertTrue(self.my_cb_cnt <= 1000) + self.assertEqual(self.my_cb_last, 20) + + # no more than 1000 times + self.my_cb_cnt = 0 + self.my_cb_last = None + s = k.get_contents_as_string(cb=callback, num_cb=1000) + self.assertTrue(self.my_cb_cnt <= 1000) + self.assertEqual(self.my_cb_last, 20) + self.assertEqual(s, content) diff --git a/tests/s3/test_multipart.py b/tests/s3/test_multipart.py index 5921889c..5c64ba73 100644 --- a/tests/s3/test_multipart.py +++ b/tests/s3/test_multipart.py @@ -90,3 +90,35 @@ class S3MultiPartUploadTest (unittest.TestCase): self.assertEqual(lmpu.key_name, key_name) # Abort using the one returned in the list lmpu.cancel_upload() + + def test_four_part_file(self): + key_name = "k" + contents = "01234567890123456789" + sfp = StringIO.StringIO(contents) + + # upload 20 bytes in 4 parts of 5 bytes each + mpu = self.bucket.initiate_multipart_upload(key_name) + mpu.upload_part_from_file(sfp, part_num=1, size=5) + mpu.upload_part_from_file(sfp, part_num=2, size=5) + mpu.upload_part_from_file(sfp, part_num=3, size=5) + mpu.upload_part_from_file(sfp, part_num=4, size=5) + sfp.close() + + etags = {} + pn = 0 + for part in mpu: + pn += 1 + self.assertEqual(5, part.size) + etags[pn] = part.etag + self.assertEqual(pn, 4) + # etags for 01234 + self.assertEqual(etags[1], etags[3]) + # etags for 56789 + self.assertEqual(etags[2], etags[4]) + # etag 01234 != etag 56789 + self.assertNotEqual(etags[1], etags[2]) + + # parts are too small to compete as each part must + # be a min of 5MB so so we'll assume that is enough + # testing and abort the upload. + mpu.cancel_upload() diff --git a/tests/test.py b/tests/test.py index bc6cd756..1eefdaa7 100755 --- a/tests/test.py +++ b/tests/test.py @@ -34,6 +34,7 @@ from s3.test_connection import S3ConnectionTest from s3.test_versioning import S3VersionTest from s3.test_encryption import S3EncryptionTest from s3.test_bucket import S3BucketTest +from s3.test_key import S3KeyTest from s3.test_multidelete import S3MultiDeleteTest from s3.test_multipart import S3MultiPartUploadTest from s3.test_gsconnection import GSConnectionTest @@ -93,21 +94,23 @@ def suite(testsuite="all"): tests.addTest(unittest.makeSuite(DynamoDBLayer2Test)) elif testsuite == "s3": tests.addTest(unittest.makeSuite(S3ConnectionTest)) + tests.addTest(unittest.makeSuite(S3BucketTest)) + tests.addTest(unittest.makeSuite(S3KeyTest)) + tests.addTest(unittest.makeSuite(S3MultiPartUploadTest)) tests.addTest(unittest.makeSuite(S3VersionTest)) tests.addTest(unittest.makeSuite(S3EncryptionTest)) tests.addTest(unittest.makeSuite(S3MultiDeleteTest)) - tests.addTest(unittest.makeSuite(S3MultiPartUploadTest)) - tests.addTest(unittest.makeSuite(S3BucketTest)) elif testsuite == "ssl": tests.addTest(unittest.makeSuite(CertValidationTest)) elif testsuite == "s3ver": tests.addTest(unittest.makeSuite(S3VersionTest)) elif testsuite == "s3nover": tests.addTest(unittest.makeSuite(S3ConnectionTest)) + tests.addTest(unittest.makeSuite(S3BucketTest)) + tests.addTest(unittest.makeSuite(S3KeyTest)) + tests.addTest(unittest.makeSuite(S3MultiPartUploadTest)) tests.addTest(unittest.makeSuite(S3EncryptionTest)) tests.addTest(unittest.makeSuite(S3MultiDeleteTest)) - tests.addTest(unittest.makeSuite(S3MultiPartUploadTest)) - tests.addTest(unittest.makeSuite(S3BucketTest)) elif testsuite == "gs": tests.addTest(unittest.makeSuite(GSConnectionTest)) elif testsuite == "sqs": -- cgit v1.2.1 From 42efefad7c7b6cd333ad75e2df69096a26d042aa Mon Sep 17 00:00:00 2001 From: Thomas O'Dowd Date: Fri, 10 Feb 2012 18:25:10 +0900 Subject: Google Storage key related fixes for file pointer - Add size parameter to gs/key/set_contents_from_file(). This makes the command work similar to s3/key/set_contents_from_file() - Add size parameter to s3/key/set_contents_from_stream(). size parameter should be useful here. - NOTE: did not add size to */key/set_content_from_(string|filename) functions as it doesn't make so much sense. String would be chopped by a user when he called it and you have the issue of multi-bytes and giving a filename normally uploads the full file also. --- boto/gs/key.py | 38 +++++++++++++++++++++++++++++--------- boto/s3/key.py | 15 +++++++++++---- 2 files changed, 40 insertions(+), 13 deletions(-) diff --git a/boto/gs/key.py b/boto/gs/key.py index de6e6f44..27edabab 100644 --- a/boto/gs/key.py +++ b/boto/gs/key.py @@ -110,7 +110,7 @@ class Key(S3Key): def set_contents_from_file(self, fp, headers=None, replace=True, cb=None, num_cb=10, policy=None, md5=None, - res_upload_handler=None): + res_upload_handler=None, size=None): """ Store an object in GS using the name of the Key object as the key in GS and the contents of the file pointed to by 'fp' as the @@ -158,12 +158,23 @@ class Key(S3Key): :param res_upload_handler: If provided, this handler will perform the upload. + :type size: int + :param size: (optional) The Maximum number of bytes to read from + the file pointer (fp). This is useful when uploading + a file in multiple parts where you are splitting the + file up into different ranges to be uploaded. If not + specified, the default behaviour is to read all bytes + from the file pointer. Less bytes may be available. + TODO: At some point we should refactor the Bucket and Key classes, to move functionality common to all providers into a parent class, and provider-specific functionality into subclasses (rather than just overriding/sharing code the way it currently works). """ provider = self.bucket.connection.provider + if res_upload_handler and size: + # could use size instead of file_length if provided but... + raise BotoClientError('Resumable Uploads with size not supported.') headers = headers or {} if policy: headers[provider.acl_header] = policy @@ -171,25 +182,34 @@ class Key(S3Key): self.path = fp.name if self.bucket != None: if not md5: - md5 = self.compute_md5(fp) + # compute_md5() and also set self.size to actual + # size of the bytes read computing the md5. + md5 = self.compute_md5(fp, size) + # adjust size if required + size = self.size + elif size: + self.size = size else: - # Even if md5 is provided, still need to set size of content. - fp.seek(0, 2) - self.size = fp.tell() - fp.seek(0) + # If md5 is provided, still need to size so + # calculate based on bytes to end of content + spos = fp.tell() + fp.seek(0, os.SEEK_END) + self.size = fp.tell() - spos + fp.seek(spos) + size = self.size self.md5 = md5[0] self.base64md5 = md5[1] + if self.name == None: self.name = self.md5 if not replace: - k = self.bucket.lookup(self.name) - if k: + if self.bucket.lookup(self.name): return if res_upload_handler: res_upload_handler.send_file(self, fp, headers, cb, num_cb) else: # Not a resumable transfer so use basic send_file mechanism. - self.send_file(fp, headers, cb, num_cb) + self.send_file(fp, headers, cb, num_cb, size=size) def set_contents_from_filename(self, filename, headers=None, replace=True, cb=None, num_cb=10, policy=None, md5=None, diff --git a/boto/s3/key.py b/boto/s3/key.py index 5ed1a8d2..206bcb6e 100644 --- a/boto/s3/key.py +++ b/boto/s3/key.py @@ -692,7 +692,8 @@ class Key(object): def set_contents_from_stream(self, fp, headers=None, replace=True, cb=None, num_cb=10, policy=None, - reduced_redundancy=False, query_args=None): + reduced_redundancy=False, query_args=None, + size=None): """ Store an object using the name of the Key object as the key in cloud and the contents of the data stream pointed to by 'fp' as @@ -738,6 +739,13 @@ class Key(object): REDUCED_REDUNDANCY. The Reduced Redundancy Storage (RRS) feature of S3, provides lower redundancy at lower storage cost. + :type size: int + :param size: (optional) The Maximum number of bytes to read from + the file pointer (fp). This is useful when uploading + a file in multiple parts where you are splitting the + file up into different ranges to be uploaded. If not + specified, the default behaviour is to read all bytes + from the file pointer. Less bytes may be available. """ provider = self.bucket.connection.provider @@ -765,7 +773,7 @@ class Key(object): if self.bucket.lookup(self.name): return self.send_file(fp, headers, cb, num_cb, query_args, - chunked_transfer=True) + chunked_transfer=True, size=size) def set_contents_from_file(self, fp, headers=None, replace=True, cb=None, num_cb=10, policy=None, md5=None, @@ -841,8 +849,7 @@ class Key(object): from the file pointer. Less bytes may be available. """ provider = self.bucket.connection.provider - if headers is None: - headers = {} + headers = headers or {} if policy: headers[provider.acl_header] = policy if encrypt_key: -- cgit v1.2.1 From 0f16d4667f7a5b16aadcaec030de8407af4a7226 Mon Sep 17 00:00:00 2001 From: Mitch Garnaat Date: Fri, 10 Feb 2012 07:30:29 -0800 Subject: Fixing some docstrings. --- boto/dynamodb/layer1.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/boto/dynamodb/layer1.py b/boto/dynamodb/layer1.py index f7606690..72d9a58d 100644 --- a/boto/dynamodb/layer1.py +++ b/boto/dynamodb/layer1.py @@ -178,7 +178,7 @@ class Layer1(AWSAuthConnection): table was created. :type table_name: str - :param table_name: The name of the table to delete. + :param table_name: The name of the table to describe. """ data = {'TableName' : table_name} json_input = json.dumps(data) @@ -193,7 +193,7 @@ class Layer1(AWSAuthConnection): table will be ACTIVE. :type table_name: str - :param table_name: The name of the table to delete. + :param table_name: The name of the table to create. :type schema: dict :param schema: A Python version of the KeySchema data structure @@ -217,7 +217,7 @@ class Layer1(AWSAuthConnection): Updates the provisioned throughput for a given table. :type table_name: str - :param table_name: The name of the table to delete. + :param table_name: The name of the table to update. :type provisioned_throughput: dict :param provisioned_throughput: A Python version of the @@ -249,7 +249,7 @@ class Layer1(AWSAuthConnection): the supplied key. :type table_name: str - :param table_name: The name of the table to delete. + :param table_name: The name of the table containing the item. :type key: dict :param key: A Python version of the Key data structure @@ -306,7 +306,7 @@ class Layer1(AWSAuthConnection): expected rule. :type table_name: str - :param table_name: The name of the table to delete. + :param table_name: The name of the table in which to put the item. :type item: dict :param item: A Python version of the Item data structure @@ -384,7 +384,7 @@ class Layer1(AWSAuthConnection): expected rule. :type table_name: str - :param table_name: The name of the table to delete. + :param table_name: The name of the table containing the item. :type key: dict :param key: A Python version of the Key data structure @@ -421,7 +421,7 @@ class Layer1(AWSAuthConnection): which is passed as is to DynamoDB. :type table_name: str - :param table_name: The name of the table to delete. + :param table_name: The name of the table to query. :type hash_key_value: dict :param key: A DynamoDB-style HashKeyValue. -- cgit v1.2.1 From 9289efed729546df092ea33e48d9726dee61126f Mon Sep 17 00:00:00 2001 From: Paul Armstrong Date: Fri, 10 Feb 2012 09:13:26 -0800 Subject: Add a prefix to b.list() so s3put/s3multiput don't list the whole bucket Listing the whole bucket to avoid overwriting keys is extremely inefficient (especially for buckets will millions or billions of objects). Use the path provided by the user to add a prefix to b.list() so that the keyspace is reduced to just the area we're trying to sync. --- bin/s3multiput | 2 +- bin/s3put | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/s3multiput b/bin/s3multiput index 095737ac..644ddc91 100755 --- a/bin/s3multiput +++ b/bin/s3multiput @@ -256,7 +256,7 @@ def main(): if not quiet: print 'Getting list of existing keys to check against' keys = [] - for key in b.list(): + for key in b.list(get_key_name(path, prefix)): keys.append(key.name) for root, dirs, files in os.walk(path): for ignore in ignore_dirs: diff --git a/bin/s3put b/bin/s3put index 972d6279..a5bf30c7 100755 --- a/bin/s3put +++ b/bin/s3put @@ -162,7 +162,7 @@ def main(): if not quiet: print 'Getting list of existing keys to check against' keys = [] - for key in b.list(): + for key in b.list(get_key_name(path, prefix)): keys.append(key.name) for root, dirs, files in os.walk(path): for ignore in ignore_dirs: -- cgit v1.2.1 From 846b2e91548bd0cc579e3b7bd8aaffb740ce18ef Mon Sep 17 00:00:00 2001 From: Thomas O'Dowd Date: Sat, 11 Feb 2012 12:56:17 +0900 Subject: fix missing BotoClientError import and more add more docs --- boto/gs/key.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/boto/gs/key.py b/boto/gs/key.py index 27edabab..a0c50a79 100644 --- a/boto/gs/key.py +++ b/boto/gs/key.py @@ -20,6 +20,7 @@ # IN THE SOFTWARE. import StringIO +from boto.exception import BotoClientError from boto.s3.key import Key as S3Key class Key(S3Key): @@ -165,6 +166,13 @@ class Key(S3Key): file up into different ranges to be uploaded. If not specified, the default behaviour is to read all bytes from the file pointer. Less bytes may be available. + Notes: + 1. The "size" parameter currently cannot be used when + a resumable upload handler is given but is still + useful for uploading part of a file as implemented + by the parent class. + 2. At present Google Cloud Storage does not support + multipart uploads. TODO: At some point we should refactor the Bucket and Key classes, to move functionality common to all providers into a parent class, @@ -174,7 +182,7 @@ class Key(S3Key): provider = self.bucket.connection.provider if res_upload_handler and size: # could use size instead of file_length if provided but... - raise BotoClientError('Resumable Uploads with size not supported.') + raise BotoClientError('"size" param not supported for resumable uploads.') headers = headers or {} if policy: headers[provider.acl_header] = policy -- cgit v1.2.1 From adeb715194a9d0e78d0e5f41638782c68c0071f8 Mon Sep 17 00:00:00 2001 From: Mitch Garnaat Date: Sat, 11 Feb 2012 07:45:04 -0800 Subject: Cleaned up the scan method on Layer2 and Table. Closes #574. --- boto/dynamodb/layer2.py | 66 +++++++++++++++++++++++++++++++++++-------- boto/dynamodb/table.py | 23 +++++++++++++-- tests/dynamodb/test_layer2.py | 7 +++++ 3 files changed, 82 insertions(+), 14 deletions(-) diff --git a/boto/dynamodb/layer2.py b/boto/dynamodb/layer2.py index 072200ea..45f13459 100644 --- a/boto/dynamodb/layer2.py +++ b/boto/dynamodb/layer2.py @@ -96,7 +96,7 @@ class Layer2(object): def dynamize_range_key_condition(self, range_key_condition): """ - Convert a range_key_condition parameter into the + Convert a layer2 range_key_condition parameter into the structure required by Layer1. """ d = None @@ -119,6 +119,34 @@ class Layer2(object): 'ComparisonOperator': range_condition} return d + def dynamize_scan_filter(self, scan_filter): + """ + Convert a layer2 scan_filter parameter into the + structure required by Layer1. + """ + d = None + if scan_filter: + d = {} + for attr_name, op, value in scan_filter: + if op == 'BETWEEN': + if isinstance(value, tuple): + avl = [self.dynamize_value(v) for v in value] + else: + msg = 'BETWEEN condition requires a tuple value' + raise TypeError(msg) + elif op == 'NULL' or op == 'NOT_NULL': + avl = None + elif isinstance(value, tuple): + msg = 'Tuple can only be supplied with BETWEEN condition' + raise TypeError(msg) + else: + avl = [self.dynamize_value(value)] + dd = {'ComparisonOperator': op} + if avl: + dd['AttributeValueList'] = avl + d[attr_name] = dd + return d + def dynamize_expected_value(self, expected_value): """ Convert an expected_value parameter into the data structure @@ -658,16 +686,31 @@ class Layer2(object): attributes_to_get=None, request_limit=None, max_results=None, count=False, exclusive_start_key=None, item_class=Item): """ - Perform a scan of DynamoDB. This version is currently punting - and expecting you to provide a full and correct JSON body - which is passed as is to DynamoDB. - - :type table: Table - :param table: The table to scan from + Perform a scan of DynamoDB. - :type scan_filter: dict - :param scan_filter: A Python version of the - ScanFilter data structure. + :type table: :class:`boto.dynamodb.table.Table` + :param table: The Table object that is being scanned. + + :type scan_filter: A list of tuples + :param scan_filter: A list of tuples where each tuple consists + of an attribute name, a comparison operator, and either + a scalar or tuple consisting of the values to compare + the attribute to. Valid comparison operators are shown below + along with the expected number of values that should be supplied. + + * EQ - equal (1) + * NE - not equal (1) + * LE - less than or equal (1) + * LT - less than (1) + * GE - greater than or equal (1) + * GT - greater than (1) + * NOT_NULL - attribute exists (0, use None) + * NULL - attribute does not exist (0, use None) + * CONTAINS - substring or value in list (1) + * NOT_CONTAINS - absence of substring or value in list (1) + * BEGINS_WITH - substring prefix (1) + * IN - exact match in list (N) + * BETWEEN - >= first value, <= second value (2) :type attributes_to_get: list :param attributes_to_get: A list of attribute names. @@ -706,6 +749,7 @@ class Layer2(object): :rtype: generator """ + sf = self.dynamize_scan_filter(scan_filter) response = True n = 0 while response: @@ -716,7 +760,7 @@ class Layer2(object): else: break - response = self.layer1.scan(table.name, scan_filter, + response = self.layer1.scan(table.name, sf, attributes_to_get,request_limit, count, exclusive_start_key, object_hook=item_object_hook) diff --git a/boto/dynamodb/table.py b/boto/dynamodb/table.py index 152c2cb8..7c8e5ea8 100644 --- a/boto/dynamodb/table.py +++ b/boto/dynamodb/table.py @@ -307,9 +307,26 @@ class Table(object): and expensive operation, and should be avoided if at all possible. - :type scan_filter: dict - :param scan_filter: A Python version of the - ScanFilter data structure. + :type scan_filter: A list of tuples + :param scan_filter: A list of tuples where each tuple consists + of an attribute name, a comparison operator, and either + a scalar or tuple consisting of the values to compare + the attribute to. Valid comparison operators are shown below + along with the expected number of values that should be supplied. + + * EQ - equal (1) + * NE - not equal (1) + * LE - less than or equal (1) + * LT - less than (1) + * GE - greater than or equal (1) + * GT - greater than (1) + * NOT_NULL - attribute exists (0, use None) + * NULL - attribute does not exist (0, use None) + * CONTAINS - substring or value in list (1) + * NOT_CONTAINS - absence of substring or value in list (1) + * BEGINS_WITH - substring prefix (1) + * IN - exact match in list (N) + * BETWEEN - >= first value, <= second value (2) :type attributes_to_get: list :param attributes_to_get: A list of attribute names. diff --git a/tests/dynamodb/test_layer2.py b/tests/dynamodb/test_layer2.py index cc608fe6..6a68b895 100644 --- a/tests/dynamodb/test_layer2.py +++ b/tests/dynamodb/test_layer2.py @@ -287,6 +287,13 @@ class DynamoDBLayer2Test (unittest.TestCase): n += 1 assert n == 2 + # Try scans + results = table.scan([('Tags', 'CONTAINS', 'table')]) + n = 0 + for item in results: + n += 1 + assert n == 2 + # Try to delete the item with the right Expected value expected = {'Views': 0} item1.delete(expected_value=expected) -- cgit v1.2.1 From 2c9dab59526ed4a6800d6f819a825f6ab6b565ce Mon Sep 17 00:00:00 2001 From: Thomas O'Dowd Date: Sun, 12 Feb 2012 18:16:24 +0900 Subject: Split MFA/versioning unit tests. This change gets us more test coverage for developers without an MFA device. - Added testsuite 's3nomfa' which runs all s3 unit tests except the MFA unit test. - Added testsuite 's3mfa' which only runs the MFA unit tests. - Removed the testsuite 's3ver' and 's3nover' as the versioning test no longer depends on MFA and can be run normally. - Removed sleeps from test_versioning as it seems to work consistently well without them for me. Feel free to re-add them if test is unreliable. --- tests/s3/test_mfa.py | 91 ++++++++++++++++++++++++++ tests/s3/test_versioning.py | 156 +++++++++++++++++--------------------------- tests/test.py | 11 ++-- 3 files changed, 157 insertions(+), 101 deletions(-) create mode 100644 tests/s3/test_mfa.py diff --git a/tests/s3/test_mfa.py b/tests/s3/test_mfa.py new file mode 100644 index 00000000..3f47e94c --- /dev/null +++ b/tests/s3/test_mfa.py @@ -0,0 +1,91 @@ +# Copyright (c) 2010 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2010, Eucalyptus Systems, Inc. +# All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Some unit tests for S3 MfaDelete with versioning +""" + +import unittest +import time +from boto.s3.connection import S3Connection +from boto.exception import S3ResponseError +from boto.s3.deletemarker import DeleteMarker + +class S3MFATest (unittest.TestCase): + + def setUp(self): + self.conn = S3Connection() + self.bucket_name = 'mfa-%d' % int(time.time()) + self.bucket = self.conn.create_bucket(self.bucket_name) + + def tearDown(self): + for k in self.bucket.list_versions(): + self.bucket.delete_key(k.name, version_id=k.version_id) + self.bucket.delete() + + def test_mfadel(self): + # Enable Versioning with MfaDelete + mfa_sn = raw_input('MFA S/N: ') + mfa_code = raw_input('MFA Code: ') + self.bucket.configure_versioning(True, mfa_delete=True, mfa_token=(mfa_sn, mfa_code)) + + # Check enabling mfa worked. + i = 0 + for i in range(1,8): + time.sleep(2**i) + d = self.bucket.get_versioning_status() + if d['Versioning'] == 'Enabled' and d['MfaDelete'] == 'Enabled': + break + self.assertEqual('Enabled', d['Versioning']) + self.assertEqual('Enabled', d['MfaDelete']) + + # Add a key to the bucket + k = self.bucket.new_key('foobar') + s1 = 'This is v1' + k.set_contents_from_string(s1) + v1 = k.version_id + + # Now try to delete v1 without the MFA token + try: + self.bucket.delete_key('foobar', version_id=v1) + self.fail("Must fail if not using MFA token") + except S3ResponseError: + pass + + # Now try delete again with the MFA token + mfa_code = raw_input('MFA Code: ') + self.bucket.delete_key('foobar', version_id=v1, mfa_token=(mfa_sn, mfa_code)) + + # Next suspend versioning and disable MfaDelete on the bucket + mfa_code = raw_input('MFA Code: ') + self.bucket.configure_versioning(False, mfa_delete=False, mfa_token=(mfa_sn, mfa_code)) + + # Lastly, check disabling mfa worked. + i = 0 + for i in range(1,8): + time.sleep(2**i) + d = self.bucket.get_versioning_status() + if d['Versioning'] == 'Suspended' and d['MfaDelete'] != 'Enabled': + break + self.assertEqual('Suspended', d['Versioning']) + self.assertNotEqual('Enabled', d['MfaDelete']) diff --git a/tests/s3/test_versioning.py b/tests/s3/test_versioning.py index 879e36b4..2d569af9 100644 --- a/tests/s3/test_versioning.py +++ b/tests/s3/test_versioning.py @@ -22,7 +22,7 @@ # IN THE SOFTWARE. """ -Some unit tests for the S3 Versioning and MfaDelete +Some unit tests for the S3 Versioning. """ import unittest @@ -33,31 +33,30 @@ from boto.s3.deletemarker import DeleteMarker class S3VersionTest (unittest.TestCase): + def setUp(self): + self.conn = S3Connection() + self.bucket_name = 'version-%d' % int(time.time()) + self.bucket = self.conn.create_bucket(self.bucket_name) + + def tearDown(self): + for k in self.bucket.list_versions(): + self.bucket.delete_key(k.name, version_id=k.version_id) + self.bucket.delete() + def test_1_versions(self): - print '--- running S3Version tests ---' - c = S3Connection() - # create a new, empty bucket - bucket_name = 'version-%d' % int(time.time()) - bucket = c.create_bucket(bucket_name) - - # now try a get_bucket call and see if it's really there - bucket = c.get_bucket(bucket_name) - - # enable versions - d = bucket.get_versioning_status() - assert not d.has_key('Versioning') - bucket.configure_versioning(versioning=True) - time.sleep(15) - d = bucket.get_versioning_status() - assert d['Versioning'] == 'Enabled' + # check versioning off + d = self.bucket.get_versioning_status() + self.assertFalse(d.has_key('Versioning')) + + # enable versioning + self.bucket.configure_versioning(versioning=True) + d = self.bucket.get_versioning_status() + self.assertEqual('Enabled', d['Versioning']) # create a new key in the versioned bucket - k = bucket.new_key() - k.name = 'foobar' - s1 = 'This is a test of s3 versioning' - s2 = 'This is the second test of s3 versioning' + k = self.bucket.new_key("foobar") + s1 = 'This is v1' k.set_contents_from_string(s1) - time.sleep(5) # remember the version id of this object v1 = k.version_id @@ -65,109 +64,77 @@ class S3VersionTest (unittest.TestCase): # now get the contents from s3 o1 = k.get_contents_as_string() - # check to make sure content read from s3 is identical to original - assert o1 == s1 + # check to make sure content read from k is identical to original + self.assertEqual(s1, o1) # now overwrite that same key with new data + s2 = 'This is v2' k.set_contents_from_string(s2) v2 = k.version_id - time.sleep(5) - # now retrieve the contents as a string and compare - s3 = k.get_contents_as_string(version_id=v2) - assert s3 == s2 + # now retrieve latest contents as a string and compare + k2 = self.bucket.new_key("foobar") + o2 = k2.get_contents_as_string() + self.assertEqual(s2, o2) + + # next retrieve explicit versions and compare + o1 = k.get_contents_as_string(version_id=v1) + o2 = k.get_contents_as_string(version_id=v2) + self.assertEqual(s1, o1) + self.assertEqual(s2, o2) # Now list all versions and compare to what we have - rs = bucket.get_all_versions() - assert rs[0].version_id == v2 - assert rs[1].version_id == v1 + rs = self.bucket.get_all_versions() + self.assertEqual(v2, rs[0].version_id) + self.assertEqual(v1, rs[1].version_id) # Now do a regular list command and make sure only the new key shows up - rs = bucket.get_all_keys() - assert len(rs) == 1 + rs = self.bucket.get_all_keys() + self.assertEqual(1, len(rs)) # Now do regular delete - bucket.delete_key('foobar') - time.sleep(5) + self.bucket.delete_key('foobar') # Now list versions and make sure old versions are there - # plus the DeleteMarker - rs = bucket.get_all_versions() - assert len(rs) == 3 - assert isinstance(rs[0], DeleteMarker) + # plus the DeleteMarker which is latest. + rs = self.bucket.get_all_versions() + self.assertEqual(3, len(rs)) + self.assertTrue(isinstance(rs[0], DeleteMarker)) # Now delete v1 of the key - bucket.delete_key('foobar', version_id=v1) - time.sleep(5) + self.bucket.delete_key('foobar', version_id=v1) # Now list versions again and make sure v1 is not there - rs = bucket.get_all_versions() + rs = self.bucket.get_all_versions() versions = [k.version_id for k in rs] - assert v1 not in versions - assert v2 in versions - - # Now try to enable MfaDelete - mfa_sn = raw_input('MFA S/N: ') - mfa_code = raw_input('MFA Code: ') - bucket.configure_versioning(True, mfa_delete=True, mfa_token=(mfa_sn, mfa_code)) - i = 0 - for i in range(1,8): - time.sleep(2**i) - d = bucket.get_versioning_status() - if d['Versioning'] == 'Enabled' and d['MfaDelete'] == 'Enabled': - break - assert d['Versioning'] == 'Enabled' - assert d['MfaDelete'] == 'Enabled' - - # Now try to delete v2 without the MFA token - try: - bucket.delete_key('foobar', version_id=v2) - except S3ResponseError: - pass - - # Now try to delete v2 with the MFA token - mfa_code = raw_input('MFA Code: ') - bucket.delete_key('foobar', version_id=v2, mfa_token=(mfa_sn, mfa_code)) - - # Now disable MfaDelete on the bucket - mfa_code = raw_input('MFA Code: ') - bucket.configure_versioning(True, mfa_delete=False, mfa_token=(mfa_sn, mfa_code)) - + self.assertTrue(v1 not in versions) + self.assertTrue(v2 in versions) + # Now suspend Versioning on the bucket - bucket.configure_versioning(False) + self.bucket.configure_versioning(False) + d = self.bucket.get_versioning_status() + self.assertEqual('Suspended', d['Versioning']) - # now delete all keys and deletemarkers in bucket - for k in bucket.list_versions(): - bucket.delete_key(k.name, version_id=k.version_id) - - # now delete bucket - c.delete_bucket(bucket) - print '--- tests completed ---' - def test_latest_version(self): - c = S3Connection() - bucket_name = 'version-%d' % int(time.time()) - bucket = c.create_bucket(bucket_name) - - bucket.configure_versioning(versioning=True) + self.bucket.configure_versioning(versioning=True) # add v1 of an object key_name = "key" - kv1 = bucket.new_key(key_name) + kv1 = self.bucket.new_key(key_name) kv1.set_contents_from_string("v1") # read list which should contain latest v1 - listed_kv1 = iter(bucket.get_all_versions()).next() + listed_kv1 = iter(self.bucket.get_all_versions()).next() self.assertEqual(listed_kv1.name, key_name) self.assertEqual(listed_kv1.version_id, kv1.version_id) self.assertEqual(listed_kv1.is_latest, True) # add v2 of the object - kv2 = bucket.new_key(key_name) + kv2 = self.bucket.new_key(key_name) kv2.set_contents_from_string("v2") # read 2 versions, confirm v2 is latest - i = iter(bucket.get_all_versions()) + i = iter(self.bucket.get_all_versions()) listed_kv2 = i.next() listed_kv1 = i.next() self.assertEqual(listed_kv2.version_id, kv2.version_id) @@ -176,8 +143,8 @@ class S3VersionTest (unittest.TestCase): self.assertEqual(listed_kv1.is_latest, False) # delete key, which creates a delete marker as latest - bucket.delete_key(key_name) - i = iter(bucket.get_all_versions()) + self.bucket.delete_key(key_name) + i = iter(self.bucket.get_all_versions()) listed_kv3 = i.next() listed_kv2 = i.next() listed_kv1 = i.next() @@ -187,8 +154,3 @@ class S3VersionTest (unittest.TestCase): self.assertEqual(listed_kv3.is_latest, True) self.assertEqual(listed_kv2.is_latest, False) self.assertEqual(listed_kv1.is_latest, False) - - # cleanup - for k in bucket.list_versions(): - bucket.delete_key(k.name, version_id=k.version_id) - c.delete_bucket(bucket) diff --git a/tests/test.py b/tests/test.py index 1eefdaa7..75c90157 100755 --- a/tests/test.py +++ b/tests/test.py @@ -32,6 +32,7 @@ import getopt from sqs.test_connection import SQSConnectionTest from s3.test_connection import S3ConnectionTest from s3.test_versioning import S3VersionTest +from s3.test_mfa import S3MFATest from s3.test_encryption import S3EncryptionTest from s3.test_bucket import S3BucketTest from s3.test_key import S3KeyTest @@ -49,7 +50,7 @@ from sts.test_session_token import SessionTokenTest def usage(): print "test.py [-t testsuite] [-v verbosity]" - print " -t run specific testsuite (s3|ssl|s3ver|s3nover|gs|sqs|ec2|sdb|dynamodb|dynamodbL1|dynamodbL2|sts|all)" + print " -t run specific testsuite (s3|ssl|s3mfa|s3nomfa|gs|sqs|ec2|sdb|dynamodb|dynamodbL1|dynamodbL2|sts|all)" print " -v verbosity (0|1|2)" def main(): @@ -100,15 +101,17 @@ def suite(testsuite="all"): tests.addTest(unittest.makeSuite(S3VersionTest)) tests.addTest(unittest.makeSuite(S3EncryptionTest)) tests.addTest(unittest.makeSuite(S3MultiDeleteTest)) + tests.addTest(unittest.makeSuite(S3MFATest)) elif testsuite == "ssl": tests.addTest(unittest.makeSuite(CertValidationTest)) - elif testsuite == "s3ver": - tests.addTest(unittest.makeSuite(S3VersionTest)) - elif testsuite == "s3nover": + elif testsuite == "s3mfa": + tests.addTest(unittest.makeSuite(S3MFATest)) + elif testsuite == "s3nomfa": tests.addTest(unittest.makeSuite(S3ConnectionTest)) tests.addTest(unittest.makeSuite(S3BucketTest)) tests.addTest(unittest.makeSuite(S3KeyTest)) tests.addTest(unittest.makeSuite(S3MultiPartUploadTest)) + tests.addTest(unittest.makeSuite(S3VersionTest)) tests.addTest(unittest.makeSuite(S3EncryptionTest)) tests.addTest(unittest.makeSuite(S3MultiDeleteTest)) elif testsuite == "gs": -- cgit v1.2.1 From 3bd1be24366f685255d80035a29576cf5ea61d4c Mon Sep 17 00:00:00 2001 From: Mitch Garnaat Date: Mon, 13 Feb 2012 08:55:24 -0800 Subject: Fixed a problem with zero-valued numeric range keys. Closes #576. --- boto/dynamodb/layer2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boto/dynamodb/layer2.py b/boto/dynamodb/layer2.py index 45f13459..8f325bef 100644 --- a/boto/dynamodb/layer2.py +++ b/boto/dynamodb/layer2.py @@ -280,7 +280,7 @@ class Layer2(object): msg = 'Hashkey must be of type: %s' % schema.hash_key_type raise TypeError(msg) dynamodb_key['HashKeyElement'] = dynamodb_value - if range_key: + if range_key is not None: dynamodb_value = self.dynamize_value(range_key) if dynamodb_value.keys()[0] != schema.range_key_type: msg = 'RangeKey must be of type: %s' % schema.range_key_type -- cgit v1.2.1 From 5c10704500ebef697116fcc38670032bc5b70d65 Mon Sep 17 00:00:00 2001 From: Mitch Garnaat Date: Mon, 13 Feb 2012 09:15:01 -0800 Subject: Remove the MFA test from the standard S3 test suite. This requires a special account and is best run separately. --- tests/test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test.py b/tests/test.py index 75c90157..5a24a06e 100755 --- a/tests/test.py +++ b/tests/test.py @@ -101,7 +101,6 @@ def suite(testsuite="all"): tests.addTest(unittest.makeSuite(S3VersionTest)) tests.addTest(unittest.makeSuite(S3EncryptionTest)) tests.addTest(unittest.makeSuite(S3MultiDeleteTest)) - tests.addTest(unittest.makeSuite(S3MFATest)) elif testsuite == "ssl": tests.addTest(unittest.makeSuite(CertValidationTest)) elif testsuite == "s3mfa": -- cgit v1.2.1 From 76a4c81c4794a330846fcfe7db40b8ff7627d14d Mon Sep 17 00:00:00 2001 From: Thomas O'Dowd Date: Tue, 14 Feb 2012 10:03:12 +0900 Subject: Remove test suite s3nomfa as it is now identical to the s3 suite. --- tests/test.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/tests/test.py b/tests/test.py index 5a24a06e..7f423b71 100755 --- a/tests/test.py +++ b/tests/test.py @@ -50,7 +50,7 @@ from sts.test_session_token import SessionTokenTest def usage(): print "test.py [-t testsuite] [-v verbosity]" - print " -t run specific testsuite (s3|ssl|s3mfa|s3nomfa|gs|sqs|ec2|sdb|dynamodb|dynamodbL1|dynamodbL2|sts|all)" + print " -t run specific testsuite (s3|ssl|s3mfa|gs|sqs|ec2|sdb|dynamodb|dynamodbL1|dynamodbL2|sts|all)" print " -v verbosity (0|1|2)" def main(): @@ -105,14 +105,6 @@ def suite(testsuite="all"): tests.addTest(unittest.makeSuite(CertValidationTest)) elif testsuite == "s3mfa": tests.addTest(unittest.makeSuite(S3MFATest)) - elif testsuite == "s3nomfa": - tests.addTest(unittest.makeSuite(S3ConnectionTest)) - tests.addTest(unittest.makeSuite(S3BucketTest)) - tests.addTest(unittest.makeSuite(S3KeyTest)) - tests.addTest(unittest.makeSuite(S3MultiPartUploadTest)) - tests.addTest(unittest.makeSuite(S3VersionTest)) - tests.addTest(unittest.makeSuite(S3EncryptionTest)) - tests.addTest(unittest.makeSuite(S3MultiDeleteTest)) elif testsuite == "gs": tests.addTest(unittest.makeSuite(GSConnectionTest)) elif testsuite == "sqs": -- cgit v1.2.1 From 18185d9b45aedee9106ba9373aa7a10653d4349b Mon Sep 17 00:00:00 2001 From: Greg Taylor Date: Mon, 13 Feb 2012 23:40:53 -0500 Subject: Adding a SES tutorial, tweaked a few minor things to match in DynamoDB tutorial. --- docs/source/dynamodb_tut.rst | 9 ++- docs/source/index.rst | 5 +- docs/source/ses_tut.rst | 171 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 180 insertions(+), 5 deletions(-) create mode 100644 docs/source/ses_tut.rst diff --git a/docs/source/dynamodb_tut.rst b/docs/source/dynamodb_tut.rst index 39405908..9f5e2412 100644 --- a/docs/source/dynamodb_tut.rst +++ b/docs/source/dynamodb_tut.rst @@ -23,9 +23,12 @@ To do so, the most straight forward way is the following:: Bear in mind that if you have your credentials in boto config in your home -directory, the two keyword arguments in the call above are not needed. At this -time, Amazon DynamoDB is available only in the US-EAST-1 region and the -above command will automatically connect to that region. +directory, the two keyword arguments in the call above are not needed. More +details on configuration can be found in :doc:`boto_config_tut`. + +.. note:: At this + time, Amazon DynamoDB is available only in the US-EAST-1 region. The + ``connect_dynamodb`` method automatically connect to that region. The :py:func:`boto.connect_dynamodb` functions returns a :py:class:`boto.dynamodb.layer2.Layer2` instance, which is a high-level API diff --git a/docs/source/index.rst b/docs/source/index.rst index 264d3d49..c669e446 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -24,7 +24,7 @@ Currently Supported Services * **Database** - * :doc:`SimpleDB `-- (:doc:`API Reference `) + * :doc:`SimpleDB ` -- (:doc:`API Reference `) * :doc:`DynamoDB ` -- (:doc:`API Reference `) * Relational Data Services (RDS) -- (:doc:`API Reference `) @@ -40,7 +40,7 @@ Currently Supported Services * :doc:`Simple Queue Service (SQS) ` -- (:doc:`API Reference `) * Simple Notification Service (SNS) -- (:doc:`API Reference `) - * Simple Email Service (SES) -- (:doc:`API Reference `) + * :doc:`Simple Email Service (SES) ` -- (:doc:`API Reference `) * **Monitoring** @@ -103,6 +103,7 @@ Additional Resources sqs_tut ref/sqs ref/sns + ses_tut ref/ses cloudwatch_tut ref/cloudwatch diff --git a/docs/source/ses_tut.rst b/docs/source/ses_tut.rst new file mode 100644 index 00000000..c71e8868 --- /dev/null +++ b/docs/source/ses_tut.rst @@ -0,0 +1,171 @@ +.. ses_tut: + +============================= +Simple Email Service Tutorial +============================= + +This tutorial focuses on the boto interface to AWS' `Simple Email Service (SES) `_. +This tutorial assumes that you have boto already downloaded and installed. + +.. _SES: http://aws.amazon.com/ses/ + +Creating a Connection +--------------------- + +The first step in accessing SES is to create a connection to the service. +To do so, the most straight forward way is the following:: + + >>> import boto + >>> conn = boto.connect_ses( + aws_access_key_id='', + aws_secret_access_key='') + >>> conn + SESConnection:email.us-east-1.amazonaws.com + +Bear in mind that if you have your credentials in boto config in your home +directory, the two keyword arguments in the call above are not needed. More +details on configuration can be fond in :doc:`boto_config_tut`. + +The :py:func:`boto.connect_ses` functions returns a +:py:class:`boto.ses.connection.SESConnection` instance, which is a the boto API +for working with SES. + +Notes on Sending +---------------- + +It is important to keep in mind that while emails appear to come "from" the +address that you specify via Reply-To, the sending is done through Amazon. +Some clients do pick up on this disparity, and leave a note on emails. + +Verifying a Sender Email Address +-------------------------------- + +Before you can send email "from" an address, you must prove that you have +access to the account. When you send a validation request, an email is sent +to the address with a link in it. Clicking on the link validates the address +and adds it to your SES account. Here's how to send the validation email:: + + >>> conn.verify_email_address('some@address.com') + { + 'VerifyEmailAddressResponse': { + 'ResponseMetadata': { + 'RequestId': '4a974fd5-56c2-11e1-ad4c-c1f08c91d554' + } + } + } + +After a short amount of time, you'll find an email with the validation +link inside. Click it, and this address may be used to send emails. + +Listing Verified Addresses +-------------------------- + +If you'd like to list the addresses that are currently verified on your +SES account, use +:py:meth:`list_verified_email_addresses `:: + + >>> conn.list_verified_email_addresses() + { + 'ListVerifiedEmailAddressesResponse': { + 'ListVerifiedEmailAddressesResult': { + 'VerifiedEmailAddresses': [ + 'some@address.com', + 'another@address.com' + ] + }, + 'ResponseMetadata': { + 'RequestId': '2ab45c18-56c3-11e1-be66-ffd2a4549d70' + } + } + } + +Deleting a Verified Address +--------------------------- + +In the event that you'd like to remove an email address from your account, +use +:py:meth:`delete_verified_email_address `:: + + >>> conn.delete_verified_email_address('another@address.com') + +Sending an Email +---------------- + +Sending an email is done via +:py:meth:`send_email `:: + + >>> conn.send_email( + 'some@address.com', + 'Your subject', + 'Body here', + ['recipient-address-1@gmail.com']) + { + 'SendEmailResponse': { + 'ResponseMetadata': { + 'RequestId': '4743c2b7-56c3-11e1-bccd-c99bd68002fd' + }, + 'SendEmailResult': { + 'MessageId': '000001357a177192-7b894025-147a-4705-8455-7c880b0c8270-000000' + } + } + } + +If you're wanting to send a multipart MIME email, see the reference for +:py:meth:`send_raw_email `, +which is a bit more of a low-level alternative. + +Checking your Send Quota +------------------------ + +Staying within your quota is critical, since the upper limit is a hard cap. +Once you have hit your quota, no further email may be sent until enough +time elapses to where your 24 hour email count (rolling continuously) is +within acceptable ranges. Use +:py:meth:`get_send_quota `:: + + >>> conn.get_send_quota() + { + 'GetSendQuotaResponse': { + 'GetSendQuotaResult': { + 'Max24HourSend': '100000.0', + 'SentLast24Hours': '181.0', + 'MaxSendRate': '28.0' + }, + 'ResponseMetadata': { + 'RequestId': u'8a629245-56c4-11e1-9c53-9d5f4d2cc8d3' + } + } + } + +Checking your Send Statistics +----------------------------- + +In order to fight spammers and ensure quality mail is being sent from SES, +Amazon tracks bounces, rejections, and complaints. This is done via +:py:meth:`get_send_statistics `. +Please be warned that the output is extremely verbose, to the point +where we'll just show a short excerpt here:: + + >>> conn.get_send_statistics() + { + 'GetSendStatisticsResponse': { + 'GetSendStatisticsResult': { + 'SendDataPoints': [ + { + 'Complaints': '0', + 'Timestamp': '2012-02-13T05:02:00Z', + 'DeliveryAttempts': '8', + 'Bounces': '0', + 'Rejects': '0' + }, + { + 'Complaints': '0', + 'Timestamp': '2012-02-13T05:17:00Z', + 'DeliveryAttempts': '12', + 'Bounces': '0', + 'Rejects': '0' + } + ] + } + } + } \ No newline at end of file -- cgit v1.2.1 From 5911676f26ab1615dd3760e7c6b392e81633f039 Mon Sep 17 00:00:00 2001 From: Greg Taylor Date: Mon, 13 Feb 2012 23:43:57 -0500 Subject: Fixing sphinx build error/warning in boto.gs.key. --- boto/gs/key.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/boto/gs/key.py b/boto/gs/key.py index a0c50a79..21532d38 100644 --- a/boto/gs/key.py +++ b/boto/gs/key.py @@ -161,18 +161,19 @@ class Key(S3Key): :type size: int :param size: (optional) The Maximum number of bytes to read from - the file pointer (fp). This is useful when uploading - a file in multiple parts where you are splitting the - file up into different ranges to be uploaded. If not - specified, the default behaviour is to read all bytes - from the file pointer. Less bytes may be available. - Notes: - 1. The "size" parameter currently cannot be used when - a resumable upload handler is given but is still - useful for uploading part of a file as implemented - by the parent class. - 2. At present Google Cloud Storage does not support - multipart uploads. + the file pointer (fp). This is useful when uploading + a file in multiple parts where you are splitting the + file up into different ranges to be uploaded. If not + specified, the default behaviour is to read all bytes + from the file pointer. Less bytes may be available. + Notes: + + 1. The "size" parameter currently cannot be used when + a resumable upload handler is given but is still + useful for uploading part of a file as implemented + by the parent class. + 2. At present Google Cloud Storage does not support + multipart uploads. TODO: At some point we should refactor the Bucket and Key classes, to move functionality common to all providers into a parent class, -- cgit v1.2.1 From dbcd044aebfb3af1938ba865412b63d4d87f1113 Mon Sep 17 00:00:00 2001 From: Thomas O'Dowd Date: Wed, 15 Feb 2012 10:29:39 +0900 Subject: Fix GS resumable uploads test case to rewind pointer between tests. --- tests/s3/test_resumable_uploads.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/s3/test_resumable_uploads.py b/tests/s3/test_resumable_uploads.py index bb0f7a93..8a4a51f3 100755 --- a/tests/s3/test_resumable_uploads.py +++ b/tests/s3/test_resumable_uploads.py @@ -202,6 +202,7 @@ class ResumableUploadTests(unittest.TestCase): """ Tests that non-resumable uploads work """ + self.small_src_file.seek(0) self.dst_key.set_contents_from_file(self.small_src_file) self.assertEqual(self.small_src_file_size, self.dst_key.size) self.assertEqual(self.small_src_file_as_string, @@ -212,6 +213,7 @@ class ResumableUploadTests(unittest.TestCase): Tests a single resumable upload, with no tracker URI persistence """ res_upload_handler = ResumableUploadHandler() + self.small_src_file.seek(0) self.dst_key.set_contents_from_file( self.small_src_file, res_upload_handler=res_upload_handler) self.assertEqual(self.small_src_file_size, self.dst_key.size) @@ -225,6 +227,7 @@ class ResumableUploadTests(unittest.TestCase): harnass = CallbackTestHarnass() res_upload_handler = ResumableUploadHandler( tracker_file_name=self.tracker_file_name, num_retries=0) + self.small_src_file.seek(0) try: self.dst_key.set_contents_from_file( self.small_src_file, cb=harnass.call, @@ -251,6 +254,7 @@ class ResumableUploadTests(unittest.TestCase): exception = ResumableUploadHandler.RETRYABLE_EXCEPTIONS[0] harnass = CallbackTestHarnass(exception=exception) res_upload_handler = ResumableUploadHandler(num_retries=1) + self.small_src_file.seek(0) self.dst_key.set_contents_from_file( self.small_src_file, cb=harnass.call, res_upload_handler=res_upload_handler) @@ -266,6 +270,7 @@ class ResumableUploadTests(unittest.TestCase): exception = IOError(errno.EPIPE, "Broken pipe") harnass = CallbackTestHarnass(exception=exception) res_upload_handler = ResumableUploadHandler(num_retries=1) + self.small_src_file.seek(0) self.dst_key.set_contents_from_file( self.small_src_file, cb=harnass.call, res_upload_handler=res_upload_handler) @@ -281,6 +286,7 @@ class ResumableUploadTests(unittest.TestCase): harnass = CallbackTestHarnass( exception=OSError(errno.EACCES, 'Permission denied')) res_upload_handler = ResumableUploadHandler(num_retries=1) + self.small_src_file.seek(0) try: self.dst_key.set_contents_from_file( self.small_src_file, cb=harnass.call, @@ -298,6 +304,7 @@ class ResumableUploadTests(unittest.TestCase): harnass = CallbackTestHarnass() res_upload_handler = ResumableUploadHandler( tracker_file_name=self.tracker_file_name, num_retries=1) + self.small_src_file.seek(0) self.dst_key.set_contents_from_file( self.small_src_file, cb=harnass.call, res_upload_handler=res_upload_handler) @@ -313,6 +320,7 @@ class ResumableUploadTests(unittest.TestCase): Tests resumable upload that fails twice in one process, then completes """ res_upload_handler = ResumableUploadHandler(num_retries=3) + self.small_src_file.seek(0) self.dst_key.set_contents_from_file( self.small_src_file, res_upload_handler=res_upload_handler) # Ensure uploaded object has correct content. @@ -332,6 +340,7 @@ class ResumableUploadTests(unittest.TestCase): fail_after_n_bytes=self.larger_src_file_size/2, num_times_to_fail=2) res_upload_handler = ResumableUploadHandler( tracker_file_name=self.tracker_file_name, num_retries=1) + self.larger_src_file.seek(0) try: self.dst_key.set_contents_from_file( self.larger_src_file, cb=harnass.call, @@ -343,6 +352,7 @@ class ResumableUploadTests(unittest.TestCase): # Ensure a tracker file survived. self.assertTrue(os.path.exists(self.tracker_file_name)) # Try it one more time; this time should succeed. + self.larger_src_file.seek(0) self.dst_key.set_contents_from_file( self.larger_src_file, cb=harnass.call, res_upload_handler=res_upload_handler) @@ -365,6 +375,7 @@ class ResumableUploadTests(unittest.TestCase): harnass = CallbackTestHarnass( fail_after_n_bytes=self.larger_src_file_size/2) res_upload_handler = ResumableUploadHandler(num_retries=1) + self.larger_src_file.seek(0) self.dst_key.set_contents_from_file( self.larger_src_file, cb=harnass.call, res_upload_handler=res_upload_handler) @@ -382,6 +393,7 @@ class ResumableUploadTests(unittest.TestCase): Tests uploading an empty file (exercises boundary conditions). """ res_upload_handler = ResumableUploadHandler() + self.empty_src_file.seek(0) self.dst_key.set_contents_from_file( self.empty_src_file, res_upload_handler=res_upload_handler) self.assertEqual(0, self.dst_key.size) @@ -393,6 +405,7 @@ class ResumableUploadTests(unittest.TestCase): res_upload_handler = ResumableUploadHandler() headers = {'Content-Type' : 'text/plain', 'Content-Encoding' : 'gzip', 'x-goog-meta-abc' : 'my meta', 'x-goog-acl' : 'public-read'} + self.small_src_file.seek(0) self.dst_key.set_contents_from_file( self.small_src_file, headers=headers, res_upload_handler=res_upload_handler) @@ -423,6 +436,7 @@ class ResumableUploadTests(unittest.TestCase): # upload server). res_upload_handler = ResumableUploadHandler( tracker_file_name=self.tracker_file_name, num_retries=0) + self.larger_src_file.seek(0) try: self.dst_key.set_contents_from_file( self.larger_src_file, cb=harnass.call, @@ -440,6 +454,7 @@ class ResumableUploadTests(unittest.TestCase): # 500 response in the next attempt. time.sleep(1) try: + self.largest_src_file.seek(0) self.dst_key.set_contents_from_file( self.largest_src_file, res_upload_handler=res_upload_handler) self.fail('Did not get expected ResumableUploadException') @@ -510,6 +525,7 @@ class ResumableUploadTests(unittest.TestCase): to set the content length when gzipping a file. """ res_upload_handler = ResumableUploadHandler() + self.small_src_file.seek(0) try: self.dst_key.set_contents_from_file( self.small_src_file, res_upload_handler=res_upload_handler, @@ -528,6 +544,7 @@ class ResumableUploadTests(unittest.TestCase): tracker_file_name=self.syntactically_invalid_tracker_file_name) # An error should be printed about the invalid URI, but then it # should run the update successfully. + self.small_src_file.seek(0) self.dst_key.set_contents_from_file( self.small_src_file, res_upload_handler=res_upload_handler) self.assertEqual(self.small_src_file_size, self.dst_key.size) @@ -542,6 +559,7 @@ class ResumableUploadTests(unittest.TestCase): tracker_file_name=self.invalid_upload_id_tracker_file_name) # An error should occur, but then the tracker URI should be # regenerated and the the update should succeed. + self.small_src_file.seek(0) self.dst_key.set_contents_from_file( self.small_src_file, res_upload_handler=res_upload_handler) self.assertEqual(self.small_src_file_size, self.dst_key.size) -- cgit v1.2.1 From f221d3974c858b9019e7bdb7b41c85382d2b493a Mon Sep 17 00:00:00 2001 From: Mitch Garnaat Date: Tue, 14 Feb 2012 19:15:24 -0800 Subject: Bumping version number and changing project home page in setup.py. --- README.markdown | 4 ++-- boto/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.markdown b/README.markdown index e8cdbc05..c636ad01 100644 --- a/README.markdown +++ b/README.markdown @@ -1,6 +1,6 @@ # boto -boto 2.2.1 -01-Feb-2012 +boto 2.2.2 +14-Feb-2012 ## Introduction diff --git a/boto/__init__.py b/boto/__init__.py index 5ae663f6..07ce9f12 100644 --- a/boto/__init__.py +++ b/boto/__init__.py @@ -31,7 +31,7 @@ import logging.config import urlparse from boto.exception import InvalidUriError -__version__ = '2.2.1-dev' +__version__ = '2.2.2' Version = __version__ # for backware compatibility UserAgent = 'Boto/%s (%s)' % (__version__, sys.platform) diff --git a/setup.py b/setup.py index e6ceddc7..9b7649ba 100644 --- a/setup.py +++ b/setup.py @@ -51,7 +51,7 @@ setup(name = "boto", "bin/bundle_image", "bin/pyami_sendmail", "bin/lss3", "bin/cq", "bin/route53", "bin/s3multiput", "bin/cwutil", "bin/instance_events", "bin/asadmin"], - url = "http://code.google.com/p/boto/", + url = "https://github.com/boto/boto/", packages = ["boto", "boto.sqs", "boto.s3", "boto.gs", "boto.file", "boto.ec2", "boto.ec2.cloudwatch", "boto.ec2.autoscale", "boto.ec2.elb", "boto.sdb", "boto.cacerts", -- cgit v1.2.1 From e2104c729e2c592c85260376a4526c37bd362688 Mon Sep 17 00:00:00 2001 From: Mitch Garnaat Date: Tue, 14 Feb 2012 19:27:06 -0800 Subject: Updating repo version after 2.2.2 release. --- boto/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boto/__init__.py b/boto/__init__.py index 07ce9f12..b8593e25 100644 --- a/boto/__init__.py +++ b/boto/__init__.py @@ -31,7 +31,7 @@ import logging.config import urlparse from boto.exception import InvalidUriError -__version__ = '2.2.2' +__version__ = '2.2.2-dev' Version = __version__ # for backware compatibility UserAgent = 'Boto/%s (%s)' % (__version__, sys.platform) -- cgit v1.2.1 From 4f713661404e4b0f62cf81e21359acd2eb43094f Mon Sep 17 00:00:00 2001 From: Mitch Garnaat Date: Tue, 14 Feb 2012 21:22:01 -0800 Subject: Adding trove classifier for python version. --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9b7649ba..dba1538d 100644 --- a/setup.py +++ b/setup.py @@ -75,6 +75,7 @@ setup(name = "boto", "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", - "Topic :: Internet"], + "Topic :: Internet", + "Programming Language :: Python :: 2.6"], **extra ) -- cgit v1.2.1 From 13462497a3f41cd16759345d1c4425e4feae593e Mon Sep 17 00:00:00 2001 From: Mike Schwartz Date: Tue, 14 Feb 2012 21:24:21 -0800 Subject: Added StorageUri methods + MockStorageService delimiter support --- boto/storage_uri.py | 112 ++++++++++++++++++++++++++++----------- tests/s3/mock_storage_service.py | 69 +++++++++++++++++++----- 2 files changed, 137 insertions(+), 44 deletions(-) diff --git a/boto/storage_uri.py b/boto/storage_uri.py index 299eb852..5f14500d 100755 --- a/boto/storage_uri.py +++ b/boto/storage_uri.py @@ -64,8 +64,11 @@ class StorageUri(object): def check_response(self, resp, level, uri): if resp is None: - raise InvalidUriError('Attempt to get %s for "%s" failed. This ' - 'probably indicates the URI is invalid' % + raise InvalidUriError('Attempt to get %s for "%s" failed.\nThis ' + 'can happen if the URI refers to a non-' + 'existent object or if you meant to\noperate ' + 'on a directory (e.g., leaving off -R option ' + 'on gsutil cp, mv, or ls of a\nbucket)' % (level, uri)) def connect(self, access_key_id=None, secret_access_key=None, **kwargs): @@ -188,6 +191,8 @@ class BucketStorageUri(StorageUri): Callers should instantiate this class by calling boto.storage_uri(). """ + delim = '/' + def __init__(self, scheme, bucket_name=None, object_name=None, debug=0, connection_args=None, suppress_consec_slashes=True): """Instantiate a BucketStorageUri from scheme,bucket,object tuple. @@ -237,7 +242,8 @@ class BucketStorageUri(StorageUri): raise InvalidUriError('clone_replace_name() on bucket-less URI %s' % self.uri) return BucketStorageUri( - self.scheme, self.bucket_name, new_name, self.debug, + self.scheme, bucket_name=self.bucket_name, object_name=new_name, + debug=self.debug, suppress_consec_slashes=self.suppress_consec_slashes) def get_acl(self, validate=True, headers=None, version_id=None): @@ -295,8 +301,8 @@ class BucketStorageUri(StorageUri): bucket.add_group_email_grant(permission, email_address, recursive, headers) else: - raise InvalidUriError('add_group_email_grant() on bucket-less URI %s' % - self.uri) + raise InvalidUriError('add_group_email_grant() on bucket-less URI ' + '%s' % self.uri) def add_email_grant(self, permission, email_address, recursive=False, validate=True, headers=None): @@ -332,21 +338,49 @@ class BucketStorageUri(StorageUri): bucket = self.get_bucket(headers) return bucket.list_grants(headers) + def is_file_uri(self): + """Returns True if this URI names a file or directory.""" + return False + + def is_cloud_uri(self): + """Returns True if this URI names a bucket or object.""" + return True + def names_container(self): - """Returns True if this URI names a bucket (vs. an object). """ - return not self.object_name + Returns True if this URI names a directory or bucket. Will return + False for bucket subdirs; providing bucket subdir semantics needs to + be done by the caller (like gsutil does). + """ + return bool(not self.object_name) def names_singleton(self): - """Returns True if this URI names an object (vs. a bucket). - """ - return self.object_name + """Returns True if this URI names a file or object.""" + return bool(self.object_name) - def is_file_uri(self): + def names_directory(self): + """Returns True if this URI names a directory.""" return False - def is_cloud_uri(self): - return True + def names_provider(self): + """Returns True if this URI names a provider.""" + return bool(not self.bucket_name) + + def names_bucket(self): + """Returns True if this URI names a bucket.""" + return self.names_container() + + def names_file(self): + """Returns True if this URI names a file.""" + return False + + def names_object(self): + """Returns True if this URI names an object.""" + return self.names_singleton() + + def is_stream(self): + """Returns True if this URI represents input/output stream.""" + return False def create_bucket(self, headers=None, location='', policy=None): if self.bucket_name is None: @@ -452,6 +486,8 @@ class FileStorageUri(StorageUri): See file/README about how we map StorageUri operations onto a file system. """ + delim = os.sep + def __init__(self, object_name, debug, is_stream=False): """Instantiate a FileStorageUri from a path name. @@ -481,34 +517,48 @@ class FileStorageUri(StorageUri): """ return FileStorageUri(new_name, self.debug, self.stream) + def is_file_uri(self): + """Returns True if this URI names a file or directory.""" + return True + + def is_cloud_uri(self): + """Returns True if this URI names a bucket or object.""" + return False + def names_container(self): - """Returns True if this URI is not representing input/output stream - and names a directory. - """ - if not self.stream: - return os.path.isdir(self.object_name) - else: - return False + """Returns True if this URI names a directory or bucket.""" + return self.names_directory() def names_singleton(self): - """Returns True if this URI names a file or - if URI represents input/output stream. - """ + """Returns True if this URI names a file (or stream) or object.""" + return not self.names_container() + + def names_directory(self): + """Returns True if this URI names a directory.""" if self.stream: - return True - else: - return os.path.isfile(self.object_name) + return False + return os.path.isdir(self.object_name) - def is_file_uri(self): - return True + def names_provider(self): + """Returns True if this URI names a provider.""" + return False - def is_cloud_uri(self): + def names_bucket(self): + """Returns True if this URI names a bucket.""" + return False + + def names_file(self): + """Returns True if this URI names a file.""" + return self.names_singleton() + + def names_object(self): + """Returns True if this URI names an object.""" return False def is_stream(self): - """Retruns True if this URI represents input/output stream. + """Returns True if this URI represents input/output stream. """ - return self.stream + return bool(self.stream) def close(self): """Closes the underlying file. diff --git a/tests/s3/mock_storage_service.py b/tests/s3/mock_storage_service.py index 2b81f5b6..3ff28cf0 100644 --- a/tests/s3/mock_storage_service.py +++ b/tests/s3/mock_storage_service.py @@ -29,7 +29,9 @@ of the optional params (which we indicate with the constant "NOT_IMPL"). import copy import boto import base64 + from boto.utils import compute_md5 +from boto.s3.prefix import Prefix try: from hashlib import md5 @@ -67,6 +69,12 @@ class MockKey(object): self.last_modified = 'Wed, 06 Oct 2010 05:11:54 GMT' self.BufferSize = 8192 + def __repr__(self): + if self.bucket: + return '' % (self.bucket.name, self.name) + else: + return '' % self.name + def get_contents_as_string(self, headers=NOT_IMPL, cb=NOT_IMPL, num_cb=NOT_IMPL, torrent=NOT_IMPL, @@ -114,10 +122,10 @@ class MockKey(object): self.size = len(s) self._handle_headers(headers) - def set_contents_from_filename(self, filename, headers=None, replace=NOT_IMPL, - cb=NOT_IMPL, num_cb=NOT_IMPL, - policy=NOT_IMPL, md5=NOT_IMPL, - res_upload_handler=NOT_IMPL): + def set_contents_from_filename(self, filename, headers=None, + replace=NOT_IMPL, cb=NOT_IMPL, + num_cb=NOT_IMPL, policy=NOT_IMPL, + md5=NOT_IMPL, res_upload_handler=NOT_IMPL): fp = open(filename, 'rb') self.set_contents_from_file(fp, headers, replace, cb, num_cb, policy, md5, res_upload_handler) @@ -174,6 +182,9 @@ class MockBucket(object): self.connection = connection self.logging = False + def __repr__(self): + return 'MockBucket: %s' % self.name + def copy_key(self, new_key_name, src_bucket_name, src_key_name, metadata=NOT_IMPL, src_version_id=NOT_IMPL, storage_class=NOT_IMPL, preserve_acl=NOT_IMPL): @@ -231,17 +242,29 @@ class MockBucket(object): return None return self.keys[key_name] - def list(self, prefix='', delimiter=NOT_IMPL, marker=NOT_IMPL, + def list(self, prefix='', delimiter='', marker=NOT_IMPL, headers=NOT_IMPL): + prefix = prefix or '' # Turn None into '' for prefix match. # Return list instead of using a generator so we don't get # 'dictionary changed size during iteration' error when performing # deletions while iterating (e.g., during test cleanup). result = [] + key_name_set = set() for k in self.keys.itervalues(): - if not prefix: - result.append(k) - elif k.name.startswith(prefix): - result.append(k) + if k.name.startswith(prefix): + k_name_past_prefix = k.name[len(prefix):] + if delimiter: + pos = k_name_past_prefix.find(delimiter) + else: + pos = -1 + if (pos != -1): + key_or_prefix = Prefix( + bucket=self, name=k.name[:len(prefix)+pos+1]) + else: + key_or_prefix = MockKey(bucket=self, name=k.name) + if key_or_prefix.name not in key_name_set: + key_name_set.add(key_or_prefix.name) + result.append(key_or_prefix) return result def set_acl(self, acl_or_str, key_name='', headers=NOT_IMPL, @@ -250,10 +273,10 @@ class MockBucket(object): # the get_acl call will just return that string name. if key_name: # Set ACL for the key. - self.acls[key_name] = acl_or_str + self.acls[key_name] = MockAcl(acl_or_str) else: # Set ACL for the bucket. - self.acls[self.name] = acl_or_str + self.acls[self.name] = MockAcl(acl_or_str) def set_def_acl(self, acl_or_str, key_name=NOT_IMPL, headers=NOT_IMPL, version_id=NOT_IMPL): @@ -313,6 +336,8 @@ mock_connection = MockConnection() class MockBucketStorageUri(object): + delim = '/' + def __init__(self, scheme, bucket_name=None, object_name=None, debug=NOT_IMPL, suppress_consec_slashes=NOT_IMPL): self.scheme = scheme @@ -395,10 +420,28 @@ class MockBucketStorageUri(object): return True def names_container(self): - return not self.object_name + return bool(not self.object_name) def names_singleton(self): - return self.object_name + return bool(self.object_name) + + def names_directory(self): + return False + + def names_provider(self): + return bool(not self.bucket_name) + + def names_bucket(self): + return self.names_container() + + def names_file(self): + return False + + def names_object(self): + return not self.names_container() + + def is_stream(self): + return False def new_key(self, validate=NOT_IMPL, headers=NOT_IMPL): bucket = self.get_bucket() -- cgit v1.2.1 From a04c47df57bf292b7eb361eed3119579968f44ab Mon Sep 17 00:00:00 2001 From: Mitch Garnaat Date: Wed, 15 Feb 2012 06:10:04 -0800 Subject: Messing around with trove classifiers again. --- setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index dba1538d..3ede0a3d 100644 --- a/setup.py +++ b/setup.py @@ -76,6 +76,9 @@ setup(name = "boto", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Topic :: Internet", - "Programming Language :: Python :: 2.6"], + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.5", + "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.7"], **extra ) -- cgit v1.2.1 From a03af102d7dedeae48eed7f08625d2682e639e64 Mon Sep 17 00:00:00 2001 From: rdodev Date: Wed, 15 Feb 2012 15:34:47 -0500 Subject: DynamoDB delete_table warning --- docs/source/dynamodb_tut.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/source/dynamodb_tut.rst b/docs/source/dynamodb_tut.rst index 9f5e2412..c49256f2 100644 --- a/docs/source/dynamodb_tut.rst +++ b/docs/source/dynamodb_tut.rst @@ -224,6 +224,10 @@ To delete items, use the Deleting Tables --------------- + +.. WARNING:: + Deleting a table will also **permanently** delete all of its contents. Use carefully. + There are two easy ways to delete a table. Through your top-level :py:class:`Layer2 ` object:: -- cgit v1.2.1 From 017f1993ad9384a581cfd6c766313b85ff8d3985 Mon Sep 17 00:00:00 2001 From: rdodev Date: Wed, 15 Feb 2012 15:37:25 -0500 Subject: Changed Phrasing a bit for clarity. --- docs/source/dynamodb_tut.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/dynamodb_tut.rst b/docs/source/dynamodb_tut.rst index c49256f2..3e641227 100644 --- a/docs/source/dynamodb_tut.rst +++ b/docs/source/dynamodb_tut.rst @@ -226,7 +226,7 @@ Deleting Tables --------------- .. WARNING:: - Deleting a table will also **permanently** delete all of its contents. Use carefully. + Deleting a table will also **permanently** delete all of its contents without prompt. Use carefully. There are two easy ways to delete a table. Through your top-level :py:class:`Layer2 ` object:: -- cgit v1.2.1 From 97c40d193a069eab55e06abdb67aaad59b4e1e43 Mon Sep 17 00:00:00 2001 From: Mike Schwartz Date: Sun, 19 Feb 2012 11:21:29 -0800 Subject: Added missing params to MockBucket/copy_key() --- tests/s3/mock_storage_service.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/s3/mock_storage_service.py b/tests/s3/mock_storage_service.py index 3ff28cf0..2bd77439 100644 --- a/tests/s3/mock_storage_service.py +++ b/tests/s3/mock_storage_service.py @@ -187,7 +187,8 @@ class MockBucket(object): def copy_key(self, new_key_name, src_bucket_name, src_key_name, metadata=NOT_IMPL, src_version_id=NOT_IMPL, - storage_class=NOT_IMPL, preserve_acl=NOT_IMPL): + storage_class=NOT_IMPL, preserve_acl=NOT_IMPL, + encrypt_key=NOT_IMPL, headers=NOT_IMPL, query_args=NOT_IMPL): new_key = self.new_key(key_name=new_key_name) src_key = mock_connection.get_bucket( src_bucket_name).get_key(src_key_name) -- cgit v1.2.1 From d5a973bd3889a1e3a257e52c8e531a767770d472 Mon Sep 17 00:00:00 2001 From: Chris Moyer Date: Mon, 20 Feb 2012 11:31:37 -0500 Subject: Allow a DateTimeProperty to take either a Date or DateTime object --- boto/sdb/db/manager/sdbmanager.py | 16 +++++++++++++--- boto/sdb/db/property.py | 9 ++++++--- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/boto/sdb/db/manager/sdbmanager.py b/boto/sdb/db/manager/sdbmanager.py index 8218f816..1a4baa8f 100644 --- a/boto/sdb/db/manager/sdbmanager.py +++ b/boto/sdb/db/manager/sdbmanager.py @@ -257,12 +257,22 @@ class SDBConverter(object): def encode_datetime(self, value): if isinstance(value, str) or isinstance(value, unicode): return value - return value.strftime(ISO8601) + if isinstance(value, date): + return value.isoformat() + else: + return value.strftime(ISO8601) def decode_datetime(self, value): + """Handles both Dates and DateTime objects""" + if value is None: + return value try: - return datetime.strptime(value, ISO8601) - except: + if "T" in value: + return datetime.strptime(value, ISO8601) + else: + value = value.split("-") + return date(int(value[0]), int(value[1]), int(value[2])) + except Exception, e: return None def encode_date(self, value): diff --git a/boto/sdb/db/property.py b/boto/sdb/db/property.py index 1929a027..698ec762 100644 --- a/boto/sdb/db/property.py +++ b/boto/sdb/db/property.py @@ -375,6 +375,9 @@ class FloatProperty(Property): return value is None class DateTimeProperty(Property): + """This class handles both the datetime.datetime object + And the datetime.date objects. It can return either one, + depending on the value stored in the database""" data_type = datetime.datetime type_name = 'DateTime' @@ -391,11 +394,11 @@ class DateTimeProperty(Property): return Property.default_value(self) def validate(self, value): - value = super(DateTimeProperty, self).validate(value) if value == None: return - if not isinstance(value, self.data_type): - raise TypeError, 'Validation Error, expecting %s, got %s' % (self.data_type, type(value)) + if isinstance(value, datetime.date): + return value + return super(DateTimeProperty, self).validate(value) def get_value_for_datastore(self, model_instance): if self.auto_now: -- cgit v1.2.1 From 5a31ba335f453ede20138e57aa8b272b927abd94 Mon Sep 17 00:00:00 2001 From: Mitch Garnaat Date: Wed, 15 Feb 2012 13:03:25 -0800 Subject: Creating helper classes to handle conditional operators for query and scan. --- boto/dynamodb/condition.py | 137 +++++++++++++++++++++++++++++++++++ boto/dynamodb/layer2.py | 161 ++++++++---------------------------------- boto/dynamodb/types.py | 88 +++++++++++++++++++++++ tests/dynamodb/test_layer2.py | 23 +++--- 4 files changed, 270 insertions(+), 139 deletions(-) create mode 100644 boto/dynamodb/condition.py create mode 100644 boto/dynamodb/types.py diff --git a/boto/dynamodb/condition.py b/boto/dynamodb/condition.py new file mode 100644 index 00000000..3df72796 --- /dev/null +++ b/boto/dynamodb/condition.py @@ -0,0 +1,137 @@ +# Copyright (c) 2012 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2012 Amazon.com, Inc. or its affiliates. All Rights Reserved +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +from boto.dynamodb.types import get_dynamodb_type, dynamize_value, convert_num + +class Condition(object): + + pass + +class ConditionNoArgs(Condition): + """ + """ + + def __repr__(self): + return '%s' % self.__class__.__name__ + + def to_dict(self): + return {'ComparisonOperator': self.__class__.__name__} + +class ConditionOneArg(Condition): + """ + """ + + def __init__(self, v1): + self.v1 = v1 + + def __repr__(self): + return '%s:%s' % (self.__class__.__name__, self.v1) + + def to_dict(self): + return {'AttributeValueList': [dynamize_value(self.v1)], + 'ComparisonOperator': self.__class__.__name__} + +class ConditionTwoArgs(Condition): + """ + """ + + def __init__(self, v1, v2): + Condition.__init__(self, v1) + self.v2 = v2 + + def __repr__(self): + return '%s(%s, %s)' % (self.__class__.__name__, self.v1, self.v2) + + def to_dict(self): + values = (self.v1, self.v2) + return {'AttributeValueList': [dynamize_value(v) for v in values], + 'ComparisonOperator': self.__class__.__name__} + +class EQ(ConditionOneArg): + + pass + +class LE(ConditionOneArg): + + pass + +class LT(ConditionOneArg): + + pass + +class GE(ConditionOneArg): + + pass + +class GT(ConditionOneArg): + + pass + +class NULL(ConditionNoArgs): + + pass + +class NOT_NULL(ConditionNoArgs): + + pass + +class CONTAINS(ConditionOneArg): + + pass + +class NOT_CONTAINS(ConditionOneArg): + + pass + +class BEGINS_WITH(ConditionOneArg): + + pass + +class IN(ConditionOneArg): + + pass + +class BEGINS_WITH(ConditionOneArg): + + pass + +class BETWEEN(ConditionTwoArgs): + """ + """ + + def __init__(self, v1, v2): + Condition.__init__(self, v1) + self.v2 = v2 + + def __repr__(self): + return '%s(%s, %s)' % (self.__class__.__name__, self.v1, self.v2) + + def to_dict(self): + values = (self.v1, self.v2) + return {'AttributeValueList': [dynamize_value(v) for v in values], + 'ComparisonOperator': self.__class__.__name__} + + + + + diff --git a/boto/dynamodb/layer2.py b/boto/dynamodb/layer2.py index 46c5059e..c701a6b2 100644 --- a/boto/dynamodb/layer2.py +++ b/boto/dynamodb/layer2.py @@ -26,24 +26,7 @@ from boto.dynamodb.table import Table from boto.dynamodb.schema import Schema from boto.dynamodb.item import Item from boto.dynamodb.batch import BatchList - -""" -Some utility functions to deal with mapping Amazon DynamoDB types to -Python types and vice-versa. -""" - -def is_num(n): - return isinstance(n, (int, long, float, bool)) - -def is_str(n): - return isinstance(n, basestring) - -def convert_num(s): - if '.' in s: - n = float(s) - else: - n = int(s) - return n +from boto.dynamodb.types import get_dynamodb_type, dynamize_value, convert_num def item_object_hook(dct): """ @@ -85,13 +68,13 @@ class Layer2(object): d[attr_name] = {"Action": action} else: d[attr_name] = {"Action": action, - "Value": self.dynamize_value(value)} + "Value": dynamize_value(value)} return d def dynamize_item(self, item): d = {} for attr_name in item: - d[attr_name] = self.dynamize_value(item[attr_name]) + d[attr_name] = dynamize_value(item[attr_name]) return d def dynamize_range_key_condition(self, range_key_condition): @@ -99,25 +82,7 @@ class Layer2(object): Convert a layer2 range_key_condition parameter into the structure required by Layer1. """ - d = None - if range_key_condition: - d = {} - for range_value in range_key_condition: - range_condition = range_key_condition[range_value] - if range_condition == 'BETWEEN': - if isinstance(range_value, tuple): - avl = [self.dynamize_value(v) for v in range_value] - else: - msg = 'BETWEEN condition requires a tuple value' - raise TypeError(msg) - elif isinstance(range_value, tuple): - msg = 'Tuple can only be supplied with BETWEEN condition' - raise TypeError(msg) - else: - avl = [self.dynamize_value(range_value)] - d = {'AttributeValueList': avl, - 'ComparisonOperator': range_condition} - return d + return range_key_condition.to_dict() def dynamize_scan_filter(self, scan_filter): """ @@ -127,24 +92,9 @@ class Layer2(object): d = None if scan_filter: d = {} - for attr_name, op, value in scan_filter: - if op == 'BETWEEN': - if isinstance(value, tuple): - avl = [self.dynamize_value(v) for v in value] - else: - msg = 'BETWEEN condition requires a tuple value' - raise TypeError(msg) - elif op == 'NULL' or op == 'NOT_NULL': - avl = None - elif isinstance(value, tuple): - msg = 'Tuple can only be supplied with BETWEEN condition' - raise TypeError(msg) - else: - avl = [self.dynamize_value(value)] - dd = {'ComparisonOperator': op} - if avl: - dd['AttributeValueList'] = avl - d[attr_name] = dd + for attr_name in scan_filter: + condition = scan_filter[attr_name] + d[attr_name] = condition.to_dict() return d def dynamize_expected_value(self, expected_value): @@ -162,7 +112,7 @@ class Layer2(object): elif attr_value is False: attr_value = {'Exists': False} else: - val = self.dynamize_value(expected_value[attr_name]) + val = dynamize_value(expected_value[attr_name]) attr_value = {'Value': val} d[attr_name] = attr_value return d @@ -175,10 +125,10 @@ class Layer2(object): d = None if last_evaluated_key: hash_key = last_evaluated_key['HashKeyElement'] - d = {'HashKeyElement': self.dynamize_value(hash_key)} + d = {'HashKeyElement': dynamize_value(hash_key)} if 'RangeKeyElement' in last_evaluated_key: range_key = last_evaluated_key['RangeKeyElement'] - d['RangeKeyElement'] = self.dynamize_value(range_key) + d['RangeKeyElement'] = dynamize_value(range_key) return d def dynamize_request_items(self, batch_list): @@ -207,54 +157,6 @@ class Layer2(object): d[batch.table.name] = batch_dict return d - def get_dynamodb_type(self, val): - """ - Take a scalar Python value and return a string representing - the corresponding Amazon DynamoDB type. If the value passed in is - not a supported type, raise a TypeError. - """ - dynamodb_type = None - if is_num(val): - dynamodb_type = 'N' - elif is_str(val): - dynamodb_type = 'S' - elif isinstance(val, (set, frozenset)): - if False not in map(is_num, val): - dynamodb_type = 'NS' - elif False not in map(is_str, val): - dynamodb_type = 'SS' - if dynamodb_type is None: - raise TypeError('Unsupported type "%s" for value "%s"' % (type(val), val)) - return dynamodb_type - - def dynamize_value(self, val): - """ - Take a scalar Python value and return a dict consisting - of the Amazon DynamoDB type specification and the value that - needs to be sent to Amazon DynamoDB. If the type of the value - is not supported, raise a TypeError - """ - def _str(val): - """ - DynamoDB stores booleans as numbers. True is 1, False is 0. - This function converts Python booleans into DynamoDB friendly - representation. - """ - if isinstance(val, bool): - return str(int(val)) - return str(val) - - dynamodb_type = self.get_dynamodb_type(val) - if dynamodb_type == 'N': - val = {dynamodb_type : _str(val)} - elif dynamodb_type == 'S': - val = {dynamodb_type : val} - elif dynamodb_type == 'NS': - val = {dynamodb_type : [ str(n) for n in val]} - elif dynamodb_type == 'SS': - val = {dynamodb_type : [ n for n in val]} - return val - def build_key_from_values(self, schema, hash_key, range_key=None): """ Build a Key structure to be used for accessing items @@ -276,13 +178,13 @@ class Layer2(object): type defined in the schema. """ dynamodb_key = {} - dynamodb_value = self.dynamize_value(hash_key) + dynamodb_value = dynamize_value(hash_key) if dynamodb_value.keys()[0] != schema.hash_key_type: msg = 'Hashkey must be of type: %s' % schema.hash_key_type raise TypeError(msg) dynamodb_key['HashKeyElement'] = dynamodb_value if range_key is not None: - dynamodb_value = self.dynamize_value(range_key) + dynamodb_value = dynamize_value(range_key) if dynamodb_value.keys()[0] != schema.range_key_type: msg = 'RangeKey must be of type: %s' % schema.range_key_type raise TypeError(msg) @@ -417,13 +319,13 @@ class Layer2(object): schema = {} hash_key = {} hash_key['AttributeName'] = hash_key_name - hash_key_type = self.get_dynamodb_type(hash_key_proto_value) + hash_key_type = get_dynamodb_type(hash_key_proto_value) hash_key['AttributeType'] = hash_key_type schema['HashKeyElement'] = hash_key if range_key_name and range_key_proto_value is not None: range_key = {} range_key['AttributeName'] = range_key_name - range_key_type = self.get_dynamodb_type(range_key_proto_value) + range_key_type = get_dynamodb_type(range_key_proto_value) range_key['AttributeType'] = range_key_type schema['RangeKeyElement'] = range_key return Schema(schema) @@ -606,18 +508,15 @@ class Layer2(object): type of the value must match the type defined in the schema for the table. - :type range_key_condition: dict - :param range_key_condition: A dict where the key is either - a scalar value appropriate for the RangeKey in the schema - of the database or a tuple of such values. The value - associated with this key in the dict will be one of the - following conditions: + :type range_key_condition: :class:`boto.dynamodb.condition.Condition` + :param range_key_condition: A Condition object. + Condition object can be one of the following types: - 'EQ'|'LE'|'LT'|'GE'|'GT'|'BEGINS_WITH'|'BETWEEN' + EQ|LE|LT|GE|GT|BEGINS_WITH|BETWEEN - The only condition which expects or will accept a tuple - of values is 'BETWEEN', otherwise a scalar value should - be used as the key in the dict. + The only condition which expects or will accept two + values is 'BETWEEN', otherwise a single value should + be passed to the Condition constructor. :type attributes_to_get: list :param attributes_to_get: A list of attribute names. @@ -660,7 +559,10 @@ class Layer2(object): :rtype: generator """ - rkc = self.dynamize_range_key_condition(range_key_condition) + if range_key_condition: + rkc = self.dynamize_range_key_condition(range_key_condition) + else: + rkc = None response = True n = 0 while response: @@ -672,7 +574,7 @@ class Layer2(object): else: break response = self.layer1.query(table.name, - self.dynamize_value(hash_key), + dynamize_value(hash_key), rkc, attributes_to_get, request_limit, consistent_read, scan_index_forward, exclusive_start_key, @@ -692,12 +594,11 @@ class Layer2(object): :type table: :class:`boto.dynamodb.table.Table` :param table: The Table object that is being scanned. - :type scan_filter: A list of tuples - :param scan_filter: A list of tuples where each tuple consists - of an attribute name, a comparison operator, and either - a scalar or tuple consisting of the values to compare - the attribute to. Valid comparison operators are shown below - along with the expected number of values that should be supplied. + :type scan_filter: A dict + :param scan_filter: A dictionary where the key is the + attribute name and the value is a + :class:`boto.dynamodb.condition.Condition` object. + Valid Condition objects include: * EQ - equal (1) * NE - not equal (1) diff --git a/boto/dynamodb/types.py b/boto/dynamodb/types.py new file mode 100644 index 00000000..723d33d8 --- /dev/null +++ b/boto/dynamodb/types.py @@ -0,0 +1,88 @@ +# Copyright (c) 2011 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +""" +Some utility functions to deal with mapping Amazon DynamoDB types to +Python types and vice-versa. +""" + +def is_num(n): + return isinstance(n, (int, long, float, bool)) + +def is_str(n): + return isinstance(n, basestring) + +def convert_num(s): + if '.' in s: + n = float(s) + else: + n = int(s) + return n + +def get_dynamodb_type(val): + """ + Take a scalar Python value and return a string representing + the corresponding Amazon DynamoDB type. If the value passed in is + not a supported type, raise a TypeError. + """ + dynamodb_type = None + if is_num(val): + dynamodb_type = 'N' + elif is_str(val): + dynamodb_type = 'S' + elif isinstance(val, (set, frozenset)): + if False not in map(is_num, val): + dynamodb_type = 'NS' + elif False not in map(is_str, val): + dynamodb_type = 'SS' + if dynamodb_type is None: + raise TypeError('Unsupported type "%s" for value "%s"' % (type(val), val)) + return dynamodb_type + +def dynamize_value(val): + """ + Take a scalar Python value and return a dict consisting + of the Amazon DynamoDB type specification and the value that + needs to be sent to Amazon DynamoDB. If the type of the value + is not supported, raise a TypeError + """ + def _str(val): + """ + DynamoDB stores booleans as numbers. True is 1, False is 0. + This function converts Python booleans into DynamoDB friendly + representation. + """ + if isinstance(val, bool): + return str(int(val)) + return str(val) + + dynamodb_type = get_dynamodb_type(val) + if dynamodb_type == 'N': + val = {dynamodb_type : _str(val)} + elif dynamodb_type == 'S': + val = {dynamodb_type : val} + elif dynamodb_type == 'NS': + val = {dynamodb_type : [ str(n) for n in val]} + elif dynamodb_type == 'SS': + val = {dynamodb_type : [ n for n in val]} + return val + diff --git a/tests/dynamodb/test_layer2.py b/tests/dynamodb/test_layer2.py index 6a68b895..9324c8d1 100644 --- a/tests/dynamodb/test_layer2.py +++ b/tests/dynamodb/test_layer2.py @@ -29,6 +29,8 @@ import time import uuid from boto.dynamodb.exceptions import DynamoDBKeyNotFoundError from boto.dynamodb.layer2 import Layer2 +from boto.dynamodb.types import get_dynamodb_type +from boto.dynamodb.condition import * class DynamoDBLayer2Test (unittest.TestCase): @@ -55,9 +57,9 @@ class DynamoDBLayer2Test (unittest.TestCase): table = c.create_table(table_name, schema, read_units, write_units) assert table.name == table_name assert table.schema.hash_key_name == hash_key_name - assert table.schema.hash_key_type == c.get_dynamodb_type(hash_key_proto_value) + assert table.schema.hash_key_type == get_dynamodb_type(hash_key_proto_value) assert table.schema.range_key_name == range_key_name - assert table.schema.range_key_type == c.get_dynamodb_type(range_key_proto_value) + assert table.schema.range_key_type == get_dynamodb_type(range_key_proto_value) assert table.read_units == read_units assert table.write_units == write_units @@ -212,15 +214,13 @@ class DynamoDBLayer2Test (unittest.TestCase): table2_item1.put() # Try a few queries - items = table.query('Amazon DynamoDB', - {'DynamoDB': 'BEGINS_WITH'}) + items = table.query('Amazon DynamoDB', BEGINS_WITH('DynamoDB')) n = 0 for item in items: n += 1 assert n == 2 - items = table.query('Amazon DynamoDB', - {'DynamoDB': 'BEGINS_WITH'}, + items = table.query('Amazon DynamoDB', BEGINS_WITH('DynamoDB'), request_limit=1, max_results=1) n = 0 for item in items: @@ -234,6 +234,12 @@ class DynamoDBLayer2Test (unittest.TestCase): n += 1 assert n == 3 + items = table.scan({'Replies': GT(0)}) + n = 0 + for item in items: + n += 1 + assert n == 1 + # Test some integer and float attributes integer_value = 42 float_value = 345.678 @@ -280,15 +286,14 @@ class DynamoDBLayer2Test (unittest.TestCase): assert len(response['Responses'][table.name]['Items']) == 2 # Try queries - results = table.query('Amazon DynamoDB', - range_key_condition={'DynamoDB': 'BEGINS_WITH'}) + results = table.query('Amazon DynamoDB', BEGINS_WITH('DynamoDB')) n = 0 for item in results: n += 1 assert n == 2 # Try scans - results = table.scan([('Tags', 'CONTAINS', 'table')]) + results = table.scan({'Tags': CONTAINS('table')}) n = 0 for item in results: n += 1 -- cgit v1.2.1 From 634f03f76cf5bb0b16b9109838a74597925c5b6c Mon Sep 17 00:00:00 2001 From: Mitch Garnaat Date: Wed, 15 Feb 2012 13:27:15 -0800 Subject: Adding missing NE condition. --- boto/dynamodb/condition.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/boto/dynamodb/condition.py b/boto/dynamodb/condition.py index 3df72796..353756c1 100644 --- a/boto/dynamodb/condition.py +++ b/boto/dynamodb/condition.py @@ -71,6 +71,10 @@ class EQ(ConditionOneArg): pass +class NE(ConditionOneArg): + + pass + class LE(ConditionOneArg): pass -- cgit v1.2.1 From 7a49a6ec293470d1668f0dc472777dafe34d83a7 Mon Sep 17 00:00:00 2001 From: Mitch Garnaat Date: Wed, 15 Feb 2012 15:03:14 -0800 Subject: Removing some duplicate code leftover from previous refactoring. --- boto/dynamodb/condition.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/boto/dynamodb/condition.py b/boto/dynamodb/condition.py index 353756c1..b79b387f 100644 --- a/boto/dynamodb/condition.py +++ b/boto/dynamodb/condition.py @@ -24,11 +24,17 @@ from boto.dynamodb.types import get_dynamodb_type, dynamize_value, convert_num class Condition(object): + """ + Base class for conditions. Doesn't do a darn thing but allows + is to test if something is a Condition instance or not. + """ pass class ConditionNoArgs(Condition): """ + Abstract class for Conditions that require no arguments, such + as NULL or NOT_NULL. """ def __repr__(self): @@ -39,6 +45,8 @@ class ConditionNoArgs(Condition): class ConditionOneArg(Condition): """ + Abstract class for Conditions that require a single argument + such as EQ or NE. """ def __init__(self, v1): @@ -53,6 +61,8 @@ class ConditionOneArg(Condition): class ConditionTwoArgs(Condition): """ + Abstract class for Conditions that require two arguments. + The only example of this currently is BETWEEN. """ def __init__(self, v1, v2): @@ -120,20 +130,8 @@ class BEGINS_WITH(ConditionOneArg): pass class BETWEEN(ConditionTwoArgs): - """ - """ - - def __init__(self, v1, v2): - Condition.__init__(self, v1) - self.v2 = v2 - def __repr__(self): - return '%s(%s, %s)' % (self.__class__.__name__, self.v1, self.v2) - - def to_dict(self): - values = (self.v1, self.v2) - return {'AttributeValueList': [dynamize_value(v) for v in values], - 'ComparisonOperator': self.__class__.__name__} + pass -- cgit v1.2.1 From 5764de0594829e123ca0699a92bd6ec27e71bd35 Mon Sep 17 00:00:00 2001 From: Christopher Roach Date: Tue, 21 Feb 2012 22:14:36 -0800 Subject: Fixing a bug in Layer2.update_throughput The update_throughput method of Layer2 calls the update_from_response method of Table, but instead of passing in the full response (as all other similar calls to update_from_response do in the codebase) it passes in response['TableDescription']. Since update_from_response looks for either the key 'Table' or 'TableDescription' in the response object, neither is found and, therefore, the table's local attributes are not updated and do not correctly reflect the changes that have taken place on the server. This fix passes in the full response object to update_from_response. --- boto/dynamodb/layer2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boto/dynamodb/layer2.py b/boto/dynamodb/layer2.py index c701a6b2..e7f771f2 100644 --- a/boto/dynamodb/layer2.py +++ b/boto/dynamodb/layer2.py @@ -282,7 +282,7 @@ class Layer2(object): response = self.layer1.update_table(table.name, {'ReadCapacityUnits': read_units, 'WriteCapacityUnits': write_units}) - table.update_from_response(response['TableDescription']) + table.update_from_response(response) def delete_table(self, table): """ -- cgit v1.2.1 From 96527e9b1f5fcb5635373fbe879e7bc68260f466 Mon Sep 17 00:00:00 2001 From: Tony Morgan Date: Wed, 22 Feb 2012 19:19:40 +0000 Subject: fixing a bug, max_results in Layer2.query --- boto/dynamodb/layer2.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/boto/dynamodb/layer2.py b/boto/dynamodb/layer2.py index e7f771f2..9cd01e9a 100644 --- a/boto/dynamodb/layer2.py +++ b/boto/dynamodb/layer2.py @@ -566,6 +566,8 @@ class Layer2(object): response = True n = 0 while response: + if max_results and n == max_results: + break if response is True: pass elif response.has_key("LastEvaluatedKey"): -- cgit v1.2.1 From 644ca93e9c099b0da926f0956a5bef8590700e94 Mon Sep 17 00:00:00 2001 From: Mitch Garnaat Date: Thu, 23 Feb 2012 07:40:42 -0800 Subject: Fixes to list_tables method in Layer1 and Layer2. Fixes #596. --- boto/dynamodb/layer1.py | 11 ++++++++--- boto/dynamodb/layer2.py | 22 ++++++++++------------ 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/boto/dynamodb/layer1.py b/boto/dynamodb/layer1.py index 72d9a58d..d7b27f41 100644 --- a/boto/dynamodb/layer1.py +++ b/boto/dynamodb/layer1.py @@ -150,14 +150,19 @@ class Layer1(AWSAuthConnection): def list_tables(self, limit=None, start_table=None): """ - Return a list of table names associated with the current account - and endpoint. + Returns a dictionary of results. The dictionary contains + a **TableNames** key whose value is a list of the table names. + The dictionary could also contain a **LastEvaluatedTableName** + key whose value would be the last table name returned if + the complete list of table names was not returned. This + value would then be passed as the ``start_table`` parameter on + a subsequent call to this method. :type limit: int :param limit: The maximum number of tables to return. :type start_table: str - :param limit: The name of the table that starts the + :param start_table: The name of the table that starts the list. If you ran a previous list_tables and not all results were returned, the response dict would include a LastEvaluatedTableName attribute. Use diff --git a/boto/dynamodb/layer2.py b/boto/dynamodb/layer2.py index e7f771f2..eba67fea 100644 --- a/boto/dynamodb/layer2.py +++ b/boto/dynamodb/layer2.py @@ -198,24 +198,22 @@ class Layer2(object): """ return BatchList(self) - def list_tables(self, limit=None, start_table=None): + def list_tables(self, limit=None): """ - Return a list of the names of all Tables associated with the + Return a list of the names of all tables associated with the current account and region. - TODO - Layer2 should probably automatically handle pagination. :type limit: int :param limit: The maximum number of tables to return. - - :type start_table: str - :param limit: The name of the table that starts the - list. If you ran a previous list_tables and not - all results were returned, the response dict would - include a LastEvaluatedTableName attribute. Use - that value here to continue the listing. """ - result = self.layer1.list_tables(limit, start_table) - return result['TableNames'] + tables = [] + while True: + result = self.layer1.list_tables(limit) + tables.extend(result.get('TableNames', [])) + start_table = result.get('LastEvaluatedTableName', None) + if not start_table: + break + return tables def describe_table(self, name): """ -- cgit v1.2.1 -- cgit v1.2.1