From 0059db720e66981336f140e0a7b16e25d15b1cdc Mon Sep 17 00:00:00 2001 From: Chris Moyer Date: Tue, 23 Jul 2013 11:59:44 -0400 Subject: Added "lookup" and "new_item" to dynamodb2.table, which makes it more compatible with dynamodb.table --- boto/dynamodb2/table.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) (limited to 'boto/dynamodb2/table.py') diff --git a/boto/dynamodb2/table.py b/boto/dynamodb2/table.py index c1de437d..92fd2713 100644 --- a/boto/dynamodb2/table.py +++ b/boto/dynamodb2/table.py @@ -418,6 +418,48 @@ class Table(object): item.load(item_data) return item + def lookup(self, hash_key=None, range_key=None, consistent=False, **kwargs): + """ + Look up an entry in DynamoDB. This is mostly backwards compatible + with boto.dynamodb. Unlike get_item, it takes hash_key and range_key first, + although you may still specify keyword arguments instead. + + Also unlike the get_item command, if the returned item has no keys + (i.e., it does not exist in DynamoDB), a None result is returned, instead + of an empty key object. + + Example:: + >>> user = users.lookup(username) + >>> user = users.lookup(username, consistent=True) + >>> app = apps.lookup('my_customer_id', 'my_app_id') + + """ + if not self.schema: + self.describe() + if hash_key: + kwargs[self.schema[0].name] = hash_key + if range_key and len(self.schema) > 1: + kwargs[self.schema[1].name] = range_key + ret = self.get_item(consistent, **kwargs) + if not ret.keys(): + return None + return ret + + def new_item(self, hash_key, range_key=None): + """ + Returns a new, blank item + + This is mostly for consistency with boto.dynamodb + """ + if not self.schema: + self.describe() + data = {} + data[self.schema[0].name] = hash_key + if range_key and len(self.schema) > 1: + data[self.schema[1].name] = range_key + return Item(self, data=data) + + def put_item(self, data, overwrite=False): """ Saves an entire item to DynamoDB. -- cgit v1.2.1 From d4a2d7979752db79995530436813a5c9822802c4 Mon Sep 17 00:00:00 2001 From: Daniel Lindsley Date: Wed, 24 Jul 2013 14:43:17 -0700 Subject: Revert "Added "lookup" and "new_item" to dynamodb2.table, which makes it more" This reverts commit 0059db720e66981336f140e0a7b16e25d15b1cdc. --- boto/dynamodb2/table.py | 42 ------------------------------------------ 1 file changed, 42 deletions(-) (limited to 'boto/dynamodb2/table.py') diff --git a/boto/dynamodb2/table.py b/boto/dynamodb2/table.py index 92fd2713..c1de437d 100644 --- a/boto/dynamodb2/table.py +++ b/boto/dynamodb2/table.py @@ -418,48 +418,6 @@ class Table(object): item.load(item_data) return item - def lookup(self, hash_key=None, range_key=None, consistent=False, **kwargs): - """ - Look up an entry in DynamoDB. This is mostly backwards compatible - with boto.dynamodb. Unlike get_item, it takes hash_key and range_key first, - although you may still specify keyword arguments instead. - - Also unlike the get_item command, if the returned item has no keys - (i.e., it does not exist in DynamoDB), a None result is returned, instead - of an empty key object. - - Example:: - >>> user = users.lookup(username) - >>> user = users.lookup(username, consistent=True) - >>> app = apps.lookup('my_customer_id', 'my_app_id') - - """ - if not self.schema: - self.describe() - if hash_key: - kwargs[self.schema[0].name] = hash_key - if range_key and len(self.schema) > 1: - kwargs[self.schema[1].name] = range_key - ret = self.get_item(consistent, **kwargs) - if not ret.keys(): - return None - return ret - - def new_item(self, hash_key, range_key=None): - """ - Returns a new, blank item - - This is mostly for consistency with boto.dynamodb - """ - if not self.schema: - self.describe() - data = {} - data[self.schema[0].name] = hash_key - if range_key and len(self.schema) > 1: - data[self.schema[1].name] = range_key - return Item(self, data=data) - - def put_item(self, data, overwrite=False): """ Saves an entire item to DynamoDB. -- cgit v1.2.1 From b111fcf264ffcbf41fa247176a584a0b2fe33e24 Mon Sep 17 00:00:00 2001 From: Daniel Lindsley Date: Mon, 29 Jul 2013 13:28:33 -0700 Subject: Altering how state tracking of ``Item`` data works in DDBv2. --- boto/dynamodb2/table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'boto/dynamodb2/table.py') diff --git a/boto/dynamodb2/table.py b/boto/dynamodb2/table.py index c1de437d..a78e3931 100644 --- a/boto/dynamodb2/table.py +++ b/boto/dynamodb2/table.py @@ -611,7 +611,7 @@ class Table(object): 'AttributeValueList': [], 'ComparisonOperator': op, } - + # Special-case the ``NULL/NOT_NULL`` case. if field_bits[-1] == 'null': del lookup['AttributeValueList'] -- cgit v1.2.1 From be0095634d52ff0e1bc5f2d7c45fd19546c2edba Mon Sep 17 00:00:00 2001 From: Daniel Lindsley Date: Fri, 16 Aug 2013 17:02:31 -0700 Subject: Fixed a typo in DDBv2's create docstring. --- boto/dynamodb2/table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'boto/dynamodb2/table.py') diff --git a/boto/dynamodb2/table.py b/boto/dynamodb2/table.py index a78e3931..35fadcbe 100644 --- a/boto/dynamodb2/table.py +++ b/boto/dynamodb2/table.py @@ -133,7 +133,7 @@ class Table(object): Example:: - >>> users = Table.create_table('users', schema=[ + >>> users = Table.create('users', schema=[ ... HashKey('username'), ... RangeKey('date_joined', data_type=NUMBER) ... ], throughput={ -- cgit v1.2.1 From 6a5302032233dcc23ef6c4a56c533072e99828d9 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Wed, 21 Aug 2013 18:26:05 -0700 Subject: Updated custom connection example. --- boto/dynamodb2/table.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'boto/dynamodb2/table.py') diff --git a/boto/dynamodb2/table.py b/boto/dynamodb2/table.py index 35fadcbe..d552e4af 100644 --- a/boto/dynamodb2/table.py +++ b/boto/dynamodb2/table.py @@ -57,7 +57,7 @@ class Table(object): >>> conn = Table('users') # The full, minimum-extra-calls case. - >>> from boto.dynamodb2.layer1 import DynamoDBConnection + >>> from boto import dynamodb2 >>> users = Table('users', schema=[ ... HashKey('username'), ... RangeKey('date_joined', data_type=NUMBER) @@ -69,11 +69,10 @@ class Table(object): ... RangeKey('date_joined') ... ]), ... ], - ... connection=DynamoDBConnection( - ... aws_access_key_id='key', - ... aws_secret_access_key='key', - ... region='us-west-2' - ... )) + ... connection=dynamodb2.connect_to_region('us-west-2', + ... aws_access_key_id='key', + ... aws_secret_access_key='key', + ... )) """ self.table_name = table_name -- cgit v1.2.1 From 2fc23695d6cbc6e5002ddd4465b1333799bd2fa7 Mon Sep 17 00:00:00 2001 From: Daniel Lindsley Date: Tue, 10 Sep 2013 16:28:38 -0700 Subject: Fixes #1679, #1566 - Altered DDBv2's ``batch_write`` to appropriately queue & retry unprocessed items. --- boto/dynamodb2/table.py | 51 +++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 6 deletions(-) (limited to 'boto/dynamodb2/table.py') diff --git a/boto/dynamodb2/table.py b/boto/dynamodb2/table.py index d552e4af..5d6803ce 100644 --- a/boto/dynamodb2/table.py +++ b/boto/dynamodb2/table.py @@ -1,3 +1,4 @@ +import boto from boto.dynamodb2 import exceptions from boto.dynamodb2.fields import (HashKey, RangeKey, AllIndex, KeysOnlyIndex, IncludeIndex) @@ -1070,17 +1071,19 @@ class BatchTable(object): self.table = table self._to_put = [] self._to_delete = [] + self._unprocessed = [] def __enter__(self): return self def __exit__(self, type, value, traceback): - if not self._to_put and not self._to_delete: - return False + if self._to_put or self._to_delete: + # Flush anything that's left. + self.flush() - # Flush anything that's left. - self.flush() - return True + if self._unprocessed: + # Finally, handle anything that wasn't processed. + self.resend_unprocessed() def put_item(self, data, overwrite=False): self._to_put.append(data) @@ -1122,7 +1125,43 @@ class BatchTable(object): } }) - self.table.connection.batch_write_item(batch_data) + resp = self.table.connection.batch_write_item(batch_data) + self.handle_unprocessed(resp) + self._to_put = [] self._to_delete = [] return True + + def handle_unprocessed(self, resp): + if len(resp.get('UnprocessedItems', [])): + table_name = self.table.table_name + unprocessed = resp['UnprocessedItems'].get(table_name, []) + + # Some items have not been processed. Stow them for now & + # re-attempt processing on ``__exit__``. + msg = "%s items were unprocessed. Storing for later." + boto.log.info(msg % len(unprocessed)) + self._unprocessed.extend(unprocessed) + + def resend_unprocessed(self): + # If there are unprocessed records (for instance, the user was over + # their throughput limitations), iterate over them & send until they're + # all there. + boto.log.info( + "Re-sending %s unprocessed items." % len(self._unprocessed) + ) + + while len(self._unprocessed): + # Again, do 25 at a time. + to_resend = self._unprocessed[:25] + # Remove them from the list. + self._unprocessed = self._unprocessed[25:] + batch_data = { + self.table.table_name: to_resend + } + boto.log.info("Sending %s items" % len(to_resend)) + resp = self.table.connection.batch_write_item(batch_data) + self.handle_unprocessed(resp) + boto.log.info( + "%s unprocessed items left" % len(self._unprocessed) + ) \ No newline at end of file -- cgit v1.2.1