Merge branch 'release-2.9.0'2.9.0

* release-2.9.0: (158 commits)
  Bump version to 2.9.0
  Added underlying DynamoDB v2 support.
  Add redshift to setup.py/docs index
  Updated requests to something more modern.
  Only use 2 metadata service calls to get credentials
  Fix #1146: return response from custom url opener
  Fixed missing import.
  Add metadata_service_num_attempts config option
  Added cleanup for the snapshots created.
  Added support for redshift.
  Let total attempts by 1 + num_retries
  Add more diagnostics to debug logs
  Change GS calls to make_request to always convert to utf-8 bytes.
  Allow kwargs to be passed through to uplaoder
  Remove whitespace, fix long line lengths
  Improve VPC and VPN support
  Added sleeps to allow amazon time to propogate
  Added error handling for out of space during downloads
  Initial integration tests for idempotent subscribe
  Removed dead code from resumable upload handler
  ...

1 files changed, 244 insertions, 0 deletions

diff --git a/tests/integration/dynamodb2/test_layer1.py b/tests/integration/dynamodb2/test_layer1.py
new file mode 100644
index 00000000..d85d8a87
--- /dev/null
+++ b/tests/integration/dynamodb2/test_layer1.py
@@ -0,0 +1,244 @@
+# Copyright (c) 2013 Amazon.com, Inc. or its affiliates.
+# All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+"""
+Tests for Layer1 of DynamoDB v2
+"""
+import time
+
+from tests.unit import unittest
+from boto.dynamodb2 import exceptions
+from boto.dynamodb2.layer1 import DynamoDBConnection
+
+
+class DynamoDBv2Layer1Test(unittest.TestCase):
+    dynamodb = True
+
+    def setUp(self):
+        self.dynamodb = DynamoDBConnection()
+        self.table_name = 'test-%d' % int(time.time())
+        self.hash_key_name = 'username'
+        self.hash_key_type = 'S'
+        self.range_key_name = 'date_joined'
+        self.range_key_type = 'N'
+        self.read_units = 5
+        self.write_units = 5
+        self.attributes = [
+            {
+                'AttributeName': self.hash_key_name,
+                'AttributeType': self.hash_key_type,
+            },
+            {
+                'AttributeName': self.range_key_name,
+                'AttributeType': self.range_key_type,
+            }
+        ]
+        self.schema = [
+            {
+                'AttributeName': self.hash_key_name,
+                'KeyType': 'HASH',
+            },
+            {
+                'AttributeName': self.range_key_name,
+                'KeyType': 'RANGE',
+            },
+        ]
+        self.provisioned_throughput = {
+            'ReadCapacityUnits': self.read_units,
+            'WriteCapacityUnits': self.write_units,
+        }
+        self.lsi = [
+            {
+                'IndexName': 'MostRecentIndex',
+                'KeySchema': [
+                    {
+                        'AttributeName': self.hash_key_name,
+                        'KeyType': 'HASH',
+                    },
+                    {
+                        'AttributeName': self.range_key_name,
+                        'KeyType': 'RANGE',
+                    },
+                ],
+                'Projection': {
+                    'ProjectionType': 'KEYS_ONLY',
+                }
+            }
+        ]
+
+    def create_table(self, table_name, attributes, schema,
+                     provisioned_throughput, lsi=None, wait=True):
+        # Note: This is a slightly different ordering that makes less sense.
+        result = self.dynamodb.create_table(
+            attributes,
+            table_name,
+            schema,
+            provisioned_throughput,
+            local_secondary_indexes=lsi
+        )
+        self.addCleanup(self.dynamodb.delete_table, table_name)
+        if wait:
+            while True:
+                description = self.dynamodb.describe_table(table_name)
+                if description['Table']['TableStatus'].lower() == 'active':
+                    return result
+                else:
+                    time.sleep(5)
+        else:
+            return result
+
+    def test_integrated(self):
+        result = self.create_table(
+            self.table_name,
+            self.attributes,
+            self.schema,
+            self.provisioned_throughput,
+            self.lsi
+        )
+        self.assertEqual(result['TableDescription']['TableName'], self.table_name)
+
+        description = self.dynamodb.describe_table(self.table_name)
+        self.assertEqual(description['Table']['ItemCount'], 0)
+
+        # Create some records.
+        record_1_data = {
+            'username': {'S': 'johndoe'},
+            'first_name': {'S': 'John'},
+            'last_name': {'S': 'Doe'},
+            'date_joined': {'N': '1366056668'},
+            'friend_count': {'N': '3'},
+            'friends': {'SS': ['alice', 'bob', 'jane']},
+        }
+        r1_result = self.dynamodb.put_item(self.table_name, record_1_data)
+
+        # Get the data.
+        record_1 = self.dynamodb.get_item(self.table_name, key={
+            'username': {'S': 'johndoe'},
+            'date_joined': {'N': '1366056668'},
+        }, consistent_read=True)
+        self.assertEqual(record_1['Item']['username']['S'], 'johndoe')
+        self.assertEqual(record_1['Item']['first_name']['S'], 'John')
+        self.assertEqual(record_1['Item']['friends']['SS'], ['alice', 'bob', 'jane'])
+
+        # Now in a batch.
+        self.dynamodb.batch_write_item({
+            self.table_name: [
+                {
+                    'PutRequest': {
+                        'Item': {
+                            'username': {'S': 'jane'},
+                            'first_name': {'S': 'Jane'},
+                            'last_name': {'S': 'Doe'},
+                            'date_joined': {'N': '1366056789'},
+                            'friend_count': {'N': '1'},
+                            'friends': {'SS': ['johndoe']},
+                        },
+                    },
+                },
+            ]
+        })
+
+        # Now a query.
+        lsi_results = self.dynamodb.query(
+            self.table_name,
+            index_name='MostRecentIndex',
+            key_conditions={
+                'username': {
+                    'AttributeValueList': [
+                        {'S': 'johndoe'},
+                    ],
+                    'ComparisonOperator': 'EQ',
+                },
+            },
+            consistent_read=True
+        )
+        self.assertEqual(lsi_results['Count'], 1)
+
+        results = self.dynamodb.query(self.table_name, key_conditions={
+            'username': {
+                'AttributeValueList': [
+                    {'S': 'jane'},
+                ],
+                'ComparisonOperator': 'EQ',
+            },
+            'date_joined': {
+                'AttributeValueList': [
+                    {'N': '1366050000'}
+                ],
+                'ComparisonOperator': 'GT',
+            }
+        }, consistent_read=True)
+        self.assertEqual(results['Count'], 1)
+
+        # Now a scan.
+        results = self.dynamodb.scan(self.table_name)
+        self.assertEqual(results['Count'], 2)
+        self.assertEqual(sorted([res['username']['S'] for res in results['Items']]), ['jane', 'johndoe'])
+
+        self.dynamodb.delete_item(self.table_name, key={
+            'username': {'S': 'johndoe'},
+            'date_joined': {'N': '1366056668'},
+        })
+
+        results = self.dynamodb.scan(self.table_name)
+        self.assertEqual(results['Count'], 1)
+
+    def test_without_range_key(self):
+        result = self.create_table(
+            self.table_name,
+            [
+                {
+                    'AttributeName': self.hash_key_name,
+                    'AttributeType': self.hash_key_type,
+                },
+            ],
+            [
+                {
+                    'AttributeName': self.hash_key_name,
+                    'KeyType': 'HASH',
+                },
+            ],
+            self.provisioned_throughput
+        )
+        self.assertEqual(result['TableDescription']['TableName'], self.table_name)
+
+        description = self.dynamodb.describe_table(self.table_name)
+        self.assertEqual(description['Table']['ItemCount'], 0)
+
+        # Create some records.
+        record_1_data = {
+            'username': {'S': 'johndoe'},
+            'first_name': {'S': 'John'},
+            'last_name': {'S': 'Doe'},
+            'date_joined': {'N': '1366056668'},
+            'friend_count': {'N': '3'},
+            'friends': {'SS': ['alice', 'bob', 'jane']},
+        }
+        r1_result = self.dynamodb.put_item(self.table_name, record_1_data)
+
+        # Now try a range-less get.
+        johndoe = self.dynamodb.get_item(self.table_name, key={
+            'username': {'S': 'johndoe'},
+        }, consistent_read=True)
+        self.assertEqual(johndoe['Item']['username']['S'], 'johndoe')
+        self.assertEqual(johndoe['Item']['first_name']['S'], 'John')
+        self.assertEqual(johndoe['Item']['friends']['SS'], ['alice', 'bob', 'jane'])