Retry download of object body

Currently the swift client retries establishing a connection to the server
(by default up to 5 times).  However, when downloading an object, once
the connection has been established and the inital headers have been
returned, no attempt is made to retry.

So, for example, if 99MB of a 100MB object have been downloaded and the
connection is then lost, the download will fail.  This patch changes the
behaviour to re-establish the connection and fetch the remaining bytes
using the 'Range' header to offset.

Data retry is not yet supported if the original request is for a subset
of the object data (ie uses the 'Range' header), or if resp_chunk_size
has not been set.

The object's etag is checked using If-Match to make sure the object data
hasn't changed since the start of the download.

Change-Id: Iab47f10081ff39f6d344dbc2479cbc3bfd1c5b29

2 files changed, 69 insertions, 0 deletions

diff --git a/tests/functional/test_swiftclient.py b/tests/functional/test_swiftclient.py
index ef1b9d5..5f9e271 100644
--- a/tests/functional/test_swiftclient.py
+++ b/tests/functional/test_swiftclient.py
@@ -333,6 +333,58 @@ class TestFunctional(testtools.TestCase):
         hdrs, body = self.conn.get_object(self.containername, self.objectname)
         self.assertEqual("should tolerate empty chunks", body)
 
+    def test_download_object_retry_chunked(self):
+        resp_chunk_size = 2
+        hdrs, body = self.conn.get_object(self.containername,
+                                          self.objectname,
+                                          resp_chunk_size=resp_chunk_size)
+        data = next(body)
+        self.assertEqual(self.test_data[:resp_chunk_size], data)
+        self.assertTrue(1, self.conn.attempts)
+        for chunk in body.resp:
+            # Flush remaining data from underlying response
+            # (simulate a dropped connection)
+            pass
+        # Trigger the retry
+        for chunk in body:
+            data += chunk
+        self.assertEqual(self.test_data, data)
+        self.assertEqual(2, self.conn.attempts)
+
+    def test_download_object_retry_chunked_auth_failure(self):
+        resp_chunk_size = 2
+        self.conn.token = 'invalid'
+        hdrs, body = self.conn.get_object(self.containername,
+                                          self.objectname,
+                                          resp_chunk_size=resp_chunk_size)
+        self.assertEqual(2, self.conn.attempts)
+        for chunk in body.resp:
+            # Flush remaining data from underlying response
+            # (simulate a dropped connection)
+            pass
+
+        self.conn.token = 'invalid'
+        data = next(body)
+        self.assertEqual(4, self.conn.attempts)
+
+        for chunk in body:
+            data += chunk
+
+        self.assertEqual(self.test_data, data)
+        self.assertEqual(4, self.conn.attempts)
+
+    def test_download_object_non_chunked(self):
+        hdrs, body = self.conn.get_object(self.containername, self.objectname)
+        data = body
+        self.assertEqual(self.test_data, data)
+        self.assertTrue(1, self.conn.attempts)
+
+        hdrs, body = self.conn.get_object(self.containername, self.objectname,
+                                          resp_chunk_size=0)
+        data = body
+        self.assertEqual(self.test_data, data)
+        self.assertTrue(1, self.conn.attempts)
+
     def test_post_account(self):
         self.conn.post_account({'x-account-meta-data': 'Something'})
         headers = self.conn.head_account()
diff --git a/tests/unit/test_swiftclient.py b/tests/unit/test_swiftclient.py
index 1f6a770..56d3ff8 100644
--- a/tests/unit/test_swiftclient.py
+++ b/tests/unit/test_swiftclient.py
@@ -835,6 +835,23 @@ class TestGetObject(MockHttpTest):
             self.assertRaises(StopIteration, next, resp)
             self.assertEqual(resp.read(), '')
 
+    def test_get_object_with_resp_chunk_size_zero(self):
+        def get_connection(self):
+            def get_auth():
+                return 'http://auth.test.com', 'token'
+
+            conn = c.Connection('http://www.test.com', 'asdf', 'asdf')
+            self.assertIs(type(conn), c.Connection)
+            conn.get_auth = get_auth
+            self.assertEqual(conn.attempts, 0)
+            return conn
+
+        with mock.patch('swiftclient.client.http_connection',
+                        self.fake_http_connection(200)):
+            conn = get_connection(self)
+            conn.get_object('container1', 'obj1', resp_chunk_size=0)
+            self.assertEqual(conn.attempts, 1)
+
 
 class TestHeadObject(MockHttpTest):