Merge ../httplib2.current

Merge to HEAD of the mercurial tree hopefully for the last time.

8 files changed, 170 insertions, 22 deletions

diff --git a/.hgignore b/.hgignore
index da08947..5c3aeaf 100644
--- a/.hgignore
+++ b/.hgignore
@@ -4,5 +4,4 @@ dist
 upload.py
 **.pyc
 .cache
-.git
 MANIFEST
diff --git a/CHANGELOG b/CHANGELOG
index e4e10c3..36c2c0c 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,13 @@
+0.8
+  More fixes for the App Engine support.
+
+  Added a new feature that allows you to supply your own provider for the
+  CA_CERTS file. Just create a module named ca_certs_locater that has a method
+  get() that returns the file location of the CA_CERTS file.
+
+  Lots of clean up of the code formatting to make it more consistent.
+
+
 0.7.7
    More fixes for App Engine, now less likely to swallow important exceptions.
    Adding proxy_info_from_* methods to Python3. Reviewed in https://codereview.appspot.com/6588078/.
diff --git a/python2/httplib2/__init__.py b/python2/httplib2/__init__.py
index 58d9ce4..8e9f0fe 100644
--- a/python2/httplib2/__init__.py
+++ b/python2/httplib2/__init__.py
@@ -22,7 +22,7 @@ __contributors__ = ["Thomas Broyer (t.broyer@ltgt.net)",
                     "Sam Ruby",
                     "Louis Nyffenegger"]
 __license__ = "MIT"
-__version__ = "0.7.7"
+__version__ = "0.8"
 
 import re
 import sys
@@ -1072,7 +1072,7 @@ try:
             raise ImportError  # Bail out; we're not actually running on App Engine.
         from google.appengine.api.urlfetch import fetch
         from google.appengine.api.urlfetch import InvalidURLError
-    except ImportError:
+    except (ImportError, AttributeError):
         from google3.apphosting.api import apiproxy_stub_map
         if apiproxy_stub_map.apiproxy.GetStub('urlfetch') is None:
             raise ImportError  # Bail out; we're not actually running on App Engine.
@@ -1083,7 +1083,7 @@ try:
         def fixed_fetch(url, payload=None, method="GET", headers={},
                         allow_truncated=False, follow_redirects=True,
                         deadline=5):
-            return fetch(url, payload=payload, method=method, headers=header,
+            return fetch(url, payload=payload, method=method, headers=headers,
                          allow_truncated=allow_truncated,
                          follow_redirects=follow_redirects, deadline=deadline,
                          validate_certificate=validate_certificate)
@@ -1118,7 +1118,7 @@ try:
         'http': AppEngineHttpConnection,
         'https': AppEngineHttpsConnection
     }
-except ImportError:
+except (ImportError, AttributeError):
     pass
 
 
@@ -1246,7 +1246,10 @@ class Http(object):
         self.authorizations = []
 
     def _conn_request(self, conn, request_uri, method, body, headers):
-        for i in range(RETRIES):
+        i = 0
+        seen_bad_status_line = False
+        while i < RETRIES:
+            i += 1
             try:
                 if hasattr(conn, 'sock') and conn.sock is None:
                     conn.connect()
@@ -1284,6 +1287,19 @@ class Http(object):
                     continue
             try:
                 response = conn.getresponse()
+            except httplib.BadStatusLine:
+                # If we get a BadStatusLine on the first try then that means
+                # the connection just went stale, so retry regardless of the
+                # number of RETRIES set.
+                if not seen_bad_status_line and i == 1:
+                    i = 0
+                    seen_bad_status_line = True
+                    conn.close()
+                    conn.connect()
+                    continue
+                else:
+                    conn.close()
+                    raise
             except (socket.error, httplib.HTTPException):
                 if i < RETRIES-1:
                     conn.close()
@@ -1364,7 +1380,10 @@ class Http(object):
                         if response.status in [302, 303]:
                             redirect_method = "GET"
                             body = None
-                        (response, content) = self.request(location, redirect_method, body=body, headers = headers, redirections = redirections - 1)
+                        (response, content) = self.request(
+                            location, method=redirect_method,
+                            body=body, headers=headers,
+                            redirections=redirections - 1)
                         response.previous = old_response
                 else:
                     raise RedirectLimit("Redirected more times than rediection_limit allows.", response, content)
@@ -1506,7 +1525,9 @@ class Http(object):
                     # Should cached permanent redirects be counted in our redirection count? For now, yes.
                     if redirections <= 0:
                         raise RedirectLimit("Redirected more times than rediection_limit allows.", {}, "")
-                    (response, new_content) = self.request(info['-x-permanent-redirect-url'], "GET", headers = headers, redirections = redirections - 1)
+                    (response, new_content) = self.request(
+                        info['-x-permanent-redirect-url'], method='GET',
+                        headers=headers, redirections=redirections - 1)
                     response.previous = Response(info)
                     response.previous.fromcache = True
                 else:
diff --git a/python2/httplib2test.py b/python2/httplib2test.py
index 3802879..104eaf7 100755
--- a/python2/httplib2test.py
+++ b/python2/httplib2test.py
@@ -144,6 +144,36 @@ class _MyHTTPConnection(object):
     def getresponse(self):
         return _MyResponse("the body", status="200")
 
+class _MyHTTPBadStatusConnection(object):
+    "Mock of httplib.HTTPConnection that raises BadStatusLine."
+
+    num_calls = 0
+
+    def __init__(self, host, port=None, key_file=None, cert_file=None,
+                 strict=None, timeout=None, proxy_info=None):
+        self.host = host
+        self.port = port
+        self.timeout = timeout
+        self.log = ""
+        self.sock = None
+        _MyHTTPBadStatusConnection.num_calls = 0
+
+    def set_debuglevel(self, level):
+        pass
+
+    def connect(self):
+        pass
+
+    def close(self):
+        pass
+
+    def request(self, method, request_uri, body, headers):
+        pass
+
+    def getresponse(self):
+        _MyHTTPBadStatusConnection.num_calls += 1
+        raise httplib.BadStatusLine("")
+
 
 class HttpTest(unittest.TestCase):
     def setUp(self):
@@ -187,6 +217,17 @@ class HttpTest(unittest.TestCase):
         self.assertEqual(response['content-location'], "http://bitworking.org")
         self.assertEqual(content, "the body")
 
+    def testBadStatusLineRetry(self):
+        old_retries = httplib2.RETRIES
+        httplib2.RETRIES = 1
+        self.http.force_exception_to_status_code = False
+        try:
+            response, content = self.http.request("http://bitworking.org",
+                connection_type=_MyHTTPBadStatusConnection)
+        except httplib.BadStatusLine:
+            self.assertEqual(2, _MyHTTPBadStatusConnection.num_calls)
+        httplib2.RETRIES = old_retries
+
     def testGetUnknownServer(self):
         self.http.force_exception_to_status_code = False
         try:
diff --git a/python2/httplib2test_appengine.py b/python2/httplib2test_appengine.py
index ea36f39..0c0bdc2 100644
--- a/python2/httplib2test_appengine.py
+++ b/python2/httplib2test_appengine.py
@@ -28,7 +28,7 @@ testbed = testbed.Testbed()
 testbed.activate()
 testbed.init_urlfetch_stub()
 
-from google.appengine.runtime import DeadlineExceededError
+import google.appengine.api
 
 import httplib2
 
@@ -37,11 +37,6 @@ class AppEngineHttpTest(unittest.TestCase):
         if os.path.exists(cacheDirName):
             [os.remove(os.path.join(cacheDirName, file)) for file in os.listdir(cacheDirName)]
 
-        if sys.version_info < (2, 6):
-            disable_cert_validation = True
-        else:
-            disable_cert_validation = False
-
     def test(self):
         h = httplib2.Http()
         response, content = h.request("http://bitworking.org")
@@ -71,12 +66,30 @@ class AppEngineHttpTest(unittest.TestCase):
     #    except DeadlineExceededError:
     #      pass
 
-
-
     def test_proxy_info_ignored(self):
         h = httplib2.Http(proxy_info='foo.txt')
         response, content = h.request("http://bitworking.org")
         self.assertEquals(response.status, 200)
 
+
+class AberrationsTest(unittest.TestCase):
+    def setUp(self):
+        self.orig_apiproxy_stub_map = google.appengine.api.apiproxy_stub_map
+
+        # Force apiproxy_stub_map to None to trigger the test condition.
+        google.appengine.api.apiproxy_stub_map = None
+        reload(httplib2)
+
+    def tearDown(self):
+        google.appengine.api.apiproxy_stub_map = self.orig_apiproxy_stub_map
+        reload(httplib2)
+
+    def test(self):
+        self.assertNotEqual(httplib2.SCHEME_TO_CONNECTION['https'],
+                            httplib2.AppEngineHttpsConnection)
+        self.assertNotEqual(httplib2.SCHEME_TO_CONNECTION['http'],
+                            httplib2.AppEngineHttpConnection)
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/python3/httplib2/__init__.py b/python3/httplib2/__init__.py
index bf6c2e9..93bd7e6 100644
--- a/python3/httplib2/__init__.py
+++ b/python3/httplib2/__init__.py
@@ -24,7 +24,7 @@ __contributors__ = ["Thomas Broyer (t.broyer@ltgt.net)",
     "Louis Nyffenegger",
     "Mark Pilgrim"]
 __license__ = "MIT"
-__version__ = "0.7.7"
+__version__ = "0.8"
 
 import re
 import sys
@@ -957,7 +957,10 @@ class Http(object):
         self.authorizations = []
 
     def _conn_request(self, conn, request_uri, method, body, headers):
-        for i in range(RETRIES):
+        i = 0
+        seen_bad_status_line = False
+        while i < RETRIES:
+            i += 1
             try:
                 if conn.sock is None:
                     conn.connect()
@@ -990,6 +993,19 @@ class Http(object):
                 pass
             try:
                 response = conn.getresponse()
+            except (http.client.BadStatusLine, http.client.ResponseNotReady):
+                # If we get a BadStatusLine on the first try then that means
+                # the connection just went stale, so retry regardless of the
+                # number of RETRIES set.
+                if not seen_bad_status_line and i == 1:
+                    i = 0
+                    seen_bad_status_line = True
+                    conn.close()
+                    conn.connect()
+                    continue
+                else:
+                    conn.close()
+                    raise
             except socket.timeout:
                 raise
             except (socket.error, http.client.HTTPException):
@@ -1073,7 +1089,9 @@ class Http(object):
                         if response.status in [302, 303]:
                           redirect_method = "GET"
                           body = None
-                        (response, content) = self.request(location, redirect_method, body=body, headers = headers, redirections = redirections - 1)
+                        (response, content) = self.request(
+                            location, method=redirect_method, body=body,
+                            headers=headers, redirections=redirections - 1)
                         response.previous = old_response
                 else:
                     raise RedirectLimit("Redirected more times than redirection_limit allows.", response, content)
@@ -1208,7 +1226,9 @@ a string that contains the response entity body.
                     # Should cached permanent redirects be counted in our redirection count? For now, yes.
                     if redirections <= 0:
                         raise RedirectLimit("Redirected more times than redirection_limit allows.", {}, "")
-                    (response, new_content) = self.request(info['-x-permanent-redirect-url'], "GET", headers = headers, redirections = redirections - 1)
+                    (response, new_content) = self.request(
+                        info['-x-permanent-redirect-url'], method='GET',
+                        headers=headers, redirections=redirections - 1)
                     response.previous = Response(info)
                     response.previous.fromcache = True
                 else:
diff --git a/python3/httplib2test.py b/python3/httplib2test.py
index 480d28e..8f7a613 100755
--- a/python3/httplib2test.py
+++ b/python3/httplib2test.py
@@ -138,6 +138,37 @@ class _MyHTTPConnection(object):
         return _MyResponse(b"the body", status="200")

 

 

+class _MyHTTPBadStatusConnection(object):

+    "Mock of httplib.HTTPConnection that raises BadStatusLine."

+

+    num_calls = 0

+

+    def __init__(self, host, port=None, key_file=None, cert_file=None,

+                 strict=None, timeout=None, proxy_info=None):

+        self.host = host

+        self.port = port

+        self.timeout = timeout

+        self.log = ""

+        self.sock = None

+        _MyHTTPBadStatusConnection.num_calls = 0

+

+    def set_debuglevel(self, level):

+        pass

+

+    def connect(self):

+        pass

+

+    def close(self):

+        pass

+

+    def request(self, method, request_uri, body, headers):

+        pass

+

+    def getresponse(self):

+        _MyHTTPBadStatusConnection.num_calls += 1

+        raise http.client.BadStatusLine("")

+

+

 class HttpTest(unittest.TestCase):

     def setUp(self):

         if os.path.exists(cacheDirName):

@@ -169,6 +200,19 @@ class HttpTest(unittest.TestCase):
         self.assertEqual(response['content-location'], "http://bitworking.org")

         self.assertEqual(content, b"the body")

 

+

+    def testBadStatusLineRetry(self):

+        old_retries = httplib2.RETRIES

+        httplib2.RETRIES = 1

+        self.http.force_exception_to_status_code = False

+        try:

+            response, content = self.http.request("http://bitworking.org",

+                connection_type=_MyHTTPBadStatusConnection)

+        except http.client.BadStatusLine:

+            self.assertEqual(2, _MyHTTPBadStatusConnection.num_calls)

+        httplib2.RETRIES = old_retries

+

+

     def testGetUnknownServer(self):

         self.http.force_exception_to_status_code = False

         try:

@@ -485,7 +529,7 @@ class HttpTest(unittest.TestCase):
 

           # Test that we get a SSLHandshakeError if we try to access

           # https://www.google.com, using a CA cert file that doesn't contain

-          # the CA Gogole uses (i.e., simulating a cert that's not signed by a

+          # the CA Google uses (i.e., simulating a cert that's not signed by a

           # trusted CA).

           other_ca_certs = os.path.join(

                   os.path.dirname(os.path.abspath(httplib2.__file__ )),

diff --git a/setup.py b/setup.py
index dcfbeab..37e7cd0 100755
--- a/setup.py
+++ b/setup.py
@@ -5,7 +5,7 @@ except ImportError:
 import sys
 
 pkgdir = {'': 'python%s' % sys.version_info[0]}
-VERSION = '0.7.7'
+VERSION = '0.8'
 
 setup(name='httplib2',
         version=VERSION,