Python 3 compat; Improve WSGI, WS, threading and testspython3-clean

This includes:
* patching more tests to pass
* removing few unit tests which I think are redundant
* repeating SSL socket reads in a loop to read all data (I suspect this
  is related to the fact that writelines is used in the server code
  there and Python 3 writelines calls write/send repeatedly while on
  Python 2 it calls it once; on one hand there's no guarantee that
  single recv/read will return all data sent by the server, on the other
  hand it's quite suspicious that the number of required reads seems to
  be connected to the number of sends on the other side of the
  connection)
* working through Python 2/Python 3 threading and thread differences;
  the lock code I used is the simplest way I could make the tests
  pass but will likely need to be modified in order to match the
  original

This commit includes 6bcb1dc3686f72b0f1af803b46883ce4f5e4b8df and closes
GH #153

3 files changed, 84 insertions, 107 deletions

diff --git a/tests/greenio_test.py b/tests/greenio_test.py
index 8993e85..0c10771 100644
--- a/tests/greenio_test.py
+++ b/tests/greenio_test.py
@@ -273,7 +273,7 @@ class TestGreenSocket(LimitedTestCase):
             # by closing the socket prior to using the made file
             try:
                 conn, addr = listener.accept()
-                fd = conn.makefile('w')
+                fd = conn.makefile('wb')
                 conn.close()
                 fd.write(b'hello\n')
                 fd.close()
@@ -287,7 +287,7 @@ class TestGreenSocket(LimitedTestCase):
             # by closing the made file and then sending a character
             try:
                 conn, addr = listener.accept()
-                fd = conn.makefile('w')
+                fd = conn.makefile('wb')
                 fd.write(b'hello')
                 fd.close()
                 conn.send(b'\n')
@@ -300,7 +300,7 @@ class TestGreenSocket(LimitedTestCase):
         def did_it_work(server):
             client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
             client.connect(('127.0.0.1', server.getsockname()[1]))
-            fd = client.makefile()
+            fd = client.makefile('rb')
             client.close()
             assert fd.readline() == b'hello\n'
             assert fd.read() == b''
@@ -329,7 +329,7 @@ class TestGreenSocket(LimitedTestCase):
             # closing the file object should close everything
             try:
                 conn, addr = listener.accept()
-                conn = conn.makefile('w')
+                conn = conn.makefile('wb')
                 conn.write(b'hello\n')
                 conn.close()
                 gc.collect()
@@ -344,7 +344,7 @@ class TestGreenSocket(LimitedTestCase):
         killer = eventlet.spawn(accept_once, server)
         client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         client.connect(('127.0.0.1', server.getsockname()[1]))
-        fd = client.makefile()
+        fd = client.makefile('rb')
         client.close()
         assert fd.read() == b'hello\n'
         assert fd.read() == b''
@@ -603,7 +603,7 @@ class TestGreenSocket(LimitedTestCase):
     def test_sockopt_interface(self):
         sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
         assert sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) == 0
-        assert sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) == '\000'
+        assert sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) == b'\000'
         sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
 
     def test_socketpair_select(self):
diff --git a/tests/websocket_test.py b/tests/websocket_test.py
index bf05555..1f7f1bf 100644
--- a/tests/websocket_test.py
+++ b/tests/websocket_test.py
@@ -4,12 +4,11 @@ import socket
 import eventlet
 from eventlet import event
 from eventlet import greenio
-from eventlet import wsgi
 from eventlet.green import httplib
-from eventlet.green import urllib2
-from eventlet.websocket import WebSocket, WebSocketWSGI
+from eventlet.support import six
+from eventlet.websocket import WebSocketWSGI
 
-from tests import mock, LimitedTestCase, certificate_file, private_key_file
+from tests import certificate_file, private_key_file
 from tests import skip_if_no_ssl
 from tests.wsgi_test import _TestBase
 
@@ -42,13 +41,10 @@ class TestWebSocket(_TestBase):
         self.site = wsapp
 
     def test_incorrect_headers(self):
-        def raiser():
-            try:
-                urllib2.urlopen("http://localhost:%s/echo" % self.port)
-            except urllib2.HTTPError as e:
-                self.assertEqual(e.code, 400)
-                raise
-        self.assertRaises(urllib2.HTTPError, raiser)
+        http = httplib.HTTPConnection('localhost', self.port)
+        http.request("GET", "/echo")
+        response = http.getresponse()
+        assert response.status == 400
 
     def test_incomplete_headers_75(self):
         headers = dict(kv.split(': ') for kv in [
@@ -113,7 +109,7 @@ class TestWebSocket(_TestBase):
         sock = eventlet.connect(
             ('localhost', self.port))
 
-        sock.sendall('\r\n'.join(connect) + '\r\n\r\n')
+        sock.sendall(six.b('\r\n'.join(connect) + '\r\n\r\n'))
         result = sock.recv(1024)
         # The server responds the correct Websocket handshake
         self.assertEqual(result, '\r\n'.join([
@@ -138,7 +134,7 @@ class TestWebSocket(_TestBase):
         sock = eventlet.connect(
             ('localhost', self.port))
 
-        sock.sendall('\r\n'.join(connect) + '\r\n\r\n^n:ds[4U')
+        sock.sendall(six.b('\r\n'.join(connect) + '\r\n\r\n^n:ds[4U'))
         result = sock.recv(1024)
         # The server responds the correct Websocket handshake
         self.assertEqual(result, '\r\n'.join([
@@ -165,7 +161,7 @@ class TestWebSocket(_TestBase):
         sock = eventlet.connect(
             ('localhost', self.port))
 
-        sock.sendall('\r\n'.join(connect) + '\r\n\r\n^n:ds[4U')
+        sock.sendall(six.b('\r\n'.join(connect) + '\r\n\r\n^n:ds[4U'))
         result = sock.recv(1024)
         self.assertEqual(result, '\r\n'.join([
             'HTTP/1.1 101 WebSocket Protocol Handshake',
@@ -191,7 +187,7 @@ class TestWebSocket(_TestBase):
         sock = eventlet.connect(
             ('localhost', self.port))
 
-        sock.sendall('\r\n'.join(connect) + '\r\n\r\n^n:ds[4U')
+        sock.sendall(six.b('\r\n'.join(connect) + '\r\n\r\n^n:ds[4U'))
         result = sock.recv(1024)
         self.assertEqual(result, '\r\n'.join([
             'HTTP/1.1 101 WebSocket Protocol Handshake',
@@ -214,16 +210,16 @@ class TestWebSocket(_TestBase):
         sock = eventlet.connect(
             ('localhost', self.port))
 
-        sock.sendall('\r\n'.join(connect) + '\r\n\r\n')
+        sock.sendall(six.b('\r\n'.join(connect) + '\r\n\r\n'))
         sock.recv(1024)
-        sock.sendall('\x00hello\xFF')
+        sock.sendall(b'\x00hello\xFF')
         result = sock.recv(1024)
         self.assertEqual(result, '\x00hello\xff')
-        sock.sendall('\x00start')
+        sock.sendall(b'\x00start')
         eventlet.sleep(0.001)
-        sock.sendall(' end\xff')
+        sock.sendall(b' end\xff')
         result = sock.recv(1024)
-        self.assertEqual(result, '\x00start end\xff')
+        self.assertEqual(result, b'\x00start end\xff')
         sock.shutdown(socket.SHUT_RDWR)
         sock.close()
         eventlet.sleep(0.01)
@@ -242,16 +238,16 @@ class TestWebSocket(_TestBase):
         sock = eventlet.connect(
             ('localhost', self.port))
 
-        sock.sendall('\r\n'.join(connect) + '\r\n\r\n^n:ds[4U')
+        sock.sendall(six.b('\r\n'.join(connect) + '\r\n\r\n^n:ds[4U'))
         sock.recv(1024)
-        sock.sendall('\x00hello\xFF')
+        sock.sendall(b'\x00hello\xFF')
         result = sock.recv(1024)
-        self.assertEqual(result, '\x00hello\xff')
-        sock.sendall('\x00start')
+        self.assertEqual(result, b'\x00hello\xff')
+        sock.sendall(b'\x00start')
         eventlet.sleep(0.001)
-        sock.sendall(' end\xff')
+        sock.sendall(b' end\xff')
         result = sock.recv(1024)
-        self.assertEqual(result, '\x00start end\xff')
+        self.assertEqual(result, b'\x00start end\xff')
         sock.shutdown(socket.SHUT_RDWR)
         sock.close()
         eventlet.sleep(0.01)
@@ -268,16 +264,16 @@ class TestWebSocket(_TestBase):
         sock = eventlet.connect(
             ('localhost', self.port))
 
-        sock.sendall('\r\n'.join(connect) + '\r\n\r\n')
+        sock.sendall(six.b('\r\n'.join(connect) + '\r\n\r\n'))
         resp = sock.recv(1024)
-        headers, result = resp.split('\r\n\r\n')
-        msgs = [result.strip('\x00\xff')]
+        headers, result = resp.split(b'\r\n\r\n')
+        msgs = [result.strip(b'\x00\xff')]
         cnt = 10
         while cnt:
-            msgs.append(sock.recv(20).strip('\x00\xff'))
+            msgs.append(sock.recv(20).strip(b'\x00\xff'))
             cnt -= 1
         # Last item in msgs is an empty string
-        self.assertEqual(msgs[:-1], ['msg %d' % i for i in range(10)])
+        self.assertEqual(msgs[:-1], [six.b('msg %d' % i) for i in range(10)])
 
     def test_getting_messages_from_websocket_76(self):
         connect = [
@@ -293,16 +289,16 @@ class TestWebSocket(_TestBase):
         sock = eventlet.connect(
             ('localhost', self.port))
 
-        sock.sendall('\r\n'.join(connect) + '\r\n\r\n^n:ds[4U')
+        sock.sendall(six.b('\r\n'.join(connect) + '\r\n\r\n^n:ds[4U'))
         resp = sock.recv(1024)
-        headers, result = resp.split('\r\n\r\n')
-        msgs = [result[16:].strip('\x00\xff')]
+        headers, result = resp.split(b'\r\n\r\n')
+        msgs = [result[16:].strip(b'\x00\xff')]
         cnt = 10
         while cnt:
-            msgs.append(sock.recv(20).strip('\x00\xff'))
+            msgs.append(sock.recv(20).strip(b'\x00\xff'))
             cnt -= 1
         # Last item in msgs is an empty string
-        self.assertEqual(msgs[:-1], ['msg %d' % i for i in range(10)])
+        self.assertEqual(msgs[:-1], [six.b('msg %d' % i) for i in range(10)])
 
     def test_breaking_the_connection_75(self):
         error_detected = [False]
@@ -330,7 +326,7 @@ class TestWebSocket(_TestBase):
         ]
         sock = eventlet.connect(
             ('localhost', self.port))
-        sock.sendall('\r\n'.join(connect) + '\r\n\r\n')
+        sock.sendall(six.b('\r\n'.join(connect) + '\r\n\r\n'))
         sock.recv(1024)  # get the headers
         sock.close()  # close while the app is running
         done_with_request.wait()
@@ -364,7 +360,7 @@ class TestWebSocket(_TestBase):
         ]
         sock = eventlet.connect(
             ('localhost', self.port))
-        sock.sendall('\r\n'.join(connect) + '\r\n\r\n^n:ds[4U')
+        sock.sendall(six.b('\r\n'.join(connect) + '\r\n\r\n^n:ds[4U'))
         sock.recv(1024)  # get the headers
         sock.close()  # close while the app is running
         done_with_request.wait()
@@ -398,9 +394,9 @@ class TestWebSocket(_TestBase):
         ]
         sock = eventlet.connect(
             ('localhost', self.port))
-        sock.sendall('\r\n'.join(connect) + '\r\n\r\n^n:ds[4U')
+        sock.sendall(six.b('\r\n'.join(connect) + '\r\n\r\n^n:ds[4U'))
         sock.recv(1024)  # get the headers
-        sock.sendall('\xff\x00')  # "Close the connection" packet.
+        sock.sendall(b'\xff\x00')  # "Close the connection" packet.
         done_with_request.wait()
         assert not error_detected[0]
 
@@ -432,9 +428,9 @@ class TestWebSocket(_TestBase):
         ]
         sock = eventlet.connect(
             ('localhost', self.port))
-        sock.sendall('\r\n'.join(connect) + '\r\n\r\n^n:ds[4U')
+        sock.sendall(six.b('\r\n'.join(connect) + '\r\n\r\n^n:ds[4U'))
         sock.recv(1024)  # get the headers
-        sock.sendall('\xef\x00')  # Weird packet.
+        sock.sendall(b'\xef\x00')  # Weird packet.
         done_with_request.wait()
         assert error_detected[0]
 
@@ -452,11 +448,11 @@ class TestWebSocket(_TestBase):
         sock = eventlet.connect(
             ('localhost', self.port))
 
-        sock.sendall('\r\n'.join(connect) + '\r\n\r\n^n:ds[4U')
+        sock.sendall(six.b('\r\n'.join(connect) + '\r\n\r\n^n:ds[4U'))
         resp = sock.recv(1024)
-        headers, result = resp.split('\r\n\r\n')
+        headers, result = resp.split(b'\r\n\r\n')
         # The remote server should have immediately closed the connection.
-        self.assertEqual(result[16:], '\xff\x00')
+        self.assertEqual(result[16:], b'\xff\x00')
 
     def test_app_socket_errors_75(self):
         error_detected = [False]
@@ -484,7 +480,7 @@ class TestWebSocket(_TestBase):
         ]
         sock = eventlet.connect(
             ('localhost', self.port))
-        sock.sendall('\r\n'.join(connect) + '\r\n\r\n')
+        sock.sendall(six.b('\r\n'.join(connect) + '\r\n\r\n'))
         sock.recv(1024)
         done_with_request.wait()
         assert error_detected[0]
@@ -517,7 +513,7 @@ class TestWebSocket(_TestBase):
         ]
         sock = eventlet.connect(
             ('localhost', self.port))
-        sock.sendall('\r\n'.join(connect) + '\r\n\r\n^n:ds[4U')
+        sock.sendall(six.b('\r\n'.join(connect) + '\r\n\r\n^n:ds[4U'))
         sock.recv(1024)
         done_with_request.wait()
         assert error_detected[0]
@@ -547,55 +543,25 @@ class TestWebSocketSSL(_TestBase):
         sock = eventlet.wrap_ssl(eventlet.connect(
             ('localhost', self.port)))
 
-        sock.sendall('\r\n'.join(connect) + '\r\n\r\n^n:ds[4U')
-        first_resp = sock.recv(1024)
+        sock.sendall(six.b('\r\n'.join(connect) + '\r\n\r\n^n:ds[4U'))
+        first_resp = b''
+        while b'\r\n\r\n' not in first_resp:
+            first_resp += sock.recv()
+            print('resp now:')
+            print(first_resp)
         # make sure it sets the wss: protocol on the location header
-        loc_line = [x for x in first_resp.split("\r\n")
-                    if x.lower().startswith('sec-websocket-location')][0]
-        self.assert_("wss://localhost" in loc_line,
+        loc_line = [x for x in first_resp.split(b"\r\n")
+                    if x.lower().startswith(b'sec-websocket-location')][0]
+        self.assert_(b"wss://localhost" in loc_line,
                      "Expecting wss protocol in location: %s" % loc_line)
-        sock.sendall('\x00hello\xFF')
+        sock.sendall(b'\x00hello\xFF')
         result = sock.recv(1024)
-        self.assertEqual(result, '\x00hello\xff')
-        sock.sendall('\x00start')
+        self.assertEqual(result, b'\x00hello\xff')
+        sock.sendall(b'\x00start')
         eventlet.sleep(0.001)
-        sock.sendall(' end\xff')
+        sock.sendall(b' end\xff')
         result = sock.recv(1024)
-        self.assertEqual(result, '\x00start end\xff')
+        self.assertEqual(result, b'\x00start end\xff')
         greenio.shutdown_safe(sock)
         sock.close()
         eventlet.sleep(0.01)
-
-
-class TestWebSocketObject(LimitedTestCase):
-
-    def setUp(self):
-        self.mock_socket = s = mock.Mock()
-        self.environ = env = dict(HTTP_ORIGIN='http://localhost', HTTP_WEBSOCKET_PROTOCOL='ws',
-                                  PATH_INFO='test')
-
-        self.test_ws = WebSocket(s, env)
-        super(TestWebSocketObject, self).setUp()
-
-    def test_recieve(self):
-        ws = self.test_ws
-        ws.socket.recv.return_value = '\x00hello\xFF'
-        self.assertEqual(ws.wait(), 'hello')
-        self.assertEqual(ws._buf, '')
-        self.assertEqual(len(ws._msgs), 0)
-        ws.socket.recv.return_value = ''
-        self.assertEqual(ws.wait(), None)
-        self.assertEqual(ws._buf, '')
-        self.assertEqual(len(ws._msgs), 0)
-
-    def test_send_to_ws(self):
-        ws = self.test_ws
-        ws.send(u'hello')
-        assert ws.socket.sendall.called_with("\x00hello\xFF")
-        ws.send(10)
-        assert ws.socket.sendall.called_with("\x0010\xFF")
-
-    def test_close_ws(self):
-        ws = self.test_ws
-        ws.close()
-        assert ws.socket.shutdown.called_with(True)
diff --git a/tests/wsgi_test.py b/tests/wsgi_test.py
index a966038..5fcfa29 100644
--- a/tests/wsgi_test.py
+++ b/tests/wsgi_test.py
@@ -145,6 +145,17 @@ hello world
 """
 
 
+def recvall(socket_):
+    result = b''
+    while True:
+        chunk = socket_.recv()
+        result += chunk
+        if chunk == b'':
+            break
+
+    return result
+
+
 class ConnectionClosed(Exception):
     pass
 
@@ -449,8 +460,8 @@ class TestHttpd(_TestBase):
         sock.write(
             b'POST /foo HTTP/1.1\r\nHost: localhost\r\n'
             b'Connection: close\r\nContent-length:3\r\n\r\nabc')
-        result = sock.read(8192)
-        self.assertEqual(result[-3:], b'abc')
+        result = recvall(sock)
+        assert result.endswith(b'abc')
 
     @tests.skip_if_no_ssl
     def test_013_empty_return(self):
@@ -469,8 +480,8 @@ class TestHttpd(_TestBase):
         sock = eventlet.connect(('localhost', server_sock.getsockname()[1]))
         sock = eventlet.wrap_ssl(sock)
         sock.write(b'GET /foo HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n')
-        result = sock.read(8192)
-        self.assertEqual(result[-4:], b'\r\n\r\n')
+        result = recvall(sock)
+        assert result[-4:] == b'\r\n\r\n'
 
     def test_014_chunked_post(self):
         self.site.application = chunked_post
@@ -964,9 +975,9 @@ class TestHttpd(_TestBase):
             try:
                 client = ssl.wrap_socket(eventlet.connect(('localhost', port)))
                 client.write(b'GET / HTTP/1.0\r\nHost: localhost\r\n\r\n')
-                result = client.read()
-                assert result.startswith('HTTP'), result
-                assert result.endswith('hello world')
+                result = recvall(client)
+                assert result.startswith(b'HTTP'), result
+                assert result.endswith(b'hello world')
             except ImportError:
                 pass  # TODO(openssl): should test with OpenSSL
             greenthread.kill(g)
@@ -1163,8 +1174,8 @@ class TestHttpd(_TestBase):
     def test_unicode_raises_error(self):
         def wsgi_app(environ, start_response):
             start_response("200 OK", [])
-            yield u"oh hai"
-            yield u"non-encodable unicode: \u0230"
+            yield b"oh hai"
+            yield u"any unicode string should raise an error"
         self.site.application = wsgi_app
         sock = eventlet.connect(('localhost', self.port))
         fd = sock.makefile('rwb')
@@ -1173,7 +1184,7 @@ class TestHttpd(_TestBase):
         result = read_http(sock)
         self.assertEqual(result.status, 'HTTP/1.1 500 Internal Server Error')
         self.assertEqual(result.headers_lower['connection'], 'close')
-        assert b'unicode' in result.body
+        assert b'bytestring' in result.body
 
     def test_path_info_decoding(self):
         def wsgi_app(environ, start_response):