Copy a subset of test.support from CPython 3.5 to no more depend on the test

module to run the asyncio test suite. The test module is rarely installed.

8 files changed, 332 insertions, 16 deletions

diff --git a/asyncio/test_support.py b/asyncio/test_support.py
new file mode 100644
index 0000000..59b3e8c
--- /dev/null
+++ b/asyncio/test_support.py
@@ -0,0 +1,290 @@
+# Subset of test.support from CPython 3.5, just what we need to run asyncio
+# test suite. The cde is copied from CPython 3.5 to not depend on the test
+# module because it is rarely installed.
+
+# Ignore symbol TEST_HOME_DIR: test_events works without it
+
+import functools
+import gc
+import os
+import platform
+import re
+import socket
+import subprocess
+import sys
+
+# A constant likely larger than the underlying OS pipe buffer size, to
+# make writes blocking.
+# Windows limit seems to be around 512 B, and many Unix kernels have a
+# 64 KiB pipe buffer size or 16 * PAGE_SIZE: take a few megs to be sure.
+# (see issue #17835 for a discussion of this number).
+PIPE_MAX_SIZE = 4 * 1024 * 1024 + 1
+
+def strip_python_stderr(stderr):
+    """Strip the stderr of a Python process from potential debug output
+    emitted by the interpreter.
+
+    This will typically be run on the result of the communicate() method
+    of a subprocess.Popen object.
+    """
+    stderr = re.sub(br"\[\d+ refs, \d+ blocks\]\r?\n?", b"", stderr).strip()
+    return stderr
+
+
+# Executing the interpreter in a subprocess
+def _assert_python(expected_success, *args, **env_vars):
+    if '__isolated' in env_vars:
+        isolated = env_vars.pop('__isolated')
+    else:
+        isolated = not env_vars
+    cmd_line = [sys.executable, '-X', 'faulthandler']
+    if isolated:
+        # isolated mode: ignore Python environment variables, ignore user
+        # site-packages, and don't add the current directory to sys.path
+        cmd_line.append('-I')
+    elif not env_vars:
+        # ignore Python environment variables
+        cmd_line.append('-E')
+    # Need to preserve the original environment, for in-place testing of
+    # shared library builds.
+    env = os.environ.copy()
+    # But a special flag that can be set to override -- in this case, the
+    # caller is responsible to pass the full environment.
+    if env_vars.pop('__cleanenv', None):
+        env = {}
+    env.update(env_vars)
+    cmd_line.extend(args)
+    p = subprocess.Popen(cmd_line, stdin=subprocess.PIPE,
+                         stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                         env=env)
+    try:
+        out, err = p.communicate()
+    finally:
+        subprocess._cleanup()
+        p.stdout.close()
+        p.stderr.close()
+    rc = p.returncode
+    err = strip_python_stderr(err)
+    if (rc and expected_success) or (not rc and not expected_success):
+        raise AssertionError(
+            "Process return code is %d, "
+            "stderr follows:\n%s" % (rc, err.decode('ascii', 'ignore')))
+    return rc, out, err
+
+
+def assert_python_ok(*args, **env_vars):
+    """
+    Assert that running the interpreter with `args` and optional environment
+    variables `env_vars` succeeds (rc == 0) and return a (return code, stdout,
+    stderr) tuple.
+
+    If the __cleanenv keyword is set, env_vars is used a fresh environment.
+
+    Python is started in isolated mode (command line option -I),
+    except if the __isolated keyword is set to False.
+    """
+    return _assert_python(True, *args, **env_vars)
+
+
+is_jython = sys.platform.startswith('java')
+
+def gc_collect():
+    """Force as many objects as possible to be collected.
+
+    In non-CPython implementations of Python, this is needed because timely
+    deallocation is not guaranteed by the garbage collector.  (Even in CPython
+    this can be the case in case of reference cycles.)  This means that __del__
+    methods may be called later than expected and weakrefs may remain alive for
+    longer than expected.  This function tries its best to force all garbage
+    objects to disappear.
+    """
+    gc.collect()
+    if is_jython:
+        time.sleep(0.1)
+    gc.collect()
+    gc.collect()
+
+
+HOST = "127.0.0.1"
+HOSTv6 = "::1"
+
+
+def _is_ipv6_enabled():
+    """Check whether IPv6 is enabled on this host."""
+    if socket.has_ipv6:
+        sock = None
+        try:
+            sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+            sock.bind((HOSTv6, 0))
+            return True
+        except OSError:
+            pass
+        finally:
+            if sock:
+                sock.close()
+    return False
+
+IPV6_ENABLED = _is_ipv6_enabled()
+
+
+def find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM):
+    """Returns an unused port that should be suitable for binding.  This is
+    achieved by creating a temporary socket with the same family and type as
+    the 'sock' parameter (default is AF_INET, SOCK_STREAM), and binding it to
+    the specified host address (defaults to 0.0.0.0) with the port set to 0,
+    eliciting an unused ephemeral port from the OS.  The temporary socket is
+    then closed and deleted, and the ephemeral port is returned.
+
+    Either this method or bind_port() should be used for any tests where a
+    server socket needs to be bound to a particular port for the duration of
+    the test.  Which one to use depends on whether the calling code is creating
+    a python socket, or if an unused port needs to be provided in a constructor
+    or passed to an external program (i.e. the -accept argument to openssl's
+    s_server mode).  Always prefer bind_port() over find_unused_port() where
+    possible.  Hard coded ports should *NEVER* be used.  As soon as a server
+    socket is bound to a hard coded port, the ability to run multiple instances
+    of the test simultaneously on the same host is compromised, which makes the
+    test a ticking time bomb in a buildbot environment. On Unix buildbots, this
+    may simply manifest as a failed test, which can be recovered from without
+    intervention in most cases, but on Windows, the entire python process can
+    completely and utterly wedge, requiring someone to log in to the buildbot
+    and manually kill the affected process.
+
+    (This is easy to reproduce on Windows, unfortunately, and can be traced to
+    the SO_REUSEADDR socket option having different semantics on Windows versus
+    Unix/Linux.  On Unix, you can't have two AF_INET SOCK_STREAM sockets bind,
+    listen and then accept connections on identical host/ports.  An EADDRINUSE
+    OSError will be raised at some point (depending on the platform and
+    the order bind and listen were called on each socket).
+
+    However, on Windows, if SO_REUSEADDR is set on the sockets, no EADDRINUSE
+    will ever be raised when attempting to bind two identical host/ports. When
+    accept() is called on each socket, the second caller's process will steal
+    the port from the first caller, leaving them both in an awkwardly wedged
+    state where they'll no longer respond to any signals or graceful kills, and
+    must be forcibly killed via OpenProcess()/TerminateProcess().
+
+    The solution on Windows is to use the SO_EXCLUSIVEADDRUSE socket option
+    instead of SO_REUSEADDR, which effectively affords the same semantics as
+    SO_REUSEADDR on Unix.  Given the propensity of Unix developers in the Open
+    Source world compared to Windows ones, this is a common mistake.  A quick
+    look over OpenSSL's 0.9.8g source shows that they use SO_REUSEADDR when
+    openssl.exe is called with the 's_server' option, for example. See
+    http://bugs.python.org/issue2550 for more info.  The following site also
+    has a very thorough description about the implications of both REUSEADDR
+    and EXCLUSIVEADDRUSE on Windows:
+    http://msdn2.microsoft.com/en-us/library/ms740621(VS.85).aspx)
+
+    XXX: although this approach is a vast improvement on previous attempts to
+    elicit unused ports, it rests heavily on the assumption that the ephemeral
+    port returned to us by the OS won't immediately be dished back out to some
+    other process when we close and delete our temporary socket but before our
+    calling code has a chance to bind the returned port.  We can deal with this
+    issue if/when we come across it.
+    """
+
+    tempsock = socket.socket(family, socktype)
+    port = bind_port(tempsock)
+    tempsock.close()
+    del tempsock
+    return port
+
+def bind_port(sock, host=HOST):
+    """Bind the socket to a free port and return the port number.  Relies on
+    ephemeral ports in order to ensure we are using an unbound port.  This is
+    important as many tests may be running simultaneously, especially in a
+    buildbot environment.  This method raises an exception if the sock.family
+    is AF_INET and sock.type is SOCK_STREAM, *and* the socket has SO_REUSEADDR
+    or SO_REUSEPORT set on it.  Tests should *never* set these socket options
+    for TCP/IP sockets.  The only case for setting these options is testing
+    multicasting via multiple UDP sockets.
+
+    Additionally, if the SO_EXCLUSIVEADDRUSE socket option is available (i.e.
+    on Windows), it will be set on the socket.  This will prevent anyone else
+    from bind()'ing to our host/port for the duration of the test.
+    """
+
+    if sock.family == socket.AF_INET and sock.type == socket.SOCK_STREAM:
+        if hasattr(socket, 'SO_REUSEADDR'):
+            if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) == 1:
+                raise TestFailed("tests should never set the SO_REUSEADDR "   \
+                                 "socket option on TCP/IP sockets!")
+        if hasattr(socket, 'SO_REUSEPORT'):
+            try:
+                if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT) == 1:
+                    raise TestFailed("tests should never set the SO_REUSEPORT "   \
+                                     "socket option on TCP/IP sockets!")
+            except OSError:
+                # Python's socket module was compiled using modern headers
+                # thus defining SO_REUSEPORT but this process is running
+                # under an older kernel that does not support SO_REUSEPORT.
+                pass
+        if hasattr(socket, 'SO_EXCLUSIVEADDRUSE'):
+            sock.setsockopt(socket.SOL_SOCKET, socket.SO_EXCLUSIVEADDRUSE, 1)
+
+    sock.bind((host, 0))
+    port = sock.getsockname()[1]
+    return port
+
+def requires_mac_ver(*min_version):
+    """Decorator raising SkipTest if the OS is Mac OS X and the OS X
+    version if less than min_version.
+
+    For example, @requires_mac_ver(10, 5) raises SkipTest if the OS X version
+    is lesser than 10.5.
+    """
+    def decorator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kw):
+            if sys.platform == 'darwin':
+                version_txt = platform.mac_ver()[0]
+                try:
+                    version = tuple(map(int, version_txt.split('.')))
+                except ValueError:
+                    pass
+                else:
+                    if version < min_version:
+                        min_version_txt = '.'.join(map(str, min_version))
+                        raise unittest.SkipTest(
+                            "Mac OS X %s or higher required, not %s"
+                            % (min_version_txt, version_txt))
+            return func(*args, **kw)
+        wrapper.min_version = min_version
+        return wrapper
+    return decorator
+
+def _requires_unix_version(sysname, min_version):
+    """Decorator raising SkipTest if the OS is `sysname` and the version is less
+    than `min_version`.
+
+    For example, @_requires_unix_version('FreeBSD', (7, 2)) raises SkipTest if
+    the FreeBSD version is less than 7.2.
+    """
+    def decorator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kw):
+            if platform.system() == sysname:
+                version_txt = platform.release().split('-', 1)[0]
+                try:
+                    version = tuple(map(int, version_txt.split('.')))
+                except ValueError:
+                    pass
+                else:
+                    if version < min_version:
+                        min_version_txt = '.'.join(map(str, min_version))
+                        raise unittest.SkipTest(
+                            "%s version %s or higher required, not %s"
+                            % (sysname, min_version_txt, version_txt))
+            return func(*args, **kw)
+        wrapper.min_version = min_version
+        return wrapper
+    return decorator
+
+def requires_freebsd_version(*min_version):
+    """Decorator raising SkipTest if the OS is FreeBSD and the FreeBSD version is
+    less than `min_version`.
+
+    For example, @requires_freebsd_version(7, 2) raises SkipTest if the FreeBSD
+    version is less than 7.2.
+    """
+    return _requires_unix_version('FreeBSD', min_version)
diff --git a/tests/test_base_events.py b/tests/test_base_events.py
index db9d732..4e5b6ca 100644
--- a/tests/test_base_events.py
+++ b/tests/test_base_events.py
@@ -8,13 +8,17 @@ import sys
 import time
 import unittest
 from unittest import mock
-from test.script_helper import assert_python_ok
-from test.support import IPV6_ENABLED, gc_collect
 
 import asyncio
 from asyncio import base_events
 from asyncio import constants
 from asyncio import test_utils
+try:
+    from test.script_helper import assert_python_ok
+    from test import support
+except ImportError:
+    from asyncio import test_support as support
+    from asyncio.test_support import assert_python_ok
 
 
 MOCK_ANY = mock.ANY
@@ -634,7 +638,7 @@ class BaseEventLoopTests(test_utils.TestCase):
         except KeyboardInterrupt:
             pass
         self.loop.close()
-        gc_collect()
+        support.gc_collect()
 
         self.assertFalse(self.loop.call_exception_handler.called)
 
@@ -1066,7 +1070,7 @@ class BaseEventLoopWithSelectorTests(test_utils.TestCase):
         self.assertRaises(
             OSError, self.loop.run_until_complete, coro)
 
-    @unittest.skipUnless(IPV6_ENABLED, 'IPv6 not supported or enabled')
+    @unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 not supported or enabled')
     def test_create_datagram_endpoint_no_matching_family(self):
         coro = self.loop.create_datagram_endpoint(
             asyncio.DatagramProtocol,
diff --git a/tests/test_events.py b/tests/test_events.py
index d7e2f34..0630292 100644
--- a/tests/test_events.py
+++ b/tests/test_events.py
@@ -20,13 +20,16 @@ import errno
 import unittest
 from unittest import mock
 import weakref
-from test import support  # find_unused_port, IPV6_ENABLED, TEST_HOME_DIR
 
 
 import asyncio
 from asyncio import proactor_events
 from asyncio import selector_events
 from asyncio import test_utils
+try:
+    from test import support  # find_unused_port, IPV6_ENABLED, TEST_HOME_DIR
+except ImportError:
+    from asyncio import test_support as support
 
 
 def data_file(filename):
diff --git a/tests/test_futures.py b/tests/test_futures.py
index 371d351..f9c3ad2 100644
--- a/tests/test_futures.py
+++ b/tests/test_futures.py
@@ -5,11 +5,14 @@ import re
 import sys
 import threading
 import unittest
-from test import support
 from unittest import mock
 
 import asyncio
 from asyncio import test_utils
+try:
+    from test import support   # gc_collect
+except ImportError:
+    from asyncio import test_support as support
 
 
 def _fakefunc(f):
diff --git a/tests/test_subprocess.py b/tests/test_subprocess.py
index 5c0a2c8..08c8ac2 100644
--- a/tests/test_subprocess.py
+++ b/tests/test_subprocess.py
@@ -1,13 +1,17 @@
-from asyncio import subprocess
-from asyncio import test_utils
-import asyncio
 import signal
 import sys
 import unittest
 from unittest import mock
-from test import support
+
+import asyncio
+from asyncio import subprocess
+from asyncio import test_utils
 if sys.platform != 'win32':
     from asyncio import unix_events
+try:
+    from test import support   # PIPE_MAX_SIZE
+except ImportError:
+    from asyncio import test_support as support
 
 # Program blocking
 PROGRAM_BLOCKED = [sys.executable, '-c', 'import time; time.sleep(3600)']
diff --git a/tests/test_tasks.py b/tests/test_tasks.py
index 770f218..25b21dc 100644
--- a/tests/test_tasks.py
+++ b/tests/test_tasks.py
@@ -6,9 +6,13 @@ import sys
 import types
 import unittest
 import weakref
-from test import support
-from test.script_helper import assert_python_ok
 from unittest import mock
+try:
+    from test import support   # gc_collect
+    from test.script_helper import assert_python_ok
+except ImportError:
+    from asyncio import test_support as support
+    from asyncio.test_support import assert_python_ok
 
 import asyncio
 from asyncio import coroutines
diff --git a/tests/test_windows_utils.py b/tests/test_windows_utils.py
index 3e7a211..b957949 100644
--- a/tests/test_windows_utils.py
+++ b/tests/test_windows_utils.py
@@ -2,11 +2,14 @@
 
 import socket
 import sys
-import test.support
 import unittest
-from test.support import IPV6_ENABLED
 from unittest import mock
 
+try:
+    from test import support  # gc_collect, IPV6_ENABLED
+except ImportError:
+    from asyncio import test_support as support
+
 if sys.platform != 'win32':
     raise unittest.SkipTest('Windows only')
 
@@ -28,7 +31,7 @@ class WinsocketpairTests(unittest.TestCase):
         ssock, csock = windows_utils.socketpair()
         self.check_winsocketpair(ssock, csock)
 
-    @unittest.skipUnless(IPV6_ENABLED, 'IPv6 not supported or enabled')
+    @unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 not supported or enabled')
     def test_winsocketpair_ipv6(self):
         ssock, csock = windows_utils.socketpair(family=socket.AF_INET6)
         self.check_winsocketpair(ssock, csock)
@@ -114,7 +117,7 @@ class PipeTests(unittest.TestCase):
 
         # check garbage collection of p closes handle
         del p
-        test.support.gc_collect()
+        support.gc_collect()
         try:
             _winapi.CloseHandle(h)
         except OSError as e:
diff --git a/update_stdlib.sh b/update_stdlib.sh
index bb6251a..0cdbb1b 100755
--- a/update_stdlib.sh
+++ b/update_stdlib.sh
@@ -40,6 +40,11 @@ maybe_copy()
 
 for i in `(cd asyncio && ls *.py)`
 do
+    if [ $i == test_support.py ]
+    then
+        continue
+    fi
+
     if [ $i == selectors.py ]
     then
         if [ "`(cd $CPYTHON; hg branch)`" == "3.4" ]