refactor(_async.py): introduce modern coroutine syntax

Python 3.5 introduced the await and async def keywords. This change imports the coroutines from two different places, depending on the running Python version.

4 files changed, 296 insertions, 135 deletions

diff --git a/pexpect/_async.py b/pexpect/_async.py
index dfbfeef..261720c 100644
--- a/pexpect/_async.py
+++ b/pexpect/_async.py
@@ -1,103 +1,28 @@
-import asyncio
-import errno
-import signal
-
-from pexpect import EOF
-
-@asyncio.coroutine
-def expect_async(expecter, timeout=None):
-    # First process data that was previously read - if it maches, we don't need
-    # async stuff.
-    idx = expecter.existing_data()
-    if idx is not None:
-        return idx
-    if not expecter.spawn.async_pw_transport:
-        pw = PatternWaiter()
-        pw.set_expecter(expecter)
-        transport, pw = yield from asyncio.get_event_loop()\
-            .connect_read_pipe(lambda: pw, expecter.spawn)
-        expecter.spawn.async_pw_transport = pw, transport
-    else:
-        pw, transport = expecter.spawn.async_pw_transport
-        pw.set_expecter(expecter)
-        transport.resume_reading()
-    try:
-        return (yield from asyncio.wait_for(pw.fut, timeout))
-    except asyncio.TimeoutError as e:
-        transport.pause_reading()
-        return expecter.timeout(e)
-
-@asyncio.coroutine
-def repl_run_command_async(repl, cmdlines, timeout=-1):
-    res = []
-    repl.child.sendline(cmdlines[0])
-    for line in cmdlines[1:]:
-        yield from repl._expect_prompt(timeout=timeout, async_=True)
-        res.append(repl.child.before)
-        repl.child.sendline(line)
-
-    # Command was fully submitted, now wait for the next prompt
-    prompt_idx = yield from repl._expect_prompt(timeout=timeout, async_=True)
-    if prompt_idx == 1:
-        # We got the continuation prompt - command was incomplete
-        repl.child.kill(signal.SIGINT)
-        yield from repl._expect_prompt(timeout=1, async_=True)
-        raise ValueError("Continuation prompt found - input was incomplete:")
-    return u''.join(res + [repl.child.before])
-
-class PatternWaiter(asyncio.Protocol):
-    transport = None
-
-    def set_expecter(self, expecter):
-        self.expecter = expecter
-        self.fut = asyncio.Future()
-
-    def found(self, result):
-        if not self.fut.done():
-            self.fut.set_result(result)
-            self.transport.pause_reading()
-
-    def error(self, exc):
-        if not self.fut.done():
-            self.fut.set_exception(exc)
-            self.transport.pause_reading()
-
-    def connection_made(self, transport):
-        self.transport = transport
-
-    def data_received(self, data):
-        spawn = self.expecter.spawn
-        s = spawn._decoder.decode(data)
-        spawn._log(s, 'read')
-
-        if self.fut.done():
-            spawn._before.write(s)
-            spawn._buffer.write(s)
-            return
-
-        try:
-            index = self.expecter.new_data(s)
-            if index is not None:
-                # Found a match
-                self.found(index)
-        except Exception as e:
-            self.expecter.errored()
-            self.error(e)
-
-    def eof_received(self):
-        # N.B. If this gets called, async will close the pipe (the spawn object)
-        # for us
-        try:
-            self.expecter.spawn.flag_eof = True
-            index = self.expecter.eof()
-        except EOF as e:
-            self.error(e)
-        else:
-            self.found(index)
-
-    def connection_lost(self, exc):
-        if isinstance(exc, OSError) and exc.errno == errno.EIO:
-            # We may get here without eof_received being called, e.g on Linux
-            self.eof_received()
-        elif exc is not None:
-            self.error(exc)
+"""Facade that provides coroutines implementation pertinent to running Py version.
+
+Python 3.5 introduced the async def/await syntax keyword.
+With later versions coroutines and methods to get the running asyncio loop are
+being deprecated, not supported anymore.
+
+For Python versions later than 3.6, coroutines and objects that are defined via
+``async def``/``await`` keywords are imported.
+
+Here the code is just imported, to provide the same interface to older code.
+"""
+# pylint: disable=unused-import
+# flake8: noqa: F401
+from sys import version_info as py_version_info
+
+# this assumes async def/await are more stable
+if py_version_info >= (3, 6):
+    from pexpect._async_w_await import (
+        PatternWaiter,
+        expect_async,
+        repl_run_command_async,
+    )
+else:
+    from pexpect._async_pre_await import (
+        PatternWaiter,
+        expect_async,
+        repl_run_command_async,
+    )
diff --git a/pexpect/_async_pre_await.py b/pexpect/_async_pre_await.py
new file mode 100644
index 0000000..81ece1b
--- /dev/null
+++ b/pexpect/_async_pre_await.py
@@ -0,0 +1,111 @@
+"""Implementation of coroutines without using ``async def``/``await`` keywords.
+
+``@asyncio.coroutine`` and ``yield from`` are  used here instead.
+"""
+import asyncio
+import errno
+import signal
+
+from pexpect import EOF
+
+
+@asyncio.coroutine
+def expect_async(expecter, timeout=None):
+    # First process data that was previously read - if it maches, we don't need
+    # async stuff.
+    idx = expecter.existing_data()
+    if idx is not None:
+        return idx
+    if not expecter.spawn.async_pw_transport:
+        pw = PatternWaiter()
+        pw.set_expecter(expecter)
+        transport, pw = yield from asyncio.get_event_loop().connect_read_pipe(
+            lambda: pw, expecter.spawn
+        )
+        expecter.spawn.async_pw_transport = pw, transport
+    else:
+        pw, transport = expecter.spawn.async_pw_transport
+        pw.set_expecter(expecter)
+        transport.resume_reading()
+    try:
+        return (yield from asyncio.wait_for(pw.fut, timeout))
+    except asyncio.TimeoutError as e:
+        transport.pause_reading()
+        return expecter.timeout(e)
+
+
+@asyncio.coroutine
+def repl_run_command_async(repl, cmdlines, timeout=-1):
+    res = []
+    repl.child.sendline(cmdlines[0])
+    for line in cmdlines[1:]:
+        yield from repl._expect_prompt(timeout=timeout, async_=True)
+        res.append(repl.child.before)
+        repl.child.sendline(line)
+
+    # Command was fully submitted, now wait for the next prompt
+    prompt_idx = yield from repl._expect_prompt(timeout=timeout, async_=True)
+    if prompt_idx == 1:
+        # We got the continuation prompt - command was incomplete
+        repl.child.kill(signal.SIGINT)
+        yield from repl._expect_prompt(timeout=1, async_=True)
+        raise ValueError("Continuation prompt found - input was incomplete:")
+    return "".join(res + [repl.child.before])
+
+
+class PatternWaiter(asyncio.Protocol):
+    transport = None
+
+    def set_expecter(self, expecter):
+        self.expecter = expecter
+        self.fut = asyncio.Future()
+
+    def found(self, result):
+        if not self.fut.done():
+            self.fut.set_result(result)
+            self.transport.pause_reading()
+
+    def error(self, exc):
+        if not self.fut.done():
+            self.fut.set_exception(exc)
+            self.transport.pause_reading()
+
+    def connection_made(self, transport):
+        self.transport = transport
+
+    def data_received(self, data):
+        spawn = self.expecter.spawn
+        s = spawn._decoder.decode(data)
+        spawn._log(s, "read")
+
+        if self.fut.done():
+            spawn._before.write(s)
+            spawn._buffer.write(s)
+            return
+
+        try:
+            index = self.expecter.new_data(s)
+            if index is not None:
+                # Found a match
+                self.found(index)
+        except Exception as e:
+            self.expecter.errored()
+            self.error(e)
+
+    def eof_received(self):
+        # N.B. If this gets called, async will close the pipe (the spawn object)
+        # for us
+        try:
+            self.expecter.spawn.flag_eof = True
+            index = self.expecter.eof()
+        except EOF as e:
+            self.error(e)
+        else:
+            self.found(index)
+
+    def connection_lost(self, exc):
+        if isinstance(exc, OSError) and exc.errno == errno.EIO:
+            # We may get here without eof_received being called, e.g on Linux
+            self.eof_received()
+        elif exc is not None:
+            self.error(exc)
diff --git a/pexpect/_async_w_await.py b/pexpect/_async_w_await.py
new file mode 100644
index 0000000..59cb1ef
--- /dev/null
+++ b/pexpect/_async_w_await.py
@@ -0,0 +1,118 @@
+"""Implementation of coroutines using ``async def``/``await`` keywords.
+
+These keywords replaced ``@asyncio.coroutine`` and ``yield from`` from
+Python 3.5 onwards.
+"""
+import asyncio
+import errno
+import signal
+from sys import version_info as py_version_info
+
+from pexpect import EOF
+
+if py_version_info >= (3, 7):
+    # get_running_loop, new in 3.7, is preferred to get_event_loop
+    _loop_getter = asyncio.get_running_loop
+else:
+    # Deprecation warning since 3.10
+    _loop_getter = asyncio.get_event_loop
+
+
+async def expect_async(expecter, timeout=None):
+    # First process data that was previously read - if it maches, we don't need
+    # async stuff.
+    idx = expecter.existing_data()
+    if idx is not None:
+        return idx
+    if not expecter.spawn.async_pw_transport:
+        pattern_waiter = PatternWaiter()
+        pattern_waiter.set_expecter(expecter)
+        transport, pattern_waiter = await _loop_getter().connect_read_pipe(
+            lambda: pattern_waiter, expecter.spawn
+        )
+        expecter.spawn.async_pw_transport = pattern_waiter, transport
+    else:
+        pattern_waiter, transport = expecter.spawn.async_pw_transport
+        pattern_waiter.set_expecter(expecter)
+        transport.resume_reading()
+    try:
+        return await asyncio.wait_for(pattern_waiter.fut, timeout)
+    except asyncio.TimeoutError as exc:
+        transport.pause_reading()
+        return expecter.timeout(exc)
+
+
+async def repl_run_command_async(repl, cmdlines, timeout=-1):
+    res = []
+    repl.child.sendline(cmdlines[0])
+    for line in cmdlines[1:]:
+        await repl._expect_prompt(timeout=timeout, async_=True)
+        res.append(repl.child.before)
+        repl.child.sendline(line)
+
+    # Command was fully submitted, now wait for the next prompt
+    prompt_idx = await repl._expect_prompt(timeout=timeout, async_=True)
+    if prompt_idx == 1:
+        # We got the continuation prompt - command was incomplete
+        repl.child.kill(signal.SIGINT)
+        await repl._expect_prompt(timeout=1, async_=True)
+        raise ValueError("Continuation prompt found - input was incomplete:")
+    return "".join(res + [repl.child.before])
+
+
+class PatternWaiter(asyncio.Protocol):
+    transport = None
+
+    def set_expecter(self, expecter):
+        self.expecter = expecter
+        self.fut = asyncio.Future()
+
+    def found(self, result):
+        if not self.fut.done():
+            self.fut.set_result(result)
+            self.transport.pause_reading()
+
+    def error(self, exc):
+        if not self.fut.done():
+            self.fut.set_exception(exc)
+            self.transport.pause_reading()
+
+    def connection_made(self, transport):
+        self.transport = transport
+
+    def data_received(self, data):
+        spawn = self.expecter.spawn
+        s = spawn._decoder.decode(data)
+        spawn._log(s, "read")
+
+        if self.fut.done():
+            spawn._before.write(s)
+            spawn._buffer.write(s)
+            return
+
+        try:
+            index = self.expecter.new_data(s)
+            if index is not None:
+                # Found a match
+                self.found(index)
+        except Exception as exc:
+            self.expecter.errored()
+            self.error(exc)
+
+    def eof_received(self):
+        # N.B. If this gets called, async will close the pipe (the spawn object)
+        # for us
+        try:
+            self.expecter.spawn.flag_eof = True
+            index = self.expecter.eof()
+        except EOF as exc:
+            self.error(exc)
+        else:
+            self.found(index)
+
+    def connection_lost(self, exc):
+        if isinstance(exc, OSError) and exc.errno == errno.EIO:
+            # We may get here without eof_received being called, e.g on Linux
+            self.eof_received()
+        elif exc is not None:
+            self.error(exc)
diff --git a/tests/test_async.py b/tests/test_async.py
index 466d56f..dd9179a 100644
--- a/tests/test_async.py
+++ b/tests/test_async.py
@@ -4,83 +4,90 @@ except ImportError:
     asyncio = None
 
 import gc
-import sys
 import unittest
+from sys import version_info as py_version_info
 
 import pexpect
 from pexpect import replwrap
+
 from .PexpectTestCase import PexpectTestCase
 
+
 def run(coro):
-    return asyncio.get_event_loop().run_until_complete(coro)
+    if py_version_info >= (3, 7):
+        # get_running_loop, new in 3.7, is preferred to get_event_loop
+        return asyncio.get_running_loop().run_until_complete(coro)
+    else:
+        return asyncio.get_event_loop().run_until_complete(coro)
+
 
 @unittest.skipIf(asyncio is None, "Requires asyncio")
 class AsyncTests(PexpectTestCase):
     def test_simple_expect(self):
-        p = pexpect.spawn('cat')
-        p.sendline('Hello asyncio')
-        coro = p.expect(['Hello', pexpect.EOF] , async_=True)
+        p = pexpect.spawn("cat")
+        p.sendline("Hello asyncio")
+        coro = p.expect(["Hello", pexpect.EOF], async_=True)
         assert run(coro) == 0
-        print('Done')
+        print("Done")
 
     def test_timeout(self):
-        p = pexpect.spawn('cat')
-        coro = p.expect('foo', timeout=1, async_=True)
+        p = pexpect.spawn("cat")
+        coro = p.expect("foo", timeout=1, async_=True)
         with self.assertRaises(pexpect.TIMEOUT):
             run(coro)
 
-        p = pexpect.spawn('cat')
-        coro = p.expect(['foo', pexpect.TIMEOUT], timeout=1, async_=True)
+        p = pexpect.spawn("cat")
+        coro = p.expect(["foo", pexpect.TIMEOUT], timeout=1, async_=True)
         assert run(coro) == 1
 
     def test_eof(self):
-        p = pexpect.spawn('cat')
-        p.sendline('Hi')
+        p = pexpect.spawn("cat")
+        p.sendline("Hi")
         coro = p.expect(pexpect.EOF, async_=True)
         p.sendeof()
         assert run(coro) == 0
 
-        p = pexpect.spawn('cat')
+        p = pexpect.spawn("cat")
         p.sendeof()
-        coro = p.expect('Blah', async_=True)
+        coro = p.expect("Blah", async_=True)
         with self.assertRaises(pexpect.EOF):
             run(coro)
 
     def test_expect_exact(self):
-        p = pexpect.spawn('%s list100.py' % self.PYTHONBIN)
-        assert run(p.expect_exact(b'5', async_=True)) == 0
-        assert run(p.expect_exact(['wpeok', b'11'], async_=True)) == 1
-        assert run(p.expect_exact([b'foo', pexpect.EOF], async_=True)) == 1
+        p = pexpect.spawn("%s list100.py" % self.PYTHONBIN)
+        assert run(p.expect_exact(b"5", async_=True)) == 0
+        assert run(p.expect_exact(["wpeok", b"11"], async_=True)) == 1
+        assert run(p.expect_exact([b"foo", pexpect.EOF], async_=True)) == 1
 
     def test_async_utf8(self):
-        p = pexpect.spawn('%s list100.py' % self.PYTHONBIN, encoding='utf8')
-        assert run(p.expect_exact(u'5', async_=True)) == 0
-        assert run(p.expect_exact([u'wpeok', u'11'], async_=True)) == 1
-        assert run(p.expect_exact([u'foo', pexpect.EOF], async_=True)) == 1
+        p = pexpect.spawn("%s list100.py" % self.PYTHONBIN, encoding="utf8")
+        assert run(p.expect_exact("5", async_=True)) == 0
+        assert run(p.expect_exact(["wpeok", "11"], async_=True)) == 1
+        assert run(p.expect_exact(["foo", pexpect.EOF], async_=True)) == 1
 
     def test_async_and_gc(self):
-        p = pexpect.spawn('%s sleep_for.py 1' % self.PYTHONBIN, encoding='utf8')
-        assert run(p.expect_exact(u'READY', async_=True)) == 0
+        p = pexpect.spawn("%s sleep_for.py 1" % self.PYTHONBIN, encoding="utf8")
+        assert run(p.expect_exact("READY", async_=True)) == 0
         gc.collect()
-        assert run(p.expect_exact(u'END', async_=True)) == 0
+        assert run(p.expect_exact("END", async_=True)) == 0
 
     def test_async_and_sync(self):
-        p = pexpect.spawn('echo 1234', encoding='utf8', maxread=1)
-        assert run(p.expect_exact(u'1', async_=True)) == 0
-        assert p.expect_exact(u'2') == 0
-        assert run(p.expect_exact(u'3', async_=True)) == 0
+        p = pexpect.spawn("echo 1234", encoding="utf8", maxread=1)
+        assert run(p.expect_exact("1", async_=True)) == 0
+        assert p.expect_exact("2") == 0
+        assert run(p.expect_exact("3", async_=True)) == 0
 
     def test_async_replwrap(self):
         bash = replwrap.bash()
         coro = bash.run_command("time", async_=True)
         res = run(coro)
-        assert 'real' in res, res
+        assert "real" in res, res
 
     def test_async_replwrap_multiline(self):
         bash = replwrap.bash()
         coro = bash.run_command("echo '1 2\n3 4'", async_=True)
         res = run(coro)
-        self.assertEqual(res.strip().splitlines(), ['1 2', '3 4'])
+        self.assertEqual(res.strip().splitlines(), ["1 2", "3 4"])
 
         # Should raise ValueError if input is incomplete
         coro = bash.run_command("echo '5 6", async_=True)
@@ -94,4 +101,4 @@ class AsyncTests(PexpectTestCase):
         # Check that the REPL was reset (SIGINT) after the incomplete input
         coro = bash.run_command("echo '1 2\n3 4'", async_=True)
         res = run(coro)
-        self.assertEqual(res.strip().splitlines(), ['1 2', '3 4'])
+        self.assertEqual(res.strip().splitlines(), ["1 2", "3 4"])