Issue #27243: Fix __aiter__ protocol

13 files changed, 291 insertions, 32 deletions

diff --git a/Doc/glossary.rst b/Doc/glossary.rst
index 75b380b1c3..e7bcb6aecb 100644
--- a/Doc/glossary.rst
+++ b/Doc/glossary.rst
@@ -76,13 +76,12 @@ Glossary
 
    asynchronous iterable
       An object, that can be used in an :keyword:`async for` statement.
-      Must return an :term:`awaitable` from its :meth:`__aiter__` method,
-      which should in turn be resolved in an :term:`asynchronous iterator`
-      object.  Introduced by :pep:`492`.
+      Must return an :term:`asyncronous iterator` from its
+      :meth:`__aiter__` method.  Introduced by :pep:`492`.
 
    asynchronous iterator
       An object that implements :meth:`__aiter__` and :meth:`__anext__`
-      methods, that must return :term:`awaitable` objects.
+      methods.  ``__anext__`` must return an :term:`awaitable` object.
       :keyword:`async for` resolves awaitable returned from asynchronous
       iterator's :meth:`__anext__` method until it raises
       :exc:`StopAsyncIteration` exception.  Introduced by :pep:`492`.
diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst
index 8047673e19..2469422565 100644
--- a/Doc/reference/compound_stmts.rst
+++ b/Doc/reference/compound_stmts.rst
@@ -726,7 +726,7 @@ The following code::
 Is semantically equivalent to::
 
     iter = (ITER)
-    iter = await type(iter).__aiter__(iter)
+    iter = type(iter).__aiter__(iter)
     running = True
     while running:
         try:
diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst
index 3ddbd622d8..493acaaa49 100644
--- a/Doc/reference/datamodel.rst
+++ b/Doc/reference/datamodel.rst
@@ -2359,6 +2359,7 @@ generators, coroutines do not directly support iteration.
    Coroutine objects are automatically closed using the above process when
    they are about to be destroyed.
 
+.. _async-iterators:
 
 Asynchronous Iterators
 ----------------------
@@ -2371,7 +2372,7 @@ Asynchronous iterators can be used in an :keyword:`async for` statement.
 
 .. method:: object.__aiter__(self)
 
-   Must return an *awaitable* resulting in an *asynchronous iterator* object.
+   Must return an *asynchronous iterator* object.
 
 .. method:: object.__anext__(self)
 
@@ -2384,7 +2385,7 @@ An example of an asynchronous iterable object::
         async def readline(self):
             ...
 
-        async def __aiter__(self):
+        def __aiter__(self):
             return self
 
         async def __anext__(self):
@@ -2395,6 +2396,49 @@ An example of an asynchronous iterable object::
 
 .. versionadded:: 3.5
 
+.. note::
+
+   .. versionchanged:: 3.5.2
+      Starting with CPython 3.5.2, ``__aiter__`` can directly return
+      :term:`asynchronous iterators <asynchronous iterator>`.  Returning
+      an :term:`awaitable` object will result in a
+      :exc:`PendingDeprecationWarning`.
+
+      The recommended way of writing backwards compatible code in
+      CPython 3.5.x is to continue returning awaitables from
+      ``__aiter__``.  If you want to avoid the PendingDeprecationWarning
+      and keep the code backwards compatible, the following decorator
+      can be used::
+
+          import functools
+          import sys
+
+          if sys.version_info < (3, 5, 2):
+              def aiter_compat(func):
+                  @functools.wraps(func)
+                  async def wrapper(self):
+                      return func(self)
+                  return wrapper
+          else:
+              def aiter_compat(func):
+                  return func
+
+      Example::
+
+          class AsyncIterator:
+
+              @aiter_compat
+              def __aiter__(self):
+                  return self
+
+              async def __anext__(self):
+                  ...
+
+      Starting with CPython 3.6, the :exc:`PendingDeprecationWarning`
+      will be replaced with the :exc:`DeprecationWarning`.
+      In CPython 3.7, returning an awaitable from ``__aiter__`` will
+      result in a :exc:`RuntimeError`.
+
 
 Asynchronous Context Managers
 -----------------------------
diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst
index 83d5ce694c..2d7f8a4266 100644
--- a/Doc/whatsnew/3.5.rst
+++ b/Doc/whatsnew/3.5.rst
@@ -247,6 +247,19 @@ be used inside a coroutine function declared with :keyword:`async def`.
 Coroutine functions are intended to be run inside a compatible event loop,
 such as the :ref:`asyncio loop <asyncio-event-loop>`.
 
+
+.. note::
+
+   .. versionchanged:: 3.5.2
+      Starting with CPython 3.5.2, ``__aiter__`` can directly return
+      :term:`asynchronous iterators <asynchronous iterator>`.  Returning
+      an :term:`awaitable` object will result in a
+      :exc:`PendingDeprecationWarning`.
+
+      See more details in the :ref:`async-iterators` documentation
+      section.
+
+
 .. seealso::
 
    :pep:`492` -- Coroutines with async and await syntax
diff --git a/Include/genobject.h b/Include/genobject.h
index 30cb023234..1ff32a8eaf 100644
--- a/Include/genobject.h
+++ b/Include/genobject.h
@@ -54,6 +54,9 @@ typedef struct {
 PyAPI_DATA(PyTypeObject) PyCoro_Type;
 PyAPI_DATA(PyTypeObject) _PyCoroWrapper_Type;
 
+PyAPI_DATA(PyTypeObject) _PyAIterWrapper_Type;
+PyObject *_PyAIterWrapper_New(PyObject *aiter);
+
 #define PyCoro_CheckExact(op) (Py_TYPE(op) == &PyCoro_Type)
 PyObject *_PyCoro_GetAwaitableIter(PyObject *o);
 PyAPI_FUNC(PyObject *) PyCoro_New(struct _frame *,
diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py
index f89bb6f04b..fc9c9f1cc1 100644
--- a/Lib/_collections_abc.py
+++ b/Lib/_collections_abc.py
@@ -156,7 +156,7 @@ class AsyncIterable(metaclass=ABCMeta):
     __slots__ = ()
 
     @abstractmethod
-    async def __aiter__(self):
+    def __aiter__(self):
         return AsyncIterator()
 
     @classmethod
@@ -176,7 +176,7 @@ class AsyncIterator(AsyncIterable):
         """Return the next item or raise StopAsyncIteration when exhausted."""
         raise StopAsyncIteration
 
-    async def __aiter__(self):
+    def __aiter__(self):
         return self
 
     @classmethod
diff --git a/Lib/asyncio/compat.py b/Lib/asyncio/compat.py
index 660b7e7e6c..4790bb4a35 100644
--- a/Lib/asyncio/compat.py
+++ b/Lib/asyncio/compat.py
@@ -4,6 +4,7 @@ import sys
 
 PY34 = sys.version_info >= (3, 4)
 PY35 = sys.version_info >= (3, 5)
+PY352 = sys.version_info >= (3, 5, 2)
 
 
 def flatten_list_bytes(list_of_data):
diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py
index 6f465afde2..c88a87cd09 100644
--- a/Lib/asyncio/streams.py
+++ b/Lib/asyncio/streams.py
@@ -689,3 +689,9 @@ class StreamReader:
             if val == b'':
                 raise StopAsyncIteration
             return val
+
+    if compat.PY352:
+        # In Python 3.5.2 and greater, __aiter__ should return
+        # the asynchronous iterator directly.
+        def __aiter__(self):
+            return self
diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py
index 187348d48e..4f725aeab2 100644
--- a/Lib/test/test_coroutines.py
+++ b/Lib/test/test_coroutines.py
@@ -1255,8 +1255,9 @@ class CoroutineTest(unittest.TestCase):
 
         buffer = []
         async def test1():
-            async for i1, i2 in AsyncIter():
-                buffer.append(i1 + i2)
+            with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"):
+                async for i1, i2 in AsyncIter():
+                    buffer.append(i1 + i2)
 
         yielded, _ = run_async(test1())
         # Make sure that __aiter__ was called only once
@@ -1268,12 +1269,13 @@ class CoroutineTest(unittest.TestCase):
         buffer = []
         async def test2():
             nonlocal buffer
-            async for i in AsyncIter():
-                buffer.append(i[0])
-                if i[0] == 20:
-                    break
-            else:
-                buffer.append('what?')
+            with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"):
+                async for i in AsyncIter():
+                    buffer.append(i[0])
+                    if i[0] == 20:
+                        break
+                else:
+                    buffer.append('what?')
             buffer.append('end')
 
         yielded, _ = run_async(test2())
@@ -1286,12 +1288,13 @@ class CoroutineTest(unittest.TestCase):
         buffer = []
         async def test3():
             nonlocal buffer
-            async for i in AsyncIter():
-                if i[0] > 20:
-                    continue
-                buffer.append(i[0])
-            else:
-                buffer.append('what?')
+            with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"):
+                async for i in AsyncIter():
+                    if i[0] > 20:
+                        continue
+                    buffer.append(i[0])
+                else:
+                    buffer.append('what?')
             buffer.append('end')
 
         yielded, _ = run_async(test3())
@@ -1338,7 +1341,7 @@ class CoroutineTest(unittest.TestCase):
 
     def test_for_4(self):
         class I:
-            async def __aiter__(self):
+            def __aiter__(self):
                 return self
 
             def __anext__(self):
@@ -1368,8 +1371,9 @@ class CoroutineTest(unittest.TestCase):
                 return 123
 
         async def foo():
-            async for i in I():
-                print('never going to happen')
+            with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"):
+                async for i in I():
+                    print('never going to happen')
 
         with self.assertRaisesRegex(
                 TypeError,
@@ -1393,7 +1397,7 @@ class CoroutineTest(unittest.TestCase):
             def __init__(self):
                 self.i = 0
 
-            async def __aiter__(self):
+            def __aiter__(self):
                 return self
 
             async def __anext__(self):
@@ -1417,7 +1421,11 @@ class CoroutineTest(unittest.TestCase):
                     I += 1
             I += 1000
 
-        run_async(main())
+        with warnings.catch_warnings():
+            warnings.simplefilter("error")
+            # Test that __aiter__ that returns an asyncronous iterator
+            # directly does not throw any warnings.
+            run_async(main())
         self.assertEqual(I, 111011)
 
         self.assertEqual(sys.getrefcount(manager), mrefs_before)
@@ -1472,13 +1480,63 @@ class CoroutineTest(unittest.TestCase):
                 1/0
         async def foo():
             nonlocal CNT
+            with self.assertWarnsRegex(PendingDeprecationWarning, "legacy"):
+                async for i in AI():
+                    CNT += 1
+            CNT += 10
+        with self.assertRaises(ZeroDivisionError):
+            run_async(foo())
+        self.assertEqual(CNT, 0)
+
+    def test_for_8(self):
+        CNT = 0
+        class AI:
+            def __aiter__(self):
+                1/0
+        async def foo():
+            nonlocal CNT
             async for i in AI():
                 CNT += 1
             CNT += 10
         with self.assertRaises(ZeroDivisionError):
-            run_async(foo())
+            with warnings.catch_warnings():
+                warnings.simplefilter("error")
+                # Test that if __aiter__ raises an exception it propagates
+                # without any kind of warning.
+                run_async(foo())
         self.assertEqual(CNT, 0)
 
+    def test_for_9(self):
+        # Test that PendingDeprecationWarning can safely be converted into
+        # an exception (__aiter__ should not have a chance to raise
+        # a ZeroDivisionError.)
+        class AI:
+            async def __aiter__(self):
+                1/0
+        async def foo():
+            async for i in AI():
+                pass
+
+        with self.assertRaises(PendingDeprecationWarning):
+            with warnings.catch_warnings():
+                warnings.simplefilter("error")
+                run_async(foo())
+
+    def test_for_10(self):
+        # Test that PendingDeprecationWarning can safely be converted into
+        # an exception.
+        class AI:
+            async def __aiter__(self):
+                pass
+        async def foo():
+            async for i in AI():
+                pass
+
+        with self.assertRaises(PendingDeprecationWarning):
+            with warnings.catch_warnings():
+                warnings.simplefilter("error")
+                run_async(foo())
+
     def test_copy(self):
         async def func(): pass
         coro = func()
diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py
index d68cc7da7c..154e3b608c 100644
--- a/Lib/test/test_grammar.py
+++ b/Lib/test/test_grammar.py
@@ -1076,7 +1076,7 @@ class GrammarTests(unittest.TestCase):
         class Done(Exception): pass
 
         class AIter:
-            async def __aiter__(self):
+            def __aiter__(self):
                 return self
             async def __anext__(self):
                 raise StopAsyncIteration
diff --git a/Misc/NEWS b/Misc/NEWS
index e16fa196da..e7abe73908 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -130,6 +130,11 @@ Core and Builtins
 - Issue #25887: Raise a RuntimeError when a coroutine object is awaited
   more than once.
 
+- Issue #27243: Update the __aiter__ protocol: instead of returning
+  an awaitable that resolves to an asynchronous iterator, the asynchronous
+  iterator should be returned directly.  Doing the former will trigger a
+  PendingDeprecationWarning.
+
 
 Library
 -------
diff --git a/Objects/genobject.c b/Objects/genobject.c
index f74d044dcf..b3e0a46e8b 100644
--- a/Objects/genobject.c
+++ b/Objects/genobject.c
@@ -992,3 +992,97 @@ PyCoro_New(PyFrameObject *f, PyObject *name, PyObject *qualname)
 {
     return gen_new_with_qualname(&PyCoro_Type, f, name, qualname);
 }
+
+
+/* __aiter__ wrapper; see http://bugs.python.org/issue27243 for details. */
+
+typedef struct {
+    PyObject_HEAD
+    PyObject *aw_aiter;
+} PyAIterWrapper;
+
+
+static PyObject *
+aiter_wrapper_iternext(PyAIterWrapper *aw)
+{
+    PyErr_SetObject(PyExc_StopIteration, aw->aw_aiter);
+    return NULL;
+}
+
+static int
+aiter_wrapper_traverse(PyAIterWrapper *aw, visitproc visit, void *arg)
+{
+    Py_VISIT((PyObject *)aw->aw_aiter);
+    return 0;
+}
+
+static void
+aiter_wrapper_dealloc(PyAIterWrapper *aw)
+{
+    _PyObject_GC_UNTRACK((PyObject *)aw);
+    Py_CLEAR(aw->aw_aiter);
+    PyObject_GC_Del(aw);
+}
+
+static PyAsyncMethods aiter_wrapper_as_async = {
+    PyObject_SelfIter,                          /* am_await */
+    0,                                          /* am_aiter */
+    0                                           /* am_anext */
+};
+
+PyTypeObject _PyAIterWrapper_Type = {
+    PyVarObject_HEAD_INIT(&PyType_Type, 0)
+    "aiter_wrapper",
+    sizeof(PyAIterWrapper),                     /* tp_basicsize */
+    0,                                          /* tp_itemsize */
+    (destructor)aiter_wrapper_dealloc,          /* destructor tp_dealloc */
+    0,                                          /* tp_print */
+    0,                                          /* tp_getattr */
+    0,                                          /* tp_setattr */
+    &aiter_wrapper_as_async,                    /* tp_as_async */
+    0,                                          /* tp_repr */
+    0,                                          /* tp_as_number */
+    0,                                          /* tp_as_sequence */
+    0,                                          /* tp_as_mapping */
+    0,                                          /* tp_hash */
+    0,                                          /* tp_call */
+    0,                                          /* tp_str */
+    PyObject_GenericGetAttr,                    /* tp_getattro */
+    0,                                          /* tp_setattro */
+    0,                                          /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,    /* tp_flags */
+    "A wrapper object for __aiter__ bakwards compatibility.",
+    (traverseproc)aiter_wrapper_traverse,       /* tp_traverse */
+    0,                                          /* tp_clear */
+    0,                                          /* tp_richcompare */
+    0,                                          /* tp_weaklistoffset */
+    PyObject_SelfIter,                          /* tp_iter */
+    (iternextfunc)aiter_wrapper_iternext,       /* tp_iternext */
+    0,                                          /* tp_methods */
+    0,                                          /* tp_members */
+    0,                                          /* tp_getset */
+    0,                                          /* tp_base */
+    0,                                          /* tp_dict */
+    0,                                          /* tp_descr_get */
+    0,                                          /* tp_descr_set */
+    0,                                          /* tp_dictoffset */
+    0,                                          /* tp_init */
+    0,                                          /* tp_alloc */
+    0,                                          /* tp_new */
+    PyObject_Del,                               /* tp_free */
+};
+
+
+PyObject *
+_PyAIterWrapper_New(PyObject *aiter)
+{
+    PyAIterWrapper *aw = PyObject_GC_New(PyAIterWrapper,
+                                         &_PyAIterWrapper_Type);
+    if (aw == NULL) {
+        return NULL;
+    }
+    Py_INCREF(aiter);
+    aw->aw_aiter = aiter;
+    _PyObject_GC_TRACK(aw);
+    return (PyObject *)aw;
+}
diff --git a/Python/ceval.c b/Python/ceval.c
index 3758b0936a..3d69038444 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -1933,8 +1933,9 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
             PyObject *obj = TOP();
             PyTypeObject *type = Py_TYPE(obj);
 
-            if (type->tp_as_async != NULL)
+            if (type->tp_as_async != NULL) {
                 getter = type->tp_as_async->am_aiter;
+            }
 
             if (getter != NULL) {
                 iter = (*getter)(obj);
@@ -1955,6 +1956,27 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
                 goto error;
             }
 
+            if (Py_TYPE(iter)->tp_as_async != NULL &&
+                    Py_TYPE(iter)->tp_as_async->am_anext != NULL) {
+
+                /* Starting with CPython 3.5.2 __aiter__ should return
+                   asynchronous iterators directly (not awaitables that
+                   resolve to asynchronous iterators.)
+
+                   Therefore, we check if the object that was returned
+                   from __aiter__ has an __anext__ method.  If it does,
+                   we wrap it in an awaitable that resolves to `iter`.
+
+                   See http://bugs.python.org/issue27243 for more
+                   details.
+                */
+
+                PyObject *wrapper = _PyAIterWrapper_New(iter);
+                Py_DECREF(iter);
+                SET_TOP(wrapper);
+                DISPATCH();
+            }
+
             awaitable = _PyCoro_GetAwaitableIter(iter);
             if (awaitable == NULL) {
                 SET_TOP(NULL);
@@ -1966,9 +1988,23 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 
                 Py_DECREF(iter);
                 goto error;
-            } else
+            } else {
                 Py_DECREF(iter);
 
+                if (PyErr_WarnFormat(
+                        PyExc_PendingDeprecationWarning, 1,
+                        "'%.100s' implements legacy __aiter__ protocol; "
+                        "__aiter__ should return an asynchronous "
+                        "iterator, not awaitable",
+                        type->tp_name))
+                {
+                    /* Warning was converted to an error. */
+                    Py_DECREF(awaitable);
+                    SET_TOP(NULL);
+                    goto error;
+                }
+            }
+
             SET_TOP(awaitable);
             DISPATCH();
         }