bpo-45711: Change exc_info related APIs to derive type and traceback from the exception instance (GH-29780)

7 files changed, 104 insertions, 36 deletions

diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst
index 5d90248f85..27feab92de 100644
--- a/Doc/c-api/exceptions.rst
+++ b/Doc/c-api/exceptions.rst
@@ -482,7 +482,6 @@ Querying the error indicator
    to an exception that was *already caught*, not to an exception that was
    freshly raised.  This function steals the references of the arguments.
    To clear the exception state, pass ``NULL`` for all three arguments.
-   For general rules about the three arguments, see :c:func:`PyErr_Restore`.
 
    .. note::
 
@@ -493,6 +492,12 @@ Querying the error indicator
 
    .. versionadded:: 3.3
 
+   .. versionchanged:: 3.11
+      The ``type`` and ``traceback`` arguments are no longer used and
+      can be NULL. The interpreter now derives them from the exception
+      instance (the ``value`` argument). The function still steals
+      references of all three arguments.
+
 
 Signal Handling
 ===============
diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst
index 175fc09165..7d1b21f05e 100644
--- a/Doc/library/sys.rst
+++ b/Doc/library/sys.rst
@@ -396,9 +396,14 @@ always available.
    ``(type, value, traceback)``.  Their meaning is: *type* gets the type of the
    exception being handled (a subclass of :exc:`BaseException`); *value* gets
    the exception instance (an instance of the exception type); *traceback* gets
-   a :ref:`traceback object <traceback-objects>` which encapsulates the call
-   stack at the point where the exception originally occurred.
-
+   a :ref:`traceback object <traceback-objects>` which typically encapsulates
+   the call stack at the point where the exception last occurred.
+
+   .. versionchanged:: 3.11
+      The ``type`` and ``traceback`` fields are now derived from the ``value``
+      (the exception instance), so when an exception is modified while it is
+      being handled, the changes are reflected in the results of subsequent
+      calls to :func:`exc_info`.
 
 .. data:: exec_prefix
 
diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst
index bb1209dfc3..3d02074960 100644
--- a/Doc/reference/simple_stmts.rst
+++ b/Doc/reference/simple_stmts.rst
@@ -655,6 +655,12 @@ and information about handling exceptions is in section :ref:`try`.
     The ``__suppress_context__`` attribute to suppress automatic display of the
     exception context.
 
+.. versionchanged:: 3.11
+    If the traceback of the active exception is modified in an :keyword:`except`
+    clause, a subsequent ``raise`` statement re-raises the exception with the
+    modified traceback. Previously, the exception was re-raised with the
+    traceback it had when it was caught.
+
 .. _break:
 
 The :keyword:`!break` statement
diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst
index 8db26cd012..6853c04143 100644
--- a/Doc/whatsnew/3.11.rst
+++ b/Doc/whatsnew/3.11.rst
@@ -181,6 +181,12 @@ Other CPython Implementation Changes
   hash-based pyc files now use ``siphash13``, too.
   (Contributed by Inada Naoki in :issue:`29410`.)
 
+* When an active exception is re-raised by a :keyword:`raise` statement with no parameters,
+  the traceback attached to this exception is now always ``sys.exc_info()[1].__traceback__``.
+  This means that changes made to the traceback in the current :keyword:`except` clause are
+  reflected in the re-raised exception.
+  (Contributed by Irit Katriel in :issue:`45711`.)
+
 New Modules
 ===========
 
@@ -266,6 +272,16 @@ sqlite3
   (Contributed by Erlend E. Aasland in :issue:`45828`.)
 
 
+sys
+---
+
+* :func:`sys.exc_info` now derives the ``type`` and ``traceback`` fields
+  from the ``value`` (the exception instance), so when an exception is
+  modified while it is being handled, the changes are reflected in
+  the results of subsequent calls to :func:`exc_info`.
+  (Contributed by Irit Katriel in :issue:`45711`.)
+
+
 threading
 ---------
 
@@ -579,6 +595,17 @@ New Features
   suspend and resume tracing and profiling.
   (Contributed by Victor Stinner in :issue:`43760`.)
 
+* :c:func:`PyErr_SetExcInfo()` no longer uses the ``type`` and ``traceback``
+  arguments, the interpreter now derives those values from the exception
+  instance (the ``value`` argument). The function still steals references
+  of all three arguments.
+  (Contributed by Irit Katriel in :issue:`45711`.)
+
+* :c:func:`PyErr_GetExcInfo()` now derives the ``type`` and ``traceback``
+  fields of the result from the exception instance (the ``value`` field).
+  (Contributed by Irit Katriel in :issue:`45711`.)
+
+
 Porting to Python 3.11
 ----------------------
 
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-11-25-17-51-29.bpo-45711.D2igmz.rst b/Misc/NEWS.d/next/Core and Builtins/2021-11-25-17-51-29.bpo-45711.D2igmz.rst
new file mode 100644
index 0000000000..c499f185d2
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2021-11-25-17-51-29.bpo-45711.D2igmz.rst
@@ -0,0 +1,6 @@
+The three values of ``exc_info`` are now always consistent with each other.
+In particular, the ``type`` and ``traceback`` fields are now derived from
+the exception instance. This impacts the return values of :func:`sys.exc_info`
+and :c:func:`PyErr_GetExcInfo()` if the exception instance is modified while
+the exception is handled, as well as :c:func:`PyErr_SetExcInfo()`, which now
+ignores the ``type`` and ``traceback`` arguments provided to it.
diff --git a/Python/ceval.c b/Python/ceval.c
index 0427361a03..c5477b30f7 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -5918,20 +5918,17 @@ do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause)
     if (exc == NULL) {
         /* Reraise */
         _PyErr_StackItem *exc_info = _PyErr_GetTopmostException(tstate);
-        PyObject *tb;
-        type = exc_info->exc_type;
         value = exc_info->exc_value;
-        tb = exc_info->exc_traceback;
-        assert(((Py_IsNone(value) || value == NULL)) ==
-               ((Py_IsNone(type) || type == NULL)));
         if (Py_IsNone(value) || value == NULL) {
             _PyErr_SetString(tstate, PyExc_RuntimeError,
                              "No active exception to reraise");
             return 0;
         }
+        assert(PyExceptionInstance_Check(value));
+        type = PyExceptionInstance_Class(value);
         Py_XINCREF(type);
         Py_XINCREF(value);
-        Py_XINCREF(tb);
+        PyObject *tb = PyException_GetTraceback(value); /* new ref */
         _PyErr_Restore(tstate, type, value, tb);
         return 1;
     }
diff --git a/Python/errors.c b/Python/errors.c
index 6e74d19b78..0a8b5a257f 100644
--- a/Python/errors.c
+++ b/Python/errors.c
@@ -470,6 +470,33 @@ PyErr_Clear(void)
     _PyErr_Clear(tstate);
 }
 
+static PyObject*
+get_exc_type(PyObject *exc_value)  /* returns a borrowed ref */
+{
+    if (exc_value == NULL || exc_value == Py_None) {
+        return Py_None;
+    }
+    else {
+        assert(PyExceptionInstance_Check(exc_value));
+        PyObject *type = PyExceptionInstance_Class(exc_value);
+        assert(type != NULL);
+        return type;
+    }
+}
+
+static PyObject*
+get_exc_traceback(PyObject *exc_value)  /* returns a borrowed ref */
+{
+    if (exc_value == NULL || exc_value == Py_None) {
+        return Py_None;
+    }
+    else {
+        assert(PyExceptionInstance_Check(exc_value));
+        PyObject *tb = PyException_GetTraceback(exc_value);
+        Py_XDECREF(tb);
+        return tb ? tb : Py_None;
+    }
+}
 
 void
 _PyErr_GetExcInfo(PyThreadState *tstate,
@@ -477,18 +504,9 @@ _PyErr_GetExcInfo(PyThreadState *tstate,
 {
     _PyErr_StackItem *exc_info = _PyErr_GetTopmostException(tstate);
 
+    *p_type = get_exc_type(exc_info->exc_value);
     *p_value = exc_info->exc_value;
-    *p_traceback = exc_info->exc_traceback;
-
-    if (*p_value == NULL || *p_value == Py_None) {
-        assert(exc_info->exc_type == NULL || exc_info->exc_type == Py_None);
-        *p_type = Py_None;
-    }
-    else {
-        assert(PyExceptionInstance_Check(*p_value));
-        assert(exc_info->exc_type == PyExceptionInstance_Class(*p_value));
-        *p_type = PyExceptionInstance_Class(*p_value);
-    }
+    *p_traceback = get_exc_traceback(exc_info->exc_value);
 
     Py_XINCREF(*p_type);
     Py_XINCREF(*p_value);
@@ -504,7 +522,7 @@ PyErr_GetExcInfo(PyObject **p_type, PyObject **p_value, PyObject **p_traceback)
 }
 
 void
-PyErr_SetExcInfo(PyObject *p_type, PyObject *p_value, PyObject *p_traceback)
+PyErr_SetExcInfo(PyObject *type, PyObject *value, PyObject *traceback)
 {
     PyObject *oldtype, *oldvalue, *oldtraceback;
     PyThreadState *tstate = _PyThreadState_GET();
@@ -513,9 +531,16 @@ PyErr_SetExcInfo(PyObject *p_type, PyObject *p_value, PyObject *p_traceback)
     oldvalue = tstate->exc_info->exc_value;
     oldtraceback = tstate->exc_info->exc_traceback;
 
-    tstate->exc_info->exc_type = p_type;
-    tstate->exc_info->exc_value = p_value;
-    tstate->exc_info->exc_traceback = p_traceback;
+
+    tstate->exc_info->exc_type = get_exc_type(value);
+    Py_XINCREF(tstate->exc_info->exc_type);
+    tstate->exc_info->exc_value = value;
+    tstate->exc_info->exc_traceback = get_exc_traceback(value);
+    Py_XINCREF(tstate->exc_info->exc_traceback);
+
+    /* These args are no longer used, but we still need to steal a ref */
+    Py_XDECREF(type);
+    Py_XDECREF(traceback);
 
     Py_XDECREF(oldtype);
     Py_XDECREF(oldvalue);
@@ -527,22 +552,19 @@ PyObject*
 _PyErr_StackItemToExcInfoTuple(_PyErr_StackItem *err_info)
 {
     PyObject *exc_value = err_info->exc_value;
-    if (exc_value == NULL) {
-        exc_value = Py_None;
-    }
 
-    assert(exc_value == Py_None || PyExceptionInstance_Check(exc_value));
+    assert(exc_value == NULL ||
+           exc_value == Py_None ||
+           PyExceptionInstance_Check(exc_value));
 
-    PyObject *exc_type = PyExceptionInstance_Check(exc_value) ?
-               PyExceptionInstance_Class(exc_value) :
-               Py_None;
+    PyObject *exc_type = get_exc_type(exc_value);
+    PyObject *exc_traceback = get_exc_traceback(exc_value);
 
     return Py_BuildValue(
         "(OOO)",
-        exc_type,
-        exc_value,
-        err_info->exc_traceback != NULL ?
-            err_info->exc_traceback : Py_None);
+        exc_type ? exc_type : Py_None,
+        exc_value ? exc_value : Py_None,
+        exc_traceback ? exc_traceback : Py_None);
 }