Issue #20189: Four additional builtin types (PyTypeObject,

PyMethodDescr_Type, _PyMethodWrapper_Type, and PyWrapperDescr_Type)
have been modified to provide introspection information for builtins.
Also: many additional Lib, test suite, and Argument Clinic fixes.

8 files changed, 137 insertions, 54 deletions

diff --git a/Lib/idlelib/idle_test/test_calltips.py b/Lib/idlelib/idle_test/test_calltips.py
index eca24ec129..5b51732b2d 100644
--- a/Lib/idlelib/idle_test/test_calltips.py
+++ b/Lib/idlelib/idle_test/test_calltips.py
@@ -55,24 +55,27 @@ class Get_signatureTest(unittest.TestCase):
         gtest(list.__new__,
                'T.__new__(S, ...) -> a new object with type S, a subtype of T')
         gtest(list.__init__,
-               'x.__init__(...) initializes x; see help(type(x)) for signature')
+               'Initializes self.  See help(type(self)) for accurate signature.')
         append_doc =  "L.append(object) -> None -- append object to end"
         gtest(list.append, append_doc)
         gtest([].append, append_doc)
         gtest(List.append, append_doc)
 
-        gtest(types.MethodType, "method(function, instance)")
+        gtest(types.MethodType, "Create a bound instance method object.")
         gtest(SB(), default_tip)
 
     def test_multiline_docstring(self):
         # Test fewer lines than max.
-        self.assertEqual(signature(list),
-                "list() -> new empty list\n"
-                "list(iterable) -> new list initialized from iterable's items")
+        self.assertEqual(signature(dict),
+                "dict(mapping) -> new dictionary initialized from a mapping object's\n"
+                "(key, value) pairs\n"
+                "dict(iterable) -> new dictionary initialized as if via:\n"
+                "d = {}\n"
+                "for k, v in iterable:"
+                )
 
         # Test max lines and line (currently) too long.
         self.assertEqual(signature(bytes),
-"bytes(iterable_of_ints) -> bytes\n"
 "bytes(string, encoding[, errors]) -> bytes\n"
 "bytes(bytes_or_buffer) -> immutable copy of bytes_or_buffer\n"
 #bytes(int) -> bytes object of size given by the parameter initialized with null bytes
diff --git a/Lib/inspect.py b/Lib/inspect.py
index d6bd8cdb62..781a5320a0 100644
--- a/Lib/inspect.py
+++ b/Lib/inspect.py
@@ -1419,9 +1419,11 @@ def getgeneratorlocals(generator):
 
 _WrapperDescriptor = type(type.__call__)
 _MethodWrapper = type(all.__call__)
+_ClassMethodWrapper = type(int.__dict__['from_bytes'])
 
 _NonUserDefinedCallables = (_WrapperDescriptor,
                             _MethodWrapper,
+                            _ClassMethodWrapper,
                             types.BuiltinFunctionType)
 
 
@@ -1443,6 +1445,13 @@ def signature(obj):
     if not callable(obj):
         raise TypeError('{!r} is not a callable object'.format(obj))
 
+    if (isinstance(obj, _NonUserDefinedCallables) or
+       ismethoddescriptor(obj) or
+       isinstance(obj, type)):
+        sig = Signature.from_builtin(obj)
+        if sig:
+            return sig
+
     if isinstance(obj, types.MethodType):
         # In this case we skip the first parameter of the underlying
         # function (usually `self` or `cls`).
@@ -1460,13 +1469,9 @@ def signature(obj):
         if sig is not None:
             return sig
 
-
     if isinstance(obj, types.FunctionType):
         return Signature.from_function(obj)
 
-    if isinstance(obj, types.BuiltinFunctionType):
-        return Signature.from_builtin(obj)
-
     if isinstance(obj, functools.partial):
         sig = signature(obj.func)
 
@@ -2033,7 +2038,7 @@ class Signature:
             name = parse_name(name_node)
             if name is invalid:
                 return None
-            if default_node:
+            if default_node and default_node is not _empty:
                 try:
                     default_node = RewriteSymbolics().visit(default_node)
                     o = ast.literal_eval(default_node)
@@ -2066,6 +2071,23 @@ class Signature:
             kind = Parameter.VAR_KEYWORD
             p(f.args.kwarg, empty)
 
+        if parameters and (hasattr(func, '__self__') or
+            isinstance(func, _WrapperDescriptor,) or
+            ismethoddescriptor(func)
+            ):
+            name = parameters[0].name
+            if name not in ('self', 'module', 'type'):
+                pass
+            elif getattr(func, '__self__', None):
+                # strip off self (it's already been bound)
+                p = parameters.pop(0)
+                if not p.name in ('self', 'module', 'type'):
+                    raise ValueError('Unexpected name ' + repr(p.name) + ', expected self/module/cls/type')
+            else:
+                # for builtins, self parameter is always positional-only!
+                p = parameters[0].replace(kind=Parameter.POSITIONAL_ONLY)
+                parameters[0] = p
+
         return cls(parameters, return_annotation=cls.empty)
 
 
diff --git a/Lib/pydoc.py b/Lib/pydoc.py
index 9f626929e2..cf164ccfca 100755
--- a/Lib/pydoc.py
+++ b/Lib/pydoc.py
@@ -925,7 +925,10 @@ class HTMLDoc(Doc):
                 anchor, name, reallink)
         argspec = None
         if inspect.isfunction(object) or inspect.isbuiltin(object):
-            signature = inspect.signature(object)
+            try:
+                signature = inspect.signature(object)
+            except (ValueError, TypeError):
+                signature = None
             if signature:
                 argspec = str(signature)
                 if realname == '<lambda>':
@@ -1319,8 +1322,12 @@ location listed above.
                 skipdocs = 1
             title = self.bold(name) + ' = ' + realname
         argspec = None
-        if inspect.isfunction(object) or inspect.isbuiltin(object):
-            signature = inspect.signature(object)
+
+        if inspect.isroutine(object):
+            try:
+                signature = inspect.signature(object)
+            except (ValueError, TypeError):
+                signature = None
             if signature:
                 argspec = str(signature)
                 if realname == '<lambda>':
diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py
index 22c8eb0709..444feb6314 100644
--- a/Lib/test/test_capi.py
+++ b/Lib/test/test_capi.py
@@ -125,7 +125,7 @@ class CAPITest(unittest.TestCase):
         self.assertEqual(_testcapi.docstring_no_signature.__text_signature__, None)
 
         self.assertEqual(_testcapi.docstring_with_invalid_signature.__doc__,
-            "docstring_with_invalid_signature (boo)\n"
+            "docstring_with_invalid_signature (module, boo)\n"
             "\n"
             "This docstring has an invalid signature."
             )
@@ -133,12 +133,12 @@ class CAPITest(unittest.TestCase):
 
         self.assertEqual(_testcapi.docstring_with_signature.__doc__,
             "This docstring has a valid signature.")
-        self.assertEqual(_testcapi.docstring_with_signature.__text_signature__, "(sig)")
+        self.assertEqual(_testcapi.docstring_with_signature.__text_signature__, "(module, sig)")
 
         self.assertEqual(_testcapi.docstring_with_signature_and_extra_newlines.__doc__,
             "This docstring has a valid signature and some extra newlines.")
         self.assertEqual(_testcapi.docstring_with_signature_and_extra_newlines.__text_signature__,
-            "(parameter)")
+            "(module, parameter)")
 
 
 @unittest.skipUnless(threading, 'Threading required for this test.')
diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py
index 4e921177a5..5b7424bcf8 100644
--- a/Lib/test/test_generators.py
+++ b/Lib/test/test_generators.py
@@ -436,8 +436,8 @@ From the Iterators list, about the types of these things.
 >>> [s for s in dir(i) if not s.startswith('_')]
 ['close', 'gi_code', 'gi_frame', 'gi_running', 'send', 'throw']
 >>> from test.support import HAVE_DOCSTRINGS
->>> print(i.__next__.__doc__ if HAVE_DOCSTRINGS else 'x.__next__() <==> next(x)')
-x.__next__() <==> next(x)
+>>> print(i.__next__.__doc__ if HAVE_DOCSTRINGS else 'Implements next(self).')
+Implements next(self).
 >>> iter(i) is i
 True
 >>> import types
diff --git a/Lib/test/test_genexps.py b/Lib/test/test_genexps.py
index 203b336fde..74957cb8f4 100644
--- a/Lib/test/test_genexps.py
+++ b/Lib/test/test_genexps.py
@@ -222,8 +222,8 @@ Check that generator attributes are present
     True
 
     >>> from test.support import HAVE_DOCSTRINGS
-    >>> print(g.__next__.__doc__ if HAVE_DOCSTRINGS else 'x.__next__() <==> next(x)')
-    x.__next__() <==> next(x)
+    >>> print(g.__next__.__doc__ if HAVE_DOCSTRINGS else 'Implements next(self).')
+    Implements next(self).
     >>> import types
     >>> isinstance(g, types.GeneratorType)
     True
diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py
index 1bfe7246f7..028eeb9caa 100644
--- a/Lib/test/test_inspect.py
+++ b/Lib/test/test_inspect.py
@@ -1,21 +1,25 @@
-import re
-import sys
-import types
-import unittest
-import inspect
-import linecache
-import datetime
+import _testcapi
 import collections
-import os
-import shutil
+import datetime
 import functools
 import importlib
+import inspect
+import io
+import linecache
+import os
 from os.path import normcase
+import _pickle
+import re
+import shutil
+import sys
+import types
+import unicodedata
+import unittest
+
 try:
     from concurrent.futures import ThreadPoolExecutor
 except ImportError:
     ThreadPoolExecutor = None
-import _testcapi
 
 from test.support import run_unittest, TESTFN, DirsOnSysPath
 from test.support import MISSING_C_DOCSTRINGS
@@ -23,8 +27,6 @@ from test.script_helper import assert_python_ok, assert_python_failure
 from test import inspect_fodder as mod
 from test import inspect_fodder2 as mod2
 
-# C module for test_findsource_binary
-import unicodedata
 
 # Functions tested in this suite:
 # ismodule, isclass, ismethod, isfunction, istraceback, isframe, iscode,
@@ -1582,23 +1584,30 @@ class TestSignatureObject(unittest.TestCase):
                           ...))
 
     def test_signature_on_unsupported_builtins(self):
-        with self.assertRaisesRegex(ValueError, 'not supported by signature'):
-            inspect.signature(type)
-        with self.assertRaisesRegex(ValueError, 'not supported by signature'):
-            # support for 'wrapper_descriptor'
-            inspect.signature(type.__call__)
-        with self.assertRaisesRegex(ValueError, 'not supported by signature'):
-            # support for 'method-wrapper'
-            inspect.signature(min.__call__)
+        with self.assertRaisesRegex(ValueError, 'no signature found'):
+            # min simply doesn't have a signature (yet)
+            inspect.signature(min)
 
     @unittest.skipIf(MISSING_C_DOCSTRINGS,
                      "Signature information for builtins requires docstrings")
     def test_signature_on_builtins(self):
-        # min doesn't have a signature (yet)
-        self.assertEqual(inspect.signature(min), None)
 
-        signature = inspect.signature(_testcapi.docstring_with_signature_with_defaults)
-        self.assertTrue(isinstance(signature, inspect.Signature))
+        def test_unbound_method(o):
+            """Use this to test unbound methods (things that should have a self)"""
+            signature = inspect.signature(o)
+            self.assertTrue(isinstance(signature, inspect.Signature))
+            self.assertEqual(list(signature.parameters.values())[0].name, 'self')
+            return signature
+
+        def test_callable(o):
+            """Use this to test bound methods or normal callables (things that don't expect self)"""
+            signature = inspect.signature(o)
+            self.assertTrue(isinstance(signature, inspect.Signature))
+            if signature.parameters:
+                self.assertNotEqual(list(signature.parameters.values())[0].name, 'self')
+            return signature
+
+        signature = test_callable(_testcapi.docstring_with_signature_with_defaults)
         def p(name): return signature.parameters[name].default
         self.assertEqual(p('s'), 'avocado')
         self.assertEqual(p('b'), b'bytes')
@@ -1611,6 +1620,41 @@ class TestSignatureObject(unittest.TestCase):
         self.assertEqual(p('sys'), sys.maxsize)
         self.assertEqual(p('exp'), sys.maxsize - 1)
 
+        test_callable(type)
+        test_callable(object)
+
+        # normal method
+        # (PyMethodDescr_Type, "method_descriptor")
+        test_unbound_method(_pickle.Pickler.dump)
+        d = _pickle.Pickler(io.StringIO())
+        test_callable(d.dump)
+
+        # static method
+        test_callable(str.maketrans)
+        test_callable('abc'.maketrans)
+
+        # class method
+        test_callable(dict.fromkeys)
+        test_callable({}.fromkeys)
+
+        # wrapper around slot (PyWrapperDescr_Type, "wrapper_descriptor")
+        test_unbound_method(type.__call__)
+        test_unbound_method(int.__add__)
+        test_callable((3).__add__)
+
+        # _PyMethodWrapper_Type
+        # support for 'method-wrapper'
+        test_callable(min.__call__)
+
+        class ThisWorksNow:
+            __call__ = type
+        test_callable(ThisWorksNow())
+
+
+    def test_signature_on_builtins_no_signature(self):
+        with self.assertRaisesRegex(ValueError, 'no signature found for builtin'):
+            inspect.signature(_testcapi.docstring_no_signature)
+
     def test_signature_on_non_function(self):
         with self.assertRaisesRegex(TypeError, 'is not a callable object'):
             inspect.signature(42)
@@ -1985,12 +2029,6 @@ class TestSignatureObject(unittest.TestCase):
                          ((('a', ..., ..., "positional_or_keyword"),),
                           ...))
 
-        class ToFail:
-            __call__ = type
-        with self.assertRaisesRegex(ValueError, "not supported by signature"):
-            inspect.signature(ToFail())
-
-
         class Wrapped:
             pass
         Wrapped.__wrapped__ = lambda a: None
diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index dc5c033739..8b76503429 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -112,11 +112,24 @@ def _check_signature(func, mock, skipfirst, instance=False):
 def _copy_func_details(func, funcopy):
     funcopy.__name__ = func.__name__
     funcopy.__doc__ = func.__doc__
+    try:
+        funcopy.__text_signature__ = func.__text_signature__
+    except AttributeError:
+        pass
     # we explicitly don't copy func.__dict__ into this copy as it would
     # expose original attributes that should be mocked
-    funcopy.__module__ = func.__module__
-    funcopy.__defaults__ = func.__defaults__
-    funcopy.__kwdefaults__ = func.__kwdefaults__
+    try:
+        funcopy.__module__ = func.__module__
+    except AttributeError:
+        pass
+    try:
+        funcopy.__defaults__ = func.__defaults__
+    except AttributeError:
+        pass
+    try:
+        funcopy.__kwdefaults__ = func.__kwdefaults__
+    except AttributeError:
+        pass
 
 
 def _callable(obj):