diff options
Diffstat (limited to 'Python')
| -rw-r--r-- | Python/ceval.c | 216 | ||||
| -rw-r--r-- | Python/compile.c | 31 | ||||
| -rw-r--r-- | Python/opcode_targets.h | 54 | ||||
| -rw-r--r-- | Python/specialize.c | 99 |
4 files changed, 280 insertions, 120 deletions
diff --git a/Python/ceval.c b/Python/ceval.c index fb19f78ba1..6d2784894f 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1673,6 +1673,22 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr CFrame cframe; + /* Variables used for making calls */ + PyObject *kwnames; + int nargs; + /* + * It is only between a PRECALL_METHOD instruction and the following instruction, + * that these two values can be anything other than their defaults. */ + int postcall_shrink = 1; + int extra_args = 0; +#define RESET_STACK_ADJUST_FOR_CALLS \ + do { \ + postcall_shrink = 1; \ + extra_args = 0; \ + } while (0) +#define STACK_ADJUST_IS_RESET \ + (postcall_shrink == 1 && extra_args == 0) + /* WARNING: Because the CFrame lives on the C stack, * but can be accessed from a heap allocated object (tstate) * strict stack discipline must be maintained. @@ -1815,11 +1831,6 @@ check_eval_breaker: switch (opcode) { #endif - /* Variables used for making calls */ - PyObject *kwnames; - int nargs; - int postcall_shrink; - /* BEWARE! It is essential that any operation that fails must goto error and that all operation that succeed call DISPATCH() ! */ @@ -4319,7 +4330,7 @@ check_eval_breaker: if (iter == NULL) goto error; PREDICT(FOR_ITER); - PREDICT(CALL_FUNCTION); + PREDICT(CALL_NO_KW); DISPATCH(); } @@ -4661,7 +4672,7 @@ check_eval_breaker: DISPATCH(); } - TARGET(CALL_METHOD) { + TARGET(PRECALL_METHOD) { /* Designed to work in tamdem with LOAD_METHOD. */ /* `meth` is NULL when LOAD_METHOD thinks that it's not a method call. @@ -4689,38 +4700,26 @@ check_eval_breaker: make it accept the `self` as a first argument. */ int is_method = (PEEK(oparg + 2) != NULL); - oparg += is_method; - nargs = oparg; - kwnames = NULL; + extra_args = is_method; postcall_shrink = 2-is_method; - goto call_function; - } - - TARGET(CALL_METHOD_KW) { - /* Designed to work in tandem with LOAD_METHOD. Same as CALL_METHOD - but pops TOS to get a tuple of keyword names. */ - kwnames = POP(); - int is_method = (PEEK(oparg + 2) != NULL); - oparg += is_method; - nargs = oparg - (int)PyTuple_GET_SIZE(kwnames); - postcall_shrink = 2-is_method; - goto call_function; + DISPATCH(); } - TARGET(CALL_FUNCTION_KW) { + TARGET(CALL_KW) { kwnames = POP(); + oparg += extra_args; + extra_args = 0; nargs = oparg - (int)PyTuple_GET_SIZE(kwnames); - postcall_shrink = 1; goto call_function; } - TARGET(CALL_FUNCTION) { - PREDICTED(CALL_FUNCTION); - STAT_INC(CALL_FUNCTION, unquickened); + TARGET(CALL_NO_KW) { PyObject *function; - nargs = oparg; + PREDICTED(CALL_NO_KW); + STAT_INC(CALL_NO_KW, unquickened); kwnames = NULL; - postcall_shrink = 1; + oparg += extra_args; + nargs = oparg; call_function: function = PEEK(oparg + 1); if (Py_TYPE(function) == &PyMethod_Type) { @@ -4748,6 +4747,7 @@ check_eval_breaker: stack_pointer, nargs, kwnames ); STACK_SHRINK(postcall_shrink); + RESET_STACK_ADJUST_FOR_CALLS; // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. Py_XDECREF(kwnames); @@ -4780,6 +4780,7 @@ check_eval_breaker: Py_DECREF(stack_pointer[i]); } STACK_SHRINK(postcall_shrink); + RESET_STACK_ADJUST_FOR_CALLS; PUSH(res); if (res == NULL) { goto error; @@ -4788,43 +4789,46 @@ check_eval_breaker: DISPATCH(); } - TARGET(CALL_FUNCTION_ADAPTIVE) { + TARGET(CALL_NO_KW_ADAPTIVE) { SpecializedCacheEntry *cache = GET_CACHE(); - nargs = cache->adaptive.original_oparg; + oparg = cache->adaptive.original_oparg; if (cache->adaptive.counter == 0) { next_instr--; - if (_Py_Specialize_CallFunction( + int nargs = oparg+extra_args; + if (_Py_Specialize_CallNoKw( PEEK(nargs + 1), next_instr, nargs, cache, BUILTINS()) < 0) { goto error; } DISPATCH(); } else { - STAT_INC(CALL_FUNCTION, deferred); + STAT_INC(CALL_NO_KW, deferred); cache->adaptive.counter--; - oparg = nargs; kwnames = NULL; - postcall_shrink = 1; + oparg += extra_args; + nargs = oparg; goto call_function; } } - TARGET(CALL_FUNCTION_PY_SIMPLE) { + TARGET(CALL_NO_KW_PY_SIMPLE) { SpecializedCacheEntry *caches = GET_CACHE(); _PyAdaptiveEntry *cache0 = &caches[0].adaptive; - int argcount = cache0->original_oparg; + int argcount = cache0->original_oparg + extra_args; + DEOPT_IF(argcount != cache0->index, CALL_NO_KW); _PyCallCache *cache1 = &caches[-1].call; PyObject *callable = PEEK(argcount+1); - DEOPT_IF(!PyFunction_Check(callable), CALL_FUNCTION); + DEOPT_IF(!PyFunction_Check(callable), CALL_NO_KW); PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != cache1->func_version, CALL_FUNCTION); + DEOPT_IF(func->func_version != cache1->func_version, CALL_NO_KW); /* PEP 523 */ - DEOPT_IF(tstate->interp->eval_frame != NULL, CALL_FUNCTION); - STAT_INC(CALL_FUNCTION, hit); + DEOPT_IF(tstate->interp->eval_frame != NULL, CALL_NO_KW); + STAT_INC(CALL_NO_KW, hit); PyCodeObject *code = (PyCodeObject *)func->func_code; size_t size = code->co_nlocalsplus + code->co_stacksize + FRAME_SPECIALS_SIZE; InterpreterFrame *new_frame = _PyThreadState_BumpFramePointer(tstate, size); if (new_frame == NULL) { + RESET_STACK_ADJUST_FOR_CALLS; goto error; } _PyFrame_InitializeSpecials(new_frame, func, @@ -4842,7 +4846,8 @@ check_eval_breaker: for (int i = argcount+deflen; i < code->co_nlocalsplus; i++) { new_frame->localsplus[i] = NULL; } - STACK_SHRINK(1); + STACK_SHRINK(postcall_shrink); + RESET_STACK_ADJUST_FOR_CALLS; Py_DECREF(func); _PyFrame_SetStackPointer(frame, stack_pointer); new_frame->previous = frame; @@ -4851,14 +4856,15 @@ check_eval_breaker: goto start_frame; } - TARGET(CALL_FUNCTION_BUILTIN_O) { + TARGET(CALL_NO_KW_BUILTIN_O) { assert(cframe.use_tracing == 0); + assert(STACK_ADJUST_IS_RESET); /* Builtin METH_O functions */ PyObject *callable = SECOND(); - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL_FUNCTION); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL_FUNCTION); - STAT_INC(CALL_FUNCTION, hit); + DEOPT_IF(!PyCFunction_CheckExact(callable), CALL_NO_KW); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL_NO_KW); + STAT_INC(CALL_NO_KW, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); // This is slower but CPython promises to check all non-vectorcall @@ -4881,18 +4887,19 @@ check_eval_breaker: DISPATCH(); } - TARGET(CALL_FUNCTION_BUILTIN_FAST) { + TARGET(CALL_NO_KW_BUILTIN_FAST) { assert(cframe.use_tracing == 0); + assert(STACK_ADJUST_IS_RESET); /* Builtin METH_FASTCALL functions, without keywords */ SpecializedCacheEntry *caches = GET_CACHE(); _PyAdaptiveEntry *cache0 = &caches[0].adaptive; int nargs = cache0->original_oparg; PyObject **pfunc = &PEEK(nargs + 1); PyObject *callable = *pfunc; - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL_FUNCTION); + DEOPT_IF(!PyCFunction_CheckExact(callable), CALL_NO_KW); DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, - CALL_FUNCTION); - STAT_INC(CALL_FUNCTION, hit); + CALL_NO_KW); + STAT_INC(CALL_NO_KW, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); /* res = func(self, args, nargs) */ @@ -4919,16 +4926,17 @@ check_eval_breaker: DISPATCH(); } - TARGET(CALL_FUNCTION_LEN) { + TARGET(CALL_NO_KW_LEN) { assert(cframe.use_tracing == 0); + assert(STACK_ADJUST_IS_RESET); /* len(o) */ SpecializedCacheEntry *caches = GET_CACHE(); assert(caches[0].adaptive.original_oparg == 1); _PyObjectCache *cache1 = &caches[-1].obj; PyObject *callable = SECOND(); - DEOPT_IF(callable != cache1->obj, CALL_FUNCTION); - STAT_INC(CALL_FUNCTION, hit); + DEOPT_IF(callable != cache1->obj, CALL_NO_KW); + STAT_INC(CALL_NO_KW, hit); Py_ssize_t len_i = PyObject_Length(TOP()); if (len_i < 0) { @@ -4947,16 +4955,17 @@ check_eval_breaker: DISPATCH(); } - TARGET(CALL_FUNCTION_ISINSTANCE) { + TARGET(CALL_NO_KW_ISINSTANCE) { assert(cframe.use_tracing == 0); + assert(STACK_ADJUST_IS_RESET); /* isinstance(o, o2) */ SpecializedCacheEntry *caches = GET_CACHE(); assert(caches[0].adaptive.original_oparg == 2); _PyObjectCache *cache1 = &caches[-1].obj; PyObject *callable = THIRD(); - DEOPT_IF(callable != cache1->obj, CALL_FUNCTION); - STAT_INC(CALL_FUNCTION, hit); + DEOPT_IF(callable != cache1->obj, CALL_NO_KW); + STAT_INC(CALL_NO_KW, hit); int retval = PyObject_IsInstance(SECOND(), TOP()); if (retval < 0) { @@ -4976,6 +4985,97 @@ check_eval_breaker: DISPATCH(); } + TARGET(CALL_NO_KW_LIST_APPEND) { + assert(_Py_OPCODE(next_instr[-2]) == PRECALL_METHOD); + assert(GET_CACHE()->adaptive.original_oparg == 1); + DEOPT_IF(extra_args == 0, CALL_NO_KW); + PyObject *list = SECOND(); + DEOPT_IF(!PyList_CheckExact(list), CALL_NO_KW); + STAT_INC(CALL_NO_KW, hit); + assert(extra_args == 1); + extra_args = 0; + assert(STACK_ADJUST_IS_RESET); + PyObject *arg = TOP(); + int err = PyList_Append(list, arg); + if (err) { + goto error; + } + PyObject *callable = THIRD(); + Py_DECREF(arg); + Py_DECREF(list); + Py_INCREF(Py_None); + STACK_SHRINK(2); + SET_TOP(Py_None); + Py_DECREF(callable); + DISPATCH(); + } + + TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { + assert(_Py_OPCODE(next_instr[-2]) == PRECALL_METHOD); + assert(GET_CACHE()->adaptive.original_oparg == 1); + DEOPT_IF(extra_args == 0, CALL_NO_KW); + assert(extra_args == 1); + PyObject *callable = THIRD(); + DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL_NO_KW); + DEOPT_IF(((PyMethodDescrObject *)callable)->d_method->ml_flags != METH_O, CALL_NO_KW); + STAT_INC(CALL_NO_KW, hit); + assert(extra_args == 1); + extra_args = 0; + assert(STACK_ADJUST_IS_RESET); + PyCFunction cfunc = ((PyMethodDescrObject *)callable)->d_method->ml_meth; + // This is slower but CPython promises to check all non-vectorcall + // function calls. + if (_Py_EnterRecursiveCall(tstate, " while calling a Python object")) { + goto error; + } + PyObject *arg = POP(); + PyObject *self = POP(); + PyObject *res = cfunc(self, arg); + _Py_LeaveRecursiveCall(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(self); + Py_DECREF(arg); + SET_TOP(res); + Py_DECREF(callable); + if (res == NULL) { + goto error; + } + DISPATCH(); + } + + TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_FAST) { + assert(_Py_OPCODE(next_instr[-2]) == PRECALL_METHOD); + /* Builtin METH_FASTCALL methods, without keywords */ + SpecializedCacheEntry *caches = GET_CACHE(); + _PyAdaptiveEntry *cache0 = &caches[0].adaptive; + DEOPT_IF(extra_args == 0, CALL_NO_KW); + assert(extra_args == 1); + int nargs = cache0->original_oparg; + PyObject *callable = PEEK(nargs + 2); + DEOPT_IF(!Py_IS_TYPE(callable, &PyMethodDescr_Type), CALL_NO_KW); + PyMethodDef *meth = ((PyMethodDescrObject *)callable)->d_method; + DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL_NO_KW); + STAT_INC(CALL_NO_KW, hit); + assert(extra_args == 1); + extra_args = 0; + assert(STACK_ADJUST_IS_RESET); + _PyCFunctionFast cfunc = (_PyCFunctionFast)(void(*)(void))meth->ml_meth; + PyObject *self = PEEK(nargs+1); + PyObject *res = cfunc(self, &PEEK(nargs), nargs); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Clear the stack of the arguments. */ + STACK_SHRINK(nargs+1); + for (int i = 0; i <= nargs; i++) { + Py_DECREF(stack_pointer[i]); + } + SET_TOP(res); + Py_DECREF(callable); + if (res == NULL) { + goto error; + } + DISPATCH(); + } + TARGET(CALL_FUNCTION_EX) { PREDICTED(CALL_FUNCTION_EX); PyObject *func, *callargs, *kwargs = NULL, *result; @@ -5289,7 +5389,7 @@ MISS_WITH_CACHE(LOAD_ATTR) MISS_WITH_CACHE(STORE_ATTR) MISS_WITH_CACHE(LOAD_GLOBAL) MISS_WITH_CACHE(LOAD_METHOD) -MISS_WITH_CACHE(CALL_FUNCTION) +MISS_WITH_CACHE(CALL_NO_KW) MISS_WITH_CACHE(BINARY_OP) MISS_WITH_CACHE(COMPARE_OP) MISS_WITH_CACHE(BINARY_SUBSCR) @@ -7509,7 +7609,7 @@ format_awaitable_error(PyThreadState *tstate, PyTypeObject *type, int prevprevop "that does not implement __await__: %.100s", type->tp_name); } - else if (prevopcode == WITH_EXCEPT_START || (prevopcode == CALL_FUNCTION && prevprevopcode == DUP_TOP)) { + else if (prevopcode == WITH_EXCEPT_START || (prevopcode == CALL_NO_KW && prevprevopcode == DUP_TOP)) { _PyErr_Format(tstate, PyExc_TypeError, "'async with' received an object from __aexit__ " "that does not implement __await__: %.100s", diff --git a/Python/compile.c b/Python/compile.c index 00e1e01696..afd9a629c0 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1155,13 +1155,11 @@ stack_effect(int opcode, int oparg, int jump) return -oparg; /* Functions and calls */ - case CALL_FUNCTION: + case PRECALL_METHOD: + return -1; + case CALL_NO_KW: return -oparg; - case CALL_METHOD: - return -oparg-1; - case CALL_METHOD_KW: - return -oparg-2; - case CALL_FUNCTION_KW: + case CALL_KW: return -oparg-1; case CALL_FUNCTION_EX: return -1 - ((oparg & 0x01) != 0); @@ -1817,7 +1815,7 @@ compiler_call_exit_with_nones(struct compiler *c) { ADDOP_LOAD_CONST(c, Py_None); ADDOP(c, DUP_TOP); ADDOP(c, DUP_TOP); - ADDOP_I(c, CALL_FUNCTION, 3); + ADDOP_I(c, CALL_NO_KW, 3); return 1; } @@ -2166,7 +2164,7 @@ compiler_apply_decorators(struct compiler *c, asdl_expr_seq* decos) int old_end_col_offset = c->u->u_end_col_offset; for (Py_ssize_t i = asdl_seq_LEN(decos) - 1; i > -1; i--) { SET_LOC(c, (expr_ty)asdl_seq_GET(decos, i)); - ADDOP_I(c, CALL_FUNCTION, 1); + ADDOP_I(c, CALL_NO_KW, 1); } c->u->u_lineno = old_lineno; c->u->u_end_lineno = old_end_lineno; @@ -3885,7 +3883,7 @@ compiler_assert(struct compiler *c, stmt_ty s) ADDOP(c, LOAD_ASSERTION_ERROR); if (s->v.Assert.msg) { VISIT(c, expr, s->v.Assert.msg); - ADDOP_I(c, CALL_FUNCTION, 1); + ADDOP_I(c, CALL_NO_KW, 1); } ADDOP_I(c, RAISE_VARARGS, 1); compiler_use_next_block(c, end); @@ -4693,10 +4691,12 @@ maybe_optimize_method_call(struct compiler *c, expr_ty e) if (!compiler_call_simple_kw_helper(c, kwds, kwdsl)) { return 0; }; - ADDOP_I(c, CALL_METHOD_KW, argsl + kwdsl); + ADDOP_I(c, PRECALL_METHOD, argsl + kwdsl+1); + ADDOP_I(c, CALL_KW, argsl + kwdsl); } else { - ADDOP_I(c, CALL_METHOD, argsl); + ADDOP_I(c, PRECALL_METHOD, argsl); + ADDOP_I(c, CALL_NO_KW, argsl); } c->u->u_lineno = old_lineno; return 1; @@ -4763,7 +4763,8 @@ compiler_joined_str(struct compiler *c, expr_ty e) VISIT(c, expr, asdl_seq_GET(e->v.JoinedStr.values, i)); ADDOP_I(c, LIST_APPEND, 1); } - ADDOP_I(c, CALL_METHOD, 1); + ADDOP_I(c, PRECALL_METHOD, 1); + ADDOP_I(c, CALL_NO_KW, 1); } else { VISIT_SEQ(c, expr, e->v.JoinedStr.values); @@ -4935,11 +4936,11 @@ compiler_call_helper(struct compiler *c, if (!compiler_call_simple_kw_helper(c, keywords, nkwelts)) { return 0; }; - ADDOP_I(c, CALL_FUNCTION_KW, n + nelts + nkwelts); + ADDOP_I(c, CALL_KW, n + nelts + nkwelts); return 1; } else { - ADDOP_I(c, CALL_FUNCTION, n + nelts); + ADDOP_I(c, CALL_NO_KW, n + nelts); return 1; } @@ -5336,7 +5337,7 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type, ADDOP(c, GET_ITER); } - ADDOP_I(c, CALL_FUNCTION, 1); + ADDOP_I(c, CALL_NO_KW, 1); if (is_async_generator && type != COMP_GENEXP) { ADDOP(c, GET_AWAITABLE); diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index aeb9830ccc..05bd9392d8 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -39,34 +39,34 @@ static void *opcode_targets[256] = { &&TARGET_POP_EXCEPT_AND_RERAISE, &&TARGET_STORE_SUBSCR_LIST_INT, &&TARGET_STORE_SUBSCR_DICT, - &&TARGET_CALL_FUNCTION_ADAPTIVE, - &&TARGET_CALL_FUNCTION_BUILTIN_O, - &&TARGET_CALL_FUNCTION_BUILTIN_FAST, - &&TARGET_CALL_FUNCTION_LEN, - &&TARGET_CALL_FUNCTION_ISINSTANCE, - &&TARGET_CALL_FUNCTION_PY_SIMPLE, - &&TARGET_JUMP_ABSOLUTE_QUICK, - &&TARGET_LOAD_ATTR_ADAPTIVE, - &&TARGET_LOAD_ATTR_INSTANCE_VALUE, + &&TARGET_CALL_NO_KW_ADAPTIVE, + &&TARGET_CALL_NO_KW_BUILTIN_O, + &&TARGET_CALL_NO_KW_BUILTIN_FAST, + &&TARGET_CALL_NO_KW_LEN, + &&TARGET_CALL_NO_KW_ISINSTANCE, + &&TARGET_CALL_NO_KW_PY_SIMPLE, + &&TARGET_CALL_NO_KW_LIST_APPEND, + &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_O, + &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_FAST, &&TARGET_WITH_EXCEPT_START, &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, &&TARGET_BEFORE_ASYNC_WITH, &&TARGET_BEFORE_WITH, &&TARGET_END_ASYNC_FOR, + &&TARGET_JUMP_ABSOLUTE_QUICK, + &&TARGET_LOAD_ATTR_ADAPTIVE, + &&TARGET_LOAD_ATTR_INSTANCE_VALUE, &&TARGET_LOAD_ATTR_WITH_HINT, &&TARGET_LOAD_ATTR_SLOT, + &&TARGET_STORE_SUBSCR, + &&TARGET_DELETE_SUBSCR, &&TARGET_LOAD_ATTR_MODULE, &&TARGET_LOAD_GLOBAL_ADAPTIVE, &&TARGET_LOAD_GLOBAL_MODULE, - &&TARGET_STORE_SUBSCR, - &&TARGET_DELETE_SUBSCR, &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_LOAD_METHOD_ADAPTIVE, &&TARGET_LOAD_METHOD_CACHED, - &&TARGET_LOAD_METHOD_CLASS, - &&TARGET_LOAD_METHOD_MODULE, - &&TARGET_LOAD_METHOD_NO_DICT, &&TARGET_GET_ITER, &&TARGET_GET_YIELD_FROM_ITER, &&TARGET_PRINT_EXPR, @@ -74,19 +74,19 @@ static void *opcode_targets[256] = { &&TARGET_YIELD_FROM, &&TARGET_GET_AWAITABLE, &&TARGET_LOAD_ASSERTION_ERROR, + &&TARGET_LOAD_METHOD_CLASS, + &&TARGET_LOAD_METHOD_MODULE, + &&TARGET_LOAD_METHOD_NO_DICT, &&TARGET_STORE_ATTR_ADAPTIVE, &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_STORE_ATTR_SLOT, &&TARGET_STORE_ATTR_WITH_HINT, - &&TARGET_LOAD_FAST__LOAD_FAST, - &&TARGET_STORE_FAST__LOAD_FAST, - &&TARGET_LOAD_FAST__LOAD_CONST, &&TARGET_LIST_TO_TUPLE, &&TARGET_RETURN_VALUE, &&TARGET_IMPORT_STAR, &&TARGET_SETUP_ANNOTATIONS, &&TARGET_YIELD_VALUE, - &&TARGET_LOAD_CONST__LOAD_FAST, + &&TARGET_LOAD_FAST__LOAD_FAST, &&TARGET_PREP_RERAISE_STAR, &&TARGET_POP_EXCEPT, &&TARGET_STORE_NAME, @@ -122,25 +122,25 @@ static void *opcode_targets[256] = { &&TARGET_COPY, &&TARGET_JUMP_IF_NOT_EXC_MATCH, &&TARGET_BINARY_OP, - &&TARGET_STORE_FAST__STORE_FAST, + &&TARGET_STORE_FAST__LOAD_FAST, &&TARGET_LOAD_FAST, &&TARGET_STORE_FAST, &&TARGET_DELETE_FAST, &&TARGET_JUMP_IF_NOT_EG_MATCH, - &&_unknown_opcode, + &&TARGET_LOAD_FAST__LOAD_CONST, &&TARGET_GEN_START, &&TARGET_RAISE_VARARGS, - &&TARGET_CALL_FUNCTION, + &&TARGET_LOAD_CONST__LOAD_FAST, &&TARGET_MAKE_FUNCTION, &&TARGET_BUILD_SLICE, - &&_unknown_opcode, + &&TARGET_STORE_FAST__STORE_FAST, &&TARGET_MAKE_CELL, &&TARGET_LOAD_CLOSURE, &&TARGET_LOAD_DEREF, &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, &&_unknown_opcode, - &&TARGET_CALL_FUNCTION_KW, + &&_unknown_opcode, &&TARGET_CALL_FUNCTION_EX, &&_unknown_opcode, &&TARGET_EXTENDED_ARG, @@ -160,16 +160,16 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&TARGET_LOAD_METHOD, - &&TARGET_CALL_METHOD, + &&_unknown_opcode, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, - &&TARGET_CALL_METHOD_KW, - &&_unknown_opcode, - &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, + &&TARGET_PRECALL_METHOD, + &&TARGET_CALL_NO_KW, + &&TARGET_CALL_KW, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, diff --git a/Python/specialize.c b/Python/specialize.c index 5121845008..bdcba46ed4 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -127,7 +127,7 @@ _Py_GetSpecializationStats(void) { err += add_stat_dict(stats, BINARY_SUBSCR, "binary_subscr"); err += add_stat_dict(stats, STORE_SUBSCR, "store_subscr"); err += add_stat_dict(stats, STORE_ATTR, "store_attr"); - err += add_stat_dict(stats, CALL_FUNCTION, "call_function"); + err += add_stat_dict(stats, CALL_NO_KW, "call_no_kw"); err += add_stat_dict(stats, BINARY_OP, "binary_op"); err += add_stat_dict(stats, COMPARE_OP, "compare_op"); if (err < 0) { @@ -186,7 +186,7 @@ _Py_PrintSpecializationStats(void) print_stats(out, &_specialization_stats[BINARY_SUBSCR], "binary_subscr"); print_stats(out, &_specialization_stats[STORE_SUBSCR], "store_subscr"); print_stats(out, &_specialization_stats[STORE_ATTR], "store_attr"); - print_stats(out, &_specialization_stats[CALL_FUNCTION], "call_function"); + print_stats(out, &_specialization_stats[CALL_NO_KW], "call_no_kw"); print_stats(out, &_specialization_stats[BINARY_OP], "binary_op"); print_stats(out, &_specialization_stats[COMPARE_OP], "compare_op"); if (out != stderr) { @@ -238,7 +238,7 @@ static uint8_t adaptive_opcodes[256] = { [LOAD_METHOD] = LOAD_METHOD_ADAPTIVE, [BINARY_SUBSCR] = BINARY_SUBSCR_ADAPTIVE, [STORE_SUBSCR] = STORE_SUBSCR_ADAPTIVE, - [CALL_FUNCTION] = CALL_FUNCTION_ADAPTIVE, + [CALL_NO_KW] = CALL_NO_KW_ADAPTIVE, [STORE_ATTR] = STORE_ATTR_ADAPTIVE, [BINARY_OP] = BINARY_OP_ADAPTIVE, [COMPARE_OP] = COMPARE_OP_ADAPTIVE, @@ -251,7 +251,7 @@ static uint8_t cache_requirements[256] = { [LOAD_METHOD] = 3, /* _PyAdaptiveEntry, _PyAttrCache and _PyObjectCache */ [BINARY_SUBSCR] = 2, /* _PyAdaptiveEntry, _PyObjectCache */ [STORE_SUBSCR] = 0, - [CALL_FUNCTION] = 2, /* _PyAdaptiveEntry and _PyObjectCache/_PyCallCache */ + [CALL_NO_KW] = 2, /* _PyAdaptiveEntry and _PyObjectCache/_PyCallCache */ [STORE_ATTR] = 2, /* _PyAdaptiveEntry and _PyAttrCache */ [BINARY_OP] = 1, // _PyAdaptiveEntry [COMPARE_OP] = 1, /* _PyAdaptiveEntry */ @@ -491,6 +491,8 @@ initial_counter_value(void) { #define SPEC_FAIL_PYCFUNCTION_NOARGS 16 #define SPEC_FAIL_BAD_CALL_FLAGS 17 #define SPEC_FAIL_CLASS 18 +#define SPEC_FAIL_C_METHOD_CALL 19 +#define SPEC_FAIL_METHDESCR_NON_METHOD 20 /* COMPARE_OP */ #define SPEC_FAIL_STRING_COMPARE 13 @@ -1261,7 +1263,51 @@ specialize_class_call( PyObject *callable, _Py_CODEUNIT *instr, int nargs, SpecializedCacheEntry *cache) { - SPECIALIZATION_FAIL(CALL_FUNCTION, SPEC_FAIL_CLASS); + SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_CLASS); + return -1; +} + +static PyMethodDescrObject *_list_append = NULL; +_Py_IDENTIFIER(append); + +static int +specialize_method_descriptor( + PyMethodDescrObject *descr, _Py_CODEUNIT *instr, + int nargs, SpecializedCacheEntry *cache) +{ + int oparg = cache->adaptive.original_oparg; + if (nargs - oparg != 1) { + SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_METHDESCR_NON_METHOD); + return -1; + } + if (_list_append == NULL) { + _list_append = (PyMethodDescrObject *)_PyType_LookupId(&PyList_Type, &PyId_append); + } + if (oparg == 1 && descr == _list_append) { + assert(_Py_OPCODE(instr[-1]) == PRECALL_METHOD); + *instr = _Py_MAKECODEUNIT(CALL_NO_KW_LIST_APPEND, _Py_OPARG(*instr)); + return 0; + } + + switch (descr->d_method->ml_flags & + (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | + METH_KEYWORDS | METH_METHOD)) { + case METH_O: { + if (oparg != 1) { + SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_OUT_OF_RANGE); + return 1; + } + *instr = _Py_MAKECODEUNIT(CALL_NO_KW_METHOD_DESCRIPTOR_O, + _Py_OPARG(*instr)); + return 0; + } + case METH_FASTCALL: { + *instr = _Py_MAKECODEUNIT(CALL_NO_KW_METHOD_DESCRIPTOR_FAST, + _Py_OPARG(*instr)); + return 0; + } + } + SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_OTHER); return -1; } @@ -1274,15 +1320,19 @@ specialize_py_call( PyCodeObject *code = (PyCodeObject *)func->func_code; int kind = function_kind(code); if (kind != SIMPLE_FUNCTION) { - SPECIALIZATION_FAIL(CALL_FUNCTION, kind); + SPECIALIZATION_FAIL(CALL_NO_KW, kind); return -1; } int argcount = code->co_argcount; + if (argcount > 0xffff) { + SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_OUT_OF_RANGE); + return -1; + } int defcount = func->func_defaults == NULL ? 0 : (int)PyTuple_GET_SIZE(func->func_defaults); assert(defcount <= argcount); int min_args = argcount-defcount; if (nargs > argcount || nargs < min_args) { - SPECIALIZATION_FAIL(CALL_FUNCTION, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); + SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); return -1; } assert(nargs <= argcount && nargs >= min_args); @@ -1291,18 +1341,19 @@ specialize_py_call( assert(defstart >= 0 && deflen >= 0); assert(deflen == 0 || func->func_defaults != NULL); if (defstart > 0xffff || deflen > 0xffff) { - SPECIALIZATION_FAIL(CALL_FUNCTION, SPEC_FAIL_OUT_OF_RANGE); + SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_OUT_OF_RANGE); return -1; } int version = _PyFunction_GetVersionForCurrentState(func); if (version == 0) { - SPECIALIZATION_FAIL(CALL_FUNCTION, SPEC_FAIL_OUT_OF_VERSIONS); + SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_OUT_OF_VERSIONS); return -1; } + cache[0].adaptive.index = nargs; cache1->func_version = version; cache1->defaults_start = defstart; cache1->defaults_len = deflen; - *instr = _Py_MAKECODEUNIT(CALL_FUNCTION_PY_SIMPLE, _Py_OPARG(*instr)); + *instr = _Py_MAKECODEUNIT(CALL_NO_KW_PY_SIMPLE, _Py_OPARG(*instr)); return 0; } @@ -1335,6 +1386,10 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, SpecializedCacheEntry *cache, PyObject *builtins) { _PyObjectCache *cache1 = &cache[-1].obj; + if (_Py_OPCODE(instr[-1]) == PRECALL_METHOD) { + SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_C_METHOD_CALL); + return -1; + } if (PyCFunction_GET_FUNCTION(callable) == NULL) { return 1; } @@ -1343,18 +1398,18 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, METH_KEYWORDS | METH_METHOD)) { case METH_O: { if (nargs != 1) { - SPECIALIZATION_FAIL(CALL_FUNCTION, SPEC_FAIL_OUT_OF_RANGE); + SPECIALIZATION_FAIL(CALL_NO_KW, SPEC_FAIL_OUT_OF_RANGE); return 1; } /* len(o) */ PyObject *builtin_len = PyDict_GetItemString(builtins, "len"); if (callable == builtin_len) { cache1->obj = builtin_len; // borrowed - *instr = _Py_MAKECODEUNIT(CALL_FUNCTION_LEN, + *instr = _Py_MAKECODEUNIT(CALL_NO_KW_LEN, _Py_OPARG(*instr)); return 0; } - *instr = _Py_MAKECODEUNIT(CALL_FUNCTION_BUILTIN_O, + *instr = _Py_MAKECODEUNIT(CALL_NO_KW_BUILTIN_O, _Py_OPARG(*instr)); return 0; } @@ -1365,17 +1420,17 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, builtins, "isinstance"); if (callable == builtin_isinstance) { cache1->obj = builtin_isinstance; // borrowed - *instr = _Py_MAKECODEUNIT(CALL_FUNCTION_ISINSTANCE, + *instr = _Py_MAKECODEUNIT(CALL_NO_KW_ISINSTANCE, _Py_OPARG(*instr)); return 0; } } - *instr = _Py_MAKECODEUNIT(CALL_FUNCTION_BUILTIN_FAST, + *instr = _Py_MAKECODEUNIT(CALL_NO_KW_BUILTIN_FAST, _Py_OPARG(*instr)); return 0; } default: - SPECIALIZATION_FAIL(CALL_FUNCTION, + SPECIALIZATION_FAIL(CALL_NO_KW, builtin_call_fail_kind(PyCFunction_GET_FLAGS(callable))); return 1; } @@ -1406,7 +1461,7 @@ call_fail_kind(PyObject *callable) - Specialize calling classes. */ int -_Py_Specialize_CallFunction( +_Py_Specialize_CallNoKw( PyObject *callable, _Py_CODEUNIT *instr, int nargs, SpecializedCacheEntry *cache, PyObject *builtins) @@ -1421,18 +1476,22 @@ _Py_Specialize_CallFunction( else if (PyType_Check(callable)) { fail = specialize_class_call(callable, instr, nargs, cache); } + else if (Py_IS_TYPE(callable, &PyMethodDescr_Type)) { + fail = specialize_method_descriptor( + (PyMethodDescrObject *)callable, instr, nargs, cache); + } else { - SPECIALIZATION_FAIL(CALL_FUNCTION, call_fail_kind(callable)); + SPECIALIZATION_FAIL(CALL_NO_KW, call_fail_kind(callable)); fail = -1; } _PyAdaptiveEntry *cache0 = &cache->adaptive; if (fail) { - STAT_INC(CALL_FUNCTION, specialization_failure); + STAT_INC(CALL_NO_KW, specialization_failure); assert(!PyErr_Occurred()); cache_backoff(cache0); } else { - STAT_INC(CALL_FUNCTION, specialization_success); + STAT_INC(CALL_NO_KW, specialization_success); assert(!PyErr_Occurred()); cache0->counter = initial_counter_value(); } |