Fixed trampoline handling

5 files changed, 161 insertions, 105 deletions

diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c
index 6f9e978a6d..32df2e41b3 100644
--- a/ext/opcache/jit/zend_jit.c
+++ b/ext/opcache/jit/zend_jit.c
@@ -3296,7 +3296,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op
 						}
 						if (!zend_jit_init_method_call(&dasm_state, opline, b, op_array, ssa, ssa_op, call_level,
 								op1_info, op1_addr, ce, ce_is_instanceof, 0, NULL,
-								NULL, 1)) {
+								NULL, 1, 0)) {
 							goto jit_failure;
 						}
 						goto done;
diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c
index d57f0d9407..b74ccaad31 100644
--- a/ext/opcache/jit/zend_jit_trace.c
+++ b/ext/opcache/jit/zend_jit_trace.c
@@ -3168,7 +3168,8 @@ static int zend_jit_trace_deoptimization(dasm_State             **Dst,
                                          int                      parent_vars_count,
                                          zend_ssa                *ssa,
                                          zend_jit_trace_stack    *stack,
-                                         zend_lifetime_interval **ra)
+                                         zend_lifetime_interval **ra,
+                                         zend_bool                polymorphic_side_trace)
 {
 	int i;
 	zend_bool has_constants = 0;
@@ -3242,7 +3243,9 @@ static int zend_jit_trace_deoptimization(dasm_State             **Dst,
 				if (reg < ZREG_NUM) {
 					/* pass */
 				} else if (reg == ZREG_THIS) {
-					if (!zend_jit_load_this(Dst, EX_NUM_TO_VAR(i))) {
+					if (polymorphic_side_trace) {
+						ssa->var_info[i].delayed_fetch_this = 1;
+					} else if (!zend_jit_load_this(Dst, EX_NUM_TO_VAR(i))) {
 						return 0;
 					}
 				} else {
@@ -3441,6 +3444,9 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
 	zend_bool ce_is_instanceof;
 	zend_bool delayed_fetch_this = 0;
 	zend_bool avoid_refcounting = 0;
+	zend_bool polymorphic_side_trace =
+		parent_trace &&
+		(zend_jit_traces[parent_trace].exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL);
 	uint32_t i;
 	zend_jit_trace_stack_frame *frame, *top, *call;
 	zend_jit_trace_stack *stack;
@@ -3603,7 +3609,8 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
 			if (!zend_jit_trace_deoptimization(&dasm_state,
 					zend_jit_traces[parent_trace].exit_info[exit_num].flags,
 					zend_jit_traces[parent_trace].exit_info[exit_num].opline,
-					parent_stack, parent_vars_count, ssa, stack, ra)) {
+					parent_stack, parent_vars_count, ssa, stack, ra,
+					polymorphic_side_trace)) {
 				goto jit_failure;
 			}
 		}
@@ -5449,7 +5456,10 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
 						} else {
 							op1_info = OP1_INFO();
 							op1_addr = OP1_REG_ADDR();
-							if (orig_op1_type != IS_UNKNOWN
+							if (polymorphic_side_trace) {
+								op1_info = MAY_BE_OBJECT;
+								op1_addr = 0;
+							} else if (orig_op1_type != IS_UNKNOWN
 							 && (orig_op1_type & IS_TRACE_REFERENCE)) {
 								if (!zend_jit_fetch_reference(&dasm_state, opline, orig_op1_type, &op1_info, &op1_addr,
 										!ssa->var_info[ssa_op->op1_use].guarded_reference, 1)) {
@@ -5479,7 +5489,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
 								op_array_ssa->cfg.map ? op_array_ssa->cfg.map[opline - op_array->opcodes] : -1,
 								op_array, ssa, ssa_op, frame->call_level,
 								op1_info, op1_addr, ce, ce_is_instanceof, delayed_fetch_this, op1_ce,
-								p + 1, used_stack < 0)) {
+								p + 1, used_stack < 0, polymorphic_side_trace)) {
 							goto jit_failure;
 						}
 						goto done;
@@ -5549,6 +5559,7 @@ generic_dynamic_call:
 			}
 
 done:
+			polymorphic_side_trace = 0;
 			switch (opline->opcode) {
 				case ZEND_DO_FCALL:
 				case ZEND_DO_ICALL:
@@ -6083,7 +6094,7 @@ done:
 	} else if (p->stop == ZEND_JIT_TRACE_STOP_LINK
 	        || p->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
 		if (!zend_jit_trace_deoptimization(&dasm_state, 0, NULL,
-				stack, op_array->last_var + op_array->T, NULL, NULL, NULL)) {
+				stack, op_array->last_var + op_array->T, NULL, NULL, NULL, 0)) {
 			goto jit_failure;
 		}
 		if (p->stop == ZEND_JIT_TRACE_STOP_LINK) {
@@ -6247,7 +6258,7 @@ static const void *zend_jit_trace_exit_to_vm(uint32_t trace_num, uint32_t exit_n
 	if (!zend_jit_trace_deoptimization(&dasm_state,
 			zend_jit_traces[trace_num].exit_info[exit_num].flags,
 			zend_jit_traces[trace_num].exit_info[exit_num].opline,
-			stack, stack_size, NULL, NULL, NULL)) {
+			stack, stack_size, NULL, NULL, NULL, 0)) {
 		goto jit_failure;
 	}
 
diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc
index c53fd4817c..f00ea8cb9c 100644
--- a/ext/opcache/jit/zend_jit_x86.dasc
+++ b/ext/opcache/jit/zend_jit_x86.dasc
@@ -9052,8 +9052,8 @@ static int zend_jit_init_fcall_guard(dasm_State **Dst, uint32_t level, const zen
 		|	mov r1, aword EX:r1->func
 		|   .if X64
 		||		if (!IS_SIGNED_32BIT(opcodes)) {
-		|			mov64 r0, ((ptrdiff_t)opcodes)
-		|			cmp aword [r1 + offsetof(zend_op_array, opcodes)], r0
+		|			mov64 r2, ((ptrdiff_t)opcodes)
+		|			cmp aword [r1 + offsetof(zend_op_array, opcodes)], r2
 		||		} else {
 		|			cmp aword [r1 + offsetof(zend_op_array, opcodes)], opcodes
 		||		}
@@ -9064,8 +9064,8 @@ static int zend_jit_init_fcall_guard(dasm_State **Dst, uint32_t level, const zen
 	} else {
 		|   .if X64
 		||		if (!IS_SIGNED_32BIT(func)) {
-		|			mov64 r0, ((ptrdiff_t)func)
-		|			cmp aword EX:r1->func, r0
+		|			mov64 r2, ((ptrdiff_t)func)
+		|			cmp aword EX:r1->func, r2
 		||		} else {
 		|			cmp aword EX:r1->func, func
 		||		}
@@ -9241,7 +9241,8 @@ static int zend_jit_init_method_call(dasm_State          **Dst,
                                      zend_bool             use_this,
                                      zend_class_entry     *trace_ce,
                                      zend_jit_trace_rec   *trace,
-                                     zend_bool             stack_check)
+                                     zend_bool             stack_check,
+                                     zend_bool             polymorphic_side_trace)
 {
 	zend_func_info *info = ZEND_FUNC_INFO(op_array);
 	zend_call_info *call_info = NULL;
@@ -9253,114 +9254,118 @@ static int zend_jit_init_method_call(dasm_State          **Dst,
 
 	function_name = RT_CONSTANT(opline, opline->op2);
 
-	if (opline->op1_type == IS_UNUSED || use_this) {
-		zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
+	if (info) {
+		call_info = info->callee_info;
+		while (call_info && call_info->caller_init_opline != opline) {
+			call_info = call_info->next_callee;
+		}
+		if (call_info && call_info->callee_func) {
+			func = call_info->callee_func;
+		}
+	}
 
-		|	GET_ZVAL_PTR FCARG1a, this_addr
+	if (polymorphic_side_trace) {
+		/* function is passed in r0 from parent_trace */
 	} else {
-	    if (op1_info & MAY_BE_REF) {
-			if (opline->op1_type == IS_CV) {
-				if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) {
+		if (opline->op1_type == IS_UNUSED || use_this) {
+			zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
+
+			|	GET_ZVAL_PTR FCARG1a, this_addr
+		} else {
+		    if (op1_info & MAY_BE_REF) {
+				if (opline->op1_type == IS_CV) {
+					if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) {
+						|	LOAD_ZVAL_ADDR FCARG1a, op1_addr
+					}
+					|	ZVAL_DEREF FCARG1a, op1_info
+					op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
+				} else {
+					/* Hack: Convert reference to regular value to simplify JIT code */
+					ZEND_ASSERT(Z_REG(op1_addr) == ZREG_FP);
+					|	IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >1
 					|	LOAD_ZVAL_ADDR FCARG1a, op1_addr
+					|	EXT_CALL zend_jit_unref_helper, r0
+					|1:
 				}
-				|	ZVAL_DEREF FCARG1a, op1_info
-				op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1a, 0);
-			} else {
-				/* Hack: Convert reference to regular value to simplify JIT code */
-				ZEND_ASSERT(Z_REG(op1_addr) == ZREG_FP);
-				|	IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >1
-				|	LOAD_ZVAL_ADDR FCARG1a, op1_addr
-				|	EXT_CALL zend_jit_unref_helper, r0
-				|1:
 			}
-		}
-		if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
-			if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
-				int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
-				const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+			if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
+				if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
+					int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
+					const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
 
-				if (!exit_addr) {
-					return 0;
-				}
-				|	IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr
-			} else {
-				|	IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1
-				|.cold_code
-				|1:
-				if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) {
-					|	LOAD_ZVAL_ADDR FCARG1a, op1_addr
-				}
-				|	SET_EX_OPLINE opline, r0
-				if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !use_this) {
-					|	EXT_CALL zend_jit_invalid_method_call_tmp, r0
+					if (!exit_addr) {
+						return 0;
+					}
+					|	IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr
 				} else {
-					|	EXT_CALL zend_jit_invalid_method_call, r0
+					|	IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1
+					|.cold_code
+					|1:
+					if (Z_REG(op1_addr) != ZREG_FCARG1a || Z_OFFSET(op1_addr) != 0) {
+						|	LOAD_ZVAL_ADDR FCARG1a, op1_addr
+					}
+					|	SET_EX_OPLINE opline, r0
+					if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !use_this) {
+						|	EXT_CALL zend_jit_invalid_method_call_tmp, r0
+					} else {
+						|	EXT_CALL zend_jit_invalid_method_call, r0
+					}
+					|	jmp ->exception_handler
+					|.code
 				}
-				|	jmp ->exception_handler
-				|.code
 			}
+			|	GET_ZVAL_PTR FCARG1a, op1_addr
 		}
-		|	GET_ZVAL_PTR FCARG1a, op1_addr
-	}
 
-	if (delayed_call_chain) {
-		if (!zend_jit_save_call_chain(Dst, delayed_call_level)) {
-			return 0;
+		if (delayed_call_chain) {
+			if (!zend_jit_save_call_chain(Dst, delayed_call_level)) {
+				return 0;
+			}
 		}
-	}
 
-	if (info) {
-		call_info = info->callee_info;
-		while (call_info && call_info->caller_init_opline != opline) {
-			call_info = call_info->next_callee;
-		}
-		if (call_info && call_info->callee_func) {
-			func = call_info->callee_func;
-		}
-	}
+		|	mov aword T1, FCARG1a // save
 
-	|	mov aword T1, FCARG1a // save
+		if (func) {
+			|	// fbc = CACHED_PTR(opline->result.num + sizeof(void*));
+			|	mov r0, EX->run_time_cache
+			|	mov r0, aword [r0 + opline->result.num + sizeof(void*)]
+			|	test r0, r0
+			|	jz >1
+		} else {
+			|	// if (CACHED_PTR(opline->result.num) == obj->ce)) {
+			|	mov r0, EX->run_time_cache
+			|	mov r2, aword [r0 + opline->result.num]
+			|	cmp r2, [FCARG1a + offsetof(zend_object, ce)]
+			|	jnz >1
+			|	// fbc = CACHED_PTR(opline->result.num + sizeof(void*));
+			|	mov r0, aword [r0 + opline->result.num + sizeof(void*)]
+		}
 
-	if (func) {
-		|	// fbc = CACHED_PTR(opline->result.num + sizeof(void*));
-		|	mov r0, EX->run_time_cache
-		|	mov r0, aword [r0 + opline->result.num + sizeof(void*)]
+		|.cold_code
+		|1:
+		|	LOAD_ADDR FCARG2a, function_name
+		|.if X64
+		|	lea CARG3, aword T1
+		|.else
+		|	lea r0, aword T1
+		|	sub r4, 12
+		|	push r0
+		|.endif
+		|	SET_EX_OPLINE opline, r0
+		if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !use_this) {
+			|	EXT_CALL zend_jit_find_method_tmp_helper, r0
+		} else {
+			|	EXT_CALL zend_jit_find_method_helper, r0
+		}
+		|.if not(X64)
+		|	add r4, 12
+		|.endif
 		|	test r0, r0
-		|	jz >1
-	} else {
-		|	// if (CACHED_PTR(opline->result.num) == obj->ce)) {
-		|	mov r0, EX->run_time_cache
-		|	mov r2, aword [r0 + opline->result.num]
-		|	cmp r2, [FCARG1a + offsetof(zend_object, ce)]
-		|	jnz >1
-		|	// fbc = CACHED_PTR(opline->result.num + sizeof(void*));
-		|	mov r0, aword [r0 + opline->result.num + sizeof(void*)]
-	}
-
-	|.cold_code
-	|1:
-	|	LOAD_ADDR FCARG2a, function_name
-	|.if X64
-	|	lea CARG3, aword T1
-	|.else
-	|	lea r0, aword T1
-	|	sub r4, 12
-	|	push r0
-	|.endif
-	|	SET_EX_OPLINE opline, r0
-	if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !use_this) {
-		|	EXT_CALL zend_jit_find_method_tmp_helper, r0
-	} else {
-		|	EXT_CALL zend_jit_find_method_helper, r0
+		|	jnz >2
+		|	jmp ->exception_handler
+		|.code
+		|2:
 	}
-	|.if not(X64)
-	|	add r4, 12
-	|.endif
-	|	test r0, r0
-	|	jnz >2
-	|	jmp ->exception_handler
-	|.code
-	|2:
 
 	if (!func
 	 && trace
diff --git a/ext/opcache/tests/jit/trampoline_001.phpt b/ext/opcache/tests/jit/trampoline_001.phpt
index 21b8b20ee0..95041257a7 100644
--- a/ext/opcache/tests/jit/trampoline_001.phpt
+++ b/ext/opcache/tests/jit/trampoline_001.phpt
@@ -1,5 +1,5 @@
 --TEST--
-JIT: trampoline cleanup
+JIT Trampoline 001: trampoline cleanup
 --INI--
 opcache.enable=1
 opcache.enable_cli=1
diff --git a/ext/opcache/tests/jit/trampoline_002.phpt b/ext/opcache/tests/jit/trampoline_002.phpt
new file mode 100644
index 0000000000..10e02e864e
--- /dev/null
+++ b/ext/opcache/tests/jit/trampoline_002.phpt
@@ -0,0 +1,40 @@
+--TEST--
+JIT Trampoline 002: trampoline cleanup
+--INI--
+opcache.enable=1
+opcache.enable_cli=1
+opcache.file_update_protection=0
+opcache.jit_buffer_size=1M
+opcache.jit=tracing
+--SKIPIF--
+<?php require_once('skipif.inc'); ?>
+--FILE--
+<?php
+class A {
+}
+class B extends A {
+	function foo() {
+		echo "B";
+	}
+}
+class C extends A {
+	function __call($name, $argd) {
+		echo "C";
+	}
+}
+class D extends A {
+	function foo() {
+		echo "D";
+	}
+}
+$b = new B;
+$c = new C;
+$d = new D;
+$a = [$b, $b, $b, $c, $c, $c, $d, $d, $d, $c, $c, $c];
+foreach ($a as $x) {
+	$x->foo();
+}
+echo "\n";
+?>
+--EXPECT--
+BBBCCCDDDCCC