Avoid unnecessary IP initiliaization on trace linking

3 files changed, 116 insertions, 41 deletions

diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h
index bcbafad147..18a3096204 100644
--- a/ext/opcache/jit/zend_jit_internal.h
+++ b/ext/opcache/jit/zend_jit_internal.h
@@ -348,6 +348,7 @@ typedef union _zend_jit_trace_stack {
 /* trace info flags */
 #define ZEND_JIT_TRACE_CHECK_INTERRUPT (1<<0)
 #define ZEND_JIT_TRACE_LOOP            (1<<1)
+#define ZEND_JIT_TRACE_USES_INITIAL_IP (1<<2)
 
 typedef struct _zend_jit_trace_info {
 	uint32_t                  id;            /* trace id */
diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c
index b922655e37..5e09b2dd7c 100644
--- a/ext/opcache/jit/zend_jit_trace.c
+++ b/ext/opcache/jit/zend_jit_trace.c
@@ -3088,7 +3088,6 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
 	zend_uchar res_type = IS_UNKNOWN;
 	const zend_op *opline, *orig_opline;
 	const zend_ssa_op *ssa_op, *orig_ssa_op;
-	const void *timeout_exit_addr = NULL;
 
 	JIT_G(current_trace) = trace_buffer;
 
@@ -3142,6 +3141,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
 
 	if (!parent_trace) {
 		zend_jit_set_last_valid_opline(opline);
+		zend_jit_track_last_valid_opline();
 	} else {
 		if (zend_jit_traces[parent_trace].exit_info[exit_num].opline == NULL) {
 			zend_jit_trace_opline_guard(&dasm_state, opline);
@@ -3273,18 +3273,16 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par
 			}
 		}
 
-		if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
-			if (ra && zend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T)) {
-				uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, ZEND_JIT_EXIT_TO_VM);
-
-				timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
-				if (!timeout_exit_addr) {
-					goto jit_failure;
-				}
-			} else {
-				timeout_exit_addr = dasm_labels[zend_lbinterrupt_handler];
-			}
-		}
+//		if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
+//			if (ra && zend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T)) {
+//				uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
+//
+//				timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+//				if (!timeout_exit_addr) {
+//					goto jit_failure;
+//				}
+//			}
+//		}
 
 		if (ra && trace_buffer->stop != ZEND_JIT_TRACE_STOP_LOOP) {
 			int last_var = op_array->last_var;
@@ -5074,11 +5072,17 @@ done:
 
 	t = &zend_jit_traces[ZEND_JIT_TRACE_NUM];
 
+	if (!parent_trace && zend_jit_trace_uses_initial_ip()) {
+		t->flags |= ZEND_JIT_TRACE_USES_INITIAL_IP;
+	}
+
 	if (p->stop == ZEND_JIT_TRACE_STOP_LOOP
 	 || p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL
 	 || p->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
 		if (p->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
-			if (!zend_jit_set_ip(&dasm_state, p->opline)) {
+			ZEND_ASSERT(!parent_trace);
+			if ((t->flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
+			 && !zend_jit_set_ip(&dasm_state, p->opline)) {
 				goto jit_failure;
 			}
 		}
@@ -5087,7 +5091,25 @@ done:
 			t->flags |= ZEND_JIT_TRACE_CHECK_INTERRUPT;
 		}
 		if (!(t->flags & ZEND_JIT_TRACE_LOOP)) {
+			const void *timeout_exit_addr = NULL;
+
 			t->flags |= ZEND_JIT_TRACE_LOOP;
+
+			if (trace_buffer->stop != ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
+				if (!(t->flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
+				 || (ra
+				  && zend_jit_trace_stack_needs_deoptimization(stack, op_array->last_var + op_array->T))) {
+					uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
+
+					timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+					if (!timeout_exit_addr) {
+						goto jit_failure;
+					}
+				} else {
+					timeout_exit_addr = dasm_labels[zend_lbinterrupt_handler];
+				}
+			}
+
 			zend_jit_trace_end_loop(&dasm_state, 0, timeout_exit_addr); /* jump back to start of the trace loop */
 		}
 	} else if (p->stop == ZEND_JIT_TRACE_STOP_LINK
@@ -5097,14 +5119,35 @@ done:
 			goto jit_failure;
 		}
 		if (p->stop == ZEND_JIT_TRACE_STOP_LINK) {
-			if (!zend_jit_set_ip(&dasm_state, p->opline)) {
+			const void *timeout_exit_addr = NULL;
+
+			t->link = zend_jit_find_trace(p->opline->handler);
+			if ((zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_USES_INITIAL_IP)
+			 && !zend_jit_set_ip(&dasm_state, p->opline)) {
 				goto jit_failure;
 			}
-			t->link = zend_jit_find_trace(p->opline->handler);
-			zend_jit_trace_link_to_root(&dasm_state, &zend_jit_traces[t->link],
-				parent_trace &&
-				(zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_CHECK_INTERRUPT) &&
-				zend_jit_traces[parent_trace].root == t->link);
+			if (!parent_trace && zend_jit_trace_uses_initial_ip()) {
+				t->flags |= ZEND_JIT_TRACE_USES_INITIAL_IP;
+			}
+			if (parent_trace
+			 && (zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_CHECK_INTERRUPT)
+			 && zend_jit_traces[parent_trace].root == t->link) {
+				if (!(zend_jit_traces[t->link].flags & ZEND_JIT_TRACE_USES_INITIAL_IP)) {
+					uint32_t exit_point;
+
+					for (i = 0; i < op_array->last_var + op_array->T; i++) {
+						SET_STACK_TYPE(stack, i, IS_UNKNOWN);
+					}
+					exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
+					timeout_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
+					if (!timeout_exit_addr) {
+						goto jit_failure;
+					}
+				} else {
+					timeout_exit_addr = dasm_labels[zend_lbinterrupt_handler];
+				}
+			}
+			zend_jit_trace_link_to_root(&dasm_state, &zend_jit_traces[t->link], timeout_exit_addr);
 		} else {
 			zend_jit_trace_return(&dasm_state, 0);
 		}
diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc
index afb51163dd..566c052181 100644
--- a/ext/opcache/jit/zend_jit_x86.dasc
+++ b/ext/opcache/jit/zend_jit_x86.dasc
@@ -1340,11 +1340,12 @@ static void* dasm_labels[zend_lb_MAX];
 
 |.macro SET_EX_OPLINE, op, tmp_reg
 ||	if (op == last_valid_opline) {
+||		zend_jit_use_last_valid_opline();
 |		SAVE_IP
 ||	} else {
 |		ADDR_OP2_2 mov, aword EX->opline, op, tmp_reg
 ||		if (!GCC_GLOBAL_REGS) {
-||			last_valid_opline = NULL;
+||			zend_jit_reset_last_valid_opline();
 ||		}
 ||	}
 |.endmacro
@@ -1578,6 +1579,7 @@ static void* dasm_labels[zend_lb_MAX];
 
 |.macro UNDEFINED_OFFSET, opline
 ||	if (opline == last_valid_opline) {
+||		zend_jit_use_last_valid_opline();
 |		call ->undefined_offset_ex
 ||	} else {
 |		SET_EX_OPLINE  opline, r0
@@ -1587,6 +1589,7 @@ static void* dasm_labels[zend_lb_MAX];
 
 |.macro UNDEFINED_INDEX, opline
 ||	if (opline == last_valid_opline) {
+||		zend_jit_use_last_valid_opline();
 |		call ->undefined_index_ex
 ||	} else {
 |		SET_EX_OPLINE opline, r0
@@ -1596,6 +1599,7 @@ static void* dasm_labels[zend_lb_MAX];
 
 |.macro CANNOT_ADD_ELEMENT, opline
 ||	if (opline == last_valid_opline) {
+||		zend_jit_use_last_valid_opline();
 |		call ->cannot_add_element_ex
 ||	} else {
 |		SET_EX_OPLINE opline, r0
@@ -1603,31 +1607,56 @@ static void* dasm_labels[zend_lb_MAX];
 ||	}
 |.endmacro
 
-static zend_bool reuse_ip;
-static zend_bool delayed_call_chain;
-static uint32_t  delayed_call_level;
-static const zend_op *last_valid_opline;
-static int jit_return_label;
-static uint32_t current_trace_num;
+static zend_bool reuse_ip = 0;
+static zend_bool delayed_call_chain = 0;
+static uint32_t  delayed_call_level = 0;
+static const zend_op *last_valid_opline = NULL;
+static zend_bool use_last_vald_opline = 0;
+static zend_bool track_last_valid_opline = 0;
+static int jit_return_label = -1;
+static uint32_t current_trace_num = 0;
+
+static void zend_jit_track_last_valid_opline(void)
+{
+	use_last_vald_opline = 0;
+	track_last_valid_opline = 1;
+}
+
+static void zend_jit_use_last_valid_opline(void)
+{
+	if (track_last_valid_opline) {
+		use_last_vald_opline = 1;
+		track_last_valid_opline = 0;
+	}
+}
+
+static zend_bool zend_jit_trace_uses_initial_ip(void)
+{
+	return use_last_vald_opline;
+}
 
 static void zend_jit_set_last_valid_opline(const zend_op *target_opline)
 {
 	if (!reuse_ip) {
+		track_last_valid_opline = 0;
 		last_valid_opline = target_opline;
 	}
 }
 
 static void zend_jit_reset_last_valid_opline(void)
 {
+	track_last_valid_opline = 0;
 	last_valid_opline = NULL;
 }
 
-static void zend_jit_start_reuse_ip(void) {
-	last_valid_opline = NULL;
+static void zend_jit_start_reuse_ip(void)
+{
+	zend_jit_reset_last_valid_opline();
 	reuse_ip = 1;
 }
 
-static void zend_jit_stop_reuse_ip(void) {
+static void zend_jit_stop_reuse_ip(void)
+{
 	reuse_ip = 0;
 }
 
@@ -2804,6 +2833,8 @@ static int zend_jit_align_func(dasm_State **Dst)
 	reuse_ip = 0;
 	delayed_call_chain = 0;
 	last_valid_opline = NULL;
+	use_last_vald_opline = 0;
+	track_last_valid_opline = 0;
 	jit_return_label = -1;
 	|.align 16
 	return 1;
@@ -2849,16 +2880,15 @@ static int zend_jit_save_call_chain(dasm_State **Dst, uint32_t call_level)
 
 static int zend_jit_set_ip(dasm_State **Dst, const zend_op *opline)
 {
-	if (!last_valid_opline) {
+	if (last_valid_opline == opline) {
+		zend_jit_use_last_valid_opline();
+	} else if (GCC_GLOBAL_REGS && last_valid_opline) {
+		zend_jit_use_last_valid_opline();
+		|	ADD_IP (opline - last_valid_opline) * sizeof(zend_op);
+	} else {
 		|	LOAD_IP_ADDR opline
-	} else if (last_valid_opline != opline) {
-		if (GCC_GLOBAL_REGS) {
-			|	ADD_IP (opline - last_valid_opline) * sizeof(zend_op);
-		} else {
-			|	LOAD_IP_ADDR opline
-		}
 	}
-	last_valid_opline = opline;
+	zend_jit_set_last_valid_opline(opline);
 
 	return 1;
 }
@@ -2890,6 +2920,7 @@ static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline, const
 	if (exit_addr) {
 		|	jne &exit_addr
 	} else if (last_valid_opline == opline) {
+		||		zend_jit_use_last_valid_opline();
 		|	jne ->interrupt_handler
 	} else {
 		|	jne >1
@@ -3154,7 +3185,7 @@ static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t jmp_
 	return zend_jit_patch(code, size, jmp_table_size, zend_jit_trace_get_exit_addr(exit_num), addr);
 }
 
-static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t, zend_bool check_interrupt)
+static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t, const void *timeout_exit_addr)
 {
 	const void *link_addr;
 	size_t prologue_size;
@@ -3188,11 +3219,11 @@ static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t,
 	}
 	link_addr = (const void*)((const char*)t->code_start + prologue_size);
 
-	if (check_interrupt) {
+	if (timeout_exit_addr) {
 		/* Check timeout for links to LOOP */
 		|	MEM_OP2_1_ZTS cmp, byte, executor_globals, vm_interrupt, 0, r0
 		|	je &link_addr
-		|	jmp ->interrupt_handler
+		|	jmp &timeout_exit_addr
 	} else {
 		|	jmp &link_addr
 	}