From 097edc86c820178da7b26c50bb87693b45e3be8e Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Tue, 15 Oct 2024 22:31:05 +0300 Subject: [PATCH] Allow JIT for passing arguments to trampolines and "bad" functions (#16365) * Better trace coverage (JIT trampoline calls) * clenup trampoline by zend_jit_free_trampoline() * Fix ZEND_JIT_TRACE_INIT_CALL/ZEND_JIT_TRACE_DO_ICALL num_args mismatch It may be caused by SEND_UNPACK/SEND_ARRAY * cleanup * cleanup * Don't record function that may be temporary * cleanup * Prevent invalid run_time_cache allocation for "bad" internal functions * Update zend_jit_trace_record_fake_init_call_ex() accordingly * Better handling of "bad" functions and fake closures --- ext/opcache/jit/zend_jit_helpers.c | 2 +- ext/opcache/jit/zend_jit_internal.h | 14 ++-- ext/opcache/jit/zend_jit_ir.c | 34 +++++++--- ext/opcache/jit/zend_jit_trace.c | 92 +++++++++++++++++++-------- ext/opcache/jit/zend_jit_vm_helpers.c | 86 ++++++++++++------------- 5 files changed, 140 insertions(+), 88 deletions(-) diff --git a/ext/opcache/jit/zend_jit_helpers.c b/ext/opcache/jit/zend_jit_helpers.c index 140f69dee062c..3eec4df0ecc7e 100644 --- a/ext/opcache/jit/zend_jit_helpers.c +++ b/ext/opcache/jit/zend_jit_helpers.c @@ -42,7 +42,7 @@ static zend_never_inline zend_op_array* ZEND_FASTCALL zend_jit_init_func_run_tim { void **run_time_cache; - if (!RUN_TIME_CACHE(op_array)) { + if (op_array->type == ZEND_USER_FUNCTION && !RUN_TIME_CACHE(op_array)) { run_time_cache = zend_arena_alloc(&CG(arena), op_array->cache_size); memset(run_time_cache, 0, op_array->cache_size); ZEND_MAP_PTR_SET(op_array->run_time_cache, run_time_cache); diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index 4fe07222d8f72..9a09be49f1bd5 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -248,8 +248,11 @@ zend_constant* ZEND_FASTCALL zend_jit_check_constant(const zval *key); _(RECURSIVE_CALL, "recursive call") \ _(RECURSIVE_RET, "recursive return") \ _(RETURN, "return") \ - _(INTERPRETER, "exit to VM interpreter") \ _(LINK, "link to another trace") \ + _(INTERPRETER, "exit to VM interpreter") \ + _(TRAMPOLINE, "trampoline call") \ + _(PROP_HOOK_CALL, "property hook call") \ + _(BAD_FUNC, "bad function call") \ /* compilation and linking successful */ \ _(COMPILED, "compiled") \ _(ALREADY_DONE, "already prcessed") \ @@ -267,9 +270,6 @@ zend_constant* ZEND_FASTCALL zend_jit_check_constant(const zval *key); _(BLACK_LIST, "trace blacklisted") \ _(INNER_LOOP, "inner loop") /* trace it */ \ _(COMPILED_LOOP, "compiled loop") \ - _(TRAMPOLINE, "trampoline call") \ - _(PROP_HOOK_CALL, "property hook call") \ - _(BAD_FUNC, "bad function call") \ _(COMPILER_ERROR, "JIT compilation error") \ /* no recoverable error (blacklist immediately) */ \ _(NO_SHM, "insufficient shared memory") \ @@ -380,6 +380,12 @@ typedef enum _zend_jit_trace_op { #define ZEND_JIT_TRACE_FAKE_INFO(level) \ (((level) << ZEND_JIT_TRACE_FAKE_LEVEL_SHIFT) | ZEND_JIT_TRACE_FAKE_INIT_CALL) +#define ZEND_JIT_TRACE_NUM_ARGS_INFO(count) \ + ((count) << ZEND_JIT_TRACE_FAKE_LEVEL_SHIFT) + +#define ZEND_JIT_TRACE_NUM_ARGS(info) \ + (((info) & ZEND_JIT_TRACE_FAKE_LEVEL_MASK) >> ZEND_JIT_TRACE_FAKE_LEVEL_SHIFT) + #define ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(_info, var) do { \ _info |= (var << ZEND_JIT_TRACE_SSA_VAR_SHIFT); \ } while (0) diff --git a/ext/opcache/jit/zend_jit_ir.c b/ext/opcache/jit/zend_jit_ir.c index e4a66a6743daf..204a667732078 100644 --- a/ext/opcache/jit/zend_jit_ir.c +++ b/ext/opcache/jit/zend_jit_ir.c @@ -8431,13 +8431,21 @@ static int zend_jit_push_call_frame(zend_jit_ctx *jit, const zend_op *opline, co used_stack_ref); if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { - int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); + bool may_be_trampoline = !func && (opline->opcode == ZEND_INIT_METHOD_CALL); + int32_t exit_point = zend_jit_trace_get_exit_point(opline, + may_be_trampoline ? + (ZEND_JIT_EXIT_TO_VM | ZEND_JIT_EXIT_METHOD_CALL) : ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } + if (may_be_trampoline) { + jit->trace->exit_info[exit_point].poly_func_ref = func_ref; + jit->trace->exit_info[exit_point].poly_this_ref = this_ref; + } + ir_GUARD(ref, ir_CONST_ADDR(exit_addr)); } else { if_enough_stack = ir_IF(ref); @@ -9064,6 +9072,14 @@ static int zend_jit_init_method_call(zend_jit_ctx *jit, jit->delayed_call_level = call_level; } + if (trace + && trace->op == ZEND_JIT_TRACE_END + && trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER) { + if (!zend_jit_set_ip(jit, opline + 1)) { + return 0; + } + } + return 1; } @@ -9324,7 +9340,7 @@ static int zend_jit_init_closure_call(zend_jit_ctx *jit, if (trace && trace->op == ZEND_JIT_TRACE_END - && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { + && trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER) { if (!zend_jit_set_ip(jit, opline + 1)) { return 0; } @@ -9933,7 +9949,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen if (trace && !func) { if (trace->op == ZEND_JIT_TRACE_DO_ICALL) { - ZEND_ASSERT(trace->func->type == ZEND_INTERNAL_FUNCTION); + ZEND_ASSERT(!trace->func || trace->func->type == ZEND_INTERNAL_FUNCTION); #ifndef ZEND_WIN32 // TODO: ASLR may cause different addresses in different workers ??? func = trace->func; @@ -10115,7 +10131,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen if (call_num_args <= func->op_array.num_args) { if (!trace || (trace->op == ZEND_JIT_TRACE_END - && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) { + && trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER)) { uint32_t num_args; if ((func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) { @@ -10149,7 +10165,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen } } else { if (!trace || (trace->op == ZEND_JIT_TRACE_END - && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) { + && trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER)) { ir_ref ip; if (zend_accel_in_shm(func->op_array.opcodes)) { @@ -10275,7 +10291,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen ir_ref observer_handler; ir_ref rx = jit_FP(jit); struct jit_observer_fcall_is_unobserved_data unobserved_data = jit_observer_fcall_is_unobserved_start(jit, func, &observer_handler, rx, func_ref); - if (trace && (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) { + if (trace && (trace->op != ZEND_JIT_TRACE_END || trace->stop < ZEND_JIT_TRACE_STOP_INTERPRETER)) { ZEND_ASSERT(trace[1].op == ZEND_JIT_TRACE_VM || trace[1].op == ZEND_JIT_TRACE_END); jit_SET_EX_OPLINE(jit, trace[1].opline); } else if (GCC_GLOBAL_REGS) { @@ -10568,7 +10584,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen jit_LOAD_IP_ADDR(jit, opline + 1); } else if (trace && trace->op == ZEND_JIT_TRACE_END - && trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { + && trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER) { jit_LOAD_IP_ADDR(jit, opline + 1); } } @@ -16908,7 +16924,7 @@ static int zend_jit_trace_handler(zend_jit_ctx *jit, const zend_op_array *op_arr if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) { if (trace->op != ZEND_JIT_TRACE_END || (trace->stop != ZEND_JIT_TRACE_STOP_RETURN && - trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) { + trace->stop < ZEND_JIT_TRACE_STOP_INTERPRETER)) { /* this check may be handled by the following OPLINE guard or jmp [IP] */ ir_GUARD(ir_NE(jit_IP(jit), ir_CONST_ADDR(zend_jit_halt_op)), jit_STUB_ADDR(jit, jit_stub_trace_halt)); @@ -16926,7 +16942,7 @@ static int zend_jit_trace_handler(zend_jit_ctx *jit, const zend_op_array *op_arr } if (trace->op != ZEND_JIT_TRACE_END || (trace->stop != ZEND_JIT_TRACE_STOP_RETURN && - trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) { + trace->stop < ZEND_JIT_TRACE_STOP_INTERPRETER)) { const zend_op *next_opline = trace->opline; const zend_op *exit_opline = NULL; diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index 02d2e62ccf4ff..a3abbc1217900 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -427,25 +427,25 @@ static zend_always_inline void zend_jit_trace_add_op_guard(zend_ssa #define CHECK_OP1_DATA_TRACE_TYPE() \ CHECK_OP_TRACE_TYPE((opline+1)->op1.var, (ssa_op+1)->op1_use, op1_data_info, op3_type) -static zend_always_inline size_t zend_jit_trace_frame_size(const zend_op_array *op_array) +static zend_always_inline size_t zend_jit_trace_frame_size(const zend_op_array *op_array, uint32_t num_args) { if (op_array && op_array->type == ZEND_USER_FUNCTION) { return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE((op_array->last_var + op_array->T) * sizeof(zend_jit_trace_stack))); } else if (op_array) { return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE(op_array->num_args * sizeof(zend_jit_trace_stack))); } else { - return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack)); + return ZEND_MM_ALIGNED_SIZE(offsetof(zend_jit_trace_stack_frame, stack) + ZEND_MM_ALIGNED_SIZE(num_args * sizeof(zend_jit_trace_stack))); } } -static zend_jit_trace_stack_frame* zend_jit_trace_call_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array) +static zend_jit_trace_stack_frame* zend_jit_trace_call_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array, uint32_t num_args) { - return (zend_jit_trace_stack_frame*)((char*)frame + zend_jit_trace_frame_size(op_array)); + return (zend_jit_trace_stack_frame*)((char*)frame + zend_jit_trace_frame_size(op_array, num_args)); } static zend_jit_trace_stack_frame* zend_jit_trace_ret_frame(zend_jit_trace_stack_frame *frame, const zend_op_array *op_array) { - return (zend_jit_trace_stack_frame*)((char*)frame - zend_jit_trace_frame_size(op_array)); + return (zend_jit_trace_stack_frame*)((char*)frame - zend_jit_trace_frame_size(op_array, 0)); } static void zend_jit_trace_send_type(const zend_op *opline, zend_jit_trace_stack_frame *call, uint8_t type) @@ -1297,6 +1297,39 @@ typedef struct _zend_tssa { static const zend_op _nop_opcode = {0}; +static uint32_t find_trampoline_num_args(zend_jit_trace_rec *start, zend_jit_trace_rec *p) +{ + int inline_level = 0, call_level = 0; + + p--; + while (p != start) { + if (p->op == ZEND_JIT_TRACE_INIT_CALL) { + if (inline_level == 0) { + if (call_level == 0) { + ZEND_ASSERT(!p->op_array); + return ZEND_JIT_TRACE_NUM_ARGS(p->info); + } else { + call_level--; + } + } + } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) { + if (inline_level == 0) { + call_level++; + } + } else if (p->op == ZEND_JIT_TRACE_ENTER) { + if (inline_level) { + inline_level--; + } else { + return 0; + } + } else if (p->op == ZEND_JIT_TRACE_BACK) { + inline_level++; + } + p--; + } + return 0; +} + static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uint32_t parent_trace, uint32_t exit_num, zend_script *script, const zend_op_array **op_arrays, int *num_op_arrays_ptr) { zend_ssa *tssa; @@ -1323,7 +1356,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin * Calculate size of abstract stack; * Construct regular SSA for involved op_array */ op_array = trace_buffer->op_array; - stack_top = stack_size = zend_jit_trace_frame_size(op_array); + stack_top = stack_size = zend_jit_trace_frame_size(op_array, 0); stack_bottom = 0; p = trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE; ssa_ops_count = 0; @@ -1363,11 +1396,12 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin ssa_ops_count += zend_jit_trace_op_len(p->opline); } else if (p->op == ZEND_JIT_TRACE_INIT_CALL) { call_level++; - stack_top += zend_jit_trace_frame_size(p->op_array); + stack_top += zend_jit_trace_frame_size(p->op_array, ZEND_JIT_TRACE_NUM_ARGS(p->info)); if (stack_top > stack_size) { stack_size = stack_top; } } else if (p->op == ZEND_JIT_TRACE_DO_ICALL) { + uint32_t num_args = 0; if (JIT_G(opt_level) < ZEND_JIT_LEVEL_OPT_FUNC) { if (p->func && p->func != (zend_function*)&zend_pass_function @@ -1377,7 +1411,11 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS; } } - frame_size = zend_jit_trace_frame_size(p->op_array); + if (!p->func) { + /* Find num_args in the corresponding ZEND_JIT_TRACE_INIT_CALL record */ + num_args = find_trampoline_num_args(trace_buffer + ZEND_JIT_TRACE_START_REC_SIZE, p); + } + frame_size = zend_jit_trace_frame_size(p->op_array, num_args); if (call_level == 0) { if (stack_top + frame_size > stack_size) { stack_size = stack_top + frame_size; @@ -1389,7 +1427,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin } else if (p->op == ZEND_JIT_TRACE_ENTER) { op_array = p->op_array; if (call_level == 0) { - stack_top += zend_jit_trace_frame_size(op_array); + stack_top += zend_jit_trace_frame_size(op_array, 0); if (stack_top > stack_size) { stack_size = stack_top; } @@ -1414,7 +1452,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin } } else if (p->op == ZEND_JIT_TRACE_BACK) { if (level == 0) { - stack_bottom += zend_jit_trace_frame_size(p->op_array); + stack_bottom += zend_jit_trace_frame_size(p->op_array, 0); jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); ssa = &jit_extension->func_info.ssa; @@ -1431,7 +1469,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin ssa = zend_jit_trace_build_ssa(op_array, script); } } else { - stack_top -= zend_jit_trace_frame_size(op_array); + stack_top -= zend_jit_trace_frame_size(op_array, 0); level--; } op_array = p->op_array; @@ -1534,7 +1572,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin len--; } } else if (p->op == ZEND_JIT_TRACE_ENTER) { - frame = zend_jit_trace_call_frame(frame, op_array); + frame = zend_jit_trace_call_frame(frame, op_array, 0); stack = frame->stack; op_array = p->op_array; level++; @@ -1754,7 +1792,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin } frame = JIT_G(current_frame); - top = zend_jit_trace_call_frame(frame, op_array); + top = zend_jit_trace_call_frame(frame, op_array, 0); TRACE_FRAME_INIT(frame, op_array, 0, 0); TRACE_FRAME_SET_RETURN_SSA_VAR(frame, -1); frame->used_stack = 0; @@ -2448,7 +2486,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin call = top; TRACE_FRAME_INIT(call, op_array, 0, 0); call->used_stack = 0; - top = zend_jit_trace_call_frame(top, op_array); + top = zend_jit_trace_call_frame(top, op_array, 0); } else { ZEND_ASSERT(&call->func->op_array == op_array); } @@ -2583,7 +2621,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin call->prev = frame->call; call->used_stack = 0; frame->call = call; - top = zend_jit_trace_call_frame(top, p->op_array); + top = zend_jit_trace_call_frame(top, p->op_array, ZEND_JIT_TRACE_NUM_ARGS(p->info)); if (p->func && p->func->type == ZEND_USER_FUNCTION) { for (i = 0; i < p->op_array->last_var + p->op_array->T; i++) { SET_STACK_INFO(call->stack, i, -1); @@ -2626,6 +2664,7 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin if (idx > 0 && ssa_ops[idx-1].result_def >= 0 + && p->func && (p->func->common.fn_flags & ZEND_ACC_HAS_RETURN_TYPE) && !(p->func->common.fn_flags & ZEND_ACC_RETURN_REFERENCE)) { ZEND_ASSERT(ssa_opcodes[idx-1] == opline); @@ -3156,7 +3195,7 @@ static zend_jit_reg_var* zend_jit_trace_allocate_registers(zend_jit_trace_rec *t } } - frame = zend_jit_trace_call_frame(frame, op_array); + frame = zend_jit_trace_call_frame(frame, op_array, 0); frame->prev = prev_frame; frame->func = (const zend_function*)p->op_array; stack = frame->stack; @@ -3306,8 +3345,7 @@ static zend_jit_reg_var* zend_jit_trace_allocate_registers(zend_jit_trace_rec *t } phi = phi->next; } - } else if (p->stop == ZEND_JIT_TRACE_STOP_LINK - || p->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { + } else if (p->stop >= ZEND_JIT_TRACE_STOP_LINK) { for (i = 0; i < op_array->last_var + op_array->T; i++) { int var = STACK_VAR(stack, i); if (var >= 0 && RA_HAS_REG(var) @@ -4123,7 +4161,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par ZEND_ASSERT(p->op == ZEND_JIT_TRACE_START); op_array = p->op_array; frame = JIT_G(current_frame); - top = zend_jit_trace_call_frame(frame, op_array); + top = zend_jit_trace_call_frame(frame, op_array, 0); TRACE_FRAME_INIT(frame, op_array, TRACE_FRAME_MASK_UNKNOWN_RETURN, -1); frame->used_stack = checked_stack = peek_checked_stack = 0; stack = frame->stack; @@ -7006,7 +7044,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par } } frame->call = call; - top = zend_jit_trace_call_frame(top, p->op_array); + top = zend_jit_trace_call_frame(top, p->op_array, ZEND_JIT_TRACE_NUM_ARGS(p->info)); if (p->func) { if (p->func->type == ZEND_USER_FUNCTION) { if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) { @@ -7192,8 +7230,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par zend_jit_trace_end_loop(&ctx, jit->trace_loop_ref, timeout_exit_addr); /* jump back to start of the trace loop */ } - } else if (p->stop == ZEND_JIT_TRACE_STOP_LINK - || p->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { + } else if (p->stop >= ZEND_JIT_TRACE_STOP_LINK) { if (ra && (p-1)->op != ZEND_JIT_TRACE_ENTER && (p-1)->op != ZEND_JIT_TRACE_BACK @@ -7303,8 +7340,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par break; } } - } else if (p->stop == ZEND_JIT_TRACE_STOP_LINK - || p->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) { + } else if (p->stop >= ZEND_JIT_TRACE_STOP_LINK) { if (opline && (opline->opcode == ZEND_DO_UCALL || opline->opcode == ZEND_DO_FCALL @@ -7929,7 +7965,7 @@ static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa level, ' ', (p->func && p->func->common.scope) ? ZSTR_VAL(p->func->common.scope->name) : "", (p->func && p->func->common.scope) ? "::" : "", - p->func ? ZSTR_VAL(p->func->common.function_name) : "???"); + (p->func && p->func->common.function_name) ? ZSTR_VAL(p->func->common.function_name) : "???"); } else { fprintf(stderr, " %*c>skip\n", level, ' '); @@ -7938,9 +7974,9 @@ static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa if (p->func != (zend_function*)&zend_pass_function) { fprintf(stderr, " %*c>call %s%s%s\n", level, ' ', - p->func->common.scope ? ZSTR_VAL(p->func->common.scope->name) : "", - p->func->common.scope ? "::" : "", - ZSTR_VAL(p->func->common.function_name)); + (p->func && p->func->common.scope) ? ZSTR_VAL(p->func->common.scope->name) : "", + (p->func && p->func->common.scope) ? "::" : "", + (p->func && p->func->common.function_name) ? ZSTR_VAL(p->func->common.function_name) : "???"); } else { fprintf(stderr, " %*c>skip\n", level, ' '); diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index d42c3c6366d4a..70122eb9736e9 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -508,32 +508,28 @@ static int zend_jit_trace_record_fake_init_call_ex(zend_execute_data *call, zend } func = call->func; - if (func->common.fn_flags & (ZEND_ACC_CALL_VIA_TRAMPOLINE|ZEND_ACC_NEVER_CACHE)) { - /* TODO: Can we continue recording ??? */ - return -1; - } - /* Function is a property hook. */ - if (func->common.prop_info) { - /* TODO: Can we continue recording ??? */ - return -1; - } if (func->type == ZEND_INTERNAL_FUNCTION && (func->op_array.fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE))) { - return -1; - } - if (func->type == ZEND_USER_FUNCTION) { + func = NULL; + } else if (func->type == ZEND_USER_FUNCTION) { jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&func->op_array); - if (UNEXPECTED(!jit_extension && (func->op_array.fn_flags & ZEND_ACC_CLOSURE)) + if (UNEXPECTED(!jit_extension && (func->op_array.fn_flags & ZEND_ACC_CLOSURE)) || (jit_extension && !(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE)) || (func->op_array.fn_flags & ZEND_ACC_FAKE_CLOSURE)) { - return -1; - } - if (func->op_array.fn_flags & ZEND_ACC_CLOSURE) { + func = NULL; + } else if (func->op_array.fn_flags & ZEND_ACC_CLOSURE) { func = (zend_function*)jit_extension->op_array; } } - if (is_megamorphic == ZEND_JIT_EXIT_POLYMORPHISM + + if (!func + || (func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) + || (func->common.fn_flags & ZEND_ACC_NEVER_CACHE) + || func->common.prop_info) { + /* continue recording */ + func = NULL; + } else if (is_megamorphic == ZEND_JIT_EXIT_POLYMORPHISM /* TODO: use more accurate check ??? */ && ((ZEND_CALL_INFO(call) & ZEND_CALL_DYNAMIC) || func->common.scope)) { @@ -914,11 +910,18 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, break; } if (EX(call)->func->type == ZEND_INTERNAL_FUNCTION) { - if (EX(call)->func->op_array.fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE)) { + zend_function *func = EX(call)->func; + + if ((func->op_array.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) + || (func->common.fn_flags & ZEND_ACC_NEVER_CACHE) + || func->common.prop_info) { + /* continue recording */ + func = NULL; + } else if (func->op_array.fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE)) { stop = ZEND_JIT_TRACE_STOP_BAD_FUNC; break; } - TRACE_RECORD(ZEND_JIT_TRACE_DO_ICALL, 0, EX(call)->func); + TRACE_RECORD(ZEND_JIT_TRACE_DO_ICALL, 0, func); } } else if (opline->opcode == ZEND_INCLUDE_OR_EVAL || opline->opcode == ZEND_CALLABLE_CONVERT) { @@ -957,7 +960,8 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); if (UNEXPECTED(!jit_extension) - || UNEXPECTED(!(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE))) { + || UNEXPECTED(!(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE)) + || (op_array->fn_flags & ZEND_ACC_FAKE_CLOSURE)) { stop = ZEND_JIT_TRACE_STOP_INTERPRETER; break; } @@ -982,13 +986,11 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, } if (EX(func)->op_array.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) { - /* TODO: Can we continue recording ??? */ stop = ZEND_JIT_TRACE_STOP_TRAMPOLINE; break; } if (EX(func)->op_array.prop_info) { - /* TODO: Can we continue recording ??? */ stop = ZEND_JIT_TRACE_STOP_PROP_HOOK_CALL; break; } @@ -1100,37 +1102,21 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, if (EX(call) && EX(call)->prev_execute_data == prev_call) { zend_function *func; + uint32_t info = 0; zend_jit_op_array_trace_extension *jit_extension; - if (EX(call)->func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) { - /* TODO: Can we continue recording ??? */ - stop = ZEND_JIT_TRACE_STOP_TRAMPOLINE; - break; - } else if (EX(call)->func->common.fn_flags & ZEND_ACC_NEVER_CACHE) { - /* TODO: Can we continue recording ??? */ - stop = ZEND_JIT_TRACE_STOP_BAD_FUNC; - break; - } else if (EX(call)->func->common.prop_info) { - /* TODO: Can we continue recording ??? */ - stop = ZEND_JIT_TRACE_STOP_PROP_HOOK_CALL; - break; - } func = EX(call)->func; if (func->type == ZEND_INTERNAL_FUNCTION && (func->op_array.fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE))) { - stop = ZEND_JIT_TRACE_STOP_BAD_FUNC; - break; - } - if (func->type == ZEND_USER_FUNCTION) { + func = NULL; + } else if (func->type == ZEND_USER_FUNCTION) { jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(&func->op_array); if (UNEXPECTED(!jit_extension && (func->op_array.fn_flags & ZEND_ACC_CLOSURE)) || (jit_extension && !(jit_extension->func_info.flags & ZEND_FUNC_JIT_ON_HOT_TRACE)) || (func->op_array.fn_flags & ZEND_ACC_FAKE_CLOSURE)) { - stop = ZEND_JIT_TRACE_STOP_INTERPRETER; - break; - } - if (func->op_array.fn_flags & ZEND_ACC_CLOSURE) { + func = NULL; + } else if (func->op_array.fn_flags & ZEND_ACC_CLOSURE) { func = (zend_function*)jit_extension->op_array; } } @@ -1139,18 +1125,26 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, opline = EX(opline); #endif - if (JIT_G(max_polymorphic_calls) == 0 + if (!func + || (func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE) + || (func->common.fn_flags & ZEND_ACC_NEVER_CACHE) + || func->common.prop_info) { + /* continue recording */ + func = NULL; + } else if (JIT_G(max_polymorphic_calls) == 0 && zend_jit_may_be_polymorphic_call(opline - 1)) { func = NULL; + ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_MEGAMORPHIC); } else if ((is_megamorphic == ZEND_JIT_EXIT_METHOD_CALL || is_megamorphic == ZEND_JIT_EXIT_CLOSURE_CALL) && trace_buffer[1].opline == opline - 1) { func = NULL; + ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_MEGAMORPHIC); } if (!func) { - ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_MEGAMORPHIC); + info = ZEND_JIT_TRACE_NUM_ARGS_INFO(ZEND_CALL_NUM_ARGS(EX(call))); } - TRACE_RECORD(ZEND_JIT_TRACE_INIT_CALL, 0, func); + TRACE_RECORD(ZEND_JIT_TRACE_INIT_CALL, info, func); } prev_call = EX(call); }