Skip to content

Commit

Permalink
Allow JIT for passing arguments to trampolines and "bad" functions (#…
Browse files Browse the repository at this point in the history 
…16365)

* Better trace coverage (JIT trampoline calls)

* clenup trampoline by zend_jit_free_trampoline()

* Fix ZEND_JIT_TRACE_INIT_CALL/ZEND_JIT_TRACE_DO_ICALL num_args mismatch

It may be caused by SEND_UNPACK/SEND_ARRAY

* cleanup

* cleanup

* Don't record function that may be temporary

* cleanup

* Prevent invalid run_time_cache allocation for "bad" internal functions

* Update zend_jit_trace_record_fake_init_call_ex() accordingly

* Better handling of "bad" functions and fake closures
  • Loading branch information
@dstogov
dstogov authored Oct 15, 2024
1 parent 2d9eb54 commit 097edc8
Show file tree
Hide file tree
Showing 5 changed files with 140 additions and 88 deletions.
2 changes: 1 addition & 1 deletion ext/opcache/jit/zend_jit_helpers.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ static zend_never_inline zend_op_array* ZEND_FASTCALL zend_jit_init_func_run_tim
{
void **run_time_cache;

if (!RUN_TIME_CACHE(op_array)) {
if (op_array->type == ZEND_USER_FUNCTION && !RUN_TIME_CACHE(op_array)) {
run_time_cache = zend_arena_alloc(&CG(arena), op_array->cache_size);
memset(run_time_cache, 0, op_array->cache_size);
ZEND_MAP_PTR_SET(op_array->run_time_cache, run_time_cache);
Expand Down
14 changes: 10 additions & 4 deletions ext/opcache/jit/zend_jit_internal.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -248,8 +248,11 @@ zend_constant* ZEND_FASTCALL zend_jit_check_constant(const zval *key);
_(RECURSIVE_CALL, "recursive call") \
_(RECURSIVE_RET, "recursive return") \
_(RETURN, "return") \
_(INTERPRETER, "exit to VM interpreter") \
_(LINK, "link to another trace") \
_(INTERPRETER, "exit to VM interpreter") \
_(TRAMPOLINE, "trampoline call") \
_(PROP_HOOK_CALL, "property hook call") \
_(BAD_FUNC, "bad function call") \
/* compilation and linking successful */ \
_(COMPILED, "compiled") \
_(ALREADY_DONE, "already prcessed") \
Expand All @@ -267,9 +270,6 @@ zend_constant* ZEND_FASTCALL zend_jit_check_constant(const zval *key);
_(BLACK_LIST, "trace blacklisted") \
_(INNER_LOOP, "inner loop") /* trace it */ \
_(COMPILED_LOOP, "compiled loop") \
_(TRAMPOLINE, "trampoline call") \
_(PROP_HOOK_CALL, "property hook call") \
_(BAD_FUNC, "bad function call") \
_(COMPILER_ERROR, "JIT compilation error") \
/* no recoverable error (blacklist immediately) */ \
_(NO_SHM, "insufficient shared memory") \
Expand Down Expand Up @@ -380,6 +380,12 @@ typedef enum _zend_jit_trace_op {
#define ZEND_JIT_TRACE_FAKE_INFO(level) \
(((level) << ZEND_JIT_TRACE_FAKE_LEVEL_SHIFT) | ZEND_JIT_TRACE_FAKE_INIT_CALL)

#define ZEND_JIT_TRACE_NUM_ARGS_INFO(count) \
((count) << ZEND_JIT_TRACE_FAKE_LEVEL_SHIFT)

#define ZEND_JIT_TRACE_NUM_ARGS(info) \
(((info) & ZEND_JIT_TRACE_FAKE_LEVEL_MASK) >> ZEND_JIT_TRACE_FAKE_LEVEL_SHIFT)

#define ZEND_JIT_TRACE_SET_FIRST_SSA_VAR(_info, var) do { \
_info |= (var << ZEND_JIT_TRACE_SSA_VAR_SHIFT); \
} while (0)
Expand Down
34 changes: 25 additions & 9 deletions ext/opcache/jit/zend_jit_ir.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8431,13 +8431,21 @@ static int zend_jit_push_call_frame(zend_jit_ctx *jit, const zend_op *opline, co
used_stack_ref);

if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
bool may_be_trampoline = !func && (opline->opcode == ZEND_INIT_METHOD_CALL);
int32_t exit_point = zend_jit_trace_get_exit_point(opline,
may_be_trampoline ?
(ZEND_JIT_EXIT_TO_VM | ZEND_JIT_EXIT_METHOD_CALL) : ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);

if (!exit_addr) {
return 0;
}

if (may_be_trampoline) {
jit->trace->exit_info[exit_point].poly_func_ref = func_ref;
jit->trace->exit_info[exit_point].poly_this_ref = this_ref;
}

ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else {
if_enough_stack = ir_IF(ref);
Expand Down Expand Up @@ -9064,6 +9072,14 @@ static int zend_jit_init_method_call(zend_jit_ctx *jit,
jit->delayed_call_level = call_level;
}

if (trace
&& trace->op == ZEND_JIT_TRACE_END
&& trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER) {
if (!zend_jit_set_ip(jit, opline + 1)) {
return 0;
}
}

return 1;
}

Expand Down Expand Up @@ -9324,7 +9340,7 @@ static int zend_jit_init_closure_call(zend_jit_ctx *jit,

if (trace
&& trace->op == ZEND_JIT_TRACE_END
&& trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
&& trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER) {
if (!zend_jit_set_ip(jit, opline + 1)) {
return 0;
}
Expand Down Expand Up @@ -9933,7 +9949,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen

if (trace && !func) {
if (trace->op == ZEND_JIT_TRACE_DO_ICALL) {
ZEND_ASSERT(trace->func->type == ZEND_INTERNAL_FUNCTION);
ZEND_ASSERT(!trace->func || trace->func->type == ZEND_INTERNAL_FUNCTION);
#ifndef ZEND_WIN32
// TODO: ASLR may cause different addresses in different workers ???
func = trace->func;
Expand Down Expand Up @@ -10115,7 +10131,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen

if (call_num_args <= func->op_array.num_args) {
if (!trace || (trace->op == ZEND_JIT_TRACE_END
&& trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) {
&& trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER)) {
uint32_t num_args;

if ((func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) {
Expand Down Expand Up @@ -10149,7 +10165,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen
}
} else {
if (!trace || (trace->op == ZEND_JIT_TRACE_END
&& trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) {
&& trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER)) {
ir_ref ip;

if (zend_accel_in_shm(func->op_array.opcodes)) {
Expand Down Expand Up @@ -10275,7 +10291,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen
ir_ref observer_handler;
ir_ref rx = jit_FP(jit);
struct jit_observer_fcall_is_unobserved_data unobserved_data = jit_observer_fcall_is_unobserved_start(jit, func, &observer_handler, rx, func_ref);
if (trace && (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) {
if (trace && (trace->op != ZEND_JIT_TRACE_END || trace->stop < ZEND_JIT_TRACE_STOP_INTERPRETER)) {
ZEND_ASSERT(trace[1].op == ZEND_JIT_TRACE_VM || trace[1].op == ZEND_JIT_TRACE_END);
jit_SET_EX_OPLINE(jit, trace[1].opline);
} else if (GCC_GLOBAL_REGS) {
Expand Down Expand Up @@ -10568,7 +10584,7 @@ static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zen
jit_LOAD_IP_ADDR(jit, opline + 1);
} else if (trace
&& trace->op == ZEND_JIT_TRACE_END
&& trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
&& trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER) {
jit_LOAD_IP_ADDR(jit, opline + 1);
}
}
Expand Down Expand Up @@ -16908,7 +16924,7 @@ static int zend_jit_trace_handler(zend_jit_ctx *jit, const zend_op_array *op_arr
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
if (trace->op != ZEND_JIT_TRACE_END ||
(trace->stop != ZEND_JIT_TRACE_STOP_RETURN &&
trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) {
trace->stop < ZEND_JIT_TRACE_STOP_INTERPRETER)) {
/* this check may be handled by the following OPLINE guard or jmp [IP] */
ir_GUARD(ir_NE(jit_IP(jit), ir_CONST_ADDR(zend_jit_halt_op)),
jit_STUB_ADDR(jit, jit_stub_trace_halt));
Expand All @@ -16926,7 +16942,7 @@ static int zend_jit_trace_handler(zend_jit_ctx *jit, const zend_op_array *op_arr
}
if (trace->op != ZEND_JIT_TRACE_END ||
(trace->stop != ZEND_JIT_TRACE_STOP_RETURN &&
trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) {
trace->stop < ZEND_JIT_TRACE_STOP_INTERPRETER)) {

const zend_op *next_opline = trace->opline;
const zend_op *exit_opline = NULL;
Expand Down
Loading

0 comments on commit 097edc8

Please sign in to comment.