From 962a455c774177c1bd081277217481a5e9242099 Mon Sep 17 00:00:00 2001 From: Vlad Brezae Date: Thu, 26 May 2022 00:55:56 +0300 Subject: [PATCH] [mono][interp] Add tiering within interpreter (#68823) * [mono][interp] Remove error argument from mono_interp_get_imethod It is unused * [mono][interp] Implement tiering at method execution start For a single MonoMethod*, we can have two InterpMethod* instances, one with optimized flag false and the other with true. When tiering is enabled, when first getting an InterpMethod* for a MonoMethod* we set the optimized flag to false. When generatig code for this method, if optimized is false we must emit a special MINT_TIER_ENTER_METHOD at the start and later in the codegen process we skip applying optimizations to method code. MINT_TIER_ENTER_METHOD opcode is invoked with every method start and it will bump a counter. Once we hit the limit, the method will be tiered up. This process consists of creating a new InterpMethod* instance which have optimized set and storing it in the interp_code_hash, changing the mapping from the old MonoMethod. The optimized and unoptimized method use the same argument space, so tiering the method up requires just to set the ip to the start of the tiered up method code. An additional problem that happens with tiering is that we have to replace all instances of the untiered method from generated code. InterpMethod* instances are stored stored inside data_items of other methods and also inside vtables. When generating code for any method, we have to store in a hash table mappings from untiered InterpMethod* instance to addresses where this instance was stored. When we tier up the unoptimized method, we will traverse the list of addresses where this references is stored and update it to the optimized version. * [mono][interp] Add option to enable/disable optimizations Some optimizations might not be enabled by default, so add option to enable them. * [mono][interp] Implement on stack replacement tiering up In unoptimized code, we add a patchpoint instruction when entering basic blocks that are targets of backward branches, if the stack state is empty. This means that when tiering up a frame we just need to jump to the equivalent basic block in the tiered up method and execution can continue. Since the arguments and IL locals reside in the same space in both versions of the method (their offsets are computed in interp_method_compute_offsets) * [mono][interp] Remove calc_section mutex We always take jit_mm lock when finishing compilation of method, use it also for publishing InterpMethod* fields. This also prevents weird races where the method can be tiered up before the we take the jit_mm lock, resulting in publishing the seq_points for the untiered method * [mono][interp] Fix incorrect linking of bblocks Once we emit a tailcall, execution in the current bblock is finished. * [mono][interp] Fix implicit conversion from i4 to native int We were doing unsigned conversion before * [mono][interp] Fix execution of clauses from il state when tiering is enabled When invoking these clauses we obtained the InterpMethod from the MonoMethod* and make use of the jit info stored during frame unwinding. However, the method might have been tiered up since storing the jit info, so the native offsets stored there will no longer be relative to the optimized imethod. Fetch again the MonoJitInfo* from the imethod that we will be executing. * [mono][interp] Enable tiering by default --- src/mono/mono/mini/CMakeLists.txt | 4 +- src/mono/mono/mini/ee.h | 6 +- src/mono/mono/mini/interp-stubs.c | 5 +- src/mono/mono/mini/interp/interp-internals.h | 29 ++- src/mono/mono/mini/interp/interp.c | 191 ++++++++++------- src/mono/mono/mini/interp/interp.h | 3 +- src/mono/mono/mini/interp/mintops.def | 4 + src/mono/mono/mini/interp/tiering.c | 214 +++++++++++++++++++ src/mono/mono/mini/interp/tiering.h | 26 +++ src/mono/mono/mini/interp/transform.c | 185 ++++++++++++---- src/mono/mono/mini/interp/transform.h | 9 + src/mono/mono/mini/mini-exceptions.c | 8 +- src/mono/mono/mini/mini-generic-sharing.c | 2 +- 13 files changed, 548 insertions(+), 138 deletions(-) create mode 100644 src/mono/mono/mini/interp/tiering.c create mode 100644 src/mono/mono/mini/interp/tiering.h diff --git a/src/mono/mono/mini/CMakeLists.txt b/src/mono/mono/mini/CMakeLists.txt index bc62225892fa76..76e1184c60d4a0 100644 --- a/src/mono/mono/mini/CMakeLists.txt +++ b/src/mono/mono/mini/CMakeLists.txt @@ -264,7 +264,9 @@ set(interp_sources interp/interp-intrins.c interp/mintops.h interp/mintops.c - interp/transform.c) + interp/transform.c + interp/tiering.h + interp/tiering.c) set(interp_stub_sources interp-stubs.c) diff --git a/src/mono/mono/mini/ee.h b/src/mono/mono/mini/ee.h index 3409d58bb8f962..7925d96f9f2534 100644 --- a/src/mono/mono/mini/ee.h +++ b/src/mono/mono/mini/ee.h @@ -14,7 +14,7 @@ #ifndef __MONO_EE_H__ #define __MONO_EE_H__ -#define MONO_EE_API_VERSION 0x16 +#define MONO_EE_API_VERSION 0x17 typedef struct _MonoInterpStackIter MonoInterpStackIter; @@ -38,7 +38,7 @@ typedef gpointer MonoInterpFrameHandle; MONO_EE_CALLBACK (void, get_resume_state, (const MonoJitTlsData *jit_tls, gboolean *has_resume_state, MonoInterpFrameHandle *interp_frame, gpointer *handler_ip)) \ MONO_EE_CALLBACK (gboolean, run_finally, (StackFrameInfo *frame, int clause_index, gpointer handler_ip, gpointer handler_ip_end)) \ MONO_EE_CALLBACK (gboolean, run_filter, (StackFrameInfo *frame, MonoException *ex, int clause_index, gpointer handler_ip, gpointer handler_ip_end)) \ - MONO_EE_CALLBACK (gboolean, run_clause_with_il_state, (gpointer il_state, int clause_index, gpointer handler_ip, gpointer handler_ip_end, MonoObject *ex, gboolean *filtered, MonoExceptionEnum clause_type)) \ + MONO_EE_CALLBACK (gboolean, run_clause_with_il_state, (gpointer il_state, int clause_index, MonoObject *ex, gboolean *filtered)) \ MONO_EE_CALLBACK (void, frame_iter_init, (MonoInterpStackIter *iter, gpointer interp_exit_data)) \ MONO_EE_CALLBACK (gboolean, frame_iter_next, (MonoInterpStackIter *iter, StackFrameInfo *frame)) \ MONO_EE_CALLBACK (MonoJitInfo*, find_jit_info, (MonoMethod *method)) \ @@ -63,7 +63,7 @@ typedef gpointer MonoInterpFrameHandle; MONO_EE_CALLBACK (void, jit_info_foreach, (InterpJitInfoFunc func, gpointer user_data)) \ MONO_EE_CALLBACK (gboolean, sufficient_stack, (gsize size)) \ MONO_EE_CALLBACK (void, entry_llvmonly, (gpointer res, gpointer *args, gpointer imethod)) \ - MONO_EE_CALLBACK (gpointer, get_interp_method, (MonoMethod *method, MonoError *error)) \ + MONO_EE_CALLBACK (gpointer, get_interp_method, (MonoMethod *method)) \ MONO_EE_CALLBACK (MonoJitInfo*, compile_interp_method, (MonoMethod *method, MonoError *error)) \ typedef struct _MonoEECallbacks { diff --git a/src/mono/mono/mini/interp-stubs.c b/src/mono/mono/mini/interp-stubs.c index 635c0a51ad2fe4..61dd40a259003e 100644 --- a/src/mono/mono/mini/interp-stubs.c +++ b/src/mono/mono/mini/interp-stubs.c @@ -115,8 +115,7 @@ stub_run_filter (StackFrameInfo *frame, MonoException *ex, int clause_index, gpo } static gboolean -stub_run_clause_with_il_state (gpointer il_state, int clause_index, gpointer handler_ip, gpointer handler_ip_end, MonoObject *ex, - gboolean *filtered, MonoExceptionEnum clause_type) +stub_run_clause_with_il_state (gpointer il_state, int clause_index, MonoObject *ex, gboolean *filtered) { g_assert_not_reached (); } @@ -240,7 +239,7 @@ stub_entry_llvmonly (gpointer res, gpointer *args, gpointer imethod) } static gpointer -stub_get_interp_method (MonoMethod *method, MonoError *error) +stub_get_interp_method (MonoMethod *method) { g_assert_not_reached (); return NULL; diff --git a/src/mono/mono/mini/interp/interp-internals.h b/src/mono/mono/mini/interp/interp-internals.h index 534f041363994e..e6121e727f1e62 100644 --- a/src/mono/mono/mini/interp/interp-internals.h +++ b/src/mono/mono/mini/interp/interp-internals.h @@ -98,9 +98,13 @@ typedef enum { #define PROFILE_INTERP 0 -#define INTERP_IMETHOD_TAG_UNBOX(im) ((gpointer)((mono_u)(im) | 1)) -#define INTERP_IMETHOD_IS_TAGGED_UNBOX(im) ((mono_u)(im) & 1) -#define INTERP_IMETHOD_UNTAG_UNBOX(im) ((InterpMethod*)((mono_u)(im) & ~1)) +#define INTERP_IMETHOD_TAG_1(im) ((gpointer)((mono_u)(im) | 1)) +#define INTERP_IMETHOD_IS_TAGGED_1(im) ((mono_u)(im) & 1) +#define INTERP_IMETHOD_UNTAG_1(im) ((InterpMethod*)((mono_u)(im) & ~1)) + +#define INTERP_IMETHOD_TAG_UNBOX(im) INTERP_IMETHOD_TAG_1(im) +#define INTERP_IMETHOD_IS_TAGGED_UNBOX(im) INTERP_IMETHOD_IS_TAGGED_1(im) +#define INTERP_IMETHOD_UNTAG_UNBOX(im) INTERP_IMETHOD_UNTAG_1(im) /* * Structure representing a method transformed for the interpreter @@ -144,8 +148,25 @@ struct InterpMethod { #ifdef ENABLE_EXPERIMENT_TIERED MiniTieredCounter tiered_counter; #endif + gint32 entry_count; + InterpMethod *optimized_imethod; + // This data is used to resolve native offsets from unoptimized method to native offsets + // in the optimized method. We rely on keys identifying a certain logical execution point + // to be equal between unoptimized and optimized method. In unoptimized method we map from + // native_offset to a key and in optimized_method we map from key to a native offset. + // + // The logical execution points that are being tracked are some basic block starts (in this + // case we don't need any tracking in the unoptimized method, just the mapping from bbindex + // to its native offset) and call handler returns. Call handler returns store the return ip + // on the stack so once we tier up the method we need to update these to IPs in the optimized + // method. The key for a call handler is its index, in appearance order in the IL, multiplied + // by -1. (So we don't collide with basic block indexes) + // + // Since we have both positive and negative keys in this array, we use G_MAXINTRE as terminator. + int *patchpoint_data; unsigned int init_locals : 1; unsigned int vararg : 1; + unsigned int optimized : 1; unsigned int needs_thread_attach : 1; #if PROFILE_INTERP long calls; @@ -265,7 +286,7 @@ void mono_interp_transform_init (void); InterpMethod * -mono_interp_get_imethod (MonoMethod *method, MonoError *error); +mono_interp_get_imethod (MonoMethod *method); void mono_interp_print_code (InterpMethod *imethod); diff --git a/src/mono/mono/mini/interp/interp.c b/src/mono/mono/mini/interp/interp.c index 768377c5bcd7bd..48b9b6758307a9 100644 --- a/src/mono/mono/mini/interp/interp.c +++ b/src/mono/mono/mini/interp/interp.c @@ -64,6 +64,7 @@ #include "interp-internals.h" #include "mintops.h" #include "interp-intrins.h" +#include "tiering.h" #include #include @@ -455,15 +456,13 @@ lookup_imethod (MonoMethod *method) } InterpMethod* -mono_interp_get_imethod (MonoMethod *method, MonoError *error) +mono_interp_get_imethod (MonoMethod *method) { InterpMethod *imethod; MonoMethodSignature *sig; MonoJitMemoryManager *jit_mm = jit_mm_for_method (method); int i; - error_init (error); - jit_mm_lock (jit_mm); imethod = (InterpMethod*)mono_internal_hash_table_lookup (&jit_mm->interp_code_hash, method); jit_mm_unlock (jit_mm); @@ -478,6 +477,10 @@ mono_interp_get_imethod (MonoMethod *method, MonoError *error) imethod->hasthis = sig->hasthis; imethod->vararg = sig->call_convention == MONO_CALL_VARARG; imethod->code_type = IMETHOD_CODE_UNKNOWN; + // always optimize code if tiering is disabled + // always optimize wrappers + if (!(mono_interp_opt & INTERP_OPT_TIERING) || method->wrapper_type != MONO_WRAPPER_NONE) + imethod->optimized = TRUE; if (imethod->method->string_ctor) imethod->rtype = m_class_get_byval_arg (mono_defaults.string_class); else @@ -572,17 +575,12 @@ static InterpMethod* get_virtual_method (InterpMethod *imethod, MonoVTable *vtable) { MonoMethod *m = imethod->method; - InterpMethod *ret = NULL; if ((m->flags & METHOD_ATTRIBUTE_FINAL) || !(m->flags & METHOD_ATTRIBUTE_VIRTUAL)) { - if (m->iflags & METHOD_IMPL_ATTRIBUTE_SYNCHRONIZED) { - ERROR_DECL (error); - ret = mono_interp_get_imethod (mono_marshal_get_synchronized_wrapper (m), error); - mono_interp_error_cleanup (error); /* FIXME: don't swallow the error */ - } else { - ret = imethod; - } - return ret; + if (m->iflags & METHOD_IMPL_ATTRIBUTE_SYNCHRONIZED) + return mono_interp_get_imethod (mono_marshal_get_synchronized_wrapper (m)); + else + return imethod; } mono_class_setup_vtable (vtable->klass); @@ -618,9 +616,7 @@ get_virtual_method (InterpMethod *imethod, MonoVTable *vtable) virtual_method = mono_marshal_get_synchronized_wrapper (virtual_method); } - ERROR_DECL (error); - InterpMethod *virtual_imethod = mono_interp_get_imethod (virtual_method, error); - mono_error_cleanup (error); /* FIXME: don't swallow the error */ + InterpMethod *virtual_imethod = mono_interp_get_imethod (virtual_method); return virtual_imethod; } @@ -643,6 +639,9 @@ append_imethod (MonoMemoryManager *memory_manager, GSList *list, InterpMethod *i ret->data = entry; ret = g_slist_concat (list, ret); + mono_interp_register_imethod_patch_site ((gpointer*)&entry->imethod); + mono_interp_register_imethod_patch_site ((gpointer*)&entry->target_imethod); + return ret; } @@ -651,6 +650,9 @@ get_target_imethod (GSList *list, InterpMethod *imethod) { while (list != NULL) { InterpVTableEntry *entry = (InterpVTableEntry*) list->data; + // We don't account for tiering here so this comparison is racy + // The side effect is that we might end up with duplicates of the same + // method in the vtable list, but this is extremely uncommon. if (entry->imethod == imethod) return entry->target_imethod; list = list->next; @@ -720,10 +722,12 @@ get_virtual_method_fast (InterpMethod *imethod, MonoVTable *vtable, int offset) /* Lazily initialize the method table slot */ mono_mem_manager_lock (memory_manager); if (!table [offset]) { - if (imethod->method->is_inflated || offset < 0) + if (imethod->method->is_inflated || offset < 0) { table [offset] = append_imethod (memory_manager, NULL, imethod, target_imethod); - else + } else { table [offset] = (gpointer) ((gsize)target_imethod | 0x1); + mono_interp_register_imethod_patch_site (&table [offset]); + } } mono_mem_manager_unlock (memory_manager); } @@ -1773,7 +1777,7 @@ interp_init_delegate (MonoDelegate *del, MonoDelegateTrampInfo **out_info, MonoE g_assert_not_reached (); } else if (del->method) { /* Delegate created dynamically */ - del->interp_method = mono_interp_get_imethod (del->method, error); + del->interp_method = mono_interp_get_imethod (del->method); } else { /* Created from JITted code */ g_assert_not_reached (); @@ -1798,8 +1802,7 @@ interp_init_delegate (MonoDelegate *del, MonoDelegateTrampInfo **out_info, MonoE * FIXME We should do this later, when we also know the delegate on which the * target method is called. */ - del->interp_method = mono_interp_get_imethod (mono_marshal_get_delegate_invoke (method, NULL), error); - mono_error_assert_ok (error); + del->interp_method = mono_interp_get_imethod (mono_marshal_get_delegate_invoke (method, NULL)); } } @@ -1836,15 +1839,13 @@ ftnptr_to_imethod (gpointer addr, gboolean *need_unbox) InterpMethod *imethod; if (mono_llvm_only) { - ERROR_DECL (error); /* Function pointers are represented by a MonoFtnDesc structure */ MonoFtnDesc *ftndesc = (MonoFtnDesc*)addr; g_assert (ftndesc); g_assert (ftndesc->method); if (!ftndesc->interp_method) { - imethod = mono_interp_get_imethod (ftndesc->method, error); - mono_error_assert_ok (error); + imethod = mono_interp_get_imethod (ftndesc->method); mono_memory_barrier (); // FIXME Handle unboxing here ? ftndesc->interp_method = imethod; @@ -2097,8 +2098,7 @@ interp_runtime_invoke (MonoMethod *method, void *obj, void **params, MonoObject sp [2].data.p = exc; sp [3].data.p = target_method; - InterpMethod *imethod = mono_interp_get_imethod (invoke_wrapper, error); - mono_error_assert_ok (error); + InterpMethod *imethod = mono_interp_get_imethod (invoke_wrapper); InterpFrame frame = {0}; frame.imethod = imethod; @@ -2174,12 +2174,10 @@ interp_entry (InterpEntryData *data) * This happens when AOT code for the invoke wrapper is not found. * Have to replace the method with the wrapper here, since the wrapper depends on the delegate. */ - ERROR_DECL (error); MonoDelegate *del = (MonoDelegate*)data->this_arg; // FIXME: This is slow method = mono_marshal_get_delegate_invoke (method, del); - data->rmethod = mono_interp_get_imethod (method, error); - mono_error_assert_ok (error); + data->rmethod = mono_interp_get_imethod (method); } sig = mono_method_signature_internal (method); @@ -3003,16 +3001,15 @@ interp_entry_llvmonly (gpointer res, gpointer *args, gpointer imethod_untyped) } static gpointer -interp_get_interp_method (MonoMethod *method, MonoError *error) +interp_get_interp_method (MonoMethod *method) { - return mono_interp_get_imethod (method, error); + return mono_interp_get_imethod (method); } static MonoJitInfo* interp_compile_interp_method (MonoMethod *method, MonoError *error) { - InterpMethod *imethod = mono_interp_get_imethod (method, error); - return_val_if_nok (error, NULL); + InterpMethod *imethod = mono_interp_get_imethod (method); if (!imethod->transformed) { mono_interp_transform_method (imethod, get_context (), error); @@ -3063,8 +3060,7 @@ interp_create_method_pointer_llvmonly (MonoMethod *method, gboolean unbox, MonoE MonoMethod *wrapper; InterpMethod *imethod; - imethod = mono_interp_get_imethod (method, error); - return_val_if_nok (error, NULL); + imethod = mono_interp_get_imethod (method); if (unbox) { if (imethod->llvmonly_unbox_entry) @@ -3145,7 +3141,7 @@ static gpointer interp_create_method_pointer (MonoMethod *method, gboolean compile, MonoError *error) { gpointer addr, entry_func, entry_wrapper = NULL; - InterpMethod *imethod = mono_interp_get_imethod (method, error); + InterpMethod *imethod = mono_interp_get_imethod (method); if (imethod->jit_entry) return imethod->jit_entry; @@ -3378,8 +3374,6 @@ mono_interp_isinst (MonoObject* object, MonoClass* klass) static MONO_NEVER_INLINE InterpMethod* mono_interp_get_native_func_wrapper (InterpMethod* imethod, MonoMethodSignature* csignature, guchar* code) { - ERROR_DECL(error); - /* Pinvoke call is missing the wrapper. See mono_get_native_calli_wrapper */ MonoMarshalSpec** mspecs = g_newa0 (MonoMarshalSpec*, csignature->param_count + 1); @@ -3398,8 +3392,7 @@ mono_interp_get_native_func_wrapper (InterpMethod* imethod, MonoMethodSignature* if (mspecs [i]) mono_metadata_free_marshal_spec (mspecs [i]); - InterpMethod *cmethod = mono_interp_get_imethod (m, error); - mono_error_cleanup (error); /* FIXME: don't swallow the error */ + InterpMethod *cmethod = mono_interp_get_imethod (m); return cmethod; } @@ -3603,6 +3596,7 @@ interp_exec_method (InterpFrame *frame, ThreadContext *context, FrameClauseArgs MINT_IN_CASE(MINT_NIY) MINT_IN_CASE(MINT_DEF) MINT_IN_CASE(MINT_DUMMY_USE) + MINT_IN_CASE(MINT_TIER_PATCHPOINT_DATA) g_assert_not_reached (); MINT_IN_BREAK; MINT_IN_CASE(MINT_BREAK) @@ -3756,25 +3750,19 @@ interp_exec_method (InterpFrame *frame, ThreadContext *context, FrameClauseArgs if (!del_imethod) { // FIXME push/pop LMF if (is_multicast) { - error_init_reuse (error); MonoMethod *invoke = mono_get_delegate_invoke_internal (del->object.vtable->klass); - del_imethod = mono_interp_get_imethod (mono_marshal_get_delegate_invoke (invoke, del), error); + del_imethod = mono_interp_get_imethod (mono_marshal_get_delegate_invoke (invoke, del)); del->interp_invoke_impl = del_imethod; - mono_error_assert_ok (error); } else if (!del->interp_method) { // Not created from interpreted code - error_init_reuse (error); g_assert (del->method); - del_imethod = mono_interp_get_imethod (del->method, error); + del_imethod = mono_interp_get_imethod (del->method); del->interp_method = del_imethod; del->interp_invoke_impl = del_imethod; - mono_error_assert_ok (error); } else { del_imethod = (InterpMethod*)del->interp_method; if (del_imethod->method->flags & METHOD_ATTRIBUTE_PINVOKE_IMPL) { - error_init_reuse (error); - del_imethod = mono_interp_get_imethod (mono_marshal_get_native_wrapper (del_imethod->method, FALSE, FALSE), error); - mono_error_assert_ok (error); + del_imethod = mono_interp_get_imethod (mono_marshal_get_native_wrapper (del_imethod->method, FALSE, FALSE)); del->interp_invoke_impl = del_imethod; } else if ((m_method_is_virtual (del_imethod->method) && !m_method_is_static (del_imethod->method)) && !del->target && !m_class_is_valuetype (del_imethod->method->klass)) { // 'this' is passed dynamically, we need to recompute the target method @@ -3785,6 +3773,12 @@ interp_exec_method (InterpFrame *frame, ThreadContext *context, FrameClauseArgs } } } + if (del_imethod->optimized_imethod) { + del_imethod = del_imethod->optimized_imethod; + // don't patch for virtual calls + if (del->interp_invoke_impl) + del->interp_invoke_impl = del_imethod; + } cmethod = del_imethod; if (!is_multicast) { if (cmethod->param_count == param_count + 1) { @@ -3820,8 +3814,7 @@ interp_exec_method (InterpFrame *frame, ThreadContext *context, FrameClauseArgs if (cmethod->method->flags & METHOD_ATTRIBUTE_PINVOKE_IMPL) { // FIXME push/pop LMF - cmethod = mono_interp_get_imethod (mono_marshal_get_native_wrapper (cmethod->method, FALSE, FALSE), error); - mono_interp_error_cleanup (error); /* FIXME: don't swallow the error */ + cmethod = mono_interp_get_imethod (mono_marshal_get_native_wrapper (cmethod->method, FALSE, FALSE)); } return_offset = ip [1]; @@ -6913,8 +6906,7 @@ MINT_IN_CASE(MINT_BRTRUE_I8_SP) ZEROP_SP(gint64, !=); MINT_IN_BREAK; gpointer addr = mini_get_interp_callbacks ()->create_method_pointer (local_cmethod, TRUE, error); LOCAL_VAR (ip [1], gpointer) = addr; } else { - InterpMethod *m = mono_interp_get_imethod (local_cmethod, error); - mono_error_assert_ok (error); + InterpMethod *m = mono_interp_get_imethod (local_cmethod); LOCAL_VAR (ip [1], gpointer) = imethod_to_ftnptr (m, FALSE); } ip += 3; @@ -6982,6 +6974,23 @@ MINT_IN_CASE(MINT_BRTRUE_I8_SP) ZEROP_SP(gint64, !=); MINT_IN_BREAK; MINT_IN_BREAK; } + MINT_IN_CASE(MINT_TIER_ENTER_METHOD) { + frame->imethod->entry_count++; + if (frame->imethod->entry_count > INTERP_TIER_ENTRY_LIMIT) + mono_interp_tier_up_frame_enter (frame, context, &ip); + else + ip++; + MINT_IN_BREAK; + } + MINT_IN_CASE(MINT_TIER_PATCHPOINT) { + frame->imethod->entry_count++; + if (frame->imethod->entry_count > INTERP_TIER_ENTRY_LIMIT) + mono_interp_tier_up_frame_patchpoint (frame, context, &ip); + else + ip += 2; + MINT_IN_BREAK; + } + MINT_IN_CASE(MINT_LDLOCA_S) LOCAL_VAR (ip [1], gpointer) = locals + ip [2]; ip += 3; @@ -7097,10 +7106,10 @@ MINT_IN_CASE(MINT_BRTRUE_I8_SP) ZEROP_SP(gint64, !=); MINT_IN_BREAK; MonoDelegate *del = LOCAL_VAR (ip [2], MonoDelegate*); if (!del->interp_method) { /* Not created from interpreted code */ - error_init_reuse (error); g_assert (del->method); - del->interp_method = mono_interp_get_imethod (del->method, error); - mono_error_assert_ok (error); + del->interp_method = mono_interp_get_imethod (del->method); + } else if (((InterpMethod*)del->interp_method)->optimized_imethod) { + del->interp_method = ((InterpMethod*)del->interp_method)->optimized_imethod; } g_assert (del->interp_method); LOCAL_VAR (ip [1], gpointer) = imethod_to_ftnptr (del->interp_method, FALSE); @@ -7291,20 +7300,41 @@ interp_parse_options (const char *options) for (ptr = args; ptr && *ptr; ptr ++) { char *arg = *ptr; - if (strncmp (arg, "jit=", 4) == 0) + if (strncmp (arg, "jit=", 4) == 0) { mono_interp_jit_classes = g_slist_prepend (mono_interp_jit_classes, arg + 4); - else if (strncmp (arg, "interp-only=", strlen ("interp-only=")) == 0) + } else if (strncmp (arg, "interp-only=", strlen ("interp-only=")) == 0) { mono_interp_only_classes = g_slist_prepend (mono_interp_only_classes, arg + strlen ("interp-only=")); - else if (strncmp (arg, "-inline", 7) == 0) - mono_interp_opt &= ~INTERP_OPT_INLINE; - else if (strncmp (arg, "-cprop", 6) == 0) - mono_interp_opt &= ~INTERP_OPT_CPROP; - else if (strncmp (arg, "-super", 6) == 0) - mono_interp_opt &= ~INTERP_OPT_SUPER_INSTRUCTIONS; - else if (strncmp (arg, "-bblocks", 8) == 0) - mono_interp_opt &= ~INTERP_OPT_BBLOCKS; - else if (strncmp (arg, "-all", 4) == 0) - mono_interp_opt = INTERP_OPT_NONE; + } else { + gboolean invert; + int opt = 0; + + if (*arg == '-') { + arg++; + invert = TRUE; + } else { + invert = FALSE; + } + + if (strncmp (arg, "inline", 6) == 0) + opt = INTERP_OPT_INLINE; + else if (strncmp (arg, "cprop", 5) == 0) + opt = INTERP_OPT_CPROP; + else if (strncmp (arg, "super", 5) == 0) + opt = INTERP_OPT_SUPER_INSTRUCTIONS; + else if (strncmp (arg, "bblocks", 7) == 0) + opt = INTERP_OPT_BBLOCKS; + else if (strncmp (arg, "tiering", 7) == 0) + opt = INTERP_OPT_TIERING; + else if (strncmp (arg, "all", 3) == 0) + opt = ~INTERP_OPT_NONE; + + if (opt) { + if (invert) + mono_interp_opt &= ~opt; + else + mono_interp_opt |= opt; + } + } } } @@ -7447,8 +7477,7 @@ interp_run_filter (StackFrameInfo *frame, MonoException *ex, int clause_index, g /* Returns TRUE if there is a pending exception */ static gboolean -interp_run_clause_with_il_state (gpointer il_state_ptr, int clause_index, gpointer handler_ip, gpointer handler_ip_end, - MonoObject *ex, gboolean *filtered, MonoExceptionEnum clause_type) +interp_run_clause_with_il_state (gpointer il_state_ptr, int clause_index, MonoObject *ex, gboolean *filtered) { MonoMethodILState *il_state = (MonoMethodILState*)il_state_ptr; MonoMethodSignature *sig; @@ -7462,8 +7491,12 @@ interp_run_clause_with_il_state (gpointer il_state_ptr, int clause_index, gpoint sig = mono_method_signature_internal (il_state->method); g_assert (sig); - imethod = mono_interp_get_imethod (il_state->method, error); - mono_error_assert_ok (error); + imethod = mono_interp_get_imethod (il_state->method); + if (!imethod->transformed) { + // In case method is in process of being tiered up, make sure it is compiled + mono_interp_transform_method (imethod, context, error); + mono_error_assert_ok (error); + } orig_sp = sp_args = sp = (stackval*)context->stack_pointer; @@ -7516,11 +7549,20 @@ interp_run_clause_with_il_state (gpointer il_state_ptr, int clause_index, gpoint } memset (&clause_args, 0, sizeof (FrameClauseArgs)); - clause_args.start_with_ip = (const guint16*)handler_ip; + MonoJitExceptionInfo *ei = &imethod->jinfo->clauses [clause_index]; + MonoExceptionEnum clause_type = ei->flags; + // For filter clauses, if filtered is set, then we run the filter, otherwise we run the catch handler + if (clause_type == MONO_EXCEPTION_CLAUSE_FILTER && !filtered) + clause_type = MONO_EXCEPTION_CLAUSE_NONE; + + if (clause_type == MONO_EXCEPTION_CLAUSE_FILTER) + clause_args.start_with_ip = (const guint16*)ei->data.filter; + else + clause_args.start_with_ip = (const guint16*)ei->handler_start; if (clause_type == MONO_EXCEPTION_CLAUSE_NONE || clause_type == MONO_EXCEPTION_CLAUSE_FILTER) clause_args.end_at_ip = (const guint16*)clause_args.start_with_ip + 0xffffff; else - clause_args.end_at_ip = (const guint16*)handler_ip_end; + clause_args.end_at_ip = (const guint16*)ei->data.handler_end; clause_args.exec_frame = &frame; if (clause_type == MONO_EXCEPTION_CLAUSE_NONE || clause_type == MONO_EXCEPTION_CLAUSE_FILTER) @@ -8052,6 +8094,9 @@ mono_ee_interp_init (const char *opts) mono_interp_opt = 0; mono_interp_transform_init (); + if (mono_interp_opt & INTERP_OPT_TIERING) + mono_interp_tiering_init (); + mini_install_interp_callbacks (&mono_interp_callbacks); register_interp_stats (); diff --git a/src/mono/mono/mini/interp/interp.h b/src/mono/mono/mini/interp/interp.h index 49261017d8f088..df813b3fead995 100644 --- a/src/mono/mono/mini/interp/interp.h +++ b/src/mono/mono/mini/interp/interp.h @@ -34,7 +34,8 @@ enum { INTERP_OPT_CPROP = 2, INTERP_OPT_SUPER_INSTRUCTIONS = 4, INTERP_OPT_BBLOCKS = 8, - INTERP_OPT_DEFAULT = INTERP_OPT_INLINE | INTERP_OPT_CPROP | INTERP_OPT_SUPER_INSTRUCTIONS | INTERP_OPT_BBLOCKS + INTERP_OPT_TIERING = 16, + INTERP_OPT_DEFAULT = INTERP_OPT_INLINE | INTERP_OPT_CPROP | INTERP_OPT_SUPER_INSTRUCTIONS | INTERP_OPT_BBLOCKS | INTERP_OPT_TIERING }; typedef struct _InterpMethodArguments InterpMethodArguments; diff --git a/src/mono/mono/mini/interp/mintops.def b/src/mono/mono/mini/interp/mintops.def index 623711ef5a1dcc..3f67b609bf8bc5 100644 --- a/src/mono/mono/mini/interp/mintops.def +++ b/src/mono/mono/mini/interp/mintops.def @@ -13,6 +13,7 @@ OPDEF(MINT_NIY, "niy", 1, 0, 0, MintOpNoArgs) OPDEF(MINT_DEF, "def", 2, 1, 0, MintOpNoArgs) OPDEF(MINT_IL_SEQ_POINT, "il_seq_point", 1, 0, 0, MintOpNoArgs) OPDEF(MINT_DUMMY_USE, "dummy_use", 2, 0, 1, MintOpNoArgs) +OPDEF(MINT_TIER_PATCHPOINT_DATA, "tier_patchpoint_data", 2, 0, 0, MintOpShortInt) OPDEF(MINT_BREAK, "break", 1, 0, 0, MintOpNoArgs) OPDEF(MINT_BREAKPOINT, "breakpoint", 1, 0, 0, MintOpNoArgs) @@ -765,6 +766,9 @@ OPDEF(MINT_PROF_EXIT, "prof_exit", 5, 0, 1, MintOpShortAndInt) OPDEF(MINT_PROF_EXIT_VOID, "prof_exit_void", 2, 0, 0, MintOpNoArgs) OPDEF(MINT_PROF_COVERAGE_STORE, "prof_coverage_store", 5, 0, 0, MintOpLongInt) +OPDEF(MINT_TIER_ENTER_METHOD, "tier_enter_method", 1, 0, 0, MintOpNoArgs) +OPDEF(MINT_TIER_PATCHPOINT, "tier_patchpoint", 2, 0, 0, MintOpShortInt) + OPDEF(MINT_INTRINS_ENUM_HASFLAG, "intrins_enum_hasflag", 5, 1, 2, MintOpClassToken) OPDEF(MINT_INTRINS_GET_HASHCODE, "intrins_get_hashcode", 3, 1, 1, MintOpNoArgs) OPDEF(MINT_INTRINS_GET_TYPE, "intrins_get_type", 3, 1, 1, MintOpNoArgs) diff --git a/src/mono/mono/mini/interp/tiering.c b/src/mono/mono/mini/interp/tiering.c new file mode 100644 index 00000000000000..a064c9ffe63393 --- /dev/null +++ b/src/mono/mono/mini/interp/tiering.c @@ -0,0 +1,214 @@ +#include "tiering.h" + +static mono_mutex_t tiering_mutex; +static GHashTable *patch_sites_table; +static gboolean enable_tiering; + +void +mono_interp_tiering_init (void) +{ + mono_os_mutex_init_recursive (&tiering_mutex); + patch_sites_table = g_hash_table_new (NULL, NULL); + enable_tiering = TRUE; +} + +gboolean +mono_interp_tiering_enabled (void) +{ + return enable_tiering; +} + +static InterpMethod* +get_tier_up_imethod (InterpMethod *imethod) +{ + MonoMethod *method = imethod->method; + MonoJitMemoryManager *jit_mm = jit_mm_for_method (method); + InterpMethod *new_imethod = (InterpMethod*)m_method_alloc0 (method, sizeof (InterpMethod)); + + new_imethod->method = imethod->method; + new_imethod->param_count = imethod->param_count; + new_imethod->hasthis = imethod->hasthis; + new_imethod->vararg = imethod->vararg; + new_imethod->code_type = imethod->code_type; + new_imethod->rtype = imethod->rtype; + new_imethod->param_types = imethod->param_types; + new_imethod->optimized = TRUE; + new_imethod->prof_flags = imethod->prof_flags; + + jit_mm_lock (jit_mm); + InterpMethod *old_imethod = mono_internal_hash_table_lookup (&jit_mm->interp_code_hash, method); + if (old_imethod->optimized) { + new_imethod = old_imethod; /* leak the newly allocated InterpMethod to the mempool */ + } else { + mono_internal_hash_table_remove (&jit_mm->interp_code_hash, method); + mono_internal_hash_table_insert (&jit_mm->interp_code_hash, method, new_imethod); + } + jit_mm_unlock (jit_mm); + + return new_imethod; +} + +static void +patch_imethod_site (gpointer data, gpointer user_data) +{ + gpointer *addr = (gpointer*)data; + // Preserve the tagging, in case the address originates in vtables + gboolean tagged = INTERP_IMETHOD_IS_TAGGED_1 (*addr); + *addr = (InterpMethod*)(tagged ? INTERP_IMETHOD_TAG_1 (user_data) : user_data); +} + +static void +patch_interp_data_items (InterpMethod *old_imethod, InterpMethod *new_imethod) +{ + GSList *sites = g_hash_table_lookup (patch_sites_table, old_imethod); + g_slist_foreach (sites, patch_imethod_site, new_imethod); + + g_hash_table_remove (patch_sites_table, sites); + g_slist_free (sites); +} + +static InterpMethod* +tier_up_method (InterpMethod *imethod, ThreadContext *context) +{ + g_assert (enable_tiering); + ERROR_DECL(error); + // This enables future code to obtain a reference to the optimized imethod + InterpMethod *new_imethod = get_tier_up_imethod (imethod); + + // In theory we can race with other threads compiling the same imethod, but this is not a problem + if (!new_imethod->transformed) + mono_interp_transform_method (new_imethod, context, error); + // Unoptimized method compiled fine, optimized method should also compile without error + mono_error_assert_ok (error); + + mono_os_mutex_lock (&tiering_mutex); + + if (!imethod->optimized_imethod) { + // patch all data items + patch_interp_data_items (imethod, new_imethod); + + // Other threads executing this imethod will be able to tier the frame up in patchpoints + imethod->optimized_imethod = new_imethod; + } + mono_os_mutex_unlock (&tiering_mutex); + + return new_imethod; +} + +static void +register_imethod_patch_site (InterpMethod *imethod, gpointer *ptr) +{ + g_assert (!imethod->optimized); + GSList *sites = g_hash_table_lookup (patch_sites_table, imethod); + sites = g_slist_prepend (sites, ptr); + g_hash_table_insert_replace (patch_sites_table, imethod, sites, TRUE); +} + +static void +register_imethod_data_item (gpointer data, gpointer user_data) +{ + gint32 index = (gint32)(gsize)data; + InterpMethod **data_items = (InterpMethod**)user_data; + + if (data_items [index]) { + if (data_items [index]->optimized_imethod) { + // We are under tiering lock, check if the method has been tiered up already + data_items [index] = data_items [index]->optimized_imethod; + return; + } + register_imethod_patch_site (data_items [index], (gpointer*)&data_items [index]); + } +} + +void +mono_interp_register_imethod_data_items (gpointer *data_items, GSList *indexes) +{ + if (!enable_tiering) + return; + mono_os_mutex_lock (&tiering_mutex); + g_slist_foreach (indexes, register_imethod_data_item, data_items); + mono_os_mutex_unlock (&tiering_mutex); +} + +// This method should be called within mem manager lock which means +// the contents of **imethod_ptr cannot modify until we register the +// patch site +void +mono_interp_register_imethod_patch_site (gpointer *imethod_ptr) +{ + gboolean tagged = INTERP_IMETHOD_IS_TAGGED_1 (*imethod_ptr); + InterpMethod *imethod = INTERP_IMETHOD_UNTAG_1 (*imethod_ptr); + if (imethod->optimized) { + return; + } else if (imethod->optimized_imethod) { + *imethod_ptr = tagged ? imethod->optimized_imethod : INTERP_IMETHOD_TAG_1 (imethod->optimized_imethod); + return; + } + + mono_os_mutex_lock (&tiering_mutex); + // We are under tiering lock, check if the method has been tiered up already + if (imethod->optimized_imethod) { + *imethod_ptr = tagged ? imethod->optimized_imethod : INTERP_IMETHOD_TAG_1 (imethod->optimized_imethod); + } else { + register_imethod_patch_site (imethod, imethod_ptr); + } + mono_os_mutex_unlock (&tiering_mutex); +} + +void +mono_interp_tier_up_frame_enter (InterpFrame *frame, ThreadContext *context, const guint16 **ip) +{ + InterpMethod *optimized_method; + if (frame->imethod->optimized_imethod) + optimized_method = frame->imethod->optimized_imethod; + else + optimized_method = tier_up_method (frame->imethod, context); + context->stack_pointer = (guchar*)frame->stack + optimized_method->alloca_size; + frame->imethod = optimized_method; + *ip = optimized_method->code; +} + +static int +lookup_patchpoint_data (InterpMethod *imethod, int data) +{ + int *position = imethod->patchpoint_data; + while (*position != G_MAXINT32) { + if (*position == data) + return *(position + 1); + position += 2; + } + return G_MAXINT32; +} + +void +mono_interp_tier_up_frame_patchpoint (InterpFrame *frame, ThreadContext *context, const guint16 **ip) +{ + InterpMethod *unoptimized_method = frame->imethod; + InterpMethod *optimized_method; + if (unoptimized_method->optimized_imethod) + optimized_method = unoptimized_method->optimized_imethod; + else + optimized_method = tier_up_method (unoptimized_method, context); + for (int i = 0; i < unoptimized_method->num_clauses; i++) { + MonoExceptionClause *clause = &unoptimized_method->clauses [i]; + if (clause->flags != MONO_EXCEPTION_CLAUSE_FINALLY) + continue; + // Patch return addresses used by MINT_CALL_HANDLER + MINT_ENDFINALLY + guint16 **ip_addr = (guint16**)((char*)frame->stack + unoptimized_method->clause_data_offsets [i]); + guint16 *ret_ip = *ip_addr; + // ret_ip could be junk on stack, do a quick check first + if (ret_ip < unoptimized_method->code) + continue; + int native_offset = (int)(ret_ip - unoptimized_method->code); + int call_handler_index = lookup_patchpoint_data (unoptimized_method, native_offset); + if (call_handler_index != G_MAXINT32) { + int offset = lookup_patchpoint_data (optimized_method, call_handler_index); + g_assert (offset != G_MAXINT32); + *ip_addr = optimized_method->code + offset; + } + } + context->stack_pointer = (guchar*)frame->stack + optimized_method->alloca_size; + frame->imethod = optimized_method; + int bb_index = (*ip) [1]; + *ip = optimized_method->code + lookup_patchpoint_data (optimized_method, bb_index); +} diff --git a/src/mono/mono/mini/interp/tiering.h b/src/mono/mono/mini/interp/tiering.h new file mode 100644 index 00000000000000..fd93e83ea7d13d --- /dev/null +++ b/src/mono/mono/mini/interp/tiering.h @@ -0,0 +1,26 @@ +#ifndef __MONO_MINI_INTERP_TIERING_H__ +#define __MONO_MINI_INTERP_TIERING_H__ + +#include "interp-internals.h" + +#define INTERP_TIER_ENTRY_LIMIT 1000 + +void +mono_interp_tiering_init (void); + +gboolean +mono_interp_tiering_enabled (void); + +void +mono_interp_register_imethod_data_items (gpointer *data_items, GSList *indexes); + +void +mono_interp_register_imethod_patch_site (gpointer *imethod_ptr); + +void +mono_interp_tier_up_frame_enter (InterpFrame *frame, ThreadContext *context, const guint16 **ip); + +void +mono_interp_tier_up_frame_patchpoint (InterpFrame *frame, ThreadContext *context, const guint16 **ip); + +#endif /* __MONO_MINI_INTERP_TIERING_H__ */ diff --git a/src/mono/mono/mini/interp/transform.c b/src/mono/mono/mini/interp/transform.c index d1766217cf6ab0..4d6aecc8b81edd 100644 --- a/src/mono/mono/mini/interp/transform.c +++ b/src/mono/mono/mini/interp/transform.c @@ -33,6 +33,7 @@ #include "interp-internals.h" #include "interp.h" #include "transform.h" +#include "tiering.h" MonoInterpStats mono_interp_stats; @@ -758,6 +759,19 @@ handle_branch (TransformData *td, int long_op, int offset) InterpBasicBlock *target_bb = td->offset_to_bb [target]; g_assert (target_bb); + if (offset < 0 && td->sp == td->stack && !td->inlined_method) { + // Backwards branch inside unoptimized method where the IL stack is empty + // This is candidate for a patchpoint + if (!td->rtm->optimized) + target_bb->emit_patchpoint = TRUE; + if (mono_interp_tiering_enabled () && !target_bb->patchpoint_data && td->rtm->optimized) { + // The optimized imethod will store mapping from bb index to native offset so it + // can resume execution in the optimized method, once we tier up in patchpoint + td->patchpoint_data_n++; + target_bb->patchpoint_data = TRUE; + } + } + if (long_op == MINT_LEAVE || long_op == MINT_LEAVE_CHECK) target_bb->eh_block = TRUE; @@ -1056,12 +1070,15 @@ store_local (TransformData *td, int local) } static guint32 -get_data_item_wide_index (TransformData *td, void *ptr) +get_data_item_wide_index (TransformData *td, void *ptr, gboolean *new_slot) { gpointer p = g_hash_table_lookup (td->data_hash, ptr); guint32 index; - if (p != NULL) + if (p != NULL) { + if (new_slot) + *new_slot = FALSE; return GPOINTER_TO_UINT (p) - 1; + } if (td->max_data_items == td->n_data_items) { td->max_data_items = td->n_data_items == 0 ? 16 : 2 * td->max_data_items; td->data_items = (gpointer*)g_realloc (td->data_items, td->max_data_items * sizeof(td->data_items [0])); @@ -1070,17 +1087,30 @@ get_data_item_wide_index (TransformData *td, void *ptr) td->data_items [index] = ptr; ++td->n_data_items; g_hash_table_insert (td->data_hash, ptr, GUINT_TO_POINTER (index + 1)); + if (new_slot) + *new_slot = TRUE; return index; } static guint16 get_data_item_index (TransformData *td, void *ptr) { - guint32 index = get_data_item_wide_index (td, ptr); + guint32 index = get_data_item_wide_index (td, ptr, NULL); g_assertf (index <= G_MAXUINT16, "Interpreter data item index 0x%x for method '%s' overflows", index, td->method->name); return (guint16)index; } +static guint16 +get_data_item_index_imethod (TransformData *td, InterpMethod *imethod) +{ + gboolean new_slot; + guint32 index = get_data_item_wide_index (td, imethod, &new_slot); + g_assertf (index <= G_MAXUINT16, "Interpreter data item index 0x%x for method '%s' overflows", index, td->method->name); + if (new_slot && imethod && !imethod->optimized) + td->imethod_items = g_slist_prepend (td->imethod_items, (gpointer)(gsize)index); + return index; +} + static gboolean is_data_item_wide_index (guint32 data_item_index) { @@ -2641,6 +2671,7 @@ interp_inline_method (TransformData *td, MonoMethod *target_method, MonoMethodHe InterpBasicBlock **prev_offset_to_bb; InterpBasicBlock *prev_cbb, *prev_entry_bb; MonoMethod *prev_inlined_method; + GSList *prev_imethod_items; MonoMethodSignature *csignature = mono_method_signature_internal (target_method); int nargs = csignature->param_count + !!csignature->hasthis; InterpInst *prev_last_ins; @@ -2663,6 +2694,7 @@ interp_inline_method (TransformData *td, MonoMethod *target_method, MonoMethodHe prev_cbb = td->cbb; prev_entry_bb = td->entry_bb; prev_aggressive_inlining = td->aggressive_inlining; + prev_imethod_items = td->imethod_items; td->inlined_method = target_method; prev_max_stack_height = td->max_stack_height; @@ -2700,6 +2732,13 @@ interp_inline_method (TransformData *td, MonoMethod *target_method, MonoMethodHe g_hash_table_remove (td->data_hash, td->data_items [i]); } td->n_data_items = prev_n_data_items; + /* Also remove any added indexes from the imethod list */ + while (td->imethod_items != prev_imethod_items) { + GSList *to_free = td->imethod_items; + td->imethod_items = td->imethod_items->next; + g_slist_free_1 (to_free); + } + td->sp = td->stack + prev_sp_offset; memcpy (&td->sp [-nargs], prev_param_area, nargs * sizeof (StackInfo)); td->last_ins = prev_last_ins; @@ -2879,6 +2918,9 @@ emit_convert (TransformData *td, StackInfo *sp, MonoType *target_type) // FIXME: Add more switch (target_type->type) { +#if SIZEOF_VOID_P == 8 + case MONO_TYPE_I: +#endif case MONO_TYPE_I8: { switch (stype) { case STACK_TYPE_I4: @@ -2890,7 +2932,6 @@ emit_convert (TransformData *td, StackInfo *sp, MonoType *target_type) break; } #if SIZEOF_VOID_P == 8 - case MONO_TYPE_I: case MONO_TYPE_U: { switch (stype) { case STACK_TYPE_I4: @@ -3127,8 +3168,7 @@ interp_transform_call (TransformData *td, MonoMethod *method, MonoMethod *target interp_add_ins (td, MINT_TAILCALL); } interp_ins_set_sreg (td->last_ins, MINT_CALL_ARGS_SREG); - td->last_ins->data [0] = get_data_item_index (td, mono_interp_get_imethod (target_method, error)); - return_val_if_nok (error, FALSE); + td->last_ins->data [0] = get_data_item_index_imethod (td, mono_interp_get_imethod (target_method)); td->last_ins->data [1] = params_stack_size; td->last_ins->flags |= INTERP_INST_FLAG_CALL; td->last_ins->info.call_args = call_args; @@ -3250,8 +3290,7 @@ interp_transform_call (TransformData *td, MonoMethod *method, MonoMethod *target interp_ins_set_dreg (td->last_ins, dreg); interp_ins_set_sreg (td->last_ins, MINT_CALL_ARGS_SREG); td->last_ins->flags |= INTERP_INST_FLAG_CALL; - td->last_ins->data [0] = get_data_item_index (td, (void *)mono_interp_get_imethod (target_method, error)); - mono_error_assert_ok (error); + td->last_ins->data [0] = get_data_item_index_imethod (td, mono_interp_get_imethod (target_method)); } else { if (is_delegate_invoke) { interp_add_ins (td, MINT_CALL_DELEGATE); @@ -3295,16 +3334,14 @@ interp_transform_call (TransformData *td, MonoMethod *method, MonoMethod *target */ if (method->wrapper_type == MONO_WRAPPER_MANAGED_TO_NATIVE) { MonoMethod *pinvoke_method = mono_marshal_method_from_wrapper (method); - if (pinvoke_method) { - imethod = mono_interp_get_imethod (pinvoke_method, error); - return_val_if_nok (error, FALSE); - } + if (pinvoke_method) + imethod = mono_interp_get_imethod (pinvoke_method); } interp_ins_set_dreg (td->last_ins, dreg); interp_ins_set_sregs2 (td->last_ins, fp_sreg, MINT_CALL_ARGS_SREG); td->last_ins->data [0] = get_data_item_index (td, csignature); - td->last_ins->data [1] = get_data_item_index (td, imethod); + td->last_ins->data [1] = get_data_item_index_imethod (td, imethod); td->last_ins->data [2] = save_last_error; /* Cache slot */ td->last_ins->data [3] = get_data_item_index_nonshared (td, NULL); @@ -3314,8 +3351,7 @@ interp_transform_call (TransformData *td, MonoMethod *method, MonoMethod *target interp_ins_set_sregs2 (td->last_ins, fp_sreg, MINT_CALL_ARGS_SREG); } } else { - InterpMethod *imethod = mono_interp_get_imethod (target_method, error); - return_val_if_nok (error, FALSE); + InterpMethod *imethod = mono_interp_get_imethod (target_method); if (csignature->call_convention == MONO_CALL_VARARG) { interp_add_ins (td, MINT_CALL_VARARG); @@ -3331,7 +3367,7 @@ interp_transform_call (TransformData *td, MonoMethod *method, MonoMethod *target } interp_ins_set_dreg (td->last_ins, dreg); interp_ins_set_sreg (td->last_ins, MINT_CALL_ARGS_SREG); - td->last_ins->data [0] = get_data_item_index (td, (void *)imethod); + td->last_ins->data [0] = get_data_item_index_imethod (td, imethod); #ifdef ENABLE_EXPERIMENT_TIERED if (MINT_IS_PATCHABLE_CALL (td->last_ins->opcode)) { @@ -4014,10 +4050,10 @@ interp_emit_sfld_access (TransformData *td, MonoClassField *field, MonoClass *fi return; } } - guint32 vtable_index = get_data_item_wide_index (td, vtable); - guint32 addr_index = get_data_item_wide_index (td, (char*)field_addr); + guint32 vtable_index = get_data_item_wide_index (td, vtable, NULL); + guint32 addr_index = get_data_item_wide_index (td, (char*)field_addr, NULL); gboolean wide_data = is_data_item_wide_index (vtable_index) || is_data_item_wide_index (addr_index); - guint32 klass_index = !wide_data ? 0 : get_data_item_wide_index (td, field_class); + guint32 klass_index = !wide_data ? 0 : get_data_item_wide_index (td, field_class, NULL); if (is_load) { if (G_UNLIKELY (wide_data)) { interp_add_ins (td, MINT_LDSFLD_W); @@ -4206,6 +4242,7 @@ generate_code (TransformData *td, MonoMethod *method, MonoMethodHeader *header, MonoMethodSignature *signature = mono_method_signature_internal (method); int num_args = signature->hasthis + signature->param_count; int arglist_local = -1; + int call_handler_count = 0; gboolean ret = TRUE; gboolean emitted_funccall_seq_point = FALSE; guint32 *arg_locals = NULL; @@ -4284,6 +4321,9 @@ generate_code (TransformData *td, MonoMethod *method, MonoMethodHeader *header, last_seq_point->flags |= INTERP_INST_FLAG_SEQ_POINT_METHOD_ENTRY; } + if (!td->optimized) + interp_add_ins (td, MINT_TIER_ENTER_METHOD); + if (mono_debugger_method_has_breakpoint (method)) { interp_add_ins (td, MINT_BREAKPOINT); } @@ -4292,7 +4332,7 @@ generate_code (TransformData *td, MonoMethod *method, MonoMethodHeader *header, if (td->verbose_level) { char *tmp = mono_disasm_code (NULL, method, td->ip, end); char *name = mono_method_full_name (method, TRUE); - g_print ("Method %s, original code:\n", name); + g_print ("Method %s, optimized %d, original code:\n", name, rtm->optimized); g_print ("%s\n", tmp); g_free (tmp); g_free (name); @@ -4708,8 +4748,7 @@ generate_code (TransformData *td, MonoMethod *method, MonoMethodHeader *header, m = mono_get_method_checked (image, token, NULL, generic_context, error); goto_if_nok (error, exit); interp_add_ins (td, MINT_JMP); - td->last_ins->data [0] = get_data_item_index (td, mono_interp_get_imethod (m, error)); - goto_if_nok (error, exit); + td->last_ins->data [0] = get_data_item_index_imethod (td, mono_interp_get_imethod (m)); td->ip += 5; break; } @@ -4740,6 +4779,11 @@ generate_code (TransformData *td, MonoMethod *method, MonoMethodHeader *header, last_seq_point->flags = INTERP_INST_FLAG_SEQ_POINT_NONEMPTY_STACK; } + if (td->last_ins->opcode == MINT_TAILCALL || td->last_ins->opcode == MINT_TAILCALL_VIRT) { + // Execution does not follow through + link_bblocks = FALSE; + } + constrained_class = NULL; readonly = FALSE; save_last_error = FALSE; @@ -5525,7 +5569,7 @@ generate_code (TransformData *td, MonoMethod *method, MonoMethodHeader *header, call_args [csignature->param_count + 1] = -1; interp_add_ins (td, MINT_NEWOBJ_STRING); - td->last_ins->data [0] = get_data_item_index (td, mono_interp_get_imethod (m, error)); + td->last_ins->data [0] = get_data_item_index_imethod (td, mono_interp_get_imethod (m)); push_type (td, stack_type [ret_mt], klass); interp_ins_set_dreg (td->last_ins, td->sp [-1].local); @@ -5605,7 +5649,7 @@ generate_code (TransformData *td, MonoMethod *method, MonoMethodHeader *header, } // Inlining failed. Set the method to be executed as part of newobj instruction - newobj_fast->data [0] = get_data_item_index (td, mono_interp_get_imethod (m, error)); + newobj_fast->data [0] = get_data_item_index_imethod (td, mono_interp_get_imethod (m)); /* The constructor was not inlined, abort inlining of current method */ if (!td->aggressive_inlining) INLINE_FAILURE; @@ -5613,7 +5657,7 @@ generate_code (TransformData *td, MonoMethod *method, MonoMethodHeader *header, interp_add_ins (td, MINT_NEWOBJ_SLOW); g_assert (!m_class_is_valuetype (klass)); interp_ins_set_dreg (td->last_ins, dreg); - td->last_ins->data [0] = get_data_item_index (td, mono_interp_get_imethod (m, error)); + td->last_ins->data [0] = get_data_item_index_imethod (td, mono_interp_get_imethod (m)); } goto_if_nok (error, exit); @@ -6714,6 +6758,17 @@ generate_code (TransformData *td, MonoMethod *method, MonoMethodHeader *header, (!MONO_OFFSET_IN_CLAUSE (clause, (target_offset + in_offset)))) { handle_branch (td, MINT_CALL_HANDLER, clause->handler_offset - in_offset); td->last_ins->data [2] = i; + + if (mono_interp_tiering_enabled ()) { + // In the optimized method we will remember the native_offset of this bb_index + // In the unoptimized method we will have to maintain the mapping between the + // native offset of this bb and its bb_index + td->patchpoint_data_n++; + interp_add_ins (td, MINT_TIER_PATCHPOINT_DATA); + call_handler_count++; + td->last_ins->data [0] = call_handler_count; + g_assert (call_handler_count < G_MAXUINT16); + } } } @@ -7135,7 +7190,7 @@ generate_code (TransformData *td, MonoMethod *method, MonoMethodHeader *header, break; } - int index = get_data_item_index (td, mono_interp_get_imethod (m, error)); + int index = get_data_item_index_imethod (td, mono_interp_get_imethod (m)); goto_if_nok (error, exit); if (*td->ip == CEE_LDVIRTFTN) { CHECK_STACK (td, 1); @@ -7469,6 +7524,9 @@ compute_native_offset_estimates (TransformData *td) for (bb = td->entry_bb; bb != NULL; bb = bb->next_bb) { InterpInst *ins; bb->native_offset_estimate = noe; + if (bb->emit_patchpoint) + noe += 2; + for (ins = bb->first_ins; ins != NULL; ins = ins->next) { int opcode = ins->opcode; // Skip dummy opcodes for more precise offset computation @@ -7693,12 +7751,26 @@ emit_compacted_instruction (TransformData *td, guint16* start_ip, InterpInst *in return ip; } +static int +add_patchpoint_data (TransformData *td, int patchpoint_data_index, int native_offset, int key) +{ + if (td->rtm->optimized) { + td->patchpoint_data [patchpoint_data_index++] = key; + td->patchpoint_data [patchpoint_data_index++] = native_offset; + } else { + td->patchpoint_data [patchpoint_data_index++] = native_offset; + td->patchpoint_data [patchpoint_data_index++] = key; + } + return patchpoint_data_index; +} + // Generates the final code, after we are done with all the passes static void generate_compacted_code (TransformData *td) { guint16 *ip; int size; + int patchpoint_data_index = 0; td->relocs = g_ptr_array_new (); InterpBasicBlock *bb; @@ -7709,12 +7781,30 @@ generate_compacted_code (TransformData *td) // Generate the compacted stream of instructions td->new_code = ip = (guint16*)mono_mem_manager_alloc0 (td->mem_manager, size * sizeof (guint16)); + if (td->patchpoint_data_n) { + g_assert (mono_interp_tiering_enabled ()); + td->patchpoint_data = (int*)mono_mem_manager_alloc0 (td->mem_manager, (td->patchpoint_data_n * 2 + 1) * sizeof (int)); + td->patchpoint_data [td->patchpoint_data_n * 2] = G_MAXINT32; + } + for (bb = td->entry_bb; bb != NULL; bb = bb->next_bb) { InterpInst *ins = bb->first_ins; bb->native_offset = ip - td->new_code; td->cbb = bb; + if (bb->patchpoint_data) + patchpoint_data_index = add_patchpoint_data (td, patchpoint_data_index, bb->native_offset, bb->index); + if (bb->emit_patchpoint) { + // Add patchpoint in unoptimized method + *ip++ = MINT_TIER_PATCHPOINT; + *ip++ = (guint16)bb->index; + } while (ins) { - ip = emit_compacted_instruction (td, ip, ins); + if (ins->opcode == MINT_TIER_PATCHPOINT_DATA) { + int native_offset = (int)(ip - td->new_code); + patchpoint_data_index = add_patchpoint_data (td, patchpoint_data_index, native_offset, -ins->data [0]); + } else { + ip = emit_compacted_instruction (td, ip, ins); + } ins = ins->next; } } @@ -9415,9 +9505,11 @@ generate (MonoMethod *method, MonoMethodHeader *header, InterpMethod *rtm, MonoG td->seq_points = g_ptr_array_new (); td->verbose_level = mono_interp_traceopt; td->prof_coverage = mono_profiler_coverage_instrumentation_enabled (method); + td->disable_inlining = !rtm->optimized; if (retry_compilation) td->disable_inlining = TRUE; rtm->data_items = td->data_items; + td->optimized = rtm->optimized; if (td->prof_coverage) td->coverage_info = mono_profiler_coverage_alloc (method, header->code_size); @@ -9457,7 +9549,8 @@ generate (MonoMethod *method, MonoMethodHeader *header, InterpMethod *rtm, MonoG if (td->has_localloc) interp_fix_localloc_ret (td); - interp_optimize_code (td); + if (td->optimized) + interp_optimize_code (td); interp_alloc_offsets (td); @@ -9519,6 +9612,9 @@ generate (MonoMethod *method, MonoMethodHeader *header, InterpMethod *rtm, MonoG rtm->data_items = (gpointer*)mono_mem_manager_alloc0 (td->mem_manager, td->n_data_items * sizeof (td->data_items [0])); memcpy (rtm->data_items, td->data_items, td->n_data_items * sizeof (td->data_items [0])); + mono_interp_register_imethod_data_items (rtm->data_items, td->imethod_items); + rtm->patchpoint_data = td->patchpoint_data; + /* Save debug info */ interp_save_debug_info (rtm, header, td, td->line_numbers); @@ -9568,6 +9664,7 @@ generate (MonoMethod *method, MonoMethodHeader *header, InterpMethod *rtm, MonoG g_ptr_array_free (td->seq_points, TRUE); if (td->line_numbers) g_array_free (td->line_numbers, TRUE); + g_slist_free (td->imethod_items); mono_mempool_destroy (td->mempool); if (retry_compilation) goto retry; @@ -9579,13 +9676,10 @@ mono_test_interp_generate_code (TransformData *td, MonoMethod *method, MonoMetho return generate_code (td, method, header, generic_context, error); } -static mono_mutex_t calc_section; - #ifdef ENABLE_EXPERIMENT_TIERED static gboolean tiered_patcher (MiniTieredPatchPointContext *ctx, gpointer patchsite) { - ERROR_DECL (error); MonoMethod *m = ctx->target_method; if (!jit_call2_supported (m, mono_method_signature_internal (m))) @@ -9593,8 +9687,7 @@ tiered_patcher (MiniTieredPatchPointContext *ctx, gpointer patchsite) /* TODO: Force compilation here. Currently the JIT will be invoked upon * first execution of `MINT_JIT_CALL2`. */ - InterpMethod *rmethod = mono_interp_get_imethod (cm, error); - mono_error_assert_ok (error); + InterpMethod *rmethod = mono_interp_get_imethod (cm); guint16 *ip = ((guint16 *) patchsite); *ip++ = MINT_JIT_CALL2; @@ -9610,8 +9703,6 @@ tiered_patcher (MiniTieredPatchPointContext *ctx, gpointer patchsite) void mono_interp_transform_init (void) { - mono_os_mutex_init_recursive(&calc_section); - #ifdef ENABLE_EXPERIMENT_TIERED mini_tiered_register_callsite_patcher (tiered_patcher, TIERED_PATCH_KIND_INTERP); #endif @@ -9692,12 +9783,13 @@ mono_interp_transform_method (InterpMethod *imethod, ThreadContext *context, Mon g_assert_not_reached (); } if (nm == NULL) { - mono_os_mutex_lock (&calc_section); + MonoJitMemoryManager *jit_mm = get_default_jit_mm (); + jit_mm_lock (jit_mm); imethod->alloca_size = sizeof (stackval); /* for tracing */ mono_memory_barrier (); imethod->transformed = TRUE; mono_interp_stats.methods_transformed++; - mono_os_mutex_unlock (&calc_section); + jit_mm_unlock (jit_mm); MONO_PROFILER_RAISE (jit_done, (method, NULL)); return; } @@ -9727,7 +9819,9 @@ mono_interp_transform_method (InterpMethod *imethod, ThreadContext *context, Mon /* Copy changes back */ imethod = real_imethod; - mono_os_mutex_lock (&calc_section); + + MonoJitMemoryManager *jit_mm = get_default_jit_mm (); + jit_mm_lock (jit_mm); if (!imethod->transformed) { // Ignore the first two fields which are unchanged. next_jit_code_hash shouldn't // be modified because it is racy with internal hash table insert. @@ -9738,22 +9832,19 @@ mono_interp_transform_method (InterpMethod *imethod, ThreadContext *context, Mon mono_interp_stats.methods_transformed++; mono_atomic_fetch_add_i32 (&mono_jit_stats.methods_with_interp, 1); + // FIXME Publishing of seq points seems to be racy with tiereing. We can have both tiered and untiered method + // running at the same time. We could therefore get the optimized imethod seq points for the unoptimized method. + gpointer seq_points = g_hash_table_lookup (jit_mm->seq_points, imethod->method); + if (!seq_points || seq_points != imethod->jinfo->seq_points) + g_hash_table_replace (jit_mm->seq_points, imethod->method, imethod->jinfo->seq_points); } - mono_os_mutex_unlock (&calc_section); + jit_mm_unlock (jit_mm); if (mono_stats_method_desc && mono_method_desc_full_match (mono_stats_method_desc, imethod->method)) { g_printf ("Printing runtime stats at method: %s\n", mono_method_get_full_name (imethod->method)); mono_runtime_print_stats (); } - MonoJitMemoryManager *jit_mm = get_default_jit_mm (); - jit_mm_lock (jit_mm); - gpointer seq_points = g_hash_table_lookup (jit_mm->seq_points, imethod->method); - if (!seq_points || seq_points != imethod->jinfo->seq_points) - g_hash_table_replace (jit_mm->seq_points, imethod->method, imethod->jinfo->seq_points); - - jit_mm_unlock (jit_mm); - // FIXME: Add a different callback ? MONO_PROFILER_RAISE (jit_done, (method, imethod->jinfo)); } diff --git a/src/mono/mono/mini/interp/transform.h b/src/mono/mono/mini/interp/transform.h index 80a2eae7486a4f..6e3599af486b72 100644 --- a/src/mono/mono/mini/interp/transform.h +++ b/src/mono/mono/mini/interp/transform.h @@ -126,6 +126,11 @@ struct _InterpBasicBlock { // This block has special semantics and it shouldn't be optimized away int eh_block : 1; int dead: 1; + // If patchpoint is set we will store mapping information between native offset and bblock index within + // InterpMethod. In the unoptimized method we will map from native offset to the bb_index while in the + // optimized method we will map the bb_index to the corresponding native offset. + int patchpoint_data: 1; + int emit_patchpoint: 1; }; typedef enum { @@ -195,6 +200,7 @@ typedef struct int max_data_items; void **data_items; GHashTable *data_hash; + GSList *imethod_items; #ifdef ENABLE_EXPERIMENT_TIERED GHashTable *patchsite_hash; #endif @@ -216,6 +222,8 @@ typedef struct MonoProfilerCoverageInfo *coverage_info; GList *dont_inline; int inline_depth; + int patchpoint_data_n; + int *patchpoint_data; int has_localloc : 1; // If method compilation fails due to certain limits being exceeded, we disable inlining // and retry compilation. @@ -223,6 +231,7 @@ typedef struct // If the current method (inlined_method) has the aggressive inlining attribute, we no longer // bail out of inlining when having to generate certain opcodes (like call, throw). int aggressive_inlining : 1; + int optimized : 1; } TransformData; #define STACK_TYPE_I4 0 diff --git a/src/mono/mono/mini/mini-exceptions.c b/src/mono/mono/mini/mini-exceptions.c index 94d3b74a1b562c..ff8a1ae950a476 100644 --- a/src/mono/mono/mini/mini-exceptions.c +++ b/src/mono/mono/mini/mini-exceptions.c @@ -2011,7 +2011,7 @@ handle_exception_first_pass (MonoContext *ctx, MonoObject *obj, gint32 *out_filt if (frame.type == FRAME_TYPE_IL_STATE) { if (mono_trace_is_enabled () && mono_trace_eval (method)) g_print ("EXCEPTION: filter clause found in AOTed code, running it with the interpreter.\n"); - gboolean res = mini_get_interp_callbacks ()->run_clause_with_il_state (frame.il_state, i, ei->data.filter, NULL, ex_obj, &filtered, MONO_EXCEPTION_CLAUSE_FILTER); + gboolean res = mini_get_interp_callbacks ()->run_clause_with_il_state (frame.il_state, i, ex_obj, &filtered); // FIXME: g_assert (!res); } else if (ji->is_interp) { @@ -2574,7 +2574,7 @@ mono_handle_exception_internal (MonoContext *ctx, MonoObject *obj, gboolean resu if (frame.type == FRAME_TYPE_IL_STATE) { if (mono_trace_is_enabled () && mono_trace_eval (method)) g_print ("EXCEPTION: finally/fault clause found in AOTed code, running it with the interpreter.\n"); - mini_get_interp_callbacks ()->run_clause_with_il_state (frame.il_state, i, ei->handler_start, ei->data.handler_end, NULL, NULL, MONO_EXCEPTION_CLAUSE_FINALLY); + mini_get_interp_callbacks ()->run_clause_with_il_state (frame.il_state, i, NULL, NULL); } else if (in_interp) { gboolean has_ex = mini_get_interp_callbacks ()->run_finally (&frame, i, ei->handler_start, ei->data.handler_end); if (has_ex) { @@ -3583,10 +3583,8 @@ mini_llvmonly_resume_exception_il_state (MonoLMF *lmf, gpointer info) /* Pop the LMF frame so the caller doesn't have to do it */ mono_set_lmf ((MonoLMF *)(((gsize)lmf->previous_lmf) & ~3)); - MonoJitInfo *ji = jit_tls->resume_state.ji; int clause_index = jit_tls->resume_state.clause_index; - MonoJitExceptionInfo *ei = &ji->clauses [clause_index]; - gboolean r = mini_get_interp_callbacks ()->run_clause_with_il_state (il_state, clause_index, ei->handler_start, NULL, ex_obj, NULL, MONO_EXCEPTION_CLAUSE_NONE); + gboolean r = mini_get_interp_callbacks ()->run_clause_with_il_state (il_state, clause_index, ex_obj, NULL); if (r) /* Another exception thrown, continue unwinding */ mono_llvm_cpp_throw_exception (); diff --git a/src/mono/mono/mini/mini-generic-sharing.c b/src/mono/mono/mini/mini-generic-sharing.c index af34a2d8779f93..642f2a59252333 100644 --- a/src/mono/mono/mini/mini-generic-sharing.c +++ b/src/mono/mono/mini/mini-generic-sharing.c @@ -2244,7 +2244,7 @@ instantiate_info (MonoMemoryManager *mem_manager, MonoRuntimeGenericContextInfoT case MONO_RGCTX_INFO_INTERP_METHOD: { MonoMethod *m = (MonoMethod*)data; - return mini_get_interp_callbacks ()->get_interp_method (m, error); + return mini_get_interp_callbacks ()->get_interp_method (m); } case MONO_RGCTX_INFO_LLVMONLY_INTERP_ENTRY: { MonoMethod *m = (MonoMethod*)data;