From 7e7568672d72f078f6562b43e29a77f521854989 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Wed, 12 Apr 2023 14:42:32 -0700 Subject: [PATCH] specialize: make specialization thread-safe --- Include/cpython/code.h | 4 +- Include/internal/pycore_code.h | 1 + Include/internal/pycore_opcode.h | 110 ++++++++-------- Include/opcode.h | 132 ++++++++++--------- Lib/importlib/_bootstrap_external.py | 2 +- Lib/opcode.py | 10 ++ Lib/test/test_embed.py | 1 + Lib/test/test_threading_local.py | 3 + Python/bytecodes.c | 108 +++++++++++++--- Python/ceval.c | 7 +- Python/generated_cases.c.h | 109 ++++++++++++++-- Python/opcode_metadata.h | 10 ++ Python/opcode_targets.h | 90 ++++++------- Python/pylifecycle.c | 3 + Python/specialize.c | 182 ++++++++++++++++++++++----- 15 files changed, 549 insertions(+), 223 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 0cf49f06c87..ab5bdf8fbb0 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -30,7 +30,9 @@ typedef union { static inline void _py_set_opcode(_Py_CODEUNIT *word, uint8_t opcode) { - word->opcode = opcode; + _Py_CODEUNIT new_word = *word; + new_word.opcode = opcode; + _Py_atomic_store_uint16(&word->cache, new_word.cache); } #define _Py_SET_OPCODE(word, opcode) _py_set_opocde(&(word), opcode) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 9e59fc98bf3..ecf4a350181 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -99,6 +99,7 @@ struct callable_cache { PyObject *len; PyObject *list_append; PyObject *object__getattribute__; + PyObject *object__class__; }; /* "Locals plus" for a code object is the set of locals + cell vars + diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 8fdf121aecc..3670a7121dc 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -60,6 +60,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [BINARY_OP_ADD_FLOAT] = BINARY_OP, [BINARY_OP_ADD_INT] = BINARY_OP, [BINARY_OP_ADD_UNICODE] = BINARY_OP, + [BINARY_OP_GENERIC] = BINARY_OP, [BINARY_OP_INPLACE_ADD_UNICODE] = BINARY_OP, [BINARY_OP_MULTIPLY_FLOAT] = BINARY_OP, [BINARY_OP_MULTIPLY_INT] = BINARY_OP, @@ -68,6 +69,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [BINARY_SLICE] = BINARY_SLICE, [BINARY_SUBSCR] = BINARY_SUBSCR, [BINARY_SUBSCR_DICT] = BINARY_SUBSCR, + [BINARY_SUBSCR_GENERIC] = BINARY_SUBSCR, [BINARY_SUBSCR_GETITEM] = BINARY_SUBSCR, [BINARY_SUBSCR_LIST_INT] = BINARY_SUBSCR, [BINARY_SUBSCR_TUPLE_INT] = BINARY_SUBSCR, @@ -84,6 +86,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [CALL_BUILTIN_CLASS] = CALL, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = CALL, [CALL_FUNCTION_EX] = CALL_FUNCTION_EX, + [CALL_GENERIC] = CALL, [CALL_INTRINSIC_1] = CALL_INTRINSIC_1, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = CALL, [CALL_NO_KW_BUILTIN_FAST] = CALL, @@ -104,6 +107,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [CLEANUP_THROW] = CLEANUP_THROW, [COMPARE_OP] = COMPARE_OP, [COMPARE_OP_FLOAT_JUMP] = COMPARE_OP, + [COMPARE_OP_GENERIC] = COMPARE_OP, [COMPARE_OP_INT_JUMP] = COMPARE_OP, [COMPARE_OP_STR_JUMP] = COMPARE_OP, [CONTAINS_OP] = CONTAINS_OP, @@ -123,6 +127,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [FORMAT_VALUE] = FORMAT_VALUE, [FOR_ITER] = FOR_ITER, [FOR_ITER_GEN] = FOR_ITER, + [FOR_ITER_GENERIC] = FOR_ITER, [FOR_ITER_LIST] = FOR_ITER, [FOR_ITER_RANGE] = FOR_ITER, [FOR_ITER_TUPLE] = FOR_ITER, @@ -147,6 +152,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [LOAD_ASSERTION_ERROR] = LOAD_ASSERTION_ERROR, [LOAD_ATTR] = LOAD_ATTR, [LOAD_ATTR_CLASS] = LOAD_ATTR, + [LOAD_ATTR_GENERIC] = LOAD_ATTR, [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = LOAD_ATTR, [LOAD_ATTR_INSTANCE_VALUE] = LOAD_ATTR, [LOAD_ATTR_METHOD_LAZY_DICT] = LOAD_ATTR, @@ -168,6 +174,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [LOAD_FAST__LOAD_FAST] = LOAD_FAST, [LOAD_GLOBAL] = LOAD_GLOBAL, [LOAD_GLOBAL_BUILTIN] = LOAD_GLOBAL, + [LOAD_GLOBAL_GENERIC] = LOAD_GLOBAL, [LOAD_GLOBAL_MODULE] = LOAD_GLOBAL, [LOAD_NAME] = LOAD_NAME, [MAKE_CELL] = MAKE_CELL, @@ -197,6 +204,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [SET_ADD] = SET_ADD, [SET_UPDATE] = SET_UPDATE, [STORE_ATTR] = STORE_ATTR, + [STORE_ATTR_GENERIC] = STORE_ATTR, [STORE_ATTR_INSTANCE_VALUE] = STORE_ATTR, [STORE_ATTR_SLOT] = STORE_ATTR, [STORE_ATTR_WITH_HINT] = STORE_ATTR, @@ -209,6 +217,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [STORE_SLICE] = STORE_SLICE, [STORE_SUBSCR] = STORE_SUBSCR, [STORE_SUBSCR_DICT] = STORE_SUBSCR, + [STORE_SUBSCR_GENERIC] = STORE_SUBSCR, [STORE_SUBSCR_LIST_INT] = STORE_SUBSCR, [SWAP] = SWAP, [UNARY_INVERT] = UNARY_INVERT, @@ -216,6 +225,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [UNARY_NOT] = UNARY_NOT, [UNPACK_EX] = UNPACK_EX, [UNPACK_SEQUENCE] = UNPACK_SEQUENCE, + [UNPACK_SEQUENCE_GENERIC] = UNPACK_SEQUENCE, [UNPACK_SEQUENCE_LIST] = UNPACK_SEQUENCE, [UNPACK_SEQUENCE_TUPLE] = UNPACK_SEQUENCE, [UNPACK_SEQUENCE_TWO_TUPLE] = UNPACK_SEQUENCE, @@ -231,39 +241,42 @@ static const char *const _PyOpcode_OpName[263] = { [PUSH_NULL] = "PUSH_NULL", [INTERPRETER_EXIT] = "INTERPRETER_EXIT", [END_FOR] = "END_FOR", + [BINARY_OP_GENERIC] = "BINARY_OP_GENERIC", [BINARY_OP_ADD_FLOAT] = "BINARY_OP_ADD_FLOAT", [BINARY_OP_ADD_INT] = "BINARY_OP_ADD_INT", [BINARY_OP_ADD_UNICODE] = "BINARY_OP_ADD_UNICODE", - [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE", [NOP] = "NOP", - [BINARY_OP_MULTIPLY_FLOAT] = "BINARY_OP_MULTIPLY_FLOAT", + [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE", [UNARY_NEGATIVE] = "UNARY_NEGATIVE", [UNARY_NOT] = "UNARY_NOT", + [BINARY_OP_MULTIPLY_FLOAT] = "BINARY_OP_MULTIPLY_FLOAT", [BINARY_OP_MULTIPLY_INT] = "BINARY_OP_MULTIPLY_INT", - [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", [UNARY_INVERT] = "UNARY_INVERT", + [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", [BINARY_OP_SUBTRACT_INT] = "BINARY_OP_SUBTRACT_INT", + [BINARY_SUBSCR_GENERIC] = "BINARY_SUBSCR_GENERIC", [BINARY_SUBSCR_DICT] = "BINARY_SUBSCR_DICT", [BINARY_SUBSCR_GETITEM] = "BINARY_SUBSCR_GETITEM", [BINARY_SUBSCR_LIST_INT] = "BINARY_SUBSCR_LIST_INT", [BINARY_SUBSCR_TUPLE_INT] = "BINARY_SUBSCR_TUPLE_INT", + [CALL_GENERIC] = "CALL_GENERIC", [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", - [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", - [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", - [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", [BINARY_SUBSCR] = "BINARY_SUBSCR", [BINARY_SLICE] = "BINARY_SLICE", [STORE_SLICE] = "STORE_SLICE", - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", + [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", [GET_LEN] = "GET_LEN", [MATCH_MAPPING] = "MATCH_MAPPING", [MATCH_SEQUENCE] = "MATCH_SEQUENCE", [MATCH_KEYS] = "MATCH_KEYS", - [CALL_NO_KW_BUILTIN_FAST] = "CALL_NO_KW_BUILTIN_FAST", + [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", [PUSH_EXC_INFO] = "PUSH_EXC_INFO", [CHECK_EXC_MATCH] = "CHECK_EXC_MATCH", [CHECK_EG_MATCH] = "CHECK_EG_MATCH", + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + [CALL_NO_KW_BUILTIN_FAST] = "CALL_NO_KW_BUILTIN_FAST", [CALL_NO_KW_BUILTIN_O] = "CALL_NO_KW_BUILTIN_O", [CALL_NO_KW_ISINSTANCE] = "CALL_NO_KW_ISINSTANCE", [CALL_NO_KW_LEN] = "CALL_NO_KW_LEN", @@ -272,9 +285,6 @@ static const char *const _PyOpcode_OpName[263] = { [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = "CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS", [CALL_NO_KW_METHOD_DESCRIPTOR_O] = "CALL_NO_KW_METHOD_DESCRIPTOR_O", [CALL_NO_KW_STR_1] = "CALL_NO_KW_STR_1", - [CALL_NO_KW_TUPLE_1] = "CALL_NO_KW_TUPLE_1", - [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", - [COMPARE_OP_FLOAT_JUMP] = "COMPARE_OP_FLOAT_JUMP", [WITH_EXCEPT_START] = "WITH_EXCEPT_START", [GET_AITER] = "GET_AITER", [GET_ANEXT] = "GET_ANEXT", @@ -282,38 +292,38 @@ static const char *const _PyOpcode_OpName[263] = { [BEFORE_WITH] = "BEFORE_WITH", [END_ASYNC_FOR] = "END_ASYNC_FOR", [CLEANUP_THROW] = "CLEANUP_THROW", + [CALL_NO_KW_TUPLE_1] = "CALL_NO_KW_TUPLE_1", + [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", + [COMPARE_OP_GENERIC] = "COMPARE_OP_GENERIC", + [COMPARE_OP_FLOAT_JUMP] = "COMPARE_OP_FLOAT_JUMP", + [STORE_SUBSCR] = "STORE_SUBSCR", + [DELETE_SUBSCR] = "DELETE_SUBSCR", [COMPARE_OP_INT_JUMP] = "COMPARE_OP_INT_JUMP", [COMPARE_OP_STR_JUMP] = "COMPARE_OP_STR_JUMP", + [FOR_ITER_GENERIC] = "FOR_ITER_GENERIC", [FOR_ITER_LIST] = "FOR_ITER_LIST", [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", - [STORE_SUBSCR] = "STORE_SUBSCR", - [DELETE_SUBSCR] = "DELETE_SUBSCR", [FOR_ITER_RANGE] = "FOR_ITER_RANGE", + [GET_ITER] = "GET_ITER", + [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", [FOR_ITER_GEN] = "FOR_ITER_GEN", + [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", + [LOAD_ATTR_GENERIC] = "LOAD_ATTR_GENERIC", [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", + [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", + [RETURN_GENERATOR] = "RETURN_GENERATOR", [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", - [GET_ITER] = "GET_ITER", - [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", - [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", - [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", - [RETURN_GENERATOR] = "RETURN_GENERATOR", [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", + [RETURN_VALUE] = "RETURN_VALUE", [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", + [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", - [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", - [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", - [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", - [RETURN_VALUE] = "RETURN_VALUE", - [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", - [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", - [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", - [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", [PREP_RERAISE_STAR] = "PREP_RERAISE_STAR", [POP_EXCEPT] = "POP_EXCEPT", [STORE_NAME] = "STORE_NAME", @@ -339,7 +349,7 @@ static const char *const _PyOpcode_OpName[263] = { [JUMP_FORWARD] = "JUMP_FORWARD", [JUMP_IF_FALSE_OR_POP] = "JUMP_IF_FALSE_OR_POP", [JUMP_IF_TRUE_OR_POP] = "JUMP_IF_TRUE_OR_POP", - [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", + [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", [LOAD_GLOBAL] = "LOAD_GLOBAL", @@ -347,7 +357,7 @@ static const char *const _PyOpcode_OpName[263] = { [CONTAINS_OP] = "CONTAINS_OP", [RERAISE] = "RERAISE", [COPY] = "COPY", - [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", + [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", [BINARY_OP] = "BINARY_OP", [SEND] = "SEND", [LOAD_FAST] = "LOAD_FAST", @@ -367,9 +377,9 @@ static const char *const _PyOpcode_OpName[263] = { [STORE_DEREF] = "STORE_DEREF", [DELETE_DEREF] = "DELETE_DEREF", [JUMP_BACKWARD] = "JUMP_BACKWARD", - [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", + [LOAD_GLOBAL_GENERIC] = "LOAD_GLOBAL_GENERIC", [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", - [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", + [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", [EXTENDED_ARG] = "EXTENDED_ARG", [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", @@ -379,30 +389,30 @@ static const char *const _PyOpcode_OpName[263] = { [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", - [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", - [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", + [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", + [STORE_ATTR_GENERIC] = "STORE_ATTR_GENERIC", [FORMAT_VALUE] = "FORMAT_VALUE", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", - [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", - [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", - [160] = "<160>", - [161] = "<161>", + [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", + [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", + [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", + [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", [LIST_EXTEND] = "LIST_EXTEND", [SET_UPDATE] = "SET_UPDATE", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", - [166] = "<166>", - [167] = "<167>", - [168] = "<168>", - [169] = "<169>", - [170] = "<170>", + [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", + [STORE_SUBSCR_GENERIC] = "STORE_SUBSCR_GENERIC", + [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", + [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", + [UNPACK_SEQUENCE_GENERIC] = "UNPACK_SEQUENCE_GENERIC", [CALL] = "CALL", [KW_NAMES] = "KW_NAMES", [CALL_INTRINSIC_1] = "CALL_INTRINSIC_1", - [174] = "<174>", - [175] = "<175>", - [176] = "<176>", + [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", + [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", + [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", [177] = "<177>", [178] = "<178>", [179] = "<179>", @@ -493,16 +503,6 @@ static const char *const _PyOpcode_OpName[263] = { #endif #define EXTRA_CASES \ - case 160: \ - case 161: \ - case 166: \ - case 167: \ - case 168: \ - case 169: \ - case 170: \ - case 174: \ - case 175: \ - case 176: \ case 177: \ case 178: \ case 179: \ diff --git a/Include/opcode.h b/Include/opcode.h index fcc46a3ede9..c38d63dcab3 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -124,67 +124,77 @@ extern "C" { #define JUMP_NO_INTERRUPT 261 #define LOAD_METHOD 262 #define MAX_PSEUDO_OPCODE 262 -#define BINARY_OP_ADD_FLOAT 5 -#define BINARY_OP_ADD_INT 6 -#define BINARY_OP_ADD_UNICODE 7 -#define BINARY_OP_INPLACE_ADD_UNICODE 8 -#define BINARY_OP_MULTIPLY_FLOAT 10 -#define BINARY_OP_MULTIPLY_INT 13 -#define BINARY_OP_SUBTRACT_FLOAT 14 -#define BINARY_OP_SUBTRACT_INT 16 -#define BINARY_SUBSCR_DICT 17 -#define BINARY_SUBSCR_GETITEM 18 -#define BINARY_SUBSCR_LIST_INT 19 -#define BINARY_SUBSCR_TUPLE_INT 20 -#define CALL_PY_EXACT_ARGS 21 -#define CALL_PY_WITH_DEFAULTS 22 -#define CALL_BOUND_METHOD_EXACT_ARGS 23 -#define CALL_BUILTIN_CLASS 24 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 28 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 29 -#define CALL_NO_KW_BUILTIN_FAST 34 -#define CALL_NO_KW_BUILTIN_O 38 -#define CALL_NO_KW_ISINSTANCE 39 -#define CALL_NO_KW_LEN 40 -#define CALL_NO_KW_LIST_APPEND 41 -#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 42 -#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 43 -#define CALL_NO_KW_METHOD_DESCRIPTOR_O 44 -#define CALL_NO_KW_STR_1 45 -#define CALL_NO_KW_TUPLE_1 46 -#define CALL_NO_KW_TYPE_1 47 -#define COMPARE_OP_FLOAT_JUMP 48 -#define COMPARE_OP_INT_JUMP 56 -#define COMPARE_OP_STR_JUMP 57 -#define FOR_ITER_LIST 58 -#define FOR_ITER_TUPLE 59 -#define FOR_ITER_RANGE 62 -#define FOR_ITER_GEN 63 -#define LOAD_ATTR_CLASS 64 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 65 -#define LOAD_ATTR_INSTANCE_VALUE 66 -#define LOAD_ATTR_MODULE 67 -#define LOAD_ATTR_PROPERTY 70 -#define LOAD_ATTR_SLOT 72 -#define LOAD_ATTR_WITH_HINT 73 -#define LOAD_ATTR_METHOD_LAZY_DICT 76 -#define LOAD_ATTR_METHOD_NO_DICT 77 -#define LOAD_ATTR_METHOD_WITH_VALUES 78 -#define LOAD_CONST__LOAD_FAST 79 -#define LOAD_FAST__LOAD_CONST 80 -#define LOAD_FAST__LOAD_FAST 81 -#define LOAD_GLOBAL_BUILTIN 82 -#define LOAD_GLOBAL_MODULE 84 -#define STORE_ATTR_INSTANCE_VALUE 86 -#define STORE_ATTR_SLOT 87 -#define STORE_ATTR_WITH_HINT 113 -#define STORE_FAST__LOAD_FAST 121 -#define STORE_FAST__STORE_FAST 141 -#define STORE_SUBSCR_DICT 143 -#define STORE_SUBSCR_LIST_INT 153 -#define UNPACK_SEQUENCE_LIST 154 -#define UNPACK_SEQUENCE_TUPLE 158 -#define UNPACK_SEQUENCE_TWO_TUPLE 159 +#define BINARY_OP_GENERIC 5 +#define BINARY_OP_ADD_FLOAT 6 +#define BINARY_OP_ADD_INT 7 +#define BINARY_OP_ADD_UNICODE 8 +#define BINARY_OP_INPLACE_ADD_UNICODE 10 +#define BINARY_OP_MULTIPLY_FLOAT 13 +#define BINARY_OP_MULTIPLY_INT 14 +#define BINARY_OP_SUBTRACT_FLOAT 16 +#define BINARY_OP_SUBTRACT_INT 17 +#define BINARY_SUBSCR_GENERIC 18 +#define BINARY_SUBSCR_DICT 19 +#define BINARY_SUBSCR_GETITEM 20 +#define BINARY_SUBSCR_LIST_INT 21 +#define BINARY_SUBSCR_TUPLE_INT 22 +#define CALL_GENERIC 23 +#define CALL_PY_EXACT_ARGS 24 +#define CALL_PY_WITH_DEFAULTS 28 +#define CALL_BOUND_METHOD_EXACT_ARGS 29 +#define CALL_BUILTIN_CLASS 34 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 38 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 39 +#define CALL_NO_KW_BUILTIN_FAST 40 +#define CALL_NO_KW_BUILTIN_O 41 +#define CALL_NO_KW_ISINSTANCE 42 +#define CALL_NO_KW_LEN 43 +#define CALL_NO_KW_LIST_APPEND 44 +#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 45 +#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 46 +#define CALL_NO_KW_METHOD_DESCRIPTOR_O 47 +#define CALL_NO_KW_STR_1 48 +#define CALL_NO_KW_TUPLE_1 56 +#define CALL_NO_KW_TYPE_1 57 +#define COMPARE_OP_GENERIC 58 +#define COMPARE_OP_FLOAT_JUMP 59 +#define COMPARE_OP_INT_JUMP 62 +#define COMPARE_OP_STR_JUMP 63 +#define FOR_ITER_GENERIC 64 +#define FOR_ITER_LIST 65 +#define FOR_ITER_TUPLE 66 +#define FOR_ITER_RANGE 67 +#define FOR_ITER_GEN 70 +#define LOAD_ATTR_GENERIC 72 +#define LOAD_ATTR_CLASS 73 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 76 +#define LOAD_ATTR_INSTANCE_VALUE 77 +#define LOAD_ATTR_MODULE 78 +#define LOAD_ATTR_PROPERTY 79 +#define LOAD_ATTR_SLOT 80 +#define LOAD_ATTR_WITH_HINT 81 +#define LOAD_ATTR_METHOD_LAZY_DICT 82 +#define LOAD_ATTR_METHOD_NO_DICT 84 +#define LOAD_ATTR_METHOD_WITH_VALUES 86 +#define LOAD_CONST__LOAD_FAST 87 +#define LOAD_FAST__LOAD_CONST 113 +#define LOAD_FAST__LOAD_FAST 121 +#define LOAD_GLOBAL_GENERIC 141 +#define LOAD_GLOBAL_BUILTIN 143 +#define LOAD_GLOBAL_MODULE 153 +#define STORE_ATTR_GENERIC 154 +#define STORE_ATTR_INSTANCE_VALUE 158 +#define STORE_ATTR_SLOT 159 +#define STORE_ATTR_WITH_HINT 160 +#define STORE_FAST__LOAD_FAST 161 +#define STORE_FAST__STORE_FAST 166 +#define STORE_SUBSCR_GENERIC 167 +#define STORE_SUBSCR_DICT 168 +#define STORE_SUBSCR_LIST_INT 169 +#define UNPACK_SEQUENCE_GENERIC 170 +#define UNPACK_SEQUENCE_LIST 174 +#define UNPACK_SEQUENCE_TUPLE 175 +#define UNPACK_SEQUENCE_TWO_TUPLE 176 #define DO_TRACING 255 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 8a21e0e4646..cc78fb6f7cd 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -440,7 +440,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3514).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3516).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/opcode.py b/Lib/opcode.py index 9ba39e2eec7..7951e6bd9ce 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -275,6 +275,7 @@ def pseudo_op(name, op, real_ops): _specializations = { "BINARY_OP": [ + "BINARY_OP_GENERIC", "BINARY_OP_ADD_FLOAT", "BINARY_OP_ADD_INT", "BINARY_OP_ADD_UNICODE", @@ -285,12 +286,14 @@ def pseudo_op(name, op, real_ops): "BINARY_OP_SUBTRACT_INT", ], "BINARY_SUBSCR": [ + "BINARY_SUBSCR_GENERIC", "BINARY_SUBSCR_DICT", "BINARY_SUBSCR_GETITEM", "BINARY_SUBSCR_LIST_INT", "BINARY_SUBSCR_TUPLE_INT", ], "CALL": [ + "CALL_GENERIC", "CALL_PY_EXACT_ARGS", "CALL_PY_WITH_DEFAULTS", "CALL_BOUND_METHOD_EXACT_ARGS", @@ -310,11 +313,13 @@ def pseudo_op(name, op, real_ops): "CALL_NO_KW_TYPE_1", ], "COMPARE_OP": [ + "COMPARE_OP_GENERIC", "COMPARE_OP_FLOAT_JUMP", "COMPARE_OP_INT_JUMP", "COMPARE_OP_STR_JUMP", ], "FOR_ITER": [ + "FOR_ITER_GENERIC", "FOR_ITER_LIST", "FOR_ITER_TUPLE", "FOR_ITER_RANGE", @@ -322,6 +327,7 @@ def pseudo_op(name, op, real_ops): ], "LOAD_ATTR": [ # These potentially push [NULL, bound method] onto the stack. + "LOAD_ATTR_GENERIC", "LOAD_ATTR_CLASS", "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", "LOAD_ATTR_INSTANCE_VALUE", @@ -342,10 +348,12 @@ def pseudo_op(name, op, real_ops): "LOAD_FAST__LOAD_FAST", ], "LOAD_GLOBAL": [ + "LOAD_GLOBAL_GENERIC", "LOAD_GLOBAL_BUILTIN", "LOAD_GLOBAL_MODULE", ], "STORE_ATTR": [ + "STORE_ATTR_GENERIC", "STORE_ATTR_INSTANCE_VALUE", "STORE_ATTR_SLOT", "STORE_ATTR_WITH_HINT", @@ -355,10 +363,12 @@ def pseudo_op(name, op, real_ops): "STORE_FAST__STORE_FAST", ], "STORE_SUBSCR": [ + "STORE_SUBSCR_GENERIC", "STORE_SUBSCR_DICT", "STORE_SUBSCR_LIST_INT", ], "UNPACK_SEQUENCE": [ + "UNPACK_SEQUENCE_GENERIC", "UNPACK_SEQUENCE_LIST", "UNPACK_SEQUENCE_TUPLE", "UNPACK_SEQUENCE_TWO_TUPLE", diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index cbe3f64b85d..fc8bd30792e 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -362,6 +362,7 @@ def is_specialized(f): opname in opcode._specialized_instructions # Exclude superinstructions: and "__" not in opname + and not opname.endswith("_PROFILE") ): return True return False diff --git a/Lib/test/test_threading_local.py b/Lib/test/test_threading_local.py index 48e302930ff..09c95cd8dc0 100644 --- a/Lib/test/test_threading_local.py +++ b/Lib/test/test_threading_local.py @@ -142,8 +142,11 @@ def f2(): # This is expected -- we haven't set obj.x in this thread yet! self._failed = "" # passed else: + sys.getfullrefcount(foo) self._failed = ('Incorrectly got value %r from class %r\n' % (foo, c)) + import dis + dis.dis(f2, adaptive=True, show_caches=True) sys.stderr.write(self._failed) t1 = threading.Thread(target=f1) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 98fbe5ea287..3ad3c96172a 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -202,6 +202,7 @@ dummy_func( family(binary_op, INLINE_CACHE_ENTRIES_BINARY_OP) = { BINARY_OP, + BINARY_OP_GENERIC, BINARY_OP_ADD_FLOAT, BINARY_OP_ADD_INT, BINARY_OP_ADD_UNICODE, @@ -333,22 +334,30 @@ dummy_func( family(binary_subscr, INLINE_CACHE_ENTRIES_BINARY_SUBSCR) = { BINARY_SUBSCR, + BINARY_SUBSCR_GENERIC, BINARY_SUBSCR_DICT, BINARY_SUBSCR_GETITEM, BINARY_SUBSCR_LIST_INT, BINARY_SUBSCR_TUPLE_INT, }; - inst(BINARY_SUBSCR, (unused/4, container, sub -- res)) { + inst(BINARY_SUBSCR, (unused/4, container, sub -- unused)) { + _PyMutex_lock(&_PyRuntime.mutex); _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_BinarySubscr(container, sub, next_instr); + _PyMutex_unlock(&_PyRuntime.mutex); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_SUBSCR, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); + _PyMutex_unlock(&_PyRuntime.mutex); + GO_TO_INSTRUCTION(BINARY_SUBSCR_GENERIC); + } + + inst(BINARY_SUBSCR_GENERIC, (unused/4, container, sub -- res)) { res = PyObject_GetItem(container, sub); DECREF_INPUTS(); ERROR_IF(res == NULL, error); @@ -475,20 +484,28 @@ dummy_func( family(store_subscr) = { STORE_SUBSCR, + STORE_SUBSCR_GENERIC, STORE_SUBSCR_DICT, STORE_SUBSCR_LIST_INT, }; - inst(STORE_SUBSCR, (counter/1, v, container, sub -- )) { - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + inst(STORE_SUBSCR, (unused/1, unused, container, sub -- )) { + _PyMutex_lock(&_PyRuntime.mutex); + _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_StoreSubscr(container, sub, next_instr); + _PyMutex_unlock(&_PyRuntime.mutex); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_SUBSCR, deferred); - _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; DECREMENT_ADAPTIVE_COUNTER(cache->counter); + _PyMutex_unlock(&_PyRuntime.mutex); + GO_TO_INSTRUCTION(STORE_SUBSCR_GENERIC); + } + + inst(STORE_SUBSCR_GENERIC, (unused/1, v, container, sub -- )) { /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); DECREF_INPUTS(); @@ -911,16 +928,24 @@ dummy_func( // stack effect: (__0 -- __array[oparg]) inst(UNPACK_SEQUENCE) { + _PyMutex_lock(&_PyRuntime.mutex); _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); PyObject *seq = TOP(); next_instr--; _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + _PyMutex_unlock(&_PyRuntime.mutex); DISPATCH_SAME_OPARG(); } STAT_INC(UNPACK_SEQUENCE, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); + _PyMutex_unlock(&_PyRuntime.mutex); + GO_TO_INSTRUCTION(UNPACK_SEQUENCE_GENERIC); + } + + // stack effect: (__0 -- __array[oparg]) + inst(UNPACK_SEQUENCE_GENERIC) { PyObject *seq = POP(); PyObject **top = stack_pointer + oparg; if (!unpack_iterable(tstate, seq, oparg, -1, top)) { @@ -989,22 +1014,30 @@ dummy_func( family(store_attr) = { STORE_ATTR, + STORE_ATTR_GENERIC, STORE_ATTR_INSTANCE_VALUE, STORE_ATTR_SLOT, STORE_ATTR_WITH_HINT, }; - inst(STORE_ATTR, (counter/1, unused/3, v, owner --)) { - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + inst(STORE_ATTR, (unused/1, unused/3, unused, owner --)) { + _PyMutex_lock(&_PyRuntime.mutex); + _PyAttrCache *cache = (_PyAttrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); PyObject *name = GETITEM(names, oparg); next_instr--; _Py_Specialize_StoreAttr(owner, next_instr, name); + _PyMutex_unlock(&_PyRuntime.mutex); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_ATTR, deferred); - _PyAttrCache *cache = (_PyAttrCache *)next_instr; DECREMENT_ADAPTIVE_COUNTER(cache->counter); + _PyMutex_unlock(&_PyRuntime.mutex); + GO_TO_INSTRUCTION(STORE_ATTR_GENERIC); + } + + inst(STORE_ATTR_GENERIC, (unused/1, unused/3, v, owner --)) { PyObject *name = GETITEM(names, oparg); int err = PyObject_SetAttr(owner, name, v); Py_DECREF(v); @@ -1103,16 +1136,24 @@ dummy_func( // error: LOAD_GLOBAL has irregular stack effect inst(LOAD_GLOBAL) { + _PyMutex_lock(&_PyRuntime.mutex); _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); PyObject *name = GETITEM(names, oparg>>1); next_instr--; _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); + _PyMutex_unlock(&_PyRuntime.mutex); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_GLOBAL, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); + _PyMutex_unlock(&_PyRuntime.mutex); + GO_TO_INSTRUCTION(LOAD_GLOBAL_GENERIC); + } + + // error: LOAD_GLOBAL has irregular stack effect + inst(LOAD_GLOBAL_GENERIC) { int push_null = oparg & 1; PEEK(0) = NULL; PyObject *name = GETITEM(names, oparg>>1); @@ -1496,6 +1537,7 @@ dummy_func( // error: LOAD_ATTR has irregular stack effect inst(LOAD_ATTR) { + _PyMutex_lock(&_PyRuntime.mutex); _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); @@ -1503,10 +1545,17 @@ dummy_func( PyObject *name = GETITEM(names, oparg>>1); next_instr--; _Py_Specialize_LoadAttr(owner, next_instr, name); + _PyMutex_unlock(&_PyRuntime.mutex); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_ATTR, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); + _PyMutex_unlock(&_PyRuntime.mutex); + GO_TO_INSTRUCTION(LOAD_ATTR_GENERIC); + } + + // error: LOAD_ATTR has irregular stack effect + inst(LOAD_ATTR_GENERIC) { PyObject *name = GETITEM(names, oparg >> 1); PyObject *owner = TOP(); if (oparg & 1) { @@ -1829,21 +1878,29 @@ dummy_func( family(compare_op) = { COMPARE_OP, + COMPARE_OP_GENERIC, _COMPARE_OP_FLOAT, _COMPARE_OP_INT, _COMPARE_OP_STR, }; - inst(COMPARE_OP, (unused/2, left, right -- res)) { + inst(COMPARE_OP, (unused/2, left, right -- unused)) { + _PyMutex_lock(&_PyRuntime.mutex); _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_CompareOp(left, right, next_instr, oparg); + _PyMutex_unlock(&_PyRuntime.mutex); DISPATCH_SAME_OPARG(); } STAT_INC(COMPARE_OP, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); + _PyMutex_unlock(&_PyRuntime.mutex); + GO_TO_INSTRUCTION(COMPARE_OP_GENERIC); + } + + inst(COMPARE_OP_GENERIC, (unused/2, left, right -- res)) { assert(oparg <= Py_GE); res = PyObject_RichCompare(left, right, oparg); Py_DECREF(left); @@ -2244,15 +2301,23 @@ dummy_func( // stack effect: ( -- __0) inst(FOR_ITER) { + _PyMutex_lock(&_PyRuntime.mutex); _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_ForIter(TOP(), next_instr, oparg); + _PyMutex_unlock(&_PyRuntime.mutex); DISPATCH_SAME_OPARG(); } STAT_INC(FOR_ITER, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); + _PyMutex_unlock(&_PyRuntime.mutex); + GO_TO_INSTRUCTION(FOR_ITER_GENERIC); + } + + // stack effect: ( -- __0) + inst(FOR_ITER_GENERIC) { /* before: [iter]; after: [iter, iter()] *or* [] */ PyObject *iter = TOP(); PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); @@ -2571,6 +2636,7 @@ dummy_func( // stack effect: (__0, __array[oparg] -- ) inst(CALL) { + _PyMutex_lock(&_PyRuntime.mutex); _PyCallCache *cache = (_PyCallCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); @@ -2579,10 +2645,17 @@ dummy_func( PyObject *callable = PEEK(nargs + 1); next_instr--; _Py_Specialize_Call(callable, next_instr, nargs, kwnames); + _PyMutex_unlock(&_PyRuntime.mutex); DISPATCH_SAME_OPARG(); } STAT_INC(CALL, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); + _PyMutex_unlock(&_PyRuntime.mutex); + GO_TO_INSTRUCTION(CALL_GENERIC); + } + + // stack effect: (__0, __array[oparg] -- ) + inst(CALL_GENERIC) { int total_args, is_meth; is_meth = is_method(stack_pointer, oparg); PyObject *function = PEEK(oparg + 1); @@ -3319,16 +3392,23 @@ dummy_func( PUSH(Py_NewRef(peek)); } - inst(BINARY_OP, (unused/1, lhs, rhs -- res)) { + inst(BINARY_OP, (unused/1, lhs, rhs -- unused)) { + _PyMutex_lock(&_PyRuntime.mutex); _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0)); + _PyMutex_unlock(&_PyRuntime.mutex); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_OP, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); + _PyMutex_unlock(&_PyRuntime.mutex); + GO_TO_INSTRUCTION(BINARY_OP_GENERIC); + } + + inst(BINARY_OP_GENERIC, (unused/1, lhs, rhs -- res)) { assert(0 <= oparg); assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); @@ -3378,7 +3458,7 @@ dummy_func( // Future families go below this point // family(call) = { - CALL, CALL_PY_EXACT_ARGS, + CALL, CALL_GENERIC, CALL_PY_EXACT_ARGS, CALL_PY_WITH_DEFAULTS, CALL_BOUND_METHOD_EXACT_ARGS, CALL_BUILTIN_CLASS, CALL_BUILTIN_FAST_WITH_KEYWORDS, CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, CALL_NO_KW_BUILTIN_FAST, CALL_NO_KW_BUILTIN_O, CALL_NO_KW_ISINSTANCE, CALL_NO_KW_LEN, @@ -3386,18 +3466,18 @@ family(call) = { CALL_NO_KW_METHOD_DESCRIPTOR_O, CALL_NO_KW_STR_1, CALL_NO_KW_TUPLE_1, CALL_NO_KW_TYPE_1 }; family(for_iter) = { - FOR_ITER, FOR_ITER_LIST, + FOR_ITER, FOR_ITER_GENERIC, FOR_ITER_LIST, FOR_ITER_RANGE }; family(load_attr) = { - LOAD_ATTR, LOAD_ATTR_CLASS, + LOAD_ATTR, LOAD_ATTR_GENERIC, LOAD_ATTR_CLASS, LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_MODULE, LOAD_ATTR_PROPERTY, LOAD_ATTR_SLOT, LOAD_ATTR_WITH_HINT, LOAD_ATTR_METHOD_LAZY_DICT, LOAD_ATTR_METHOD_NO_DICT, LOAD_ATTR_METHOD_WITH_VALUES }; family(load_global) = { - LOAD_GLOBAL, LOAD_GLOBAL_BUILTIN, + LOAD_GLOBAL, LODA_GLOBAL_GENERIC, LOAD_GLOBAL_BUILTIN, LOAD_GLOBAL_MODULE }; family(store_fast) = { STORE_FAST, STORE_FAST__LOAD_FAST, STORE_FAST__STORE_FAST }; family(unpack_sequence) = { - UNPACK_SEQUENCE, UNPACK_SEQUENCE_LIST, + UNPACK_SEQUENCE, UNPACK_SEQUENCE_GENERIC, UNPACK_SEQUENCE_LIST, UNPACK_SEQUENCE_TUPLE, UNPACK_SEQUENCE_TWO_TUPLE }; diff --git a/Python/ceval.c b/Python/ceval.c index aa6f0bbc293..7b7ca3a9cc0 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -753,7 +753,8 @@ GETITEM(PyObject *v, Py_ssize_t i) { /* The integer overflow is checked by an assertion below. */ #define INSTR_OFFSET() ((int)(next_instr - _PyCode_CODE(frame->f_code))) #define NEXTOPARG() do { \ - _Py_CODEUNIT word = *next_instr; \ + _Py_CODEUNIT word; \ + word.cache = _Py_atomic_load_uint16(&next_instr->cache); \ opcode = _Py_OPCODE(word); \ oparg = _Py_OPARG(word); \ } while (0) @@ -886,7 +887,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { /* This is only a single jump on release builds! */ \ UPDATE_MISS_STATS((INSTNAME)); \ assert(_PyOpcode_Deopt[opcode] == (INSTNAME)); \ - GO_TO_INSTRUCTION(INSTNAME); \ + GO_TO_INSTRUCTION(INSTNAME ## _GENERIC); \ } #define DEOPT_UNLOCK_IF(COND, INSTNAME) \ @@ -895,7 +896,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { UPDATE_MISS_STATS((INSTNAME)); \ assert(_PyOpcode_Deopt[opcode] == (INSTNAME)); \ _Py_critical_section_end(&_cs); \ - GO_TO_INSTRUCTION(INSTNAME); \ + GO_TO_INSTRUCTION(INSTNAME ## _GENERIC); \ } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 9639baebe6f..2ac596740d9 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -409,16 +409,26 @@ static_assert(INLINE_CACHE_ENTRIES_BINARY_SUBSCR == 4, "incorrect cache size"); PyObject *sub = PEEK(1); PyObject *container = PEEK(2); - PyObject *res; + _PyMutex_lock(&_PyRuntime.mutex); _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_BinarySubscr(container, sub, next_instr); + _PyMutex_unlock(&_PyRuntime.mutex); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_SUBSCR, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); + _PyMutex_unlock(&_PyRuntime.mutex); + GO_TO_INSTRUCTION(BINARY_SUBSCR_GENERIC); + } + + TARGET(BINARY_SUBSCR_GENERIC) { + PREDICTED(BINARY_SUBSCR_GENERIC); + PyObject *sub = PEEK(1); + PyObject *container = PEEK(2); + PyObject *res; res = PyObject_GetItem(container, sub); Py_DECREF(container); Py_DECREF(sub); @@ -595,17 +605,26 @@ PREDICTED(STORE_SUBSCR); PyObject *sub = PEEK(1); PyObject *container = PEEK(2); - PyObject *v = PEEK(3); - uint16_t counter = read_u16(&next_instr[0].cache); - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + _PyMutex_lock(&_PyRuntime.mutex); + _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_StoreSubscr(container, sub, next_instr); + _PyMutex_unlock(&_PyRuntime.mutex); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_SUBSCR, deferred); - _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr; DECREMENT_ADAPTIVE_COUNTER(cache->counter); + _PyMutex_unlock(&_PyRuntime.mutex); + GO_TO_INSTRUCTION(STORE_SUBSCR_GENERIC); + } + + TARGET(STORE_SUBSCR_GENERIC) { + PREDICTED(STORE_SUBSCR_GENERIC); + PyObject *sub = PEEK(1); + PyObject *container = PEEK(2); + PyObject *v = PEEK(3); /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); Py_DECREF(v); @@ -1091,16 +1110,24 @@ TARGET(UNPACK_SEQUENCE) { PREDICTED(UNPACK_SEQUENCE); + _PyMutex_lock(&_PyRuntime.mutex); _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); PyObject *seq = TOP(); next_instr--; _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + _PyMutex_unlock(&_PyRuntime.mutex); DISPATCH_SAME_OPARG(); } STAT_INC(UNPACK_SEQUENCE, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); + _PyMutex_unlock(&_PyRuntime.mutex); + GO_TO_INSTRUCTION(UNPACK_SEQUENCE_GENERIC); + } + + TARGET(UNPACK_SEQUENCE_GENERIC) { + PREDICTED(UNPACK_SEQUENCE_GENERIC); PyObject *seq = POP(); PyObject **top = stack_pointer + oparg; if (!unpack_iterable(tstate, seq, oparg, -1, top)) { @@ -1171,18 +1198,26 @@ TARGET(STORE_ATTR) { PREDICTED(STORE_ATTR); PyObject *owner = PEEK(1); - PyObject *v = PEEK(2); - uint16_t counter = read_u16(&next_instr[0].cache); - if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + _PyMutex_lock(&_PyRuntime.mutex); + _PyAttrCache *cache = (_PyAttrCache *)next_instr; + if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); PyObject *name = GETITEM(names, oparg); next_instr--; _Py_Specialize_StoreAttr(owner, next_instr, name); + _PyMutex_unlock(&_PyRuntime.mutex); DISPATCH_SAME_OPARG(); } STAT_INC(STORE_ATTR, deferred); - _PyAttrCache *cache = (_PyAttrCache *)next_instr; DECREMENT_ADAPTIVE_COUNTER(cache->counter); + _PyMutex_unlock(&_PyRuntime.mutex); + GO_TO_INSTRUCTION(STORE_ATTR_GENERIC); + } + + TARGET(STORE_ATTR_GENERIC) { + PREDICTED(STORE_ATTR_GENERIC); + PyObject *owner = PEEK(1); + PyObject *v = PEEK(2); PyObject *name = GETITEM(names, oparg); int err = PyObject_SetAttr(owner, name, v); Py_DECREF(v); @@ -1295,16 +1330,24 @@ TARGET(LOAD_GLOBAL) { PREDICTED(LOAD_GLOBAL); + _PyMutex_lock(&_PyRuntime.mutex); _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); PyObject *name = GETITEM(names, oparg>>1); next_instr--; _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); + _PyMutex_unlock(&_PyRuntime.mutex); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_GLOBAL, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); + _PyMutex_unlock(&_PyRuntime.mutex); + GO_TO_INSTRUCTION(LOAD_GLOBAL_GENERIC); + } + + TARGET(LOAD_GLOBAL_GENERIC) { + PREDICTED(LOAD_GLOBAL_GENERIC); int push_null = oparg & 1; PEEK(0) = NULL; PyObject *name = GETITEM(names, oparg>>1); @@ -1721,6 +1764,7 @@ TARGET(LOAD_ATTR) { PREDICTED(LOAD_ATTR); + _PyMutex_lock(&_PyRuntime.mutex); _PyAttrCache *cache = (_PyAttrCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); @@ -1728,10 +1772,17 @@ PyObject *name = GETITEM(names, oparg>>1); next_instr--; _Py_Specialize_LoadAttr(owner, next_instr, name); + _PyMutex_unlock(&_PyRuntime.mutex); DISPATCH_SAME_OPARG(); } STAT_INC(LOAD_ATTR, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); + _PyMutex_unlock(&_PyRuntime.mutex); + GO_TO_INSTRUCTION(LOAD_ATTR_GENERIC); + } + + TARGET(LOAD_ATTR_GENERIC) { + PREDICTED(LOAD_ATTR_GENERIC); PyObject *name = GETITEM(names, oparg >> 1); PyObject *owner = TOP(); if (oparg & 1) { @@ -2076,16 +2127,26 @@ PREDICTED(COMPARE_OP); PyObject *right = PEEK(1); PyObject *left = PEEK(2); - PyObject *res; + _PyMutex_lock(&_PyRuntime.mutex); _PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_CompareOp(left, right, next_instr, oparg); + _PyMutex_unlock(&_PyRuntime.mutex); DISPATCH_SAME_OPARG(); } STAT_INC(COMPARE_OP, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); + _PyMutex_unlock(&_PyRuntime.mutex); + GO_TO_INSTRUCTION(COMPARE_OP_GENERIC); + } + + TARGET(COMPARE_OP_GENERIC) { + PREDICTED(COMPARE_OP_GENERIC); + PyObject *right = PEEK(1); + PyObject *left = PEEK(2); + PyObject *res; assert(oparg <= Py_GE); res = PyObject_RichCompare(left, right, oparg); Py_DECREF(left); @@ -2574,15 +2635,23 @@ TARGET(FOR_ITER) { PREDICTED(FOR_ITER); + _PyMutex_lock(&_PyRuntime.mutex); _PyForIterCache *cache = (_PyForIterCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_ForIter(TOP(), next_instr, oparg); + _PyMutex_unlock(&_PyRuntime.mutex); DISPATCH_SAME_OPARG(); } STAT_INC(FOR_ITER, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); + _PyMutex_unlock(&_PyRuntime.mutex); + GO_TO_INSTRUCTION(FOR_ITER_GENERIC); + } + + TARGET(FOR_ITER_GENERIC) { + PREDICTED(FOR_ITER_GENERIC); /* before: [iter]; after: [iter, iter()] *or* [] */ PyObject *iter = TOP(); PyObject *next = (*Py_TYPE(iter)->tp_iternext)(iter); @@ -2907,6 +2976,7 @@ TARGET(CALL) { PREDICTED(CALL); + _PyMutex_lock(&_PyRuntime.mutex); _PyCallCache *cache = (_PyCallCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); @@ -2915,10 +2985,17 @@ PyObject *callable = PEEK(nargs + 1); next_instr--; _Py_Specialize_Call(callable, next_instr, nargs, kwnames); + _PyMutex_unlock(&_PyRuntime.mutex); DISPATCH_SAME_OPARG(); } STAT_INC(CALL, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); + _PyMutex_unlock(&_PyRuntime.mutex); + GO_TO_INSTRUCTION(CALL_GENERIC); + } + + TARGET(CALL_GENERIC) { + PREDICTED(CALL_GENERIC); int total_args, is_meth; is_meth = is_method(stack_pointer, oparg); PyObject *function = PEEK(oparg + 1); @@ -3660,16 +3737,26 @@ static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 1, "incorrect cache size"); PyObject *rhs = PEEK(1); PyObject *lhs = PEEK(2); - PyObject *res; + _PyMutex_lock(&_PyRuntime.mutex); _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { assert(cframe.use_tracing == 0); next_instr--; _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0)); + _PyMutex_unlock(&_PyRuntime.mutex); DISPATCH_SAME_OPARG(); } STAT_INC(BINARY_OP, deferred); DECREMENT_ADAPTIVE_COUNTER(cache->counter); + _PyMutex_unlock(&_PyRuntime.mutex); + GO_TO_INSTRUCTION(BINARY_OP_GENERIC); + } + + TARGET(BINARY_OP_GENERIC) { + PREDICTED(BINARY_OP_GENERIC); + PyObject *rhs = PEEK(1); + PyObject *lhs = PEEK(2); + PyObject *res; assert(0 <= oparg); assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 34f8145828c..39f09bea674 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -39,6 +39,7 @@ static const struct { [BINARY_OP_ADD_FLOAT] = { 2, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC }, [BINARY_OP_ADD_INT] = { 2, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC }, [BINARY_SUBSCR] = { 2, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, + [BINARY_SUBSCR_GENERIC] = { 2, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [BINARY_SLICE] = { 3, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [STORE_SLICE] = { 4, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [BINARY_SUBSCR_LIST_INT] = { 2, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, @@ -48,6 +49,7 @@ static const struct { [LIST_APPEND] = { 1, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [SET_ADD] = { 1, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [STORE_SUBSCR] = { 3, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC }, + [STORE_SUBSCR_GENERIC] = { 3, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC }, [STORE_SUBSCR_LIST_INT] = { 3, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC }, [STORE_SUBSCR_DICT] = { 3, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC }, [DELETE_SUBSCR] = { 2, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, @@ -70,16 +72,19 @@ static const struct { [STORE_NAME] = { 1, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [DELETE_NAME] = { 0, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [UNPACK_SEQUENCE] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, + [UNPACK_SEQUENCE_GENERIC] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [UNPACK_SEQUENCE_TWO_TUPLE] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [UNPACK_SEQUENCE_TUPLE] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [UNPACK_SEQUENCE_LIST] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [UNPACK_EX] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [STORE_ATTR] = { 2, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, + [STORE_ATTR_GENERIC] = { 2, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [DELETE_ATTR] = { 1, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [STORE_GLOBAL] = { 1, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [DELETE_GLOBAL] = { 0, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [LOAD_NAME] = { 0, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [LOAD_GLOBAL] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, + [LOAD_GLOBAL_GENERIC] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [LOAD_GLOBAL_MODULE] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [LOAD_GLOBAL_BUILTIN] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [DELETE_FAST] = { 0, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, @@ -102,6 +107,7 @@ static const struct { [DICT_MERGE] = { 1, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [MAP_ADD] = { 2, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [LOAD_ATTR] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, + [LOAD_ATTR_GENERIC] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [LOAD_ATTR_INSTANCE_VALUE] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [LOAD_ATTR_MODULE] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [LOAD_ATTR_WITH_HINT] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, @@ -113,6 +119,7 @@ static const struct { [STORE_ATTR_WITH_HINT] = { 2, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [STORE_ATTR_SLOT] = { 2, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC000 }, [COMPARE_OP] = { 2, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC0 }, + [COMPARE_OP_GENERIC] = { 2, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC0 }, [COMPARE_OP_FLOAT_JUMP] = { 3, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC0IB }, [COMPARE_OP_INT_JUMP] = { 3, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC0IB }, [COMPARE_OP_STR_JUMP] = { 3, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC0IB }, @@ -139,6 +146,7 @@ static const struct { [GET_ITER] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [GET_YIELD_FROM_ITER] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [FOR_ITER] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, + [FOR_ITER_GENERIC] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [FOR_ITER_LIST] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [FOR_ITER_TUPLE] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [FOR_ITER_RANGE] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, @@ -153,6 +161,7 @@ static const struct { [CALL_BOUND_METHOD_EXACT_ARGS] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [KW_NAMES] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, + [CALL_GENERIC] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_PY_EXACT_ARGS] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_PY_WITH_DEFAULTS] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CALL_NO_KW_TYPE_1] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, @@ -176,6 +185,7 @@ static const struct { [FORMAT_VALUE] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [COPY] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [BINARY_OP] = { 2, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC }, + [BINARY_OP_GENERIC] = { 2, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IBC }, [SWAP] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [EXTENDED_ARG] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [CACHE] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 83b3af7c0b1..3c61faddbcf 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -4,39 +4,42 @@ static void *opcode_targets[256] = { &&TARGET_PUSH_NULL, &&TARGET_INTERPRETER_EXIT, &&TARGET_END_FOR, + &&TARGET_BINARY_OP_GENERIC, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, &&TARGET_BINARY_OP_ADD_UNICODE, - &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, &&TARGET_NOP, - &&TARGET_BINARY_OP_MULTIPLY_FLOAT, + &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, &&TARGET_UNARY_NEGATIVE, &&TARGET_UNARY_NOT, + &&TARGET_BINARY_OP_MULTIPLY_FLOAT, &&TARGET_BINARY_OP_MULTIPLY_INT, - &&TARGET_BINARY_OP_SUBTRACT_FLOAT, &&TARGET_UNARY_INVERT, + &&TARGET_BINARY_OP_SUBTRACT_FLOAT, &&TARGET_BINARY_OP_SUBTRACT_INT, + &&TARGET_BINARY_SUBSCR_GENERIC, &&TARGET_BINARY_SUBSCR_DICT, &&TARGET_BINARY_SUBSCR_GETITEM, &&TARGET_BINARY_SUBSCR_LIST_INT, &&TARGET_BINARY_SUBSCR_TUPLE_INT, + &&TARGET_CALL_GENERIC, &&TARGET_CALL_PY_EXACT_ARGS, - &&TARGET_CALL_PY_WITH_DEFAULTS, - &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, - &&TARGET_CALL_BUILTIN_CLASS, &&TARGET_BINARY_SUBSCR, &&TARGET_BINARY_SLICE, &&TARGET_STORE_SLICE, - &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, - &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + &&TARGET_CALL_PY_WITH_DEFAULTS, + &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, &&TARGET_GET_LEN, &&TARGET_MATCH_MAPPING, &&TARGET_MATCH_SEQUENCE, &&TARGET_MATCH_KEYS, - &&TARGET_CALL_NO_KW_BUILTIN_FAST, + &&TARGET_CALL_BUILTIN_CLASS, &&TARGET_PUSH_EXC_INFO, &&TARGET_CHECK_EXC_MATCH, &&TARGET_CHECK_EG_MATCH, + &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, + &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + &&TARGET_CALL_NO_KW_BUILTIN_FAST, &&TARGET_CALL_NO_KW_BUILTIN_O, &&TARGET_CALL_NO_KW_ISINSTANCE, &&TARGET_CALL_NO_KW_LEN, @@ -45,9 +48,6 @@ static void *opcode_targets[256] = { &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_O, &&TARGET_CALL_NO_KW_STR_1, - &&TARGET_CALL_NO_KW_TUPLE_1, - &&TARGET_CALL_NO_KW_TYPE_1, - &&TARGET_COMPARE_OP_FLOAT_JUMP, &&TARGET_WITH_EXCEPT_START, &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, @@ -55,38 +55,38 @@ static void *opcode_targets[256] = { &&TARGET_BEFORE_WITH, &&TARGET_END_ASYNC_FOR, &&TARGET_CLEANUP_THROW, + &&TARGET_CALL_NO_KW_TUPLE_1, + &&TARGET_CALL_NO_KW_TYPE_1, + &&TARGET_COMPARE_OP_GENERIC, + &&TARGET_COMPARE_OP_FLOAT_JUMP, + &&TARGET_STORE_SUBSCR, + &&TARGET_DELETE_SUBSCR, &&TARGET_COMPARE_OP_INT_JUMP, &&TARGET_COMPARE_OP_STR_JUMP, + &&TARGET_FOR_ITER_GENERIC, &&TARGET_FOR_ITER_LIST, &&TARGET_FOR_ITER_TUPLE, - &&TARGET_STORE_SUBSCR, - &&TARGET_DELETE_SUBSCR, &&TARGET_FOR_ITER_RANGE, + &&TARGET_GET_ITER, + &&TARGET_GET_YIELD_FROM_ITER, &&TARGET_FOR_ITER_GEN, + &&TARGET_LOAD_BUILD_CLASS, + &&TARGET_LOAD_ATTR_GENERIC, &&TARGET_LOAD_ATTR_CLASS, + &&TARGET_LOAD_ASSERTION_ERROR, + &&TARGET_RETURN_GENERATOR, &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, &&TARGET_LOAD_ATTR_INSTANCE_VALUE, &&TARGET_LOAD_ATTR_MODULE, - &&TARGET_GET_ITER, - &&TARGET_GET_YIELD_FROM_ITER, &&TARGET_LOAD_ATTR_PROPERTY, - &&TARGET_LOAD_BUILD_CLASS, &&TARGET_LOAD_ATTR_SLOT, &&TARGET_LOAD_ATTR_WITH_HINT, - &&TARGET_LOAD_ASSERTION_ERROR, - &&TARGET_RETURN_GENERATOR, &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, + &&TARGET_RETURN_VALUE, &&TARGET_LOAD_ATTR_METHOD_NO_DICT, + &&TARGET_SETUP_ANNOTATIONS, &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, &&TARGET_LOAD_CONST__LOAD_FAST, - &&TARGET_LOAD_FAST__LOAD_CONST, - &&TARGET_LOAD_FAST__LOAD_FAST, - &&TARGET_LOAD_GLOBAL_BUILTIN, - &&TARGET_RETURN_VALUE, - &&TARGET_LOAD_GLOBAL_MODULE, - &&TARGET_SETUP_ANNOTATIONS, - &&TARGET_STORE_ATTR_INSTANCE_VALUE, - &&TARGET_STORE_ATTR_SLOT, &&TARGET_PREP_RERAISE_STAR, &&TARGET_POP_EXCEPT, &&TARGET_STORE_NAME, @@ -112,7 +112,7 @@ static void *opcode_targets[256] = { &&TARGET_JUMP_FORWARD, &&TARGET_JUMP_IF_FALSE_OR_POP, &&TARGET_JUMP_IF_TRUE_OR_POP, - &&TARGET_STORE_ATTR_WITH_HINT, + &&TARGET_LOAD_FAST__LOAD_CONST, &&TARGET_POP_JUMP_IF_FALSE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_LOAD_GLOBAL, @@ -120,7 +120,7 @@ static void *opcode_targets[256] = { &&TARGET_CONTAINS_OP, &&TARGET_RERAISE, &&TARGET_COPY, - &&TARGET_STORE_FAST__LOAD_FAST, + &&TARGET_LOAD_FAST__LOAD_FAST, &&TARGET_BINARY_OP, &&TARGET_SEND, &&TARGET_LOAD_FAST, @@ -140,9 +140,9 @@ static void *opcode_targets[256] = { &&TARGET_STORE_DEREF, &&TARGET_DELETE_DEREF, &&TARGET_JUMP_BACKWARD, - &&TARGET_STORE_FAST__STORE_FAST, + &&TARGET_LOAD_GLOBAL_GENERIC, &&TARGET_CALL_FUNCTION_EX, - &&TARGET_STORE_SUBSCR_DICT, + &&TARGET_LOAD_GLOBAL_BUILTIN, &&TARGET_EXTENDED_ARG, &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, @@ -152,30 +152,30 @@ static void *opcode_targets[256] = { &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, - &&TARGET_STORE_SUBSCR_LIST_INT, - &&TARGET_UNPACK_SEQUENCE_LIST, + &&TARGET_LOAD_GLOBAL_MODULE, + &&TARGET_STORE_ATTR_GENERIC, &&TARGET_FORMAT_VALUE, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, - &&TARGET_UNPACK_SEQUENCE_TUPLE, - &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, - &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_STORE_ATTR_INSTANCE_VALUE, + &&TARGET_STORE_ATTR_SLOT, + &&TARGET_STORE_ATTR_WITH_HINT, + &&TARGET_STORE_FAST__LOAD_FAST, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, &&TARGET_DICT_UPDATE, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_STORE_FAST__STORE_FAST, + &&TARGET_STORE_SUBSCR_GENERIC, + &&TARGET_STORE_SUBSCR_DICT, + &&TARGET_STORE_SUBSCR_LIST_INT, + &&TARGET_UNPACK_SEQUENCE_GENERIC, &&TARGET_CALL, &&TARGET_KW_NAMES, &&TARGET_CALL_INTRINSIC_1, - &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_UNPACK_SEQUENCE_LIST, + &&TARGET_UNPACK_SEQUENCE_TUPLE, + &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 5d7eb289a09..264b2c0f420 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -846,6 +846,9 @@ pycore_init_builtins(PyThreadState *tstate) PyObject *object__getattribute__ = _PyType_Lookup(&PyBaseObject_Type, &_Py_ID(__getattribute__)); assert(object__getattribute__); interp->callable_cache.object__getattribute__ = object__getattribute__; + PyObject *object__class__ = _PyType_Lookup(&PyBaseObject_Type, &_Py_ID(__class__)); + assert(object__class__); + interp->callable_cache.object__class__ = object__class__; interp->interpreter_trampoline = _Py_MakeShimCode(&INTERPRETER_TRAMPOLINE_CODEDEF); if (interp->interpreter_trampoline == NULL) { return _PyStatus_ERR("failed to create interpreter trampoline."); diff --git a/Python/specialize.c b/Python/specialize.c index 48814dad671..7b38aff378e 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -271,6 +271,13 @@ _PyCode_Quicken(PyCodeObject *code) for (int i = 0; i < Py_SIZE(code); i++) { int opcode = _PyOpcode_Deopt[_Py_OPCODE(instructions[i])]; int caches = _PyOpcode_Caches[opcode]; + // if (opcode == LOAD_ATTR) { + // instructions[i].opcode = LOAD_ATTR_PROFILE; + // memset(&instructions[i + 1], 0, caches * sizeof(_Py_CODEUNIT)); + // previous_opcode = 0; + // i += caches; + // continue; + // } if (caches) { instructions[i + 1].cache = adaptive_counter_warmup(); previous_opcode = 0; @@ -311,6 +318,7 @@ _PyCode_Quicken(PyCodeObject *code) #define SPEC_FAIL_WRONG_NUMBER_ARGUMENTS 6 #define SPEC_FAIL_CODE_COMPLEX_PARAMETERS 7 #define SPEC_FAIL_CODE_NOT_OPTIMIZED 8 +#define SPEC_FAIL_NOT_CACHED 34 #define SPEC_FAIL_LOAD_GLOBAL_NON_DICT 17 @@ -516,7 +524,8 @@ typedef enum { ABSENT, /* Attribute is not present on the class */ DUNDER_CLASS, /* __class__ attribute */ GETSET_OVERRIDDEN, /* __getattribute__ or __setattr__ has been overridden */ - GETATTRIBUTE_IS_PYTHON_FUNCTION /* Descriptor requires calling a Python __getattribute__ */ + GETATTRIBUTE_IS_PYTHON_FUNCTION, /* Descriptor requires calling a Python __getattribute__ */ + NOT_CACHED, /* Attribute is not cached */ } DescriptorClassification; @@ -540,15 +549,19 @@ analyze_descriptor(PyTypeObject *type, PyObject *name, PyObject **descr, int sto getattro_slot == _Py_slot_tp_getattro) { /* One or both of __getattribute__ or __getattr__ may have been overridden See typeobject.c for why these functions are special. */ - PyObject *getattribute = _PyType_Lookup(type, - &_Py_ID(__getattribute__)); + _Py_mro_cache_result r = _Py_mro_cache_lookup(&type->tp_mro_cache, + &_Py_ID(__getattribute__)); + if (!r.hit) { + *descr = NULL; + return NOT_CACHED; + } + PyObject *getattribute = r.value; PyInterpreterState *interp = _PyInterpreterState_GET(); bool has_custom_getattribute = getattribute != NULL && getattribute != interp->callable_cache.object__getattribute__; - has_getattr = _PyType_Lookup(type, &_Py_ID(__getattr__)) != NULL; if (has_custom_getattribute) { + /* getattro_slot == _Py_slot_tp_getattro implies no __getattr__ */ if (getattro_slot == _Py_slot_tp_getattro && - !has_getattr && Py_IS_TYPE(getattribute, &PyFunction_Type)) { *descr = getattribute; return GETATTRIBUTE_IS_PYTHON_FUNCTION; @@ -558,6 +571,14 @@ analyze_descriptor(PyTypeObject *type, PyObject *name, PyObject **descr, int sto *descr = NULL; return GETSET_OVERRIDDEN; } + if (getattro_slot == _Py_slot_tp_getattro) { + r = _Py_mro_cache_lookup(&type->tp_mro_cache, &_Py_ID(__getattr__)); + if (!r.hit) { + *descr = NULL; + return NOT_CACHED; + } + has_getattr = r.value != NULL; + } /* Potentially has __getattr__ but no custom __getattribute__. Fall through to usual descriptor analysis. Usual attribute lookup should only be allowed at runtime @@ -571,8 +592,12 @@ analyze_descriptor(PyTypeObject *type, PyObject *name, PyObject **descr, int sto return GETSET_OVERRIDDEN; } } - PyObject *descriptor = _PyType_Lookup(type, name); + _Py_mro_cache_result r = _Py_mro_cache_lookup(&type->tp_mro_cache, name); + PyObject *descriptor = r.value; *descr = descriptor; + if (!r.hit) { + return NOT_CACHED; + } if (descriptor == NULL) { return ABSENT; } @@ -595,8 +620,9 @@ analyze_descriptor(PyTypeObject *type, PyObject *name, PyObject **descr, int sto __getattr__. */ return has_getattr ? GETSET_OVERRIDDEN : PROPERTY; } - if (PyUnicode_CompareWithASCIIString(name, "__class__") == 0) { - if (descriptor == _PyType_Lookup(&PyBaseObject_Type, name)) { + if (name == &_Py_ID(__class__)) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (descriptor == interp->callable_cache.object__class__) { return DUNDER_CLASS; } } @@ -633,7 +659,7 @@ specialize_dict_access( return 0; } _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + PyDictOrValues dorv = _PyObject_DictOrValues(owner); if (_PyDictOrValues_IsValues(dorv)) { // Virtual dictionary PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys; @@ -658,8 +684,12 @@ specialize_dict_access( return 0; } // We found an instance with a __dict__. - Py_ssize_t index = - _PyDict_LookupIndex(dict, name); + PyDictKeysObject *keys = _Py_atomic_load_ptr(&dict->ma_keys); + if (!DK_IS_UNICODE(keys)) { + SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT); + return 0; + } + Py_ssize_t index = _PyDictKeys_StringLookup(keys, name); if (index != (uint16_t)index) { SPECIALIZATION_FAIL(base_op, index == DKIX_EMPTY ? @@ -668,6 +698,7 @@ specialize_dict_access( return 0; } cache->index = (uint16_t)index; + assert(type->tp_version_tag != 0); write_u32(cache->version, type->tp_version_tag); _py_set_opcode(instr, hint_op); } @@ -681,6 +712,11 @@ static int specialize_class_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyOb void _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) { + if (instr->opcode != LOAD_ATTR) { + // another thread concurrently specialized this instruction + STAT_INC(LOAD_ATTR, failure); + return; + } assert(_PyOpcode_Caches[LOAD_ATTR] == INLINE_CACHE_ENTRIES_LOAD_ATTR); _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); PyTypeObject *type = Py_TYPE(owner); @@ -704,9 +740,13 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) } goto success; } + if (type->tp_version_tag == 0) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER); + goto fail; + } PyObject *descr = NULL; DescriptorClassification kind = analyze_descriptor(type, name, &descr, 0); - assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN); + assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN || kind == NOT_CACHED); switch(kind) { case OVERRIDING: SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR); @@ -794,6 +834,9 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) case GETSET_OVERRIDDEN: SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OVERRIDDEN); goto fail; + case NOT_CACHED: + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_NOT_CACHED); + goto fail; case GETATTRIBUTE_IS_PYTHON_FUNCTION: { assert(type->tp_getattro == _Py_slot_tp_getattro); @@ -841,7 +884,7 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) fail: STAT_INC(LOAD_ATTR, failure); assert(!PyErr_Occurred()); - _py_set_opcode(instr, LOAD_ATTR); + _py_set_opcode(instr, LOAD_ATTR_GENERIC); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -853,6 +896,11 @@ _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) void _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) { + if (instr->opcode != STORE_ATTR) { + // another thread concurrently specialized this instruction + STAT_INC(STORE_ATTR, failure); + return; + } assert(_PyOpcode_Caches[STORE_ATTR] == INLINE_CACHE_ENTRIES_STORE_ATTR); _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); PyTypeObject *type = Py_TYPE(owner); @@ -867,6 +915,10 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OVERRIDDEN); goto fail; } + if (type->tp_version_tag == 0) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OTHER); + goto fail; + } PyObject *descr; DescriptorClassification kind = analyze_descriptor(type, name, &descr, 1); switch(kind) { @@ -914,6 +966,9 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) case GETSET_OVERRIDDEN: SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OVERRIDDEN); goto fail; + case NOT_CACHED: + SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_NOT_CACHED); + goto fail; case BUILTIN_CLASSMETHOD: SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_BUILTIN_CLASS_METHOD_OBJ); goto fail; @@ -936,7 +991,7 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) fail: STAT_INC(STORE_ATTR, failure); assert(!PyErr_Occurred()); - _py_set_opcode(instr, STORE_ATTR); + _py_set_opcode(instr, STORE_ATTR_GENERIC); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -988,7 +1043,16 @@ specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name) { _PyLoadMethodCache *cache = (_PyLoadMethodCache *)(instr + 1); - if (!PyType_CheckExact(owner) || _PyType_Lookup(Py_TYPE(owner), name)) { + if (!PyType_CheckExact(owner)) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METACLASS_ATTRIBUTE); + return -1; + } + _Py_mro_cache_result r = _Py_mro_cache_lookup(&Py_TYPE(owner)->tp_mro_cache, name); + if (!r.hit) { + SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_NOT_CACHED); + return -1; + } + if (r.value != NULL) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METACLASS_ATTRIBUTE); return -1; } @@ -1035,7 +1099,7 @@ PyObject *descr, DescriptorClassification kind) ObjectDictKind dictkind; PyDictKeysObject *keys; if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) { - PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); + PyDictOrValues dorv = _PyObject_DictOrValues(owner); keys = ((PyHeapTypeObject *)owner_cls)->ht_cached_keys; if (_PyDictOrValues_IsValues(dorv)) { dictkind = MANAGED_VALUES; @@ -1080,12 +1144,13 @@ PyObject *descr, DescriptorClassification kind) } write_u32(cache->keys_version, keys_version); } + int opcode = -1; switch(dictkind) { case NO_DICT: - _py_set_opcode(instr, LOAD_ATTR_METHOD_NO_DICT); + opcode = LOAD_ATTR_METHOD_NO_DICT; break; case MANAGED_VALUES: - _py_set_opcode(instr, LOAD_ATTR_METHOD_WITH_VALUES); + opcode = LOAD_ATTR_METHOD_WITH_VALUES; break; case MANAGED_DICT: SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_HAS_MANAGED_DICT); @@ -1095,7 +1160,7 @@ PyObject *descr, DescriptorClassification kind) goto fail; case LAZY_DICT: assert(owner_cls->tp_dictoffset > 0 && owner_cls->tp_dictoffset <= INT16_MAX); - _py_set_opcode(instr, LOAD_ATTR_METHOD_LAZY_DICT); + opcode =LOAD_ATTR_METHOD_LAZY_DICT; break; } /* `descr` is borrowed. This is safe for methods (even inherited ones from @@ -1112,8 +1177,10 @@ PyObject *descr, DescriptorClassification kind) * PyType_Modified usages in typeobject.c). The MCACHE has been * working since Python 2.6 and it's battle-tested. */ + assert(opcode != -1); write_u32(cache->type_version, owner_cls->tp_version_tag); write_obj(cache->descr, descr); + _py_set_opcode(instr, opcode); return 1; fail: return 0; @@ -1124,6 +1191,11 @@ _Py_Specialize_LoadGlobal( PyObject *globals, PyObject *builtins, _Py_CODEUNIT *instr, PyObject *name) { + if (instr->opcode != LOAD_GLOBAL) { + // another thread concurrently specialized this instruction + STAT_INC(LOAD_GLOBAL, failure); + return; + } assert(_PyOpcode_Caches[LOAD_GLOBAL] == INLINE_CACHE_ENTRIES_LOAD_GLOBAL); /* Use inline cache */ _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)(instr + 1); @@ -1197,7 +1269,7 @@ _Py_Specialize_LoadGlobal( fail: STAT_INC(LOAD_GLOBAL, failure); assert(!PyErr_Occurred()); - _py_set_opcode(instr, LOAD_GLOBAL); + _py_set_opcode(instr, LOAD_GLOBAL_GENERIC); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -1295,6 +1367,11 @@ void _Py_Specialize_BinarySubscr( PyObject *container, PyObject *sub, _Py_CODEUNIT *instr) { + if (instr->opcode != BINARY_SUBSCR) { + // another thread concurrently specialized this instruction + STAT_INC(BINARY_SUBSCR, failure); + return; + } assert(_PyOpcode_Caches[BINARY_SUBSCR] == INLINE_CACHE_ENTRIES_BINARY_SUBSCR); _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)(instr + 1); @@ -1330,7 +1407,12 @@ _Py_Specialize_BinarySubscr( goto success; } PyTypeObject *cls = Py_TYPE(container); - PyObject *descriptor = _PyType_Lookup(cls, &_Py_ID(__getitem__)); + _Py_mro_cache_result r = _Py_mro_cache_lookup(&cls->tp_mro_cache, &_Py_ID(__getitem__)); + if (!r.hit) { + SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_NOT_CACHED); + goto fail; + } + PyObject *descriptor = r.value; if (descriptor && Py_TYPE(descriptor) == &PyFunction_Type) { if (!(container_type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_SUBSCR_NOT_HEAP_TYPE); @@ -1365,7 +1447,7 @@ _Py_Specialize_BinarySubscr( fail: STAT_INC(BINARY_SUBSCR, failure); assert(!PyErr_Occurred()); - _py_set_opcode(instr, BINARY_SUBSCR); + _py_set_opcode(instr, BINARY_SUBSCR_GENERIC); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -1377,6 +1459,11 @@ _Py_Specialize_BinarySubscr( void _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *instr) { + if (instr->opcode != STORE_SUBSCR) { + // another thread concurrently specialized this instruction + STAT_INC(STORE_SUBSCR, failure); + return; + } _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)(instr + 1); PyTypeObject *container_type = Py_TYPE(container); if (container_type == &PyList_Type) { @@ -1451,7 +1538,13 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins } goto fail; } - PyObject *descriptor = _PyType_Lookup(container_type, &_Py_ID(__setitem__)); + _Py_mro_cache_result r; + r = _Py_mro_cache_lookup(&container_type->tp_mro_cache, &_Py_ID(__setitem__)); + if (!r.hit) { + SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_NOT_CACHED); + goto fail; + } + PyObject *descriptor = r.value; if (descriptor && Py_TYPE(descriptor) == &PyFunction_Type) { PyFunctionObject *func = (PyFunctionObject *)descriptor; PyCodeObject *code = (PyCodeObject *)func->func_code; @@ -1469,7 +1562,7 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins fail: STAT_INC(STORE_SUBSCR, failure); assert(!PyErr_Occurred()); - _py_set_opcode(instr, STORE_SUBSCR); + _py_set_opcode(instr, STORE_SUBSCR_GENERIC); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -1756,6 +1849,11 @@ void _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, PyObject *kwnames) { + if (instr->opcode != CALL) { + // another thread concurrently specialized this instruction + STAT_INC(CALL, failure); + return; + } assert(_PyOpcode_Caches[CALL] == INLINE_CACHE_ENTRIES_CALL); _PyCallCache *cache = (_PyCallCache *)(instr + 1); int fail; @@ -1790,7 +1888,7 @@ _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs, if (fail) { STAT_INC(CALL, failure); assert(!PyErr_Occurred()); - _py_set_opcode(instr, CALL); + _py_set_opcode(instr, CALL_GENERIC); cache->counter = adaptive_counter_backoff(cache->counter); } else { @@ -1874,6 +1972,11 @@ void _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, int oparg, PyObject **locals) { + if (instr->opcode != BINARY_OP) { + // another thread concurrently specialized this instruction + STAT_INC(BINARY_OP, failure); + return; + } assert(_PyOpcode_Caches[BINARY_OP] == INLINE_CACHE_ENTRIES_BINARY_OP); _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(instr + 1); switch (oparg) { @@ -1933,7 +2036,7 @@ _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, } SPECIALIZATION_FAIL(BINARY_OP, binary_op_fail_kind(oparg, lhs, rhs)); STAT_INC(BINARY_OP, failure); - _py_set_opcode(instr, BINARY_OP); + _py_set_opcode(instr, BINARY_OP_GENERIC); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -1995,6 +2098,11 @@ void _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, int oparg) { + if (instr->opcode != COMPARE_OP) { + // another thread concurrently specialized this instruction + STAT_INC(COMPARE_OP, failure); + return; + } assert(_PyOpcode_Caches[COMPARE_OP] == INLINE_CACHE_ENTRIES_COMPARE_OP); _PyCompareOpCache *cache = (_PyCompareOpCache *)(instr + 1); int next_opcode = _Py_OPCODE(instr[INLINE_CACHE_ENTRIES_COMPARE_OP + 1]); @@ -2016,14 +2124,14 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, goto failure; } if (PyFloat_CheckExact(lhs)) { - _py_set_opcode(instr, COMPARE_OP_FLOAT_JUMP); cache->mask = when_to_jump_mask; + _py_set_opcode(instr, COMPARE_OP_FLOAT_JUMP); goto success; } if (PyLong_CheckExact(lhs)) { if (Py_ABS(Py_SIZE(lhs)) <= 1 && Py_ABS(Py_SIZE(rhs)) <= 1) { - _py_set_opcode(instr, COMPARE_OP_INT_JUMP); cache->mask = when_to_jump_mask; + _py_set_opcode(instr, COMPARE_OP_INT_JUMP); goto success; } else { @@ -2037,15 +2145,15 @@ _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, goto failure; } else { - _py_set_opcode(instr, COMPARE_OP_STR_JUMP); cache->mask = (when_to_jump_mask & 8) == 0; + _py_set_opcode(instr, COMPARE_OP_STR_JUMP); goto success; } } SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs)); failure: STAT_INC(COMPARE_OP, failure); - _py_set_opcode(instr, COMPARE_OP); + _py_set_opcode(instr, COMPARE_OP_GENERIC); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -2070,6 +2178,11 @@ unpack_sequence_fail_kind(PyObject *seq) void _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg) { + if (instr->opcode != UNPACK_SEQUENCE) { + // another thread concurrently specialized this instruction + STAT_INC(UNPACK_SEQUENCE, failure); + return; + } assert(_PyOpcode_Caches[UNPACK_SEQUENCE] == INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE); _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)(instr + 1); @@ -2096,7 +2209,7 @@ _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg) SPECIALIZATION_FAIL(UNPACK_SEQUENCE, unpack_sequence_fail_kind(seq)); failure: STAT_INC(UNPACK_SEQUENCE, failure); - _py_set_opcode(instr, UNPACK_SEQUENCE); + _py_set_opcode(instr, UNPACK_SEQUENCE_GENERIC); cache->counter = adaptive_counter_backoff(cache->counter); return; success: @@ -2179,6 +2292,11 @@ int void _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg) { + if (instr->opcode != FOR_ITER) { + // another thread concurrently specialized this instruction + STAT_INC(FOR_ITER, failure); + return; + } assert(_PyOpcode_Caches[FOR_ITER] == INLINE_CACHE_ENTRIES_FOR_ITER); _PyForIterCache *cache = (_PyForIterCache *)(instr + 1); PyTypeObject *tp = Py_TYPE(iter); @@ -2204,7 +2322,7 @@ _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg) SPECIALIZATION_FAIL(FOR_ITER, _PySpecialization_ClassifyIterator(iter)); STAT_INC(FOR_ITER, failure); - _py_set_opcode(instr, FOR_ITER); + _py_set_opcode(instr, FOR_ITER_GENERIC); cache->counter = adaptive_counter_backoff(cache->counter); return; success: