From 0630b4ebae3b81eec4484b6bc22ea7756cedd8c2 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sat, 6 May 2023 12:43:44 +0100 Subject: [PATCH 1/4] move compiler_enter_scope out of compiler_codegen --- Python/compile.c | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/Python/compile.c b/Python/compile.c index cbe5403aafbc87..968afbc0d29798 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1655,16 +1655,10 @@ compiler_body(struct compiler *c, location loc, asdl_stmt_seq *stmts) static int compiler_codegen(struct compiler *c, mod_ty mod) { - _Py_DECLARE_STR(anon_module, ""); - RETURN_IF_ERROR( - compiler_enter_scope(c, &_Py_STR(anon_module), COMPILER_SCOPE_MODULE, - mod, 1)); - location loc = LOCATION(1, 1, 0, 0); switch (mod->kind) { case Module_kind: if (compiler_body(c, loc, mod->v.Module.body) < 0) { - compiler_exit_scope(c); return ERROR; } break; @@ -1691,6 +1685,11 @@ static PyCodeObject * compiler_mod(struct compiler *c, mod_ty mod) { int addNone = mod->kind != Expression_kind; + _Py_DECLARE_STR(anon_module, ""); + if (compiler_enter_scope(c, &_Py_STR(anon_module), COMPILER_SCOPE_MODULE, + mod, 1) < 0) { + return NULL; + } if (compiler_codegen(c, mod) < 0) { return NULL; } @@ -7261,27 +7260,33 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, int optimize) { PyObject *res = NULL; + struct compiler *c = NULL; + PyArena *arena = NULL; if (!PyAST_Check(ast)) { PyErr_SetString(PyExc_TypeError, "expected an AST"); return NULL; } - PyArena *arena = _PyArena_New(); + arena = _PyArena_New(); if (arena == NULL) { - return NULL; + goto end; } mod_ty mod = PyAST_obj2mod(ast, arena, 0 /* exec */); if (mod == NULL || !_PyAST_Validate(mod)) { - _PyArena_Free(arena); - return NULL; + goto end; } - struct compiler *c = new_compiler(mod, filename, pflags, optimize, arena); + c = new_compiler(mod, filename, pflags, optimize, arena); if (c == NULL) { - _PyArena_Free(arena); - return NULL; + goto end; + } + + _Py_DECLARE_STR(anon_module, ""); + if (compiler_enter_scope(c, &_Py_STR(anon_module), COMPILER_SCOPE_MODULE, + mod, 1) < 0) { + goto end; } if (compiler_codegen(c, mod) < 0) { @@ -7292,8 +7297,13 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, finally: compiler_exit_scope(c); - compiler_free(c); - _PyArena_Free(arena); +end: + if (c != NULL) { + compiler_free(c); + } + if (arena != NULL) { + _PyArena_Free(arena); + } return res; } From 52f3a59a3f273d8c44cd621479b4e09d6ef45b8a Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sat, 6 May 2023 17:42:10 +0100 Subject: [PATCH 2/4] support different compilation modes in _PyCompile_CodeGen --- Include/internal/pycore_compile.h | 3 +- .../pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + .../internal/pycore_runtime_init_generated.h | 1 + .../internal/pycore_unicodeobject_generated.h | 3 ++ Modules/_testinternalcapi.c | 8 +++-- Modules/clinic/_testinternalcapi.c.h | 29 +++++++++++++------ Python/compile.c | 8 +++-- 8 files changed, 39 insertions(+), 15 deletions(-) diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 4bd4ef57238f98..d2b12c91fe7a00 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -97,7 +97,8 @@ PyAPI_FUNC(PyObject*) _PyCompile_CodeGen( PyObject *ast, PyObject *filename, PyCompilerFlags *flags, - int optimize); + int optimize, + int compile_mode); PyAPI_FUNC(PyObject*) _PyCompile_OptimizeCfg( PyObject *instructions, diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 9377fd8526e3a2..7e495817981f06 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -847,6 +847,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(code)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(command)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(comment_factory)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(compile_mode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(consts)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(context)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cookie)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index ed9b2bb44ddffc..8ebfee85c87c23 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -335,6 +335,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(code) STRUCT_FOR_ID(command) STRUCT_FOR_ID(comment_factory) + STRUCT_FOR_ID(compile_mode) STRUCT_FOR_ID(consts) STRUCT_FOR_ID(context) STRUCT_FOR_ID(cookie) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 6ade8fb6eade03..7b9c73dd1edf3b 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -841,6 +841,7 @@ extern "C" { INIT_ID(code), \ INIT_ID(command), \ INIT_ID(comment_factory), \ + INIT_ID(compile_mode), \ INIT_ID(consts), \ INIT_ID(context), \ INIT_ID(cookie), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 0b33ea187e60ff..8e086edbdf8193 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -858,6 +858,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(comment_factory); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(compile_mode); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(consts); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index f35e3b48df9321..40ad6f88868daf 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -593,17 +593,19 @@ _testinternalcapi.compiler_codegen -> object ast: object filename: object optimize: int + compile_mode: int = 0 Apply compiler code generation to an AST. [clinic start generated code]*/ static PyObject * _testinternalcapi_compiler_codegen_impl(PyObject *module, PyObject *ast, - PyObject *filename, int optimize) -/*[clinic end generated code: output=fbbbbfb34700c804 input=e9fbe6562f7f75e4]*/ + PyObject *filename, int optimize, + int compile_mode) +/*[clinic end generated code: output=40a68f6e13951cc8 input=a0e00784f1517cd7]*/ { PyCompilerFlags *flags = NULL; - return _PyCompile_CodeGen(ast, filename, flags, optimize); + return _PyCompile_CodeGen(ast, filename, flags, optimize, compile_mode); } diff --git a/Modules/clinic/_testinternalcapi.c.h b/Modules/clinic/_testinternalcapi.c.h index 89573222572594..41dd50437956c4 100644 --- a/Modules/clinic/_testinternalcapi.c.h +++ b/Modules/clinic/_testinternalcapi.c.h @@ -9,7 +9,7 @@ preserve PyDoc_STRVAR(_testinternalcapi_compiler_codegen__doc__, -"compiler_codegen($module, /, ast, filename, optimize)\n" +"compiler_codegen($module, /, ast, filename, optimize, compile_mode=0)\n" "--\n" "\n" "Apply compiler code generation to an AST."); @@ -19,7 +19,8 @@ PyDoc_STRVAR(_testinternalcapi_compiler_codegen__doc__, static PyObject * _testinternalcapi_compiler_codegen_impl(PyObject *module, PyObject *ast, - PyObject *filename, int optimize); + PyObject *filename, int optimize, + int compile_mode); static PyObject * _testinternalcapi_compiler_codegen(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -27,14 +28,14 @@ _testinternalcapi_compiler_codegen(PyObject *module, PyObject *const *args, Py_s PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 3 + #define NUM_KEYWORDS 4 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(ast), &_Py_ID(filename), &_Py_ID(optimize), }, + .ob_item = { &_Py_ID(ast), &_Py_ID(filename), &_Py_ID(optimize), &_Py_ID(compile_mode), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -43,19 +44,21 @@ _testinternalcapi_compiler_codegen(PyObject *module, PyObject *const *args, Py_s # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"ast", "filename", "optimize", NULL}; + static const char * const _keywords[] = {"ast", "filename", "optimize", "compile_mode", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "compiler_codegen", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[3]; + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; PyObject *ast; PyObject *filename; int optimize; + int compile_mode = 0; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 4, 0, argsbuf); if (!args) { goto exit; } @@ -65,7 +68,15 @@ _testinternalcapi_compiler_codegen(PyObject *module, PyObject *const *args, Py_s if (optimize == -1 && PyErr_Occurred()) { goto exit; } - return_value = _testinternalcapi_compiler_codegen_impl(module, ast, filename, optimize); + if (!noptargs) { + goto skip_optional_pos; + } + compile_mode = _PyLong_AsInt(args[3]); + if (compile_mode == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_pos: + return_value = _testinternalcapi_compiler_codegen_impl(module, ast, filename, optimize, compile_mode); exit: return return_value; @@ -190,4 +201,4 @@ _testinternalcapi_assemble_code_object(PyObject *module, PyObject *const *args, exit: return return_value; } -/*[clinic end generated code: output=d5e08c9d67f9721f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ab661d56a14b1a1c input=a9049054013a1b77]*/ diff --git a/Python/compile.c b/Python/compile.c index 968afbc0d29798..5263849481cff1 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -7257,7 +7257,7 @@ cfg_to_instructions(cfg_builder *g) PyObject * _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, - int optimize) + int optimize, int compile_mode) { PyObject *res = NULL; struct compiler *c = NULL; @@ -7273,7 +7273,7 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, goto end; } - mod_ty mod = PyAST_obj2mod(ast, arena, 0 /* exec */); + mod_ty mod = PyAST_obj2mod(ast, arena, compile_mode); if (mod == NULL || !_PyAST_Validate(mod)) { goto end; } @@ -7292,6 +7292,10 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, if (compiler_codegen(c, mod) < 0) { goto finally; } + int addNone = mod->kind != Expression_kind; + if (add_return_at_end(c, addNone) < 0) { + return NULL; + } res = instr_sequence_to_instructions(INSTR_SEQUENCE(c)); From 7f654d7f22bc3b60d89f74ca4996884152d29f73 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sat, 6 May 2023 19:08:42 +0100 Subject: [PATCH 3/4] update codegen test (we add return None now) --- Lib/test/test_compiler_codegen.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/test/test_compiler_codegen.py b/Lib/test/test_compiler_codegen.py index 022753e0c99483..ea57df9cd2400b 100644 --- a/Lib/test/test_compiler_codegen.py +++ b/Lib/test/test_compiler_codegen.py @@ -25,6 +25,8 @@ def test_if_expression(self): ('LOAD_CONST', 2, 1), exit_lbl, ('POP_TOP', None), + ('LOAD_CONST', 3), + ('RETURN_VALUE', None), ] self.codegen_test(snippet, expected) @@ -46,5 +48,7 @@ def test_for_loop(self): ('JUMP', loop_lbl), exit_lbl, ('END_FOR', None), + ('LOAD_CONST', 0), + ('RETURN_VALUE', None), ] self.codegen_test(snippet, expected) From de8b2bac5e80cad9fd02e7c1123673787a6484bc Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sun, 7 May 2023 17:56:27 +0100 Subject: [PATCH 4/4] Revert "move compiler_enter_scope out of compiler_codegen" This reverts commit 0630b4ebae3b81eec4484b6bc22ea7756cedd8c2. --- Python/compile.c | 40 +++++++++++++++------------------------- 1 file changed, 15 insertions(+), 25 deletions(-) diff --git a/Python/compile.c b/Python/compile.c index 5263849481cff1..f875e4e17e0a9f 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1655,10 +1655,16 @@ compiler_body(struct compiler *c, location loc, asdl_stmt_seq *stmts) static int compiler_codegen(struct compiler *c, mod_ty mod) { + _Py_DECLARE_STR(anon_module, ""); + RETURN_IF_ERROR( + compiler_enter_scope(c, &_Py_STR(anon_module), COMPILER_SCOPE_MODULE, + mod, 1)); + location loc = LOCATION(1, 1, 0, 0); switch (mod->kind) { case Module_kind: if (compiler_body(c, loc, mod->v.Module.body) < 0) { + compiler_exit_scope(c); return ERROR; } break; @@ -1685,11 +1691,6 @@ static PyCodeObject * compiler_mod(struct compiler *c, mod_ty mod) { int addNone = mod->kind != Expression_kind; - _Py_DECLARE_STR(anon_module, ""); - if (compiler_enter_scope(c, &_Py_STR(anon_module), COMPILER_SCOPE_MODULE, - mod, 1) < 0) { - return NULL; - } if (compiler_codegen(c, mod) < 0) { return NULL; } @@ -7260,33 +7261,27 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, int optimize, int compile_mode) { PyObject *res = NULL; - struct compiler *c = NULL; - PyArena *arena = NULL; if (!PyAST_Check(ast)) { PyErr_SetString(PyExc_TypeError, "expected an AST"); return NULL; } - arena = _PyArena_New(); + PyArena *arena = _PyArena_New(); if (arena == NULL) { - goto end; + return NULL; } mod_ty mod = PyAST_obj2mod(ast, arena, compile_mode); if (mod == NULL || !_PyAST_Validate(mod)) { - goto end; + _PyArena_Free(arena); + return NULL; } - c = new_compiler(mod, filename, pflags, optimize, arena); + struct compiler *c = new_compiler(mod, filename, pflags, optimize, arena); if (c == NULL) { - goto end; - } - - _Py_DECLARE_STR(anon_module, ""); - if (compiler_enter_scope(c, &_Py_STR(anon_module), COMPILER_SCOPE_MODULE, - mod, 1) < 0) { - goto end; + _PyArena_Free(arena); + return NULL; } if (compiler_codegen(c, mod) < 0) { @@ -7301,13 +7296,8 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, finally: compiler_exit_scope(c); -end: - if (c != NULL) { - compiler_free(c); - } - if (arena != NULL) { - _PyArena_Free(arena); - } + compiler_free(c); + _PyArena_Free(arena); return res; }