From f533f216e6aaba3f36639ae27210420e7dcf9de1 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Mon, 6 Mar 2023 14:41:53 +0100 Subject: [PATCH 01/38] gh-102416: Do not memoize incorrectly loop rules in the parser (#102467) --- ...-03-06-13-05-33.gh-issue-102416.dz6K5f.rst | 1 + Parser/parser.c | 216 ------------------ Tools/peg_generator/pegen/c_generator.py | 5 +- 3 files changed, 4 insertions(+), 218 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-06-13-05-33.gh-issue-102416.dz6K5f.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-06-13-05-33.gh-issue-102416.dz6K5f.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-06-13-05-33.gh-issue-102416.dz6K5f.rst new file mode 100644 index 00000000000000..9ffc67cfb7ed56 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-03-06-13-05-33.gh-issue-102416.dz6K5f.rst @@ -0,0 +1 @@ +Do not memoize incorrectly automatically generated loop rules in the parser. Patch by Pablo Galindo. diff --git a/Parser/parser.c b/Parser/parser.c index 845c1739d63a60..e0a88a9cc72c8b 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -23777,7 +23777,6 @@ _loop0_1_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -23828,7 +23827,6 @@ _loop0_1_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_1_type, _seq); p->level--; return _seq; } @@ -23847,7 +23845,6 @@ _loop0_2_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -23898,7 +23895,6 @@ _loop0_2_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_2_type, _seq); p->level--; return _seq; } @@ -23917,7 +23913,6 @@ _loop1_3_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -23973,7 +23968,6 @@ _loop1_3_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_3_type, _seq); p->level--; return _seq; } @@ -23992,7 +23986,6 @@ _loop0_5_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -24052,7 +24045,6 @@ _loop0_5_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_5_type, _seq); p->level--; return _seq; } @@ -24585,7 +24577,6 @@ _loop1_14_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -24641,7 +24632,6 @@ _loop1_14_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_14_type, _seq); p->level--; return _seq; } @@ -24823,7 +24813,6 @@ _loop0_19_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -24883,7 +24872,6 @@ _loop0_19_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_19_type, _seq); p->level--; return _seq; } @@ -24944,7 +24932,6 @@ _loop0_21_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -25004,7 +24991,6 @@ _loop0_21_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_21_type, _seq); p->level--; return _seq; } @@ -25170,7 +25156,6 @@ _loop0_24_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -25221,7 +25206,6 @@ _loop0_24_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_24_type, _seq); p->level--; return _seq; } @@ -25240,7 +25224,6 @@ _loop1_25_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -25296,7 +25279,6 @@ _loop1_25_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_25_type, _seq); p->level--; return _seq; } @@ -25315,7 +25297,6 @@ _loop0_27_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -25375,7 +25356,6 @@ _loop0_27_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_27_type, _seq); p->level--; return _seq; } @@ -25483,7 +25463,6 @@ _loop0_30_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -25543,7 +25522,6 @@ _loop0_30_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_30_type, _seq); p->level--; return _seq; } @@ -25651,7 +25629,6 @@ _loop1_32_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -25707,7 +25684,6 @@ _loop1_32_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_32_type, _seq); p->level--; return _seq; } @@ -25870,7 +25846,6 @@ _loop0_36_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -25921,7 +25896,6 @@ _loop0_36_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_36_type, _seq); p->level--; return _seq; } @@ -25940,7 +25914,6 @@ _loop0_37_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -25991,7 +25964,6 @@ _loop0_37_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_37_type, _seq); p->level--; return _seq; } @@ -26010,7 +25982,6 @@ _loop0_38_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26061,7 +26032,6 @@ _loop0_38_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_38_type, _seq); p->level--; return _seq; } @@ -26080,7 +26050,6 @@ _loop1_39_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26136,7 +26105,6 @@ _loop1_39_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_39_type, _seq); p->level--; return _seq; } @@ -26155,7 +26123,6 @@ _loop0_40_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26206,7 +26173,6 @@ _loop0_40_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_40_type, _seq); p->level--; return _seq; } @@ -26225,7 +26191,6 @@ _loop1_41_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26281,7 +26246,6 @@ _loop1_41_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_41_type, _seq); p->level--; return _seq; } @@ -26300,7 +26264,6 @@ _loop1_42_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26356,7 +26319,6 @@ _loop1_42_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_42_type, _seq); p->level--; return _seq; } @@ -26375,7 +26337,6 @@ _loop1_43_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26431,7 +26392,6 @@ _loop1_43_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_43_type, _seq); p->level--; return _seq; } @@ -26450,7 +26410,6 @@ _loop0_44_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26501,7 +26460,6 @@ _loop0_44_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_44_type, _seq); p->level--; return _seq; } @@ -26520,7 +26478,6 @@ _loop1_45_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26576,7 +26533,6 @@ _loop1_45_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_45_type, _seq); p->level--; return _seq; } @@ -26595,7 +26551,6 @@ _loop0_46_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26646,7 +26601,6 @@ _loop0_46_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_46_type, _seq); p->level--; return _seq; } @@ -26665,7 +26619,6 @@ _loop1_47_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26721,7 +26674,6 @@ _loop1_47_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_47_type, _seq); p->level--; return _seq; } @@ -26740,7 +26692,6 @@ _loop0_48_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26791,7 +26742,6 @@ _loop0_48_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_48_type, _seq); p->level--; return _seq; } @@ -26810,7 +26760,6 @@ _loop0_49_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26861,7 +26810,6 @@ _loop0_49_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_49_type, _seq); p->level--; return _seq; } @@ -26880,7 +26828,6 @@ _loop1_50_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -26936,7 +26883,6 @@ _loop1_50_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_50_type, _seq); p->level--; return _seq; } @@ -26955,7 +26901,6 @@ _loop0_52_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -27015,7 +26960,6 @@ _loop0_52_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_52_type, _seq); p->level--; return _seq; } @@ -27076,7 +27020,6 @@ _loop0_54_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -27136,7 +27079,6 @@ _loop0_54_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_54_type, _seq); p->level--; return _seq; } @@ -27197,7 +27139,6 @@ _loop0_56_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -27257,7 +27198,6 @@ _loop0_56_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_56_type, _seq); p->level--; return _seq; } @@ -27318,7 +27258,6 @@ _loop0_58_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -27378,7 +27317,6 @@ _loop0_58_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_58_type, _seq); p->level--; return _seq; } @@ -27516,7 +27454,6 @@ _loop1_60_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -27572,7 +27509,6 @@ _loop1_60_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_60_type, _seq); p->level--; return _seq; } @@ -27591,7 +27527,6 @@ _loop1_61_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -27647,7 +27582,6 @@ _loop1_61_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_61_type, _seq); p->level--; return _seq; } @@ -27760,7 +27694,6 @@ _loop1_64_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -27816,7 +27749,6 @@ _loop1_64_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_64_type, _seq); p->level--; return _seq; } @@ -27835,7 +27767,6 @@ _loop0_66_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -27895,7 +27826,6 @@ _loop0_66_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_66_type, _seq); p->level--; return _seq; } @@ -28226,7 +28156,6 @@ _loop0_72_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -28286,7 +28215,6 @@ _loop0_72_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_72_type, _seq); p->level--; return _seq; } @@ -28347,7 +28275,6 @@ _loop0_74_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -28407,7 +28334,6 @@ _loop0_74_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_74_type, _seq); p->level--; return _seq; } @@ -28526,7 +28452,6 @@ _loop0_77_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -28586,7 +28511,6 @@ _loop0_77_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_77_type, _seq); p->level--; return _seq; } @@ -28647,7 +28571,6 @@ _loop0_79_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -28707,7 +28630,6 @@ _loop0_79_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_79_type, _seq); p->level--; return _seq; } @@ -28768,7 +28690,6 @@ _loop1_80_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -28824,7 +28745,6 @@ _loop1_80_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_80_type, _seq); p->level--; return _seq; } @@ -28843,7 +28763,6 @@ _loop1_81_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -28899,7 +28818,6 @@ _loop1_81_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_81_type, _seq); p->level--; return _seq; } @@ -28918,7 +28836,6 @@ _loop0_83_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -28978,7 +28895,6 @@ _loop0_83_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_83_type, _seq); p->level--; return _seq; } @@ -29039,7 +28955,6 @@ _loop1_84_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -29095,7 +29010,6 @@ _loop1_84_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_84_type, _seq); p->level--; return _seq; } @@ -29114,7 +29028,6 @@ _loop1_85_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -29170,7 +29083,6 @@ _loop1_85_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_85_type, _seq); p->level--; return _seq; } @@ -29189,7 +29101,6 @@ _loop1_86_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -29245,7 +29156,6 @@ _loop1_86_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_86_type, _seq); p->level--; return _seq; } @@ -29308,7 +29218,6 @@ _loop0_89_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -29368,7 +29277,6 @@ _loop0_89_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_89_type, _seq); p->level--; return _seq; } @@ -29765,7 +29673,6 @@ _loop0_95_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -29816,7 +29723,6 @@ _loop0_95_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_95_type, _seq); p->level--; return _seq; } @@ -29835,7 +29741,6 @@ _loop0_96_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -29886,7 +29791,6 @@ _loop0_96_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_96_type, _seq); p->level--; return _seq; } @@ -29905,7 +29809,6 @@ _loop0_97_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -29956,7 +29859,6 @@ _loop0_97_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_97_type, _seq); p->level--; return _seq; } @@ -29975,7 +29877,6 @@ _loop1_98_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30031,7 +29932,6 @@ _loop1_98_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_98_type, _seq); p->level--; return _seq; } @@ -30050,7 +29950,6 @@ _loop0_99_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30101,7 +30000,6 @@ _loop0_99_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_99_type, _seq); p->level--; return _seq; } @@ -30120,7 +30018,6 @@ _loop1_100_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30176,7 +30073,6 @@ _loop1_100_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_100_type, _seq); p->level--; return _seq; } @@ -30195,7 +30091,6 @@ _loop1_101_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30251,7 +30146,6 @@ _loop1_101_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_101_type, _seq); p->level--; return _seq; } @@ -30270,7 +30164,6 @@ _loop1_102_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30326,7 +30219,6 @@ _loop1_102_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_102_type, _seq); p->level--; return _seq; } @@ -30345,7 +30237,6 @@ _loop0_103_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30396,7 +30287,6 @@ _loop0_103_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_103_type, _seq); p->level--; return _seq; } @@ -30415,7 +30305,6 @@ _loop1_104_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30471,7 +30360,6 @@ _loop1_104_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_104_type, _seq); p->level--; return _seq; } @@ -30490,7 +30378,6 @@ _loop0_105_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30541,7 +30428,6 @@ _loop0_105_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_105_type, _seq); p->level--; return _seq; } @@ -30560,7 +30446,6 @@ _loop1_106_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30616,7 +30501,6 @@ _loop1_106_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_106_type, _seq); p->level--; return _seq; } @@ -30635,7 +30519,6 @@ _loop0_107_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30686,7 +30569,6 @@ _loop0_107_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_107_type, _seq); p->level--; return _seq; } @@ -30705,7 +30587,6 @@ _loop1_108_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30761,7 +30642,6 @@ _loop1_108_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_108_type, _seq); p->level--; return _seq; } @@ -30780,7 +30660,6 @@ _loop1_109_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30836,7 +30715,6 @@ _loop1_109_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_109_type, _seq); p->level--; return _seq; } @@ -30905,7 +30783,6 @@ _loop0_112_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -30965,7 +30842,6 @@ _loop0_112_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_112_type, _seq); p->level--; return _seq; } @@ -31026,7 +30902,6 @@ _loop1_113_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -31082,7 +30957,6 @@ _loop1_113_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_113_type, _seq); p->level--; return _seq; } @@ -31101,7 +30975,6 @@ _loop0_114_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -31152,7 +31025,6 @@ _loop0_114_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_114_type, _seq); p->level--; return _seq; } @@ -31171,7 +31043,6 @@ _loop0_115_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -31222,7 +31093,6 @@ _loop0_115_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_115_type, _seq); p->level--; return _seq; } @@ -31301,7 +31171,6 @@ _loop0_118_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -31361,7 +31230,6 @@ _loop0_118_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_118_type, _seq); p->level--; return _seq; } @@ -31470,7 +31338,6 @@ _loop0_121_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -31530,7 +31397,6 @@ _loop0_121_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_121_type, _seq); p->level--; return _seq; } @@ -31591,7 +31457,6 @@ _loop0_123_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -31651,7 +31516,6 @@ _loop0_123_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_123_type, _seq); p->level--; return _seq; } @@ -31712,7 +31576,6 @@ _loop0_125_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -31772,7 +31635,6 @@ _loop0_125_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_125_type, _seq); p->level--; return _seq; } @@ -31833,7 +31695,6 @@ _loop0_127_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -31893,7 +31754,6 @@ _loop0_127_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_127_type, _seq); p->level--; return _seq; } @@ -31954,7 +31814,6 @@ _loop0_128_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -32005,7 +31864,6 @@ _loop0_128_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_128_type, _seq); p->level--; return _seq; } @@ -32024,7 +31882,6 @@ _loop0_130_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -32084,7 +31941,6 @@ _loop0_130_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_130_type, _seq); p->level--; return _seq; } @@ -32145,7 +32001,6 @@ _loop1_131_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -32201,7 +32056,6 @@ _loop1_131_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_131_type, _seq); p->level--; return _seq; } @@ -32261,7 +32115,6 @@ _loop0_134_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -32321,7 +32174,6 @@ _loop0_134_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_134_type, _seq); p->level--; return _seq; } @@ -32382,7 +32234,6 @@ _loop0_136_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -32442,7 +32293,6 @@ _loop0_136_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_136_type, _seq); p->level--; return _seq; } @@ -32503,7 +32353,6 @@ _loop0_138_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -32563,7 +32412,6 @@ _loop0_138_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_138_type, _seq); p->level--; return _seq; } @@ -32624,7 +32472,6 @@ _loop0_140_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -32684,7 +32531,6 @@ _loop0_140_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_140_type, _seq); p->level--; return _seq; } @@ -32745,7 +32591,6 @@ _loop0_142_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -32805,7 +32650,6 @@ _loop0_142_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_142_type, _seq); p->level--; return _seq; } @@ -33557,7 +33401,6 @@ _loop0_154_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -33608,7 +33451,6 @@ _loop0_154_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_154_type, _seq); p->level--; return _seq; } @@ -33627,7 +33469,6 @@ _loop0_155_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -33678,7 +33519,6 @@ _loop0_155_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_155_type, _seq); p->level--; return _seq; } @@ -33697,7 +33537,6 @@ _loop0_156_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -33748,7 +33587,6 @@ _loop0_156_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_156_type, _seq); p->level--; return _seq; } @@ -34076,7 +33914,6 @@ _loop0_162_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -34127,7 +33964,6 @@ _loop0_162_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_162_type, _seq); p->level--; return _seq; } @@ -34146,7 +33982,6 @@ _loop0_163_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -34197,7 +34032,6 @@ _loop0_163_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_163_type, _seq); p->level--; return _seq; } @@ -34216,7 +34050,6 @@ _loop0_164_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -34267,7 +34100,6 @@ _loop0_164_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_164_type, _seq); p->level--; return _seq; } @@ -34286,7 +34118,6 @@ _loop1_165_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -34342,7 +34173,6 @@ _loop1_165_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_165_type, _seq); p->level--; return _seq; } @@ -34419,7 +34249,6 @@ _loop0_167_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -34470,7 +34299,6 @@ _loop0_167_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_167_type, _seq); p->level--; return _seq; } @@ -34547,7 +34375,6 @@ _loop0_169_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -34598,7 +34425,6 @@ _loop0_169_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_169_type, _seq); p->level--; return _seq; } @@ -34617,7 +34443,6 @@ _loop1_170_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -34673,7 +34498,6 @@ _loop1_170_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_170_type, _seq); p->level--; return _seq; } @@ -34869,7 +34693,6 @@ _loop0_174_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -34920,7 +34743,6 @@ _loop0_174_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_174_type, _seq); p->level--; return _seq; } @@ -35074,7 +34896,6 @@ _loop1_177_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -35130,7 +34951,6 @@ _loop1_177_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_177_type, _seq); p->level--; return _seq; } @@ -35207,7 +35027,6 @@ _loop0_179_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -35258,7 +35077,6 @@ _loop0_179_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_179_type, _seq); p->level--; return _seq; } @@ -35277,7 +35095,6 @@ _loop0_180_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -35328,7 +35145,6 @@ _loop0_180_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_180_type, _seq); p->level--; return _seq; } @@ -35347,7 +35163,6 @@ _loop0_181_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -35398,7 +35213,6 @@ _loop0_181_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_181_type, _seq); p->level--; return _seq; } @@ -35417,7 +35231,6 @@ _loop0_183_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -35477,7 +35290,6 @@ _loop0_183_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_183_type, _seq); p->level--; return _seq; } @@ -35596,7 +35408,6 @@ _loop0_185_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -35647,7 +35458,6 @@ _loop0_185_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_185_type, _seq); p->level--; return _seq; } @@ -35724,7 +35534,6 @@ _loop0_187_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -35775,7 +35584,6 @@ _loop0_187_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_187_type, _seq); p->level--; return _seq; } @@ -35794,7 +35602,6 @@ _loop1_188_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -35850,7 +35657,6 @@ _loop1_188_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_188_type, _seq); p->level--; return _seq; } @@ -35869,7 +35675,6 @@ _loop1_189_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -35925,7 +35730,6 @@ _loop1_189_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_189_type, _seq); p->level--; return _seq; } @@ -36063,7 +35867,6 @@ _loop0_192_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -36114,7 +35917,6 @@ _loop0_192_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_192_type, _seq); p->level--; return _seq; } @@ -36345,7 +36147,6 @@ _loop0_197_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -36405,7 +36206,6 @@ _loop0_197_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_197_type, _seq); p->level--; return _seq; } @@ -36466,7 +36266,6 @@ _loop0_199_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -36526,7 +36325,6 @@ _loop0_199_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_199_type, _seq); p->level--; return _seq; } @@ -36587,7 +36385,6 @@ _loop0_201_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -36647,7 +36444,6 @@ _loop0_201_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_201_type, _seq); p->level--; return _seq; } @@ -36708,7 +36504,6 @@ _loop0_203_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -36768,7 +36563,6 @@ _loop0_203_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_203_type, _seq); p->level--; return _seq; } @@ -36887,7 +36681,6 @@ _loop0_205_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -36938,7 +36731,6 @@ _loop0_205_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_205_type, _seq); p->level--; return _seq; } @@ -36957,7 +36749,6 @@ _loop1_206_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -37013,7 +36804,6 @@ _loop1_206_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_206_type, _seq); p->level--; return _seq; } @@ -37074,7 +36864,6 @@ _loop0_208_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -37125,7 +36914,6 @@ _loop0_208_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_208_type, _seq); p->level--; return _seq; } @@ -37144,7 +36932,6 @@ _loop1_209_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -37200,7 +36987,6 @@ _loop1_209_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop1_209_type, _seq); p->level--; return _seq; } @@ -37664,7 +37450,6 @@ _loop0_221_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - int _start_mark = p->mark; void **_children = PyMem_Malloc(sizeof(void *)); if (!_children) { p->error_indicator = 1; @@ -37724,7 +37509,6 @@ _loop0_221_rule(Parser *p) } for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); PyMem_Free(_children); - _PyPegen_insert_memo(p, _start_mark, _loop0_221_type, _seq); p->level--; return _seq; } diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py index c41d87b74d1c49..e72ce7afdc4796 100644 --- a/Tools/peg_generator/pegen/c_generator.py +++ b/Tools/peg_generator/pegen/c_generator.py @@ -619,7 +619,8 @@ def _handle_loop_rule_body(self, node: Rule, rhs: Rhs) -> None: self.add_return("_res") self.print("}") self.print("int _mark = p->mark;") - self.print("int _start_mark = p->mark;") + if memoize: + self.print("int _start_mark = p->mark;") self.print("void **_children = PyMem_Malloc(sizeof(void *));") self.out_of_memory_return(f"!_children") self.print("Py_ssize_t _children_capacity = 1;") @@ -642,7 +643,7 @@ def _handle_loop_rule_body(self, node: Rule, rhs: Rhs) -> None: self.out_of_memory_return(f"!_seq", cleanup_code="PyMem_Free(_children);") self.print("for (int i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]);") self.print("PyMem_Free(_children);") - if node.name: + if memoize and node.name: self.print(f"_PyPegen_insert_memo(p, _start_mark, {node.name}_type, _seq);") self.add_return("_seq") From d3ca042c99b2e86a2bb927a877fdfcbacdc22f89 Mon Sep 17 00:00:00 2001 From: Hyunkyun Moon Date: Mon, 6 Mar 2023 22:56:19 +0900 Subject: [PATCH 02/38] gh-95672: Fix versionadded indentation of get_pagesize in test.rst (gh-102455) --- Doc/library/test.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 3c759648e4e626..c60b4da75d1acb 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -540,7 +540,7 @@ The :mod:`test.support` module defines the following functions: Get size of a page in bytes. - .. versionadded:: 3.12 + .. versionadded:: 3.12 .. function:: setswitchinterval(interval) From f105fe4f0a704bedee21d95f1a08bc14a1fcea2a Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Mon, 6 Mar 2023 17:49:31 +0000 Subject: [PATCH 03/38] gh-102192: Replace PyErr_Fetch/Restore etc by more efficient alternatives in sub interpreters module (#102472) --- Modules/_xxsubinterpretersmodule.c | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 461c505c092c70..79dbe3474ba9e8 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -61,9 +61,9 @@ add_new_exception(PyObject *mod, const char *name, PyObject *base) static int _release_xid_data(_PyCrossInterpreterData *data, int ignoreexc) { - PyObject *exctype, *excval, *exctb; + PyObject *exc; if (ignoreexc) { - PyErr_Fetch(&exctype, &excval, &exctb); + exc = PyErr_GetRaisedException(); } int res = _PyCrossInterpreterData_Release(data); if (res < 0) { @@ -84,7 +84,7 @@ _release_xid_data(_PyCrossInterpreterData *data, int ignoreexc) } } if (ignoreexc) { - PyErr_Restore(exctype, excval, exctb); + PyErr_SetRaisedException(exc); } return res; } @@ -294,9 +294,9 @@ _sharedexception_free(_sharedexception *exc) } static _sharedexception * -_sharedexception_bind(PyObject *exctype, PyObject *exc, PyObject *tb) +_sharedexception_bind(PyObject *exc) { - assert(exctype != NULL); + assert(exc != NULL); char *failure = NULL; _sharedexception *err = _sharedexception_new(); @@ -304,7 +304,7 @@ _sharedexception_bind(PyObject *exctype, PyObject *exc, PyObject *tb) goto finally; } - PyObject *name = PyUnicode_FromFormat("%S", exctype); + PyObject *name = PyUnicode_FromFormat("%S", Py_TYPE(exc)); if (name == NULL) { failure = "unable to format exception type name"; goto finally; @@ -432,10 +432,7 @@ static int _run_script(PyInterpreterState *interp, const char *codestr, _sharedns *shared, _sharedexception **exc) { - PyObject *exctype = NULL; PyObject *excval = NULL; - PyObject *tb = NULL; - PyObject *main_mod = _PyInterpreterState_GetMainModule(interp); if (main_mod == NULL) { goto error; @@ -469,12 +466,9 @@ _run_script(PyInterpreterState *interp, const char *codestr, return 0; error: - PyErr_Fetch(&exctype, &excval, &tb); - - _sharedexception *sharedexc = _sharedexception_bind(exctype, excval, tb); - Py_XDECREF(exctype); + excval = PyErr_GetRaisedException(); + _sharedexception *sharedexc = _sharedexception_bind(excval); Py_XDECREF(excval); - Py_XDECREF(tb); if (sharedexc == NULL) { fprintf(stderr, "RunFailedError: script raised an uncaught exception"); PyErr_Clear(); From d959bcd4a0393a120fa12c034de4041037d171c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Lapeyre?= Date: Mon, 6 Mar 2023 22:20:52 +0100 Subject: [PATCH 04/38] Add gettext support to tools/extensions/c_annotations.py (#101989) --- Doc/tools/extensions/c_annotations.py | 7 ++++--- Doc/tools/templates/dummy.html | 5 +++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Doc/tools/extensions/c_annotations.py b/Doc/tools/extensions/c_annotations.py index e5dc82cf0dc2d9..5af56433f41573 100644 --- a/Doc/tools/extensions/c_annotations.py +++ b/Doc/tools/extensions/c_annotations.py @@ -24,6 +24,7 @@ from docutils.parsers.rst import directives from docutils.parsers.rst import Directive from docutils.statemachine import StringList +from sphinx.locale import _ as sphinx_gettext import csv from sphinx import addnodes @@ -168,11 +169,11 @@ def add_annotations(self, app, doctree): elif not entry.result_type.endswith("Object*"): continue if entry.result_refs is None: - rc = 'Return value: Always NULL.' + rc = sphinx_gettext('Return value: Always NULL.') elif entry.result_refs: - rc = 'Return value: New reference.' + rc = sphinx_gettext('Return value: New reference.') else: - rc = 'Return value: Borrowed reference.' + rc = sphinx_gettext('Return value: Borrowed reference.') node.insert(0, nodes.emphasis(rc, rc, classes=['refcount'])) diff --git a/Doc/tools/templates/dummy.html b/Doc/tools/templates/dummy.html index 3438b44377fcb9..bab4aaeb4604b8 100644 --- a/Doc/tools/templates/dummy.html +++ b/Doc/tools/templates/dummy.html @@ -7,6 +7,11 @@ {% trans %}Deprecated since version {deprecated}, will be removed in version {removed}{% endtrans %} {% trans %}Deprecated since version {deprecated}, removed in version {removed}{% endtrans %} +In extensions/c_annotations.py: + +{% trans %}Return value: Always NULL.{% endtrans %} +{% trans %}Return value: New reference.{% endtrans %} +{% trans %}Return value: Borrowed reference.{% endtrans %} In docsbuild-scripts, when rewriting indexsidebar.html with actual versions: From c84e6f32df989908685ea8b6cd49ddde9f428524 Mon Sep 17 00:00:00 2001 From: JosephSBoyle <48555120+JosephSBoyle@users.noreply.github.com> Date: Mon, 6 Mar 2023 22:02:19 +0000 Subject: [PATCH 05/38] Remove unused import of `warnings` from `unittest.loader` (#102479) --- Lib/unittest/loader.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/unittest/loader.py b/Lib/unittest/loader.py index 80d4fbdd8e3606..b989284a640e14 100644 --- a/Lib/unittest/loader.py +++ b/Lib/unittest/loader.py @@ -6,7 +6,6 @@ import traceback import types import functools -import warnings from fnmatch import fnmatch, fnmatchcase From f9cdeb7b99d408a2e884101ede576952510bcc9b Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Mon, 6 Mar 2023 20:02:24 -0500 Subject: [PATCH 06/38] gh-101759: Update macOS installer SQLite 3.40.1 checksum (gh-102485) --- Mac/BuildScript/build-installer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 5ea2cfc5183ef5..63fa21b2b33d17 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -361,7 +361,7 @@ def library_recipes(): dict( name="SQLite 3.40.1", url="https://sqlite.org/2022/sqlite-autoconf-3400100.tar.gz", - checksum="5498af3a357753d473ee713e363fa5b7", + checksum="42175b1a1d23529cb133bbd2b5900afd", extra_cflags=('-Os ' '-DSQLITE_ENABLE_FTS5 ' '-DSQLITE_ENABLE_FTS4 ' From 8606697f49dc58ff7e18147401ac65a09c38cf57 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 6 Mar 2023 19:40:09 -0700 Subject: [PATCH 07/38] gh-90110: Fix the c-analyzer Tool (#102483) Some incompatible changes had gone in, and the "ignore" lists weren't properly undated. This change fixes that. It's necessary prior to enabling test_check_c_globals, which I hope to do soon. Note that this does include moving last_resort_memory_error to PyInterpreterState. https://github.com/python/cpython/issues/90110 --- Include/internal/pycore_global_objects.h | 1 + Include/internal/pycore_intrinsics.h | 4 +-- Include/internal/pycore_runtime_init.h | 6 ++++ Objects/exceptions.c | 13 +++----- Python/intrinsics.c | 4 +-- Tools/c-analyzer/c_parser/parser/_common.py | 22 ++++++++++---- .../c-analyzer/c_parser/parser/_func_body.py | 25 +++++++++------- Tools/c-analyzer/c_parser/preprocessor/gcc.py | 3 +- Tools/c-analyzer/cpython/_parser.py | 9 +++++- Tools/c-analyzer/cpython/globals-to-fix.tsv | 2 +- Tools/c-analyzer/cpython/ignored.tsv | 30 +++++++++++++++++-- 11 files changed, 85 insertions(+), 34 deletions(-) diff --git a/Include/internal/pycore_global_objects.h b/Include/internal/pycore_global_objects.h index 30c7c4e3bbd067..9957da1fc5f22a 100644 --- a/Include/internal/pycore_global_objects.h +++ b/Include/internal/pycore_global_objects.h @@ -86,6 +86,7 @@ struct _Py_interp_static_objects { // hamt_empty is here instead of global because of its weakreflist. _PyGC_Head_UNUSED _hamt_empty_gc_not_used; PyHamtObject hamt_empty; + PyBaseExceptionObject last_resort_memory_error; } singletons; }; diff --git a/Include/internal/pycore_intrinsics.h b/Include/internal/pycore_intrinsics.h index deac145fff7627..46a52740eb8a0c 100644 --- a/Include/internal/pycore_intrinsics.h +++ b/Include/internal/pycore_intrinsics.h @@ -21,6 +21,6 @@ typedef PyObject *(*instrinsic_func1)(PyThreadState* tstate, PyObject *value); typedef PyObject *(*instrinsic_func2)(PyThreadState* tstate, PyObject *value1, PyObject *value2); -extern instrinsic_func1 _PyIntrinsics_UnaryFunctions[]; -extern instrinsic_func2 _PyIntrinsics_BinaryFunctions[]; +extern const instrinsic_func1 _PyIntrinsics_UnaryFunctions[]; +extern const instrinsic_func2 _PyIntrinsics_BinaryFunctions[]; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index b54adf04761d4e..a2cc7c87c2f3e2 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -14,6 +14,9 @@ extern "C" { #include "pycore_obmalloc_init.h" +extern PyTypeObject _PyExc_MemoryError; + + /* The static initializers defined here should only be used in the runtime init code (in pystate.c and pylifecycle.c). */ @@ -120,6 +123,9 @@ extern "C" { .ob_base = _PyObject_IMMORTAL_INIT(&_PyHamt_Type), \ .h_root = (PyHamtNode*)&_Py_SINGLETON(hamt_bitmap_node_empty), \ }, \ + .last_resort_memory_error = { \ + _PyObject_IMMORTAL_INIT(&_PyExc_MemoryError), \ + }, \ }, \ }, \ ._initial_thread = _PyThreadState_INIT, \ diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 976f84dbf63c93..a473cbdfeda7fc 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -3207,8 +3207,6 @@ SimpleExtendsException(PyExc_Exception, ReferenceError, #define MEMERRORS_SAVE 16 -static PyBaseExceptionObject last_resort_memory_error; - static PyObject * get_memory_error(int allow_allocation, PyObject *args, PyObject *kwds) { @@ -3216,7 +3214,9 @@ get_memory_error(int allow_allocation, PyObject *args, PyObject *kwds) struct _Py_exc_state *state = get_exc_state(); if (state->memerrors_freelist == NULL) { if (!allow_allocation) { - return Py_NewRef(&last_resort_memory_error); + PyInterpreterState *interp = _PyInterpreterState_GET(); + return Py_NewRef( + &_Py_INTERP_SINGLETON(interp, last_resort_memory_error)); } PyObject *result = BaseException_new((PyTypeObject *)PyExc_MemoryError, args, kwds); return result; @@ -3239,8 +3239,6 @@ get_memory_error(int allow_allocation, PyObject *args, PyObject *kwds) return (PyObject *)self; } -static PyBaseExceptionObject last_resort_memory_error; - static PyObject * MemoryError_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { @@ -3325,7 +3323,7 @@ free_preallocated_memerrors(struct _Py_exc_state *state) } -static PyTypeObject _PyExc_MemoryError = { +PyTypeObject _PyExc_MemoryError = { PyVarObject_HEAD_INIT(NULL, 0) "MemoryError", sizeof(PyBaseExceptionObject), @@ -3339,9 +3337,6 @@ static PyTypeObject _PyExc_MemoryError = { }; PyObject *PyExc_MemoryError = (PyObject *) &_PyExc_MemoryError; -static PyBaseExceptionObject last_resort_memory_error = { - _PyObject_IMMORTAL_INIT(&_PyExc_MemoryError) -}; /* * BufferError extends Exception diff --git a/Python/intrinsics.c b/Python/intrinsics.c index 9e90ef32130f1d..cca29d859902a4 100644 --- a/Python/intrinsics.c +++ b/Python/intrinsics.c @@ -199,7 +199,7 @@ list_to_tuple(PyThreadState* unused, PyObject *v) return _PyTuple_FromArray(((PyListObject *)v)->ob_item, Py_SIZE(v)); } -instrinsic_func1 +const instrinsic_func1 _PyIntrinsics_UnaryFunctions[] = { [0] = no_intrinsic, [INTRINSIC_PRINT] = print_expr, @@ -221,7 +221,7 @@ prep_reraise_star(PyThreadState* unused, PyObject *orig, PyObject *excs) return _PyExc_PrepReraiseStar(orig, excs); } -instrinsic_func2 +const instrinsic_func2 _PyIntrinsics_BinaryFunctions[] = { [INTRINSIC_PREP_RERAISE_STAR] = prep_reraise_star, }; diff --git a/Tools/c-analyzer/c_parser/parser/_common.py b/Tools/c-analyzer/c_parser/parser/_common.py index d468d5442a939f..2eacace2c001df 100644 --- a/Tools/c-analyzer/c_parser/parser/_common.py +++ b/Tools/c-analyzer/c_parser/parser/_common.py @@ -7,13 +7,25 @@ ) -def log_match(group, m): +def log_match(group, m, depth_before=None, depth_after=None): from . import _logger - text = m.group(0) - if text.startswith(('(', ')')) or text.endswith(('(', ')')): - _logger.debug(f'matched <{group}> ({text!r})') + + if m is not None: + text = m.group(0) + if text.startswith(('(', ')')) or text.endswith(('(', ')')): + _logger.debug(f'matched <{group}> ({text!r})') + else: + _logger.debug(f'matched <{group}> ({text})') + + elif depth_before is not None or depth_after is not None: + if depth_before is None: + depth_before = '???' + elif depth_after is None: + depth_after = '???' + _logger.log(1, f'depth: %s -> %s', depth_before, depth_after) + else: - _logger.debug(f'matched <{group}> ({text})') + raise NotImplementedError('this should not have been hit') ############################# diff --git a/Tools/c-analyzer/c_parser/parser/_func_body.py b/Tools/c-analyzer/c_parser/parser/_func_body.py index 42fd459e111d2c..25f2f5807ae827 100644 --- a/Tools/c-analyzer/c_parser/parser/_func_body.py +++ b/Tools/c-analyzer/c_parser/parser/_func_body.py @@ -65,11 +65,11 @@ def parse_function_body(name, text, resolve, source, anon_name, parent): ) = m.groups() if empty: - log_match('', m) + log_match('', m, depth) resolve(None, None, None, text) yield None, text elif inline_kind: - log_match('', m) + log_match('', m, depth) kind = inline_kind name = inline_name or anon_name('inline-') data = [] # members @@ -92,7 +92,7 @@ def parse_function_body(name, text, resolve, source, anon_name, parent): # XXX Should "parent" really be None for inline type decls? yield resolve(kind, data, name, text, None), text elif block_close: - log_match('', m) + log_match('', m, depth) depth -= 1 resolve(None, None, None, text) # XXX This isn't great. Calling resolve() should have @@ -101,13 +101,13 @@ def parse_function_body(name, text, resolve, source, anon_name, parent): # needs to be fixed. yield None, text elif compound_bare: - log_match('', m) + log_match('', m, depth) yield resolve('statement', compound_bare, None, text, parent), text elif compound_labeled: - log_match('', m) + log_match('', m, depth) yield resolve('statement', compound_labeled, None, text, parent), text elif compound_paren: - log_match('', m) + log_match('', m, depth) try: pos = match_paren(text) except ValueError: @@ -132,7 +132,7 @@ def parse_function_body(name, text, resolve, source, anon_name, parent): } yield resolve('statement', data, None, text, parent), text elif block_open: - log_match('', m) + log_match('', m, depth) depth += 1 if block_leading: # An inline block: the last evaluated expression is used @@ -144,10 +144,10 @@ def parse_function_body(name, text, resolve, source, anon_name, parent): resolve(None, None, None, text) yield None, text elif simple_ending: - log_match('', m) + log_match('', m, depth) yield resolve('statement', simple_stmt, None, text, parent), text elif var_ending: - log_match('', m) + log_match('', m, depth) kind = 'variable' _, name, vartype = parse_var_decl(decl) data = { @@ -220,7 +220,7 @@ def _parse_next_local_static(m, srcinfo, anon_name, func, depth): remainder = srcinfo.text[m.end():] if inline_kind: - log_match('func inline', m) + log_match('func inline', m, depth, depth) kind = inline_kind name = inline_name or anon_name('inline-') # Immediately emit a forward declaration. @@ -249,7 +249,7 @@ def parse_body(source): yield parse_body, depth elif static_decl: - log_match('local variable', m) + log_match('local variable', m, depth, depth) _, name, data = parse_var_decl(static_decl) yield srcinfo.resolve('variable', data, name, parent=func), depth @@ -266,10 +266,13 @@ def parse_body(source): else: log_match('func other', m) if block_open: + log_match('func other', None, depth, depth + 1) depth += 1 elif block_close: + log_match('func other', None, depth, depth - 1) depth -= 1 elif stmt_end: + log_match('func other', None, depth, depth) pass else: # This should be unreachable. diff --git a/Tools/c-analyzer/c_parser/preprocessor/gcc.py b/Tools/c-analyzer/c_parser/preprocessor/gcc.py index 770802253792d8..7ef1a8afc3b135 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/gcc.py +++ b/Tools/c-analyzer/c_parser/preprocessor/gcc.py @@ -29,7 +29,7 @@ [^()]* )* ) # - ( [)] [)] )? # + ( [)] [)] ) # ''', re.VERBOSE) POST_ARGS = ( @@ -156,6 +156,7 @@ def _iter_top_include_lines(lines, topfile, cwd, if name != 'pragma': raise Exception(line) else: + line = re.sub(r'__inline__', 'inline', line) if not raw: line, partial = _strip_directives(line, partial=partial) yield _common.SourceLine( diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index ab1d6257f1b1a9..e7764165d36c4c 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -105,9 +105,14 @@ def clean_lines(text): * ./Include * ./Include/internal +Modules/_decimal/**/*.c Modules/_decimal/libmpdec +Modules/_hacl/*.c Modules/_hacl/include +Modules/_hacl/*.h Modules/_hacl/include Modules/_tkinter.c /usr/include/tcl8.6 +Modules/md5module.c Modules/_hacl/include +Modules/sha1module.c Modules/_hacl/include +Modules/sha2module.c Modules/_hacl/include Modules/tkappinit.c /usr/include/tcl -Modules/_decimal/**/*.c Modules/_decimal/libmpdec Objects/stringlib/*.h Objects # @end=tsv@ @@ -173,6 +178,7 @@ def clean_lines(text): Modules/_functoolsmodule.c Py_BUILD_CORE 1 Modules/_heapqmodule.c Py_BUILD_CORE 1 Modules/_io/*.c Py_BUILD_CORE 1 +Modules/_io/*.h Py_BUILD_CORE 1 Modules/_localemodule.c Py_BUILD_CORE 1 Modules/_operator.c Py_BUILD_CORE 1 Modules/_posixsubprocess.c Py_BUILD_CORE 1 @@ -296,6 +302,7 @@ def clean_lines(text): # First match wins. _abs('Modules/_ctypes/ctypes.h'): (5_000, 500), _abs('Modules/_datetimemodule.c'): (20_000, 300), + _abs('Modules/_hacl/*.c'): (200_000, 500), _abs('Modules/posixmodule.c'): (20_000, 500), _abs('Modules/termios.c'): (10_000, 800), _abs('Modules/_testcapimodule.c'): (20_000, 400), diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 2e28c50c6ff69a..57b8542fb46482 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -539,7 +539,7 @@ Modules/_tkinter.c - command_mutex - Modules/_tkinter.c - HeadFHCD - Modules/_tkinter.c - stdin_ready - Modules/_tkinter.c - event_tstate - -Modules/_xxsubinterpretersmodule.c - _globals - +Modules/_xxinterpchannelsmodule.c - _globals - Modules/readline.c - completer_word_break_characters - Modules/readline.c - _history_length - Modules/readline.c - should_auto_add_history - diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 849e20a1b0f4eb..700ddf2851839e 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -206,6 +206,10 @@ Modules/_decimal/_decimal.c - invalid_signals_err - Modules/_decimal/_decimal.c - signal_map - Modules/_decimal/_decimal.c - ssize_constants - Modules/_elementtree.c - ExpatMemoryHandler - +Modules/_hashopenssl.c - py_hashes - +Modules/_hacl/Hacl_Hash_SHA1.c - _h0 - +Modules/_hacl/Hacl_Hash_MD5.c - _h0 - +Modules/_hacl/Hacl_Hash_MD5.c - _t - Modules/_io/_iomodule.c - static_types - Modules/_io/textio.c - encodefuncs - Modules/_io/winconsoleio.c - _PyWindowsConsoleIO_Type - @@ -218,9 +222,10 @@ Modules/_sre/sre_targets.h - sre_targets - Modules/_sre.c pattern_repr flag_names - Modules/_struct.c - bigendian_table - Modules/_struct.c - lilendian_table - +Modules/_struct.c - native_table - Modules/_tkinter.c - state_key - -Modules/_xxsubinterpretersmodule.c - _channelid_end_recv - -Modules/_xxsubinterpretersmodule.c - _channelid_end_send - +Modules/_xxinterpchannelsmodule.c - _channelid_end_recv - +Modules/_xxinterpchannelsmodule.c - _channelid_end_send - Modules/_zoneinfo.c - DAYS_BEFORE_MONTH - Modules/_zoneinfo.c - DAYS_IN_MONTH - Modules/arraymodule.c - descriptors - @@ -332,12 +337,15 @@ Python/frozen.c - _PyImport_FrozenTest - Python/getopt.c - longopts - Python/import.c - _PyImport_Inittab - Python/import.c - _PySys_ImplCacheTag - +Python/intrinsics.c - _PyIntrinsics_UnaryFunctions - +Python/intrinsics.c - _PyIntrinsics_BinaryFunctions - Python/opcode_targets.h - opcode_targets - Python/perf_trampoline.c - _Py_perfmap_callbacks - Python/pyhash.c - PyHash_Func - Python/pylifecycle.c - _C_LOCALE_WARNING - Python/pylifecycle.c - _PyOS_mystrnicmp_hack - Python/pylifecycle.c - _TARGET_LOCALES - +Python/pylifecycle.c - INTERPRETER_TRAMPOLINE_CODEDEF - Python/pystate.c - initial - Python/specialize.c - adaptive_opcodes - Python/specialize.c - cache_requirements - @@ -392,8 +400,23 @@ Modules/_testbuffer.c ndarray_memoryview_from_buffer strides - Modules/_testbuffer.c ndarray_memoryview_from_buffer suboffsets - Modules/_testbuffer.c ndarray_push kwlist - Modules/_testbuffer.c staticarray_init kwlist - +Modules/_testcapi/code.c get_code_extra_index key - +Modules/_testcapi/datetime.c - test_run_counter - +Modules/_testcapi/exceptions.c - PyRecursingInfinitelyError_Type - Modules/_testcapi/heaptype.c - _testcapimodule - +Modules/_testcapi/mem.c - FmData - +Modules/_testcapi/mem.c - FmHook - +Modules/_testcapi/structmember.c - test_structmembersType_OldAPI - Modules/_testcapi/unicode.c - _testcapimodule - +Modules/_testcapi/watchers.c - g_dict_watch_events - +Modules/_testcapi/watchers.c - g_dict_watchers_installed - +Modules/_testcapi/watchers.c - g_type_modified_events - +Modules/_testcapi/watchers.c - g_type_watchers_installed - +Modules/_testcapi/watchers.c - num_code_object_created_events - +Modules/_testcapi/watchers.c - num_code_object_destroyed_events - +Modules/_testcapi/watchers.c - pyfunc_watchers - +Modules/_testcapi/watchers.c - func_watcher_ids - +Modules/_testcapi/watchers.c - func_watcher_callbacks - Modules/_testcapimodule.c - ContainerNoGC_members - Modules/_testcapimodule.c - ContainerNoGC_type - Modules/_testcapimodule.c - FmData - @@ -460,11 +483,13 @@ Modules/_testcapimodule.c - meth_static_methods - Modules/_testcapimodule.c - ml - Modules/_testcapimodule.c - str1 - Modules/_testcapimodule.c - str2 - +Modules/_testcapimodule.c - test_c_thread - Modules/_testcapimodule.c - test_members - Modules/_testcapimodule.c - test_run_counter - Modules/_testcapimodule.c - test_structmembersType - Modules/_testcapimodule.c - thread_done - Modules/_testcapimodule.c - x - +Modules/_testcapimodule.c - wait_done - Modules/_testcapimodule.c getargs_keyword_only keywords - Modules/_testcapimodule.c getargs_keywords keywords - Modules/_testcapimodule.c getargs_positional_only_and_keywords keywords - @@ -526,6 +551,7 @@ Modules/_testmultiphase.c - slots_exec_unreported_exception - Modules/_testmultiphase.c - slots_nonmodule_with_exec_slots - Modules/_testmultiphase.c - testexport_methods - Modules/_testmultiphase.c - uninitialized_def - +Modules/_testsinglephase.c - global_state - Modules/_xxtestfuzz/_xxtestfuzz.c - _fuzzmodule - Modules/_xxtestfuzz/_xxtestfuzz.c - module_methods - Modules/_xxtestfuzz/fuzzer.c - SRE_FLAG_DEBUG - From 80b19a30c0d5f9f8a8651e7f8847c0e68671c89a Mon Sep 17 00:00:00 2001 From: "C.A.M. Gerlach" Date: Mon, 6 Mar 2023 20:45:52 -0600 Subject: [PATCH 08/38] gh-95913: Edit Faster CPython section in 3.11 WhatsNew (GH-98429) Co-authored-by: C.A.M. Gerlach --- Doc/whatsnew/3.11.rst | 186 +++++++++++++++++++++++++----------------- 1 file changed, 109 insertions(+), 77 deletions(-) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index bffb8d03aa7cbb..391ea531281a48 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -1317,14 +1317,17 @@ This section covers specific optimizations independent of the Faster CPython ============== -CPython 3.11 is on average `25% faster `_ -than CPython 3.10 when measured with the +CPython 3.11 is an average of +`25% faster `_ +than CPython 3.10 as measured with the `pyperformance `_ benchmark suite, -and compiled with GCC on Ubuntu Linux. Depending on your workload, the speedup -could be up to 10-60% faster. +when compiled with GCC on Ubuntu Linux. +Depending on your workload, the overall speedup could be 10-60%. -This project focuses on two major areas in Python: faster startup and faster -runtime. Other optimizations not under this project are listed in `Optimizations`_. +This project focuses on two major areas in Python: +:ref:`whatsnew311-faster-startup` and :ref:`whatsnew311-faster-runtime`. +Optimizations not covered by this project are listed separately under +:ref:`whatsnew311-optimizations`. .. _whatsnew311-faster-startup: @@ -1337,8 +1340,8 @@ Faster Startup Frozen imports / Static code objects ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Python caches bytecode in the :ref:`__pycache__` directory to -speed up module loading. +Python caches :term:`bytecode` in the :ref:`__pycache__ ` +directory to speed up module loading. Previously in 3.10, Python module execution looked like this: @@ -1347,8 +1350,9 @@ Previously in 3.10, Python module execution looked like this: Read __pycache__ -> Unmarshal -> Heap allocated code object -> Evaluate In Python 3.11, the core modules essential for Python startup are "frozen". -This means that their code objects (and bytecode) are statically allocated -by the interpreter. This reduces the steps in module execution process to this: +This means that their :ref:`codeobjects` (and bytecode) +are statically allocated by the interpreter. +This reduces the steps in module execution process to: .. code-block:: text @@ -1357,7 +1361,7 @@ by the interpreter. This reduces the steps in module execution process to this: Interpreter startup is now 10-15% faster in Python 3.11. This has a big impact for short-running programs using Python. -(Contributed by Eric Snow, Guido van Rossum and Kumar Aditya in numerous issues.) +(Contributed by Eric Snow, Guido van Rossum and Kumar Aditya in many issues.) .. _whatsnew311-faster-runtime: @@ -1370,17 +1374,19 @@ Faster Runtime Cheaper, lazy Python frames ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Python frames are created whenever Python calls a Python function. This frame -holds execution information. The following are new frame optimizations: +Python frames, holding execution information, +are created whenever Python calls a Python function. +The following are new frame optimizations: - Streamlined the frame creation process. - Avoided memory allocation by generously re-using frame space on the C stack. - Streamlined the internal frame struct to contain only essential information. Frames previously held extra debugging and memory management information. -Old-style frame objects are now created only when requested by debuggers or -by Python introspection functions such as ``sys._getframe`` or -``inspect.currentframe``. For most user code, no frame objects are +Old-style :ref:`frame objects ` +are now created only when requested by debuggers +or by Python introspection functions such as :func:`sys._getframe` and +:func:`inspect.currentframe`. For most user code, no frame objects are created at all. As a result, nearly all Python functions calls have sped up significantly. We measured a 3-7% speedup in pyperformance. @@ -1401,10 +1407,11 @@ In 3.11, when CPython detects Python code calling another Python function, it sets up a new frame, and "jumps" to the new code inside the new frame. This avoids calling the C interpreting function altogether. -Most Python function calls now consume no C stack space. This speeds up -most of such calls. In simple recursive functions like fibonacci or -factorial, a 1.7x speedup was observed. This also means recursive functions -can recurse significantly deeper (if the user increases the recursion limit). +Most Python function calls now consume no C stack space, speeding them up. +In simple recursive functions like fibonacci or +factorial, we observed a 1.7x speedup. This also means recursive functions +can recurse significantly deeper +(if the user increases the recursion limit with :func:`sys.setrecursionlimit`). We measured a 1-3% improvement in pyperformance. (Contributed by Pablo Galindo and Mark Shannon in :issue:`45256`.) @@ -1415,7 +1422,7 @@ We measured a 1-3% improvement in pyperformance. PEP 659: Specializing Adaptive Interpreter ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -:pep:`659` is one of the key parts of the faster CPython project. The general +:pep:`659` is one of the key parts of the Faster CPython project. The general idea is that while Python is a dynamic language, most code has regions where objects and types rarely change. This concept is known as *type stability*. @@ -1424,17 +1431,18 @@ in the executing code. Python will then replace the current operation with a more specialized one. This specialized operation uses fast paths available only to those use cases/types, which generally outperform their generic counterparts. This also brings in another concept called *inline caching*, where -Python caches the results of expensive operations directly in the bytecode. +Python caches the results of expensive operations directly in the +:term:`bytecode`. The specializer will also combine certain common instruction pairs into one -superinstruction. This reduces the overhead during execution. +superinstruction, reducing the overhead during execution. Python will only specialize when it sees code that is "hot" (executed multiple times). This prevents Python -from wasting time for run-once code. Python can also de-specialize when code is +from wasting time on run-once code. Python can also de-specialize when code is too dynamic or when the use changes. Specialization is attempted periodically, -and specialization attempts are not too expensive. This allows specialization -to adapt to new circumstances. +and specialization attempts are not too expensive, +allowing specialization to adapt to new circumstances. (PEP written by Mark Shannon, with ideas inspired by Stefan Brunthaler. See :pep:`659` for more information. Implementation by Mark Shannon and Brandt @@ -1447,32 +1455,32 @@ Bucher, with additional help from Irit Katriel and Dennis Sweeney.) | Operation | Form | Specialization | Operation speedup | Contributor(s) | | | | | (up to) | | +===============+====================+=======================================================+===================+===================+ -| Binary | ``x+x; x*x; x-x;`` | Binary add, multiply and subtract for common types | 10% | Mark Shannon, | -| operations | | such as ``int``, ``float``, and ``str`` take custom | | Dong-hee Na, | -| | | fast paths for their underlying types. | | Brandt Bucher, | +| Binary | ``x + x`` | Binary add, multiply and subtract for common types | 10% | Mark Shannon, | +| operations | | such as :class:`int`, :class:`float` and :class:`str` | | Dong-hee Na, | +| | ``x - x`` | take custom fast paths for their underlying types. | | Brandt Bucher, | | | | | | Dennis Sweeney | +| | ``x * x`` | | | | +---------------+--------------------+-------------------------------------------------------+-------------------+-------------------+ -| Subscript | ``a[i]`` | Subscripting container types such as ``list``, | 10-25% | Irit Katriel, | -| | | ``tuple`` and ``dict`` directly index the underlying | | Mark Shannon | -| | | data structures. | | | +| Subscript | ``a[i]`` | Subscripting container types such as :class:`list`, | 10-25% | Irit Katriel, | +| | | :class:`tuple` and :class:`dict` directly index | | Mark Shannon | +| | | the underlying data structures. | | | | | | | | | -| | | Subscripting custom ``__getitem__`` | | | +| | | Subscripting custom :meth:`~object.__getitem__` | | | | | | is also inlined similar to :ref:`inline-calls`. | | | +---------------+--------------------+-------------------------------------------------------+-------------------+-------------------+ | Store | ``a[i] = z`` | Similar to subscripting specialization above. | 10-25% | Dennis Sweeney | | subscript | | | | | +---------------+--------------------+-------------------------------------------------------+-------------------+-------------------+ | Calls | ``f(arg)`` | Calls to common builtin (C) functions and types such | 20% | Mark Shannon, | -| | ``C(arg)`` | as ``len`` and ``str`` directly call their underlying | | Ken Jin | -| | | C version. This avoids going through the internal | | | -| | | calling convention. | | | -| | | | | | +| | | as :func:`len` and :class:`str` directly call their | | Ken Jin | +| | ``C(arg)`` | underlying C version. This avoids going through the | | | +| | | internal calling convention. | | | +---------------+--------------------+-------------------------------------------------------+-------------------+-------------------+ -| Load | ``print`` | The object's index in the globals/builtins namespace | [1]_ | Mark Shannon | -| global | ``len`` | is cached. Loading globals and builtins require | | | -| variable | | zero namespace lookups. | | | +| Load | ``print`` | The object's index in the globals/builtins namespace | [#load-global]_ | Mark Shannon | +| global | | is cached. Loading globals and builtins require | | | +| variable | ``len`` | zero namespace lookups. | | | +---------------+--------------------+-------------------------------------------------------+-------------------+-------------------+ -| Load | ``o.attr`` | Similar to loading global variables. The attribute's | [2]_ | Mark Shannon | +| Load | ``o.attr`` | Similar to loading global variables. The attribute's | [#load-attr]_ | Mark Shannon | | attribute | | index inside the class/object's namespace is cached. | | | | | | In most cases, attribute loading will require zero | | | | | | namespace lookups. | | | @@ -1484,14 +1492,15 @@ Bucher, with additional help from Irit Katriel and Dennis Sweeney.) | Store | ``o.attr = z`` | Similar to load attribute optimization. | 2% | Mark Shannon | | attribute | | | in pyperformance | | +---------------+--------------------+-------------------------------------------------------+-------------------+-------------------+ -| Unpack | ``*seq`` | Specialized for common containers such as ``list`` | 8% | Brandt Bucher | -| Sequence | | and ``tuple``. Avoids internal calling convention. | | | +| Unpack | ``*seq`` | Specialized for common containers such as | 8% | Brandt Bucher | +| Sequence | | :class:`list` and :class:`tuple`. | | | +| | | Avoids internal calling convention. | | | +---------------+--------------------+-------------------------------------------------------+-------------------+-------------------+ -.. [1] A similar optimization already existed since Python 3.8. 3.11 - specializes for more forms and reduces some overhead. +.. [#load-global] A similar optimization already existed since Python 3.8. + 3.11 specializes for more forms and reduces some overhead. -.. [2] A similar optimization already existed since Python 3.10. +.. [#load-attr] A similar optimization already existed since Python 3.10. 3.11 specializes for more forms. Furthermore, all attribute loads should be sped up by :issue:`45947`. @@ -1501,49 +1510,72 @@ Bucher, with additional help from Irit Katriel and Dennis Sweeney.) Misc ---- -* Objects now require less memory due to lazily created object namespaces. Their - namespace dictionaries now also share keys more freely. +* Objects now require less memory due to lazily created object namespaces. + Their namespace dictionaries now also share keys more freely. (Contributed Mark Shannon in :issue:`45340` and :issue:`40116`.) +* "Zero-cost" exceptions are implemented, eliminating the cost + of :keyword:`try` statements when no exception is raised. + (Contributed by Mark Shannon in :issue:`40222`.) + * A more concise representation of exceptions in the interpreter reduced the time required for catching an exception by about 10%. (Contributed by Irit Katriel in :issue:`45711`.) +* :mod:`re`'s regular expression matching engine has been partially refactored, + and now uses computed gotos (or "threaded code") on supported platforms. As a + result, Python 3.11 executes the `pyperformance regular expression benchmarks + `_ up to 10% + faster than Python 3.10. + (Contributed by Brandt Bucher in :gh:`91404`.) + .. _whatsnew311-faster-cpython-faq: FAQ --- -| Q: How should I write my code to utilize these speedups? -| -| A: You don't have to change your code. Write Pythonic code that follows common - best practices. The Faster CPython project optimizes for common code - patterns we observe. -| -| -| Q: Will CPython 3.11 use more memory? -| -| A: Maybe not. We don't expect memory use to exceed 20% more than 3.10. - This is offset by memory optimizations for frame objects and object - dictionaries as mentioned above. -| -| -| Q: I don't see any speedups in my workload. Why? -| -| A: Certain code won't have noticeable benefits. If your code spends most of - its time on I/O operations, or already does most of its - computation in a C extension library like numpy, there won't be significant - speedup. This project currently benefits pure-Python workloads the most. -| -| Furthermore, the pyperformance figures are a geometric mean. Even within the - pyperformance benchmarks, certain benchmarks have slowed down slightly, while - others have sped up by nearly 2x! -| -| -| Q: Is there a JIT compiler? -| -| A: No. We're still exploring other optimizations. +.. _faster-cpython-faq-my-code: + +How should I write my code to utilize these speedups? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Write Pythonic code that follows common best practices; +you don't have to change your code. +The Faster CPython project optimizes for common code patterns we observe. + + +.. _faster-cpython-faq-memory: + +Will CPython 3.11 use more memory? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Maybe not; we don't expect memory use to exceed 20% higher than 3.10. +This is offset by memory optimizations for frame objects and object +dictionaries as mentioned above. + + +.. _faster-cpython-ymmv: + +I don't see any speedups in my workload. Why? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Certain code won't have noticeable benefits. If your code spends most of +its time on I/O operations, or already does most of its +computation in a C extension library like NumPy, there won't be significant +speedups. This project currently benefits pure-Python workloads the most. + +Furthermore, the pyperformance figures are a geometric mean. Even within the +pyperformance benchmarks, certain benchmarks have slowed down slightly, while +others have sped up by nearly 2x! + + +.. _faster-cpython-jit: + +Is there a JIT compiler? +^^^^^^^^^^^^^^^^^^^^^^^^ + +No. We're still exploring other optimizations. .. _whatsnew311-faster-cpython-about: From d8485d6c8bd7ad8191fde337d81c01f2192855eb Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Tue, 7 Mar 2023 08:44:55 +0530 Subject: [PATCH 09/38] Remove redundant `_ensure_future` in favor of `ensure_future` in `asyncio` (#102398) --- Lib/asyncio/tasks.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index a2e06d5ef72f42..1c20754b839b69 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -630,10 +630,6 @@ def ensure_future(coro_or_future, *, loop=None): If the argument is a Future, it is returned directly. """ - return _ensure_future(coro_or_future, loop=loop) - - -def _ensure_future(coro_or_future, *, loop=None): if futures.isfuture(coro_or_future): if loop is not None and loop is not futures._get_loop(coro_or_future): raise ValueError('The future belongs to a different loop than ' @@ -798,7 +794,7 @@ def _done_callback(fut): outer = None # bpo-46672 for arg in coros_or_futures: if arg not in arg_to_fut: - fut = _ensure_future(arg, loop=loop) + fut = ensure_future(arg, loop=loop) if loop is None: loop = futures._get_loop(fut) if fut is not arg: @@ -855,7 +851,7 @@ def shield(arg): weak references to tasks. A task that isn't referenced elsewhere may get garbage collected at any time, even before it's done. """ - inner = _ensure_future(arg) + inner = ensure_future(arg) if inner.done(): # Shortcut. return inner From 4a3ea1fdd890e5e2ec26540dc3c958a52fba6556 Mon Sep 17 00:00:00 2001 From: "C.A.M. Gerlach" Date: Tue, 7 Mar 2023 08:38:31 -0600 Subject: [PATCH 10/38] gh-95913: Consolidate build requirements changes in 3.11 WhatsNew (GH-98781) Apply suggestion to combine build requirements changes in 3.11 WhatsNew Co-authored-by: Petr Viktorin --- Doc/whatsnew/3.11.rst | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 391ea531281a48..10fcfb6a0b5639 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -2114,30 +2114,22 @@ Build Changes and WASI contributed by Christian Heimes in :gh:`90473`; platforms promoted in :gh:`95085`) -* Building Python now requires: +* Building CPython now requires: - * A `C11 `_ compiler. + * A `C11 `_ compiler and standard library. `Optional C11 features `_ are not required. - (Contributed by Victor Stinner in :issue:`46656`.) + (Contributed by Victor Stinner in :issue:`46656`, + :issue:`45440` and :issue:`46640`.) * Support for `IEEE 754 `_ floating point numbers. (Contributed by Victor Stinner in :issue:`46917`.) - * Support for `floating point Not-a-Number (NaN) - `_, - as the :c:macro:`!Py_NO_NAN` macro has been removed. - (Contributed by Victor Stinner in :issue:`46656`.) - - * A `C99 `_ - ```` header file providing the - :c:func:`!copysign`, :c:func:`!hypot`, :c:func:`!isfinite`, - :c:func:`!isinf`, :c:func:`!isnan`, and :c:func:`!round` functions - (contributed by Victor Stinner in :issue:`45440`); - and a :c:data:`!NAN` constant or the :c:func:`!__builtin_nan` function - (Contributed by Victor Stinner in :issue:`46640`). +* The :c:macro:`!Py_NO_NAN` macro has been removed. + Since CPython now requires IEEE 754 floats, NaN values are always available. + (Contributed by Victor Stinner in :issue:`46656`.) * The :mod:`tkinter` package now requires `Tcl/Tk `_ version 8.5.12 or newer. From 54060ae91da2df44b3f6e6c698694d40284687e9 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Tue, 7 Mar 2023 18:16:32 +0000 Subject: [PATCH 11/38] gh-87092: compiler's CFG construction moved to after codegen stage (#102320) --- Python/compile.c | 520 ++++++++++++++++++++++++++++++----------------- 1 file changed, 336 insertions(+), 184 deletions(-) diff --git a/Python/compile.c b/Python/compile.c index 2e60a8157533ad..45c97b4f8ef0e6 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -175,19 +175,18 @@ static struct jump_target_label_ NO_LABEL = {-1}; #define IS_LABEL(L) (!SAME_LABEL((L), (NO_LABEL))) #define NEW_JUMP_TARGET_LABEL(C, NAME) \ - jump_target_label NAME = cfg_new_label(CFG_BUILDER(C)); \ + jump_target_label NAME = instr_sequence_new_label(INSTR_SEQUENCE(C)); \ if (!IS_LABEL(NAME)) { \ return ERROR; \ } #define USE_LABEL(C, LBL) \ - RETURN_IF_ERROR(cfg_builder_use_label(CFG_BUILDER(C), LBL)) + RETURN_IF_ERROR(instr_sequence_use_label(INSTR_SEQUENCE(C), (LBL).id)) -struct instr { +struct cfg_instr { int i_opcode; int i_oparg; location i_loc; - /* The following fields should not be set by the front-end: */ struct basicblock_ *i_target; /* target block (if jump instruction) */ struct basicblock_ *i_except; /* target block when exception is raised */ }; @@ -196,7 +195,7 @@ struct instr { #define INSTR_SET_OP1(I, OP, ARG) \ do { \ assert(HAS_ARG(OP)); \ - struct instr *_instr__ptr_ = (I); \ + struct cfg_instr *_instr__ptr_ = (I); \ _instr__ptr_->i_opcode = (OP); \ _instr__ptr_->i_oparg = (ARG); \ } while (0); @@ -205,7 +204,7 @@ struct instr { #define INSTR_SET_OP0(I, OP) \ do { \ assert(!HAS_ARG(OP)); \ - struct instr *_instr__ptr_ = (I); \ + struct cfg_instr *_instr__ptr_ = (I); \ _instr__ptr_->i_opcode = (OP); \ _instr__ptr_->i_oparg = 0; \ } while (0); @@ -235,25 +234,25 @@ is_bit_set_in_table(const uint32_t *table, int bitindex) { } static inline int -is_relative_jump(struct instr *i) +is_relative_jump(struct cfg_instr *i) { return is_bit_set_in_table(_PyOpcode_RelativeJump, i->i_opcode); } static inline int -is_block_push(struct instr *i) +is_block_push(struct cfg_instr *i) { return IS_BLOCK_PUSH_OPCODE(i->i_opcode); } static inline int -is_jump(struct instr *i) +is_jump(struct cfg_instr *i) { return IS_JUMP_OPCODE(i->i_opcode); } static int -instr_size(struct instr *instruction) +instr_size(struct cfg_instr *instruction) { int opcode = instruction->i_opcode; assert(!IS_PSEUDO_OPCODE(opcode)); @@ -265,7 +264,7 @@ instr_size(struct instr *instruction) } static void -write_instr(_Py_CODEUNIT *codestr, struct instr *instruction, int ilen) +write_instr(_Py_CODEUNIT *codestr, struct cfg_instr *instruction, int ilen) { int opcode = instruction->i_opcode; assert(!IS_PSEUDO_OPCODE(opcode)); @@ -313,7 +312,7 @@ typedef struct basicblock_ { /* Exception stack at start of block, used by assembler to create the exception handling table */ ExceptStack *b_exceptstack; /* pointer to an array of instructions, initially NULL */ - struct instr *b_instr; + struct cfg_instr *b_instr; /* If b_next is non-NULL, it is a pointer to the next block reached by normal control flow. */ struct basicblock_ *b_next; @@ -342,7 +341,7 @@ typedef struct basicblock_ { } basicblock; -static struct instr * +static struct cfg_instr * basicblock_last_instr(const basicblock *b) { assert(b->b_iused >= 0); if (b->b_iused > 0) { @@ -352,21 +351,15 @@ basicblock_last_instr(const basicblock *b) { return NULL; } -static inline int -basicblock_returns(const basicblock *b) { - struct instr *last = basicblock_last_instr(b); - return last && (last->i_opcode == RETURN_VALUE || last->i_opcode == RETURN_CONST); -} - static inline int basicblock_exits_scope(const basicblock *b) { - struct instr *last = basicblock_last_instr(b); + struct cfg_instr *last = basicblock_last_instr(b); return last && IS_SCOPE_EXIT_OPCODE(last->i_opcode); } static inline int basicblock_nofallthrough(const basicblock *b) { - struct instr *last = basicblock_last_instr(b); + struct cfg_instr *last = basicblock_last_instr(b); return (last && (IS_SCOPE_EXIT_OPCODE(last->i_opcode) || IS_UNCONDITIONAL_JUMP_OPCODE(last->i_opcode))); @@ -415,10 +408,208 @@ typedef struct cfg_builder_ { basicblock *g_curblock; /* label for the next instruction to be placed */ jump_target_label g_current_label; - /* next free label id */ - int g_next_free_label; } cfg_builder; +typedef struct { + int i_opcode; + int i_oparg; + location i_loc; +} instruction; + + +typedef struct instr_sequence_ { + instruction *s_instrs; + int s_allocated; + int s_used; + + int *s_labelmap; /* label id --> instr offset */ + int s_labelmap_size; + int s_next_free_label; /* next free label id */ +} instr_sequence; + +#define INITIAL_INSTR_SEQUENCE_SIZE 100 +#define INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE 10 + +/* + * Resize the array if index is out of range. + * + * idx: the index we want to access + * arr: pointer to the array + * alloc: pointer to the capacity of the array + * default_alloc: initial number of items + * item_size: size of each item + * + */ +static int +ensure_array_large_enough(int idx, void **arr_, int *alloc, int default_alloc, size_t item_size) +{ + void *arr = *arr_; + if (arr == NULL) { + int new_alloc = default_alloc; + if (idx >= new_alloc) { + new_alloc = idx + default_alloc; + } + arr = PyObject_Calloc(new_alloc, item_size); + if (arr == NULL) { + PyErr_NoMemory(); + return ERROR; + } + *alloc = new_alloc; + } + else if (idx >= *alloc) { + size_t oldsize = *alloc * item_size; + int new_alloc = *alloc << 1; + if (idx >= new_alloc) { + new_alloc = idx + default_alloc; + } + size_t newsize = new_alloc * item_size; + + if (oldsize > (SIZE_MAX >> 1)) { + PyErr_NoMemory(); + return ERROR; + } + + assert(newsize > 0); + void *tmp = PyObject_Realloc(arr, newsize); + if (tmp == NULL) { + PyErr_NoMemory(); + return ERROR; + } + *alloc = new_alloc; + arr = tmp; + memset((char *)arr + oldsize, 0, newsize - oldsize); + } + + *arr_ = arr; + return SUCCESS; +} + +static int +instr_sequence_next_inst(instr_sequence *seq) { + assert(seq->s_instrs != NULL || seq->s_used == 0); + + RETURN_IF_ERROR( + ensure_array_large_enough(seq->s_used + 1, + (void**)&seq->s_instrs, + &seq->s_allocated, + INITIAL_INSTR_SEQUENCE_SIZE, + sizeof(instruction))); + assert(seq->s_used < seq->s_allocated); + return seq->s_used++; +} + +static jump_target_label +instr_sequence_new_label(instr_sequence *seq) +{ + jump_target_label lbl = {seq->s_next_free_label++}; + return lbl; +} + +static int +instr_sequence_use_label(instr_sequence *seq, int lbl) { + int old_size = seq->s_labelmap_size; + RETURN_IF_ERROR( + ensure_array_large_enough(lbl, + (void**)&seq->s_labelmap, + &seq->s_labelmap_size, + INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE, + sizeof(int))); + + for(int i = old_size; i < seq->s_labelmap_size; i++) { + seq->s_labelmap[i] = -111; /* something weird, for debugging */ + } + seq->s_labelmap[lbl] = seq->s_used; /* label refers to the next instruction */ + return SUCCESS; +} + +static int +instr_sequence_addop(instr_sequence *seq, int opcode, int oparg, location loc) +{ + assert(IS_WITHIN_OPCODE_RANGE(opcode)); + assert(!IS_ASSEMBLER_OPCODE(opcode)); + assert(HAS_ARG(opcode) || HAS_TARGET(opcode) || oparg == 0); + assert(0 <= oparg && oparg < (1 << 30)); + + int idx = instr_sequence_next_inst(seq); + RETURN_IF_ERROR(idx); + instruction *ci = &seq->s_instrs[idx]; + ci->i_opcode = opcode; + ci->i_oparg = oparg; + ci->i_loc = loc; + return SUCCESS; +} + +static int +instr_sequence_insert_instruction(instr_sequence *seq, int pos, + int opcode, int oparg, location loc) +{ + assert(pos >= 0 && pos <= seq->s_used); + int last_idx = instr_sequence_next_inst(seq); + RETURN_IF_ERROR(last_idx); + for (int i=last_idx-1; i >= pos; i--) { + seq->s_instrs[i+1] = seq->s_instrs[i]; + } + instruction *ci = &seq->s_instrs[pos]; + ci->i_opcode = opcode; + ci->i_oparg = oparg; + ci->i_loc = loc; + + /* fix the labels map */ + for(int lbl=0; lbl < seq->s_labelmap_size; lbl++) { + if (seq->s_labelmap[lbl] >= pos) { + seq->s_labelmap[lbl]++; + } + } + return SUCCESS; +} + +static void +instr_sequence_fini(instr_sequence *seq) { + PyObject_Free(seq->s_labelmap); + seq->s_labelmap = NULL; + + PyObject_Free(seq->s_instrs); + seq->s_instrs = NULL; +} + +static int basicblock_addop(basicblock *b, int opcode, int oparg, location loc); +static int cfg_builder_maybe_start_new_block(cfg_builder *g); + +static int +cfg_builder_use_label(cfg_builder *g, jump_target_label lbl) +{ + g->g_current_label = lbl; + return cfg_builder_maybe_start_new_block(g); +} + +static int +cfg_builder_addop(cfg_builder *g, int opcode, int oparg, location loc) +{ + RETURN_IF_ERROR(cfg_builder_maybe_start_new_block(g)); + return basicblock_addop(g->g_curblock, opcode, oparg, loc); +} + +static int cfg_builder_init(cfg_builder *g); + +static int +instr_sequence_to_cfg(instr_sequence *seq, cfg_builder *g) { + memset(g, 0, sizeof(cfg_builder)); + RETURN_IF_ERROR(cfg_builder_init(g)); + /* Note: there can be more than one label for the same offset */ + for (int i = 0; i < seq->s_used; i++) { + for (int j=0; j < seq->s_labelmap_size; j++) { + if (seq->s_labelmap[j] == i) { + jump_target_label lbl = {j}; + RETURN_IF_ERROR(cfg_builder_use_label(g, lbl)); + } + } + instruction *instr = &seq->s_instrs[i]; + RETURN_IF_ERROR(cfg_builder_addop(g, instr->i_opcode, instr->i_oparg, instr->i_loc)); + } + return SUCCESS; +} + + /* The following items change on entry and exit of code blocks. They must be saved and restored when returning to a block. */ @@ -445,7 +636,7 @@ struct compiler_unit { Py_ssize_t u_posonlyargcount; /* number of positional only arguments for block */ Py_ssize_t u_kwonlyargcount; /* number of keyword only arguments for block */ - cfg_builder u_cfg_builder; /* The control flow graph */ + instr_sequence u_instr_sequence; /* codegen output */ int u_nfblocks; struct fblockinfo u_fblock[CO_MAXBLOCKS]; @@ -481,7 +672,7 @@ struct compiler { PyArena *c_arena; /* pointer to memory allocation arena */ }; -#define CFG_BUILDER(C) (&((C)->u->u_cfg_builder)) +#define INSTR_SEQUENCE(C) (&((C)->u->u_instr_sequence)) typedef struct { @@ -511,9 +702,7 @@ typedef struct { static int basicblock_next_instr(basicblock *); -static basicblock *cfg_builder_new_block(cfg_builder *g); -static int cfg_builder_maybe_start_new_block(cfg_builder *g); -static int cfg_builder_addop_i(cfg_builder *g, int opcode, Py_ssize_t oparg, location loc); +static int codegen_addop_i(instr_sequence *seq, int opcode, Py_ssize_t oparg, location loc); static void compiler_free(struct compiler *); static int compiler_error(struct compiler *, location loc, const char *, ...); @@ -744,7 +933,8 @@ dictbytype(PyObject *src, int scope_type, int flag, Py_ssize_t offset) return dest; } -static void +#ifndef NDEBUG +static bool cfg_builder_check(cfg_builder *g) { for (basicblock *block = g->g_block_list; block != NULL; block = block->b_list) { @@ -759,7 +949,11 @@ cfg_builder_check(cfg_builder *g) assert (block->b_ialloc == 0); } } + return true; } +#endif + +static basicblock *cfg_builder_new_block(cfg_builder *g); static int cfg_builder_init(cfg_builder *g) @@ -777,7 +971,7 @@ cfg_builder_init(cfg_builder *g) static void cfg_builder_fini(cfg_builder* g) { - cfg_builder_check(g); + assert(cfg_builder_check(g)); basicblock *b = g->g_block_list; while (b != NULL) { if (b->b_instr) { @@ -792,7 +986,7 @@ cfg_builder_fini(cfg_builder* g) static void compiler_unit_free(struct compiler_unit *u) { - cfg_builder_fini(&u->u_cfg_builder); + instr_sequence_fini(&u->u_instr_sequence); Py_CLEAR(u->u_ste); Py_CLEAR(u->u_name); Py_CLEAR(u->u_qualname); @@ -878,13 +1072,6 @@ compiler_set_qualname(struct compiler *c) return SUCCESS; } -static jump_target_label -cfg_new_label(cfg_builder *g) -{ - jump_target_label lbl = {g->g_next_free_label++}; - return lbl; -} - /* Allocate a new block and return a pointer to it. Returns NULL on error. */ @@ -912,13 +1099,6 @@ cfg_builder_use_next_block(cfg_builder *g, basicblock *block) return block; } -static int -cfg_builder_use_label(cfg_builder *g, jump_target_label lbl) -{ - g->g_current_label = lbl; - return cfg_builder_maybe_start_new_block(g); -} - static inline int basicblock_append_instructions(basicblock *target, basicblock *source) { @@ -958,40 +1138,15 @@ static int basicblock_next_instr(basicblock *b) { assert(b != NULL); - if (b->b_instr == NULL) { - b->b_instr = (struct instr *)PyObject_Calloc( - DEFAULT_BLOCK_SIZE, sizeof(struct instr)); - if (b->b_instr == NULL) { - PyErr_NoMemory(); - return ERROR; - } - b->b_ialloc = DEFAULT_BLOCK_SIZE; - } - else if (b->b_iused == b->b_ialloc) { - struct instr *tmp; - size_t oldsize, newsize; - oldsize = b->b_ialloc * sizeof(struct instr); - newsize = oldsize << 1; - if (oldsize > (SIZE_MAX >> 1)) { - PyErr_NoMemory(); - return ERROR; - } + RETURN_IF_ERROR( + ensure_array_large_enough( + b->b_iused + 1, + (void**)&b->b_instr, + &b->b_ialloc, + DEFAULT_BLOCK_SIZE, + sizeof(struct cfg_instr))); - if (newsize == 0) { - PyErr_NoMemory(); - return ERROR; - } - b->b_ialloc <<= 1; - tmp = (struct instr *)PyObject_Realloc( - (void *)b->b_instr, newsize); - if (tmp == NULL) { - PyErr_NoMemory(); - return ERROR; - } - b->b_instr = tmp; - memset((char *)b->b_instr + oldsize, 0, newsize - oldsize); - } return b->b_iused++; } @@ -1100,7 +1255,7 @@ basicblock_addop(basicblock *b, int opcode, int oparg, location loc) if (off < 0) { return ERROR; } - struct instr *i = &b->b_instr[off]; + struct cfg_instr *i = &b->b_instr[off]; i->i_opcode = opcode; i->i_oparg = oparg; i->i_target = NULL; @@ -1115,7 +1270,7 @@ cfg_builder_current_block_is_terminated(cfg_builder *g) if (IS_LABEL(g->g_current_label)) { return true; } - struct instr *last = basicblock_last_instr(g->g_curblock); + struct cfg_instr *last = basicblock_last_instr(g->g_curblock); return last && IS_TERMINATOR_OPCODE(last->i_opcode); } @@ -1135,17 +1290,10 @@ cfg_builder_maybe_start_new_block(cfg_builder *g) } static int -cfg_builder_addop(cfg_builder *g, int opcode, int oparg, location loc) -{ - RETURN_IF_ERROR(cfg_builder_maybe_start_new_block(g)); - return basicblock_addop(g->g_curblock, opcode, oparg, loc); -} - -static int -cfg_builder_addop_noarg(cfg_builder *g, int opcode, location loc) +codegen_addop_noarg(instr_sequence *seq, int opcode, location loc) { assert(!HAS_ARG(opcode)); - return cfg_builder_addop(g, opcode, 0, loc); + return instr_sequence_addop(seq, opcode, 0, loc); } static Py_ssize_t @@ -1303,7 +1451,7 @@ compiler_addop_load_const(struct compiler *c, location loc, PyObject *o) if (arg < 0) { return ERROR; } - return cfg_builder_addop_i(CFG_BUILDER(c), LOAD_CONST, arg, loc); + return codegen_addop_i(INSTR_SEQUENCE(c), LOAD_CONST, arg, loc); } static int @@ -1314,7 +1462,7 @@ compiler_addop_o(struct compiler *c, location loc, if (arg < 0) { return ERROR; } - return cfg_builder_addop_i(CFG_BUILDER(c), opcode, arg, loc); + return codegen_addop_i(INSTR_SEQUENCE(c), opcode, arg, loc); } static int @@ -1340,12 +1488,12 @@ compiler_addop_name(struct compiler *c, location loc, arg <<= 1; arg |= 1; } - return cfg_builder_addop_i(CFG_BUILDER(c), opcode, arg, loc); + return codegen_addop_i(INSTR_SEQUENCE(c), opcode, arg, loc); } /* Add an opcode with an integer argument */ static int -cfg_builder_addop_i(cfg_builder *g, int opcode, Py_ssize_t oparg, location loc) +codegen_addop_i(instr_sequence *seq, int opcode, Py_ssize_t oparg, location loc) { /* oparg value is unsigned, but a signed C int is usually used to store it in the C code (like Python/ceval.c). @@ -1356,23 +1504,23 @@ cfg_builder_addop_i(cfg_builder *g, int opcode, Py_ssize_t oparg, location loc) EXTENDED_ARG is used for 16, 24, and 32-bit arguments. */ int oparg_ = Py_SAFE_DOWNCAST(oparg, Py_ssize_t, int); - return cfg_builder_addop(g, opcode, oparg_, loc); + return instr_sequence_addop(seq, opcode, oparg_, loc); } static int -cfg_builder_addop_j(cfg_builder *g, location loc, +codegen_addop_j(instr_sequence *seq, location loc, int opcode, jump_target_label target) { assert(IS_LABEL(target)); assert(IS_JUMP_OPCODE(opcode) || IS_BLOCK_PUSH_OPCODE(opcode)); - return cfg_builder_addop(g, opcode, target.id, loc); + return instr_sequence_addop(seq, opcode, target.id, loc); } #define ADDOP(C, LOC, OP) \ - RETURN_IF_ERROR(cfg_builder_addop_noarg(CFG_BUILDER(C), (OP), (LOC))) + RETURN_IF_ERROR(codegen_addop_noarg(INSTR_SEQUENCE(C), (OP), (LOC))) #define ADDOP_IN_SCOPE(C, LOC, OP) { \ - if (cfg_builder_addop_noarg(CFG_BUILDER(C), (OP), (LOC)) < 0) { \ + if (codegen_addop_noarg(INSTR_SEQUENCE(C), (OP), (LOC)) < 0) { \ compiler_exit_scope(C); \ return ERROR; \ } \ @@ -1407,10 +1555,10 @@ cfg_builder_addop_j(cfg_builder *g, location loc, RETURN_IF_ERROR(compiler_addop_name((C), (LOC), (OP), (C)->u->u_ ## TYPE, (O))) #define ADDOP_I(C, LOC, OP, O) \ - RETURN_IF_ERROR(cfg_builder_addop_i(CFG_BUILDER(C), (OP), (O), (LOC))) + RETURN_IF_ERROR(codegen_addop_i(INSTR_SEQUENCE(C), (OP), (O), (LOC))) #define ADDOP_JUMP(C, LOC, OP, O) \ - RETURN_IF_ERROR(cfg_builder_addop_j(CFG_BUILDER(C), (LOC), (OP), (O))) + RETURN_IF_ERROR(codegen_addop_j(INSTR_SEQUENCE(C), (LOC), (OP), (O))) #define ADDOP_COMPARE(C, LOC, CMP) \ RETURN_IF_ERROR(compiler_addcompare((C), (LOC), (cmpop_ty)(CMP))) @@ -1546,9 +1694,6 @@ compiler_enter_scope(struct compiler *c, identifier name, c->c_nestlevel++; - cfg_builder *g = CFG_BUILDER(c); - RETURN_IF_ERROR(cfg_builder_init(g)); - if (u->u_scope_type == COMPILER_SCOPE_MODULE) { loc.lineno = 0; } @@ -1582,7 +1727,6 @@ compiler_exit_scope(struct compiler *c) _PyErr_WriteUnraisableMsg("on removing the last compiler " "stack item", NULL); } - cfg_builder_check(CFG_BUILDER(c)); } else { c->u = NULL; @@ -2321,7 +2465,7 @@ compiler_check_debug_args(struct compiler *c, arguments_ty args) } static inline int -insert_instruction(basicblock *block, int pos, struct instr *instr) { +insert_instruction(basicblock *block, int pos, struct cfg_instr *instr) { RETURN_IF_ERROR(basicblock_next_instr(block)); for (int i = block->b_iused - 1; i > pos; i--) { block->b_instr[i] = block->b_instr[i-1]; @@ -2336,14 +2480,10 @@ wrap_in_stopiteration_handler(struct compiler *c) NEW_JUMP_TARGET_LABEL(c, handler); /* Insert SETUP_CLEANUP at start */ - struct instr setup = { - .i_opcode = SETUP_CLEANUP, - .i_oparg = handler.id, - .i_loc = NO_LOCATION, - .i_target = NULL, - }; RETURN_IF_ERROR( - insert_instruction(c->u->u_cfg_builder.g_entryblock, 0, &setup)); + instr_sequence_insert_instruction( + INSTR_SEQUENCE(c), 0, + SETUP_CLEANUP, handler.id, NO_LOCATION)); ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); ADDOP(c, NO_LOCATION, RETURN_VALUE); @@ -4032,7 +4172,7 @@ compiler_nameop(struct compiler *c, location loc, if (op == LOAD_GLOBAL) { arg <<= 1; } - return cfg_builder_addop_i(CFG_BUILDER(c), op, arg, loc); + return codegen_addop_i(INSTR_SEQUENCE(c), op, arg, loc); } static int @@ -6051,7 +6191,7 @@ emit_and_reset_fail_pop(struct compiler *c, location loc, } while (--pc->fail_pop_size) { USE_LABEL(c, pc->fail_pop[pc->fail_pop_size]); - if (cfg_builder_addop_noarg(CFG_BUILDER(c), POP_TOP, loc) < 0) { + if (codegen_addop_noarg(INSTR_SEQUENCE(c), POP_TOP, loc) < 0) { pc->fail_pop_size = 0; PyObject_Free(pc->fail_pop); pc->fail_pop = NULL; @@ -6491,7 +6631,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) pc->fail_pop = NULL; pc->fail_pop_size = 0; pc->on_top = 0; - if (cfg_builder_addop_i(CFG_BUILDER(c), COPY, 1, LOC(alt)) < 0 || + if (codegen_addop_i(INSTR_SEQUENCE(c), COPY, 1, LOC(alt)) < 0 || compiler_pattern(c, alt, pc) < 0) { goto error; } @@ -6554,7 +6694,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) } } assert(control); - if (cfg_builder_addop_j(CFG_BUILDER(c), LOC(alt), JUMP, end) < 0 || + if (codegen_addop_j(INSTR_SEQUENCE(c), LOC(alt), JUMP, end) < 0 || emit_and_reset_fail_pop(c, LOC(alt), pc) < 0) { goto error; @@ -6566,7 +6706,7 @@ compiler_pattern_or(struct compiler *c, pattern_ty p, pattern_context *pc) // Need to NULL this for the PyObject_Free call in the error block. old_pc.fail_pop = NULL; // No match. Pop the remaining copy of the subject and fail: - if (cfg_builder_addop_noarg(CFG_BUILDER(c), POP_TOP, LOC(p)) < 0 || + if (codegen_addop_noarg(INSTR_SEQUENCE(c), POP_TOP, LOC(p)) < 0 || jump_to_fail_pop(c, LOC(p), pc, JUMP) < 0) { goto error; } @@ -6884,7 +7024,7 @@ stackdepth(basicblock *entryblock, int code_flags) assert(depth >= 0); basicblock *next = b->b_next; for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; int effect = stack_effect(instr->i_opcode, instr->i_oparg, 0); if (effect == PY_INVALID_STACK_EFFECT) { PyErr_Format(PyExc_SystemError, @@ -6973,7 +7113,7 @@ blocksize(basicblock *b) } static basicblock * -push_except_block(ExceptStack *stack, struct instr *setup) { +push_except_block(ExceptStack *stack, struct cfg_instr *setup) { assert(is_block_push(setup)); int opcode = setup->i_opcode; basicblock * target = setup->i_target; @@ -7045,7 +7185,7 @@ label_exception_targets(basicblock *entryblock) { b->b_exceptstack = NULL; handler = except_stack_top(except_stack); for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; if (is_block_push(instr)) { if (!instr->i_target->b_visited) { ExceptStack *copy = copy_except_stack(except_stack); @@ -7123,7 +7263,7 @@ mark_except_handlers(basicblock *entryblock) { #endif for (basicblock *b = entryblock; b != NULL; b = b->b_next) { for (int i=0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; if (is_block_push(instr)) { instr->i_target->b_except_handler = 1; } @@ -7152,7 +7292,7 @@ mark_warm(basicblock *entryblock) { next->b_visited = 1; } for (int i=0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; if (is_jump(instr) && !instr->i_target->b_visited) { *sp++ = instr->i_target; instr->i_target->b_visited = 1; @@ -7197,7 +7337,7 @@ mark_cold(basicblock *entryblock) { } } for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; if (is_jump(instr)) { assert(i == b->b_iused - 1); basicblock *target = b->b_instr[i].i_target; @@ -7238,7 +7378,7 @@ push_cold_blocks_to_end(cfg_builder *g, int code_flags) { b->b_next = explicit_jump; /* set target */ - struct instr *last = basicblock_last_instr(explicit_jump); + struct cfg_instr *last = basicblock_last_instr(explicit_jump); last->i_target = explicit_jump->b_next; } } @@ -7293,7 +7433,7 @@ static void convert_exception_handlers_to_nops(basicblock *entryblock) { for (basicblock *b = entryblock; b != NULL; b = b->b_next) { for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; if (is_block_push(instr) || instr->i_opcode == POP_BLOCK) { INSTR_SET_OP0(instr, NOP); } @@ -7372,7 +7512,7 @@ assemble_exception_table(struct assembler *a, basicblock *entryblock) for (b = entryblock; b != NULL; b = b->b_next) { ioffset = b->b_offset; for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; if (instr->i_except != handler) { if (handler != NULL) { RETURN_IF_ERROR( @@ -7542,7 +7682,7 @@ assemble_emit_location(struct assembler* a, location loc, int isize) */ static int -assemble_emit(struct assembler *a, struct instr *i) +assemble_emit(struct assembler *a, struct cfg_instr *i) { Py_ssize_t len = PyBytes_GET_SIZE(a->a_bytecode); _Py_CODEUNIT *code; @@ -7562,7 +7702,7 @@ assemble_emit(struct assembler *a, struct instr *i) static int normalize_jumps_in_block(cfg_builder *g, basicblock *b) { - struct instr *last = basicblock_last_instr(b); + struct cfg_instr *last = basicblock_last_instr(b); if (last == NULL || !is_jump(last)) { return SUCCESS; } @@ -7663,7 +7803,7 @@ assemble_jump_offsets(basicblock *entryblock) for (basicblock *b = entryblock; b != NULL; b = b->b_next) { bsize = b->b_offset; for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; int isize = instr_size(instr); /* Relative jumps are computed relative to the instruction pointer after fetching @@ -7735,7 +7875,7 @@ scan_block_for_locals(basicblock *b, basicblock ***sp) // bit i is set if local i is potentially uninitialized uint64_t unsafe_mask = b->b_unsafe_locals_mask; for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; assert(instr->i_opcode != EXTENDED_ARG); assert(!IS_SUPERINSTRUCTION_OPCODE(instr->i_opcode)); if (instr->i_except != NULL) { @@ -7768,7 +7908,7 @@ scan_block_for_locals(basicblock *b, basicblock ***sp) if (b->b_next && BB_HAS_FALLTHROUGH(b)) { maybe_push(b->b_next, unsafe_mask, sp); } - struct instr *last = basicblock_last_instr(b); + struct cfg_instr *last = basicblock_last_instr(b); if (last && is_jump(last)) { assert(last->i_target != NULL); maybe_push(last->i_target, unsafe_mask, sp); @@ -7791,7 +7931,7 @@ fast_scan_many_locals(basicblock *entryblock, int nlocals) for (basicblock *b = entryblock; b != NULL; b = b->b_next) { blocknum++; for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; assert(instr->i_opcode != EXTENDED_ARG); assert(!IS_SUPERINSTRUCTION_OPCODE(instr->i_opcode)); int arg = instr->i_oparg; @@ -8035,7 +8175,6 @@ makecode(struct compiler *c, struct assembler *a, PyObject *constslist, PyObject *consts = NULL; PyObject *localsplusnames = NULL; PyObject *localspluskinds = NULL; - names = dict_keys_inorder(c->u->u_names, 0); if (!names) { goto error; @@ -8121,7 +8260,7 @@ makecode(struct compiler *c, struct assembler *a, PyObject *constslist, /* For debugging purposes only */ #if 0 static void -dump_instr(struct instr *i) +dump_instr(struct cfg_instr *i) { const char *jrel = (is_relative_jump(i)) ? "jrel " : ""; const char *jabs = (is_jump(i) && !is_relative_jump(i))? "jabs " : ""; @@ -8139,6 +8278,12 @@ dump_instr(struct instr *i) i->i_loc.lineno, i->i_opcode, arg, jabs, jrel); } +static inline int +basicblock_returns(const basicblock *b) { + struct cfg_instr *last = basicblock_last_instr(b); + return last && (last->i_opcode == RETURN_VALUE || last->i_opcode == RETURN_CONST); +} + static void dump_basicblock(const basicblock *b) { @@ -8211,14 +8356,14 @@ insert_prefix_instructions(struct compiler *c, basicblock *entryblock, /* Add the generator prefix instructions. */ if (code_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) { - struct instr make_gen = { + struct cfg_instr make_gen = { .i_opcode = RETURN_GENERATOR, .i_oparg = 0, .i_loc = LOCATION(c->u->u_firstlineno, c->u->u_firstlineno, -1, -1), .i_target = NULL, }; RETURN_IF_ERROR(insert_instruction(entryblock, 0, &make_gen)); - struct instr pop_top = { + struct cfg_instr pop_top = { .i_opcode = POP_TOP, .i_oparg = 0, .i_loc = NO_LOCATION, @@ -8247,7 +8392,7 @@ insert_prefix_instructions(struct compiler *c, basicblock *entryblock, if (oldindex == -1) { continue; } - struct instr make_cell = { + struct cfg_instr make_cell = { .i_opcode = MAKE_CELL, // This will get fixed in offset_derefs(). .i_oparg = oldindex, @@ -8261,7 +8406,7 @@ insert_prefix_instructions(struct compiler *c, basicblock *entryblock, } if (nfreevars) { - struct instr copy_frees = { + struct cfg_instr copy_frees = { .i_opcode = COPY_FREE_VARS, .i_oparg = nfreevars, .i_loc = NO_LOCATION, @@ -8282,7 +8427,7 @@ guarantee_lineno_for_exits(basicblock *entryblock, int firstlineno) { int lineno = firstlineno; assert(lineno > 0); for (basicblock *b = entryblock; b != NULL; b = b->b_next) { - struct instr *last = basicblock_last_instr(b); + struct cfg_instr *last = basicblock_last_instr(b); if (last == NULL) { continue; } @@ -8324,7 +8469,7 @@ fix_cell_offsets(struct compiler *c, basicblock *entryblock, int *fixedmap) // Then update offsets, either relative to locals or by cell2arg. for (basicblock *b = entryblock; b != NULL; b = b->b_next) { for (int i = 0; i < b->b_iused; i++) { - struct instr *inst = &b->b_instr[i]; + struct cfg_instr *inst = &b->b_instr[i]; // This is called before extended args are generated. assert(inst->i_opcode != EXTENDED_ARG); int oldoffset = inst->i_oparg; @@ -8364,7 +8509,7 @@ no_redundant_nops(cfg_builder *g) { static bool no_redundant_jumps(cfg_builder *g) { for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - struct instr *last = basicblock_last_instr(b); + struct cfg_instr *last = basicblock_last_instr(b); if (last != NULL) { if (IS_UNCONDITIONAL_JUMP_OPCODE(last->i_opcode)) { assert(last->i_target != b->b_next); @@ -8382,7 +8527,7 @@ opcode_metadata_is_sane(cfg_builder *g) { bool result = true; for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; int opcode = instr->i_opcode; int oparg = instr->i_oparg; assert(opcode <= MAX_REAL_OPCODE); @@ -8426,7 +8571,7 @@ remove_redundant_jumps(cfg_builder *g) { */ assert(no_empty_basic_blocks(g)); for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - struct instr *last = basicblock_last_instr(b); + struct cfg_instr *last = basicblock_last_instr(b); assert(last != NULL); assert(!IS_ASSEMBLER_OPCODE(last->i_opcode)); if (IS_UNCONDITIONAL_JUMP_OPCODE(last->i_opcode)) { @@ -8444,7 +8589,7 @@ remove_redundant_jumps(cfg_builder *g) { } static int -prepare_localsplus(struct compiler* c, int code_flags) +prepare_localsplus(struct compiler* c, cfg_builder *g, int code_flags) { assert(PyDict_GET_SIZE(c->u->u_varnames) < INT_MAX); assert(PyDict_GET_SIZE(c->u->u_cellvars) < INT_MAX); @@ -8460,7 +8605,6 @@ prepare_localsplus(struct compiler* c, int code_flags) return ERROR; } - cfg_builder* g = CFG_BUILDER(c); // This must be called before fix_cell_offsets(). if (insert_prefix_instructions(c, g->g_entryblock, cellfixedoffsets, nfreevars, code_flags)) { @@ -8474,20 +8618,21 @@ prepare_localsplus(struct compiler* c, int code_flags) if (numdropped < 0) { return ERROR; } + nlocalsplus -= numdropped; return nlocalsplus; } static int -add_return_at_end_of_block(struct compiler *c, int addNone) +add_return_at_end(struct compiler *c, int addNone) { - /* Make sure every block that falls off the end returns None. */ - if (!basicblock_returns(CFG_BUILDER(c)->g_curblock)) { - if (addNone) { - ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); - } - ADDOP(c, NO_LOCATION, RETURN_VALUE); + /* Make sure every instruction stream that falls off the end returns None. + * This also ensures that no jump target offsets are out of bounds. + */ + if (addNone) { + ADDOP_LOAD_CONST(c, NO_LOCATION, Py_None); } + ADDOP(c, NO_LOCATION, RETURN_VALUE); return SUCCESS; } @@ -8496,6 +8641,8 @@ assemble(struct compiler *c, int addNone) { PyCodeObject *co = NULL; PyObject *consts = NULL; + cfg_builder g_; + cfg_builder *g = &g_; struct assembler a; memset(&a, 0, sizeof(struct assembler)); @@ -8504,12 +8651,18 @@ assemble(struct compiler *c, int addNone) return NULL; } - if (add_return_at_end_of_block(c, addNone) < 0) { + if (add_return_at_end(c, addNone) < 0) { return NULL; } + /** Preprocessing **/ + if (instr_sequence_to_cfg(INSTR_SEQUENCE(c), g) < 0) { + goto error; + } + assert(cfg_builder_check(g)); + int nblocks = 0; - for (basicblock *b = CFG_BUILDER(c)->g_block_list; b != NULL; b = b->b_list) { + for (basicblock *b = g->g_block_list; b != NULL; b = b->b_list) { nblocks++; } if ((size_t)nblocks > SIZE_MAX / sizeof(basicblock *)) { @@ -8517,9 +8670,6 @@ assemble(struct compiler *c, int addNone) goto error; } - cfg_builder *g = CFG_BUILDER(c); - assert(g->g_entryblock != NULL); - /* Set firstlineno if it wasn't explicitly set. */ if (!c->u->u_firstlineno) { if (g->g_entryblock->b_instr && g->g_entryblock->b_instr->i_loc.lineno) { @@ -8530,9 +8680,8 @@ assemble(struct compiler *c, int addNone) } } - /** Preprocessing **/ /* Map labels to targets and mark exception handlers */ - if (translate_jump_labels_to_targets(g->g_entryblock)) { + if (translate_jump_labels_to_targets(g->g_entryblock) < 0) { goto error; } if (mark_except_handlers(g->g_entryblock) < 0) { @@ -8550,10 +8699,10 @@ assemble(struct compiler *c, int addNone) if (optimize_cfg(g, consts, c->c_const_cache)) { goto error; } - if (add_checks_for_loads_of_uninitialized_variables(g->g_entryblock, c) < 0) { + if (remove_unused_consts(g->g_entryblock, consts) < 0) { goto error; } - if (remove_unused_consts(g->g_entryblock, consts) < 0) { + if (add_checks_for_loads_of_uninitialized_variables(g->g_entryblock, c) < 0) { goto error; } @@ -8570,7 +8719,7 @@ assemble(struct compiler *c, int addNone) /** Assembly **/ - int nlocalsplus = prepare_localsplus(c, code_flags); + int nlocalsplus = prepare_localsplus(c, g, code_flags); if (nlocalsplus < 0) { goto error; } @@ -8587,7 +8736,6 @@ assemble(struct compiler *c, int addNone) if (normalize_jumps(g) < 0) { goto error; } - assert(no_redundant_jumps(g)); assert(opcode_metadata_is_sane(g)); @@ -8655,6 +8803,7 @@ assemble(struct compiler *c, int addNone) co = makecode(c, &a, consts, maxdepth, nlocalsplus, code_flags); error: Py_XDECREF(consts); + cfg_builder_fini(g); assemble_free(&a); return co; } @@ -8684,7 +8833,7 @@ get_const_value(int opcode, int oparg, PyObject *co_consts) */ static int fold_tuple_on_constants(PyObject *const_cache, - struct instr *inst, + struct cfg_instr *inst, int n, PyObject *consts) { /* Pre-conditions */ @@ -8753,7 +8902,7 @@ swaptimize(basicblock *block, int *ix) // NOTE: "./python -m test test_patma" serves as a good, quick stress test // for this function. Make sure to blow away cached *.pyc files first! assert(*ix < block->b_iused); - struct instr *instructions = &block->b_instr[*ix]; + struct cfg_instr *instructions = &block->b_instr[*ix]; // Find the length of the current sequence of SWAPs and NOPs, and record the // maximum depth of the stack manipulations: assert(instructions[0].i_opcode == SWAP); @@ -8854,7 +9003,7 @@ static int next_swappable_instruction(basicblock *block, int i, int lineno) { while (++i < block->b_iused) { - struct instr *instruction = &block->b_instr[i]; + struct cfg_instr *instruction = &block->b_instr[i]; if (0 <= lineno && instruction->i_loc.lineno != lineno) { // Optimizing across this instruction could cause user-visible // changes in the names bound between line tracing events! @@ -8880,7 +9029,7 @@ apply_static_swaps(basicblock *block, int i) // SWAPs are to our left, and potential swaperands are to our right: for (; 0 <= i; i--) { assert(i < block->b_iused); - struct instr *swap = &block->b_instr[i]; + struct cfg_instr *swap = &block->b_instr[i]; if (swap->i_opcode != SWAP) { if (swap->i_opcode == NOP || SWAPPABLE(swap->i_opcode)) { // Nope, but we know how to handle these. Keep looking: @@ -8903,7 +9052,7 @@ apply_static_swaps(basicblock *block, int i) } // Success! INSTR_SET_OP0(swap, NOP); - struct instr temp = block->b_instr[j]; + struct cfg_instr temp = block->b_instr[j]; block->b_instr[j] = block->b_instr[k]; block->b_instr[k] = temp; } @@ -8913,7 +9062,7 @@ apply_static_swaps(basicblock *block, int i) // target->i_target using the provided opcode. Return whether or not the // optimization was successful. static bool -jump_thread(struct instr *inst, struct instr *target, int opcode) +jump_thread(struct cfg_instr *inst, struct cfg_instr *target, int opcode) { assert(is_jump(inst)); assert(is_jump(target)); @@ -8938,11 +9087,11 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) { assert(PyDict_CheckExact(const_cache)); assert(PyList_CheckExact(consts)); - struct instr nop; + struct cfg_instr nop; INSTR_SET_OP0(&nop, NOP); - struct instr *target; + struct cfg_instr *target; for (int i = 0; i < bb->b_iused; i++) { - struct instr *inst = &bb->b_instr[i]; + struct cfg_instr *inst = &bb->b_instr[i]; int oparg = inst->i_oparg; int nextop = i+1 < bb->b_iused ? bb->b_instr[i+1].i_opcode : 0; if (HAS_TARGET(inst->i_opcode)) { @@ -9179,7 +9328,7 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) */ static int inline_small_exit_blocks(basicblock *bb) { - struct instr *last = basicblock_last_instr(bb); + struct cfg_instr *last = basicblock_last_instr(bb); if (last == NULL) { return 0; } @@ -9287,7 +9436,7 @@ mark_reachable(basicblock *entryblock) { } for (int i = 0; i < b->b_iused; i++) { basicblock *target; - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; if (is_jump(instr) || is_block_push(instr)) { target = instr->i_target; if (!target->b_visited) { @@ -9318,7 +9467,7 @@ eliminate_empty_basic_blocks(cfg_builder *g) { for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { assert(b->b_iused > 0); for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; if (HAS_TARGET(instr->i_opcode)) { basicblock *target = instr->i_target; while (target->b_iused == 0) { @@ -9342,7 +9491,7 @@ eliminate_empty_basic_blocks(cfg_builder *g) { static void propagate_line_numbers(basicblock *entryblock) { for (basicblock *b = entryblock; b != NULL; b = b->b_next) { - struct instr *last = basicblock_last_instr(b); + struct cfg_instr *last = basicblock_last_instr(b); if (last == NULL) { continue; } @@ -9398,7 +9547,7 @@ translate_jump_labels_to_targets(basicblock *entryblock) } for (basicblock *b = entryblock; b != NULL; b = b->b_next) { for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; assert(instr->i_target == NULL); if (HAS_TARGET(instr->i_opcode)) { int lbl = instr->i_oparg; @@ -9584,7 +9733,7 @@ duplicate_exits_without_lineno(cfg_builder *g) */ basicblock *entryblock = g->g_entryblock; for (basicblock *b = entryblock; b != NULL; b = b->b_next) { - struct instr *last = basicblock_last_instr(b); + struct cfg_instr *last = basicblock_last_instr(b); assert(last != NULL); if (is_jump(last)) { basicblock *target = last->i_target; @@ -9608,7 +9757,7 @@ duplicate_exits_without_lineno(cfg_builder *g) for (basicblock *b = entryblock; b != NULL; b = b->b_next) { if (BB_HAS_FALLTHROUGH(b) && b->b_next && b->b_iused > 0) { if (is_exit_without_lineno(b->b_next)) { - struct instr *last = basicblock_last_instr(b); + struct cfg_instr *last = basicblock_last_instr(b); assert(last != NULL); b->b_next->b_instr[0].i_loc = last->i_loc; } @@ -9724,7 +9873,7 @@ cfg_to_instructions(cfg_builder *g) } for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { for (int i = 0; i < b->b_iused; i++) { - struct instr *instr = &b->b_instr[i]; + struct cfg_instr *instr = &b->b_instr[i]; location loc = instr->i_loc; int arg = HAS_TARGET(instr->i_opcode) ? instr->i_target->b_label : instr->i_oparg; @@ -9783,16 +9932,19 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, goto finally; } - cfg_builder *g = CFG_BUILDER(c); - - if (translate_jump_labels_to_targets(g->g_entryblock) < 0) { + cfg_builder g; + if (instr_sequence_to_cfg(INSTR_SEQUENCE(c), &g) < 0) { + goto finally; + } + if (translate_jump_labels_to_targets(g.g_entryblock) < 0) { goto finally; } - res = cfg_to_instructions(g); + res = cfg_to_instructions(&g); finally: compiler_exit_scope(c); + cfg_builder_fini(&g); compiler_free(c); _PyArena_Free(arena); return res; @@ -9815,7 +9967,7 @@ _PyCompile_OptimizeCfg(PyObject *instructions, PyObject *consts) if (const_cache == NULL) { goto error; } - if (translate_jump_labels_to_targets(g.g_entryblock)) { + if (translate_jump_labels_to_targets(g.g_entryblock) < 0) { goto error; } if (optimize_cfg(&g, consts, const_cache) < 0) { From a33ca2ad1fcf857817cba505a788e15cf9d6ed0c Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Tue, 7 Mar 2023 21:27:46 +0000 Subject: [PATCH 12/38] gh-102493: fix normalization in PyErr_SetObject (#102502) Co-authored-by: Jelle Zijlstra --- Lib/test/test_capi/test_exceptions.py | 28 +++++++++++++++++++ ...-03-07-16-56-28.gh-issue-102493.gTXrcD.rst | 1 + Modules/_testcapi/exceptions.c | 15 ++++++++++ Python/errors.c | 16 ++++++++--- 4 files changed, 56 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-07-16-56-28.gh-issue-102493.gTXrcD.rst diff --git a/Lib/test/test_capi/test_exceptions.py b/Lib/test/test_capi/test_exceptions.py index b543a1a565a56f..55f131699a2567 100644 --- a/Lib/test/test_capi/test_exceptions.py +++ b/Lib/test/test_capi/test_exceptions.py @@ -140,6 +140,34 @@ def test_err_restore(self): self.assertEqual(1, v.args[0]) self.assertIs(tb, v.__traceback__.tb_next) + def test_set_object(self): + + # new exception as obj is not an exception + with self.assertRaises(ValueError) as e: + _testcapi.exc_set_object(ValueError, 42) + self.assertEqual(e.exception.args, (42,)) + + # wraps the exception because unrelated types + with self.assertRaises(ValueError) as e: + _testcapi.exc_set_object(ValueError, TypeError(1,2,3)) + wrapped = e.exception.args[0] + self.assertIsInstance(wrapped, TypeError) + self.assertEqual(wrapped.args, (1, 2, 3)) + + # is superclass, so does not wrap + with self.assertRaises(PermissionError) as e: + _testcapi.exc_set_object(OSError, PermissionError(24)) + self.assertEqual(e.exception.args, (24,)) + + class Meta(type): + def __subclasscheck__(cls, sub): + 1/0 + + class Broken(Exception, metaclass=Meta): + pass + + with self.assertRaises(ZeroDivisionError) as e: + _testcapi.exc_set_object(Broken, Broken()) if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-07-16-56-28.gh-issue-102493.gTXrcD.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-07-16-56-28.gh-issue-102493.gTXrcD.rst new file mode 100644 index 00000000000000..4c4e88ca4e7c3c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-03-07-16-56-28.gh-issue-102493.gTXrcD.rst @@ -0,0 +1 @@ +Fix regression in semantics of normalisation in ``PyErr_SetObject``. diff --git a/Modules/_testcapi/exceptions.c b/Modules/_testcapi/exceptions.c index 43b88ccf261d98..a0575213987ffc 100644 --- a/Modules/_testcapi/exceptions.c +++ b/Modules/_testcapi/exceptions.c @@ -78,6 +78,20 @@ make_exception_with_doc(PyObject *self, PyObject *args, PyObject *kwargs) return PyErr_NewExceptionWithDoc(name, doc, base, dict); } +static PyObject * +exc_set_object(PyObject *self, PyObject *args) +{ + PyObject *exc; + PyObject *obj; + + if (!PyArg_ParseTuple(args, "OO:exc_set_object", &exc, &obj)) { + return NULL; + } + + PyErr_SetObject(exc, obj); + return NULL; +} + static PyObject * raise_exception(PyObject *self, PyObject *args) { @@ -247,6 +261,7 @@ static PyMethodDef test_methods[] = { PyDoc_STR("fatal_error(message, release_gil=False): call Py_FatalError(message)")}, {"make_exception_with_doc", _PyCFunction_CAST(make_exception_with_doc), METH_VARARGS | METH_KEYWORDS}, + {"exc_set_object", exc_set_object, METH_VARARGS}, {"raise_exception", raise_exception, METH_VARARGS}, {"raise_memoryerror", raise_memoryerror, METH_NOARGS}, {"set_exc_info", test_set_exc_info, METH_VARARGS}, diff --git a/Python/errors.c b/Python/errors.c index f573bed3d63ef0..bbf6d397ce8097 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -149,9 +149,16 @@ _PyErr_SetObject(PyThreadState *tstate, PyObject *exception, PyObject *value) exception); return; } - Py_XINCREF(value); /* Normalize the exception */ - if (value == NULL || (PyObject *)Py_TYPE(value) != exception) { + int is_subclass = 0; + if (value != NULL && PyExceptionInstance_Check(value)) { + is_subclass = PyObject_IsSubclass((PyObject *)Py_TYPE(value), exception); + if (is_subclass < 0) { + return; + } + } + Py_XINCREF(value); + if (!is_subclass) { /* We must normalize the value right now */ PyObject *fixed_value; @@ -206,9 +213,10 @@ _PyErr_SetObject(PyThreadState *tstate, PyObject *exception, PyObject *value) Py_DECREF(exc_value); } } - if (value != NULL && PyExceptionInstance_Check(value)) + assert(value != NULL); + if (PyExceptionInstance_Check(value)) tb = PyException_GetTraceback(value); - _PyErr_Restore(tstate, Py_XNewRef(exception), value, tb); + _PyErr_Restore(tstate, Py_NewRef(Py_TYPE(value)), value, tb); } void From f9774e57d84162ff0cba0b17a3dcdb93dfbce45e Mon Sep 17 00:00:00 2001 From: Thomas Wouters Date: Tue, 7 Mar 2023 22:41:50 +0100 Subject: [PATCH 13/38] Python 3.12.0a6 --- Include/patchlevel.h | 4 +- Lib/pydoc_data/topics.py | 95 +- Misc/NEWS.d/3.12.0a6.rst | 821 ++++++++++++++++++ ...2-09-14-10-38-15.gh-issue-96821.Zk2a9c.rst | 3 - ...-12-18-08-33-28.gh-issue-100221.K94Ct3.rst | 2 - ...2-12-20-01-06-17.gh-issue-99942.lbmzYj.rst | 2 - ...3-01-12-00-49-16.gh-issue-99942.DUR8b4.rst | 2 - ...2-04-21-17-25-22.gh-issue-91744.FgvaMi.rst | 3 - ...-02-06-16-14-30.gh-issue-101578.PW5fA9.rst | 10 - ...3-02-09-10-38-20.gh-issue-99293.mFqfpp.rst | 2 - ...-02-14-15-53-01.gh-issue-101907.HgF1N2.rst | 1 - ...2-11-02-20-23-47.gh-issue-98627.VJkdRM.rst | 5 - ...-01-04-12-49-33.gh-issue-100719.uRPccL.rst | 3 - ...-02-07-14-56-43.gh-issue-101632.Fd1yxk.rst | 1 - ...-02-08-17-13-31.gh-issue-101696.seJhTt.rst | 1 - ...-02-10-01-15-57.gh-issue-101430.T3Gegb.rst | 2 - ...-02-10-07-21-47.gh-issue-101765.MO5LlC.rst | 1 - ...3-02-10-15-54-57.gh-issue-87849.IUVvPz.rst | 3 - ...3-02-11-23-14-06.gh-issue-84783._P5sMa.rst | 1 - ...-02-12-22-40-22.gh-issue-101857._bribG.rst | 1 - ...-02-13-18-21-14.gh-issue-101799.wpHbCn.rst | 2 - ...3-02-13-22-21-58.gh-issue-74895.esMNtq.rst | 5 - ...-02-16-16-57-23.gh-issue-101952.Zo1dlq.rst | 1 - ...-02-16-23-19-01.gh-issue-101967.Kqr1dz.rst | 1 - ...-02-17-10-12-13.gh-issue-100982.mJGJQw.rst | 2 - ...-02-20-15-18-33.gh-issue-102056.uHKuwH.rst | 1 - ...-02-22-15-15-32.gh-issue-102027.Km4G-d.rst | 2 - ...-02-24-17-59-39.gh-issue-102126.HTT8Vc.rst | 1 - ...-02-26-23-10-32.gh-issue-102250.7MUKoC.rst | 1 - ...-02-28-21-17-03.gh-issue-102336.-wL3Tm.rst | 1 - ...-03-04-20-56-12.gh-issue-102356.07KvUd.rst | 2 - ...-03-06-13-05-33.gh-issue-102416.dz6K5f.rst | 1 - ...-03-07-16-56-28.gh-issue-102493.gTXrcD.rst | 1 - ...3-02-07-21-43-24.gh-issue-97725.cuY7Cd.rst | 2 - ...3-02-19-10-33-01.gh-issue-85417.kYO8u3.rst | 1 - .../2018-06-20-09-12-21.bpo-23224.zxCQ13.rst | 6 - ...2-09-05-12-17-34.gh-issue-88233.gff9qJ.rst | 2 - ...2-10-22-09-26-43.gh-issue-96764.Dh9Y5L.rst | 1 - ...3-01-02-22-41-44.gh-issue-99138.17hp9U.rst | 1 - ...-01-06-21-14-41.gh-issue-100809.I697UT.rst | 3 - ...-01-25-00-14-52.gh-issue-101277.FceHX7.rst | 2 - ...-01-27-02-53-50.gh-issue-101360.bPB7SL.rst | 3 - ...3-02-01-10-42-16.gh-issue-63301.XNxSFh.rst | 1 - ...-02-04-16-35-46.gh-issue-101561.Xo6pIZ.rst | 1 - ...3-02-05-21-40-15.gh-issue-85984.Kfzbb2.rst | 4 - ...-02-07-20-46-08.gh-issue-101362.2ckZ6R.rst | 2 - ...-02-07-21-16-41.gh-issue-101362.KMQllM.rst | 2 - ...-02-07-22-20-32.gh-issue-101362.Jlk6mt.rst | 4 - ...-02-07-22-21-46.gh-issue-101446.-c0FdK.rst | 2 - ...-02-08-18-20-58.gh-issue-101693.4_LPXj.rst | 6 - ...-02-10-11-59-13.gh-issue-101773.J_kI7y.rst | 2 - ...-02-10-16-02-29.gh-issue-101517.r7S2u8.rst | 1 - ...3-02-11-13-23-29.gh-issue-97786.QjvQ1B.rst | 2 - ...3-02-13-12-55-48.gh-issue-87634.q-SBhJ.rst | 1 - ...-02-14-09-08-48.gh-issue-101892.FMos8l.rst | 3 - ...3-02-15-01-54-06.gh-issue-99108.rjTSic.rst | 3 - ...-02-17-18-44-27.gh-issue-101997.A6_blD.rst | 1 - ...3-02-17-19-00-58.gh-issue-97930.C_nQjb.rst | 4 - ...-02-17-20-24-15.gh-issue-101566.FjgWBt.rst | 4 - ...-02-21-07-15-41.gh-issue-101936.QVOxHH.rst | 2 - ...-02-21-10-05-33.gh-issue-101961.7e56jh.rst | 2 - ...-02-23-15-06-01.gh-issue-102179.P6KQ4c.rst | 1 - ...3-02-23-20-39-52.gh-issue-81652.Vxz0Mr.rst | 2 - ...3-02-26-12-37-17.gh-issue-91038.S4rFH_.rst | 1 - ...-02-28-09-52-25.gh-issue-101979.or3hXV.rst | 2 - ...-03-04-14-46-47.gh-issue-102302.-b_s6Z.rst | 1 - ...-01-24-16-12-00.gh-issue-101283.9tqu39.rst | 3 - ...3-02-08-12-57-35.gh-issue-99108.6tnmhA.rst | 4 - ...-02-08-22-03-04.gh-issue-101727.9P5eZz.rst | 4 - ...3-02-17-10-42-48.gh-issue-99108.MKA8-f.rst | 2 - ...3-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst | 4 - ...3-02-11-22-36-10.gh-issue-85984.EVXjT9.rst | 1 - ...-02-18-10-51-02.gh-issue-102019.0797SJ.rst | 2 - ...-01-25-11-33-54.gh-issue-101196.wAX_2g.rst | 3 - ...-02-07-18-22-54.gh-issue-101614.NjVP0n.rst | 1 - ...-02-09-22-09-27.gh-issue-101759.zFlqSH.rst | 1 - ...-02-10-14-26-05.gh-issue-101763.RPaj7r.rst | 1 - ...-02-13-16-32-50.gh-issue-101849.7lm_53.rst | 1 - ...-02-13-18-05-49.gh-issue-101881._TnHzN.rst | 1 - ...-02-15-11-08-10.gh-issue-101881.fScr3m.rst | 1 - ...-03-01-01-36-39.gh-issue-102344.Dgfux4.rst | 2 - ...-02-09-22-07-17.gh-issue-101759.B0JP2H.rst | 1 - README.rst | 2 +- 83 files changed, 864 insertions(+), 229 deletions(-) create mode 100644 Misc/NEWS.d/3.12.0a6.rst delete mode 100644 Misc/NEWS.d/next/Build/2022-09-14-10-38-15.gh-issue-96821.Zk2a9c.rst delete mode 100644 Misc/NEWS.d/next/Build/2022-12-18-08-33-28.gh-issue-100221.K94Ct3.rst delete mode 100644 Misc/NEWS.d/next/Build/2022-12-20-01-06-17.gh-issue-99942.lbmzYj.rst delete mode 100644 Misc/NEWS.d/next/Build/2023-01-12-00-49-16.gh-issue-99942.DUR8b4.rst delete mode 100644 Misc/NEWS.d/next/C API/2022-04-21-17-25-22.gh-issue-91744.FgvaMi.rst delete mode 100644 Misc/NEWS.d/next/C API/2023-02-06-16-14-30.gh-issue-101578.PW5fA9.rst delete mode 100644 Misc/NEWS.d/next/C API/2023-02-09-10-38-20.gh-issue-99293.mFqfpp.rst delete mode 100644 Misc/NEWS.d/next/C API/2023-02-14-15-53-01.gh-issue-101907.HgF1N2.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-11-02-20-23-47.gh-issue-98627.VJkdRM.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-01-04-12-49-33.gh-issue-100719.uRPccL.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-07-14-56-43.gh-issue-101632.Fd1yxk.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-08-17-13-31.gh-issue-101696.seJhTt.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-10-01-15-57.gh-issue-101430.T3Gegb.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-10-07-21-47.gh-issue-101765.MO5LlC.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-10-15-54-57.gh-issue-87849.IUVvPz.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-11-23-14-06.gh-issue-84783._P5sMa.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-12-22-40-22.gh-issue-101857._bribG.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-13-18-21-14.gh-issue-101799.wpHbCn.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-13-22-21-58.gh-issue-74895.esMNtq.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-16-16-57-23.gh-issue-101952.Zo1dlq.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-16-23-19-01.gh-issue-101967.Kqr1dz.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-17-10-12-13.gh-issue-100982.mJGJQw.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-20-15-18-33.gh-issue-102056.uHKuwH.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-22-15-15-32.gh-issue-102027.Km4G-d.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-24-17-59-39.gh-issue-102126.HTT8Vc.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-26-23-10-32.gh-issue-102250.7MUKoC.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-28-21-17-03.gh-issue-102336.-wL3Tm.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-04-20-56-12.gh-issue-102356.07KvUd.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-06-13-05-33.gh-issue-102416.dz6K5f.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-07-16-56-28.gh-issue-102493.gTXrcD.rst delete mode 100644 Misc/NEWS.d/next/Documentation/2023-02-07-21-43-24.gh-issue-97725.cuY7Cd.rst delete mode 100644 Misc/NEWS.d/next/Documentation/2023-02-19-10-33-01.gh-issue-85417.kYO8u3.rst delete mode 100644 Misc/NEWS.d/next/Library/2018-06-20-09-12-21.bpo-23224.zxCQ13.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-09-05-12-17-34.gh-issue-88233.gff9qJ.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-10-22-09-26-43.gh-issue-96764.Dh9Y5L.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-01-02-22-41-44.gh-issue-99138.17hp9U.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-01-06-21-14-41.gh-issue-100809.I697UT.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-01-25-00-14-52.gh-issue-101277.FceHX7.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-01-27-02-53-50.gh-issue-101360.bPB7SL.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-01-10-42-16.gh-issue-63301.XNxSFh.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-04-16-35-46.gh-issue-101561.Xo6pIZ.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-05-21-40-15.gh-issue-85984.Kfzbb2.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-07-20-46-08.gh-issue-101362.2ckZ6R.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-07-21-16-41.gh-issue-101362.KMQllM.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-07-22-20-32.gh-issue-101362.Jlk6mt.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-07-22-21-46.gh-issue-101446.-c0FdK.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-08-18-20-58.gh-issue-101693.4_LPXj.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-10-11-59-13.gh-issue-101773.J_kI7y.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-10-16-02-29.gh-issue-101517.r7S2u8.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-11-13-23-29.gh-issue-97786.QjvQ1B.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-13-12-55-48.gh-issue-87634.q-SBhJ.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-14-09-08-48.gh-issue-101892.FMos8l.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-15-01-54-06.gh-issue-99108.rjTSic.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-17-18-44-27.gh-issue-101997.A6_blD.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-17-19-00-58.gh-issue-97930.C_nQjb.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-17-20-24-15.gh-issue-101566.FjgWBt.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-21-07-15-41.gh-issue-101936.QVOxHH.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-21-10-05-33.gh-issue-101961.7e56jh.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-23-15-06-01.gh-issue-102179.P6KQ4c.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-23-20-39-52.gh-issue-81652.Vxz0Mr.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-26-12-37-17.gh-issue-91038.S4rFH_.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-28-09-52-25.gh-issue-101979.or3hXV.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-04-14-46-47.gh-issue-102302.-b_s6Z.rst delete mode 100644 Misc/NEWS.d/next/Security/2023-01-24-16-12-00.gh-issue-101283.9tqu39.rst delete mode 100644 Misc/NEWS.d/next/Security/2023-02-08-12-57-35.gh-issue-99108.6tnmhA.rst delete mode 100644 Misc/NEWS.d/next/Security/2023-02-08-22-03-04.gh-issue-101727.9P5eZz.rst delete mode 100644 Misc/NEWS.d/next/Security/2023-02-17-10-42-48.gh-issue-99108.MKA8-f.rst delete mode 100644 Misc/NEWS.d/next/Tests/2023-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst delete mode 100644 Misc/NEWS.d/next/Tests/2023-02-11-22-36-10.gh-issue-85984.EVXjT9.rst delete mode 100644 Misc/NEWS.d/next/Tests/2023-02-18-10-51-02.gh-issue-102019.0797SJ.rst delete mode 100644 Misc/NEWS.d/next/Windows/2023-01-25-11-33-54.gh-issue-101196.wAX_2g.rst delete mode 100644 Misc/NEWS.d/next/Windows/2023-02-07-18-22-54.gh-issue-101614.NjVP0n.rst delete mode 100644 Misc/NEWS.d/next/Windows/2023-02-09-22-09-27.gh-issue-101759.zFlqSH.rst delete mode 100644 Misc/NEWS.d/next/Windows/2023-02-10-14-26-05.gh-issue-101763.RPaj7r.rst delete mode 100644 Misc/NEWS.d/next/Windows/2023-02-13-16-32-50.gh-issue-101849.7lm_53.rst delete mode 100644 Misc/NEWS.d/next/Windows/2023-02-13-18-05-49.gh-issue-101881._TnHzN.rst delete mode 100644 Misc/NEWS.d/next/Windows/2023-02-15-11-08-10.gh-issue-101881.fScr3m.rst delete mode 100644 Misc/NEWS.d/next/Windows/2023-03-01-01-36-39.gh-issue-102344.Dgfux4.rst delete mode 100644 Misc/NEWS.d/next/macOS/2023-02-09-22-07-17.gh-issue-101759.B0JP2H.rst diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 7957220ed7cf9f..fd16421eace94a 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -20,10 +20,10 @@ #define PY_MINOR_VERSION 12 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 5 +#define PY_RELEASE_SERIAL 6 /* Version as a string */ -#define PY_VERSION "3.12.0a5+" +#define PY_VERSION "3.12.0a6" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index e7f403d3ffbf12..573065b4b714d9 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Tue Feb 7 13:18:04 2023 +# Autogenerated by Sphinx on Tue Mar 7 22:42:28 2023 topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' @@ -2499,42 +2499,33 @@ 'alive\n' 'until the next garbage collection occurs.\n' '\n' - 'Before an "except" clause’s suite is executed, details about ' - 'the\n' - 'exception are stored in the "sys" module and can be accessed ' - 'via\n' - '"sys.exc_info()". "sys.exc_info()" returns a 3-tuple consisting ' - 'of the\n' - 'exception class, the exception instance and a traceback object ' - '(see\n' - 'section The standard type hierarchy) identifying the point in ' - 'the\n' - 'program where the exception occurred. The details about the ' - 'exception\n' - 'accessed via "sys.exc_info()" are restored to their previous ' - 'values\n' - 'when leaving an exception handler:\n' + 'Before an "except" clause’s suite is executed, the exception is ' + 'stored\n' + 'in the "sys" module, where it can be accessed from within the ' + 'body of\n' + 'the "except" clause by calling "sys.exception()". When leaving ' + 'an\n' + 'exception handler, the exception stored in the "sys" module is ' + 'reset\n' + 'to its previous value:\n' '\n' - ' >>> print(sys.exc_info())\n' - ' (None, None, None)\n' + ' >>> print(sys.exception())\n' + ' None\n' ' >>> try:\n' ' ... raise TypeError\n' ' ... except:\n' - ' ... print(sys.exc_info())\n' + ' ... print(repr(sys.exception()))\n' ' ... try:\n' ' ... raise ValueError\n' ' ... except:\n' - ' ... print(sys.exc_info())\n' - ' ... print(sys.exc_info())\n' + ' ... print(repr(sys.exception()))\n' + ' ... print(repr(sys.exception()))\n' ' ...\n' - " (, TypeError(), )\n' - " (, ValueError(), )\n' - " (, TypeError(), )\n' - ' >>> print(sys.exc_info())\n' - ' (None, None, None)\n' + ' TypeError()\n' + ' ValueError()\n' + ' TypeError()\n' + ' >>> print(sys.exception())\n' + ' None\n' '\n' '\n' '"except*" clause\n' @@ -4533,7 +4524,7 @@ 'objects and\n' ' implements an "__eq__()" method, it should not ' 'implement\n' - ' "__hash__()", since the implementation of hashable ' + ' "__hash__()", since the implementation of *hashable* ' 'collections\n' ' requires that a key’s hash value is immutable (if the ' 'object’s hash\n' @@ -9453,7 +9444,7 @@ ' hashable collections. If a class defines mutable objects ' 'and\n' ' implements an "__eq__()" method, it should not implement\n' - ' "__hash__()", since the implementation of hashable ' + ' "__hash__()", since the implementation of *hashable* ' 'collections\n' ' requires that a key’s hash value is immutable (if the ' 'object’s hash\n' @@ -12644,37 +12635,31 @@ 'cycle with the stack frame, keeping all locals in that frame alive\n' 'until the next garbage collection occurs.\n' '\n' - 'Before an "except" clause’s suite is executed, details about the\n' - 'exception are stored in the "sys" module and can be accessed via\n' - '"sys.exc_info()". "sys.exc_info()" returns a 3-tuple consisting of ' - 'the\n' - 'exception class, the exception instance and a traceback object (see\n' - 'section The standard type hierarchy) identifying the point in the\n' - 'program where the exception occurred. The details about the ' - 'exception\n' - 'accessed via "sys.exc_info()" are restored to their previous values\n' - 'when leaving an exception handler:\n' + 'Before an "except" clause’s suite is executed, the exception is ' + 'stored\n' + 'in the "sys" module, where it can be accessed from within the body ' + 'of\n' + 'the "except" clause by calling "sys.exception()". When leaving an\n' + 'exception handler, the exception stored in the "sys" module is reset\n' + 'to its previous value:\n' '\n' - ' >>> print(sys.exc_info())\n' - ' (None, None, None)\n' + ' >>> print(sys.exception())\n' + ' None\n' ' >>> try:\n' ' ... raise TypeError\n' ' ... except:\n' - ' ... print(sys.exc_info())\n' + ' ... print(repr(sys.exception()))\n' ' ... try:\n' ' ... raise ValueError\n' ' ... except:\n' - ' ... print(sys.exc_info())\n' - ' ... print(sys.exc_info())\n' + ' ... print(repr(sys.exception()))\n' + ' ... print(repr(sys.exception()))\n' ' ...\n' - " (, TypeError(), )\n' - " (, ValueError(), )\n' - " (, TypeError(), )\n' - ' >>> print(sys.exc_info())\n' - ' (None, None, None)\n' + ' TypeError()\n' + ' ValueError()\n' + ' TypeError()\n' + ' >>> print(sys.exception())\n' + ' None\n' '\n' '\n' '"except*" clause\n' @@ -14408,7 +14393,7 @@ ' New in version 3.10.\n' '\n' 'Keys views are set-like since their entries are unique and ' - 'hashable.\n' + '*hashable*.\n' 'If all values are hashable, so that "(key, value)" pairs are ' 'unique\n' 'and hashable, then the items view is also set-like. (Values ' diff --git a/Misc/NEWS.d/3.12.0a6.rst b/Misc/NEWS.d/3.12.0a6.rst new file mode 100644 index 00000000000000..2bcb4c8c854d4e --- /dev/null +++ b/Misc/NEWS.d/3.12.0a6.rst @@ -0,0 +1,821 @@ +.. date: 2023-02-17-10-42-48 +.. gh-issue: 99108 +.. nonce: MKA8-f +.. release date: 2023-03-07 +.. section: Security + +Replace builtin hashlib implementations of MD5 and SHA1 with verified ones +from the HACL* project. + +.. + +.. date: 2023-02-08-22-03-04 +.. gh-issue: 101727 +.. nonce: 9P5eZz +.. section: Security + +Updated the OpenSSL version used in Windows and macOS binary release builds +to 1.1.1t to address CVE-2023-0286, CVE-2022-4303, and CVE-2022-4303 per +`the OpenSSL 2023-02-07 security advisory +`_. + +.. + +.. date: 2023-02-08-12-57-35 +.. gh-issue: 99108 +.. nonce: 6tnmhA +.. section: Security + +Replace the builtin :mod:`hashlib` implementations of SHA2-384 and SHA2-512 +originally from LibTomCrypt with formally verified, side-channel resistant +code from the `HACL* `_ project. +The builtins remain a fallback only used when OpenSSL does not provide them. + +.. + +.. date: 2023-01-24-16-12-00 +.. gh-issue: 101283 +.. nonce: 9tqu39 +.. section: Security + +:class:`subprocess.Popen` now uses a safer approach to find ``cmd.exe`` when +launching with ``shell=True``. Patch by Eryk Sun, based on a patch by Oleg +Iarygin. + +.. + +.. date: 2023-03-07-16-56-28 +.. gh-issue: 102493 +.. nonce: gTXrcD +.. section: Core and Builtins + +Fix regression in semantics of normalisation in ``PyErr_SetObject``. + +.. + +.. date: 2023-03-06-13-05-33 +.. gh-issue: 102416 +.. nonce: dz6K5f +.. section: Core and Builtins + +Do not memoize incorrectly automatically generated loop rules in the parser. +Patch by Pablo Galindo. + +.. + +.. date: 2023-03-04-20-56-12 +.. gh-issue: 102356 +.. nonce: 07KvUd +.. section: Core and Builtins + +Fix a bug that caused a crash when deallocating deeply nested filter +objects. Patch by Marta Gómez Macías. + +.. + +.. date: 2023-02-28-21-17-03 +.. gh-issue: 102336 +.. nonce: -wL3Tm +.. section: Core and Builtins + +Cleanup Windows 7 specific special handling. Patch by Max Bachmann. + +.. + +.. date: 2023-02-26-23-10-32 +.. gh-issue: 102250 +.. nonce: 7MUKoC +.. section: Core and Builtins + +Fixed a segfault occurring when the interpreter calls a ``__bool__`` method +that raises. + +.. + +.. date: 2023-02-24-17-59-39 +.. gh-issue: 102126 +.. nonce: HTT8Vc +.. section: Core and Builtins + +Fix deadlock at shutdown when clearing thread states if any finalizer tries +to acquire the runtime head lock. Patch by Kumar Aditya. + +.. + +.. date: 2023-02-22-15-15-32 +.. gh-issue: 102027 +.. nonce: Km4G-d +.. section: Core and Builtins + +Use ``GetCurrentProcessId`` on Windows when ``getpid`` is unavailable. Patch +by Max Bachmann. + +.. + +.. date: 2023-02-20-15-18-33 +.. gh-issue: 102056 +.. nonce: uHKuwH +.. section: Core and Builtins + +Fix error handling bugs in interpreter's exception printing code, which +could cause a crash on infinite recursion. + +.. + +.. date: 2023-02-17-10-12-13 +.. gh-issue: 100982 +.. nonce: mJGJQw +.. section: Core and Builtins + +Restrict the scope of the :opcode:`FOR_ITER_RANGE` instruction to the scope +of the original :opcode:`FOR_ITER` instruction, to allow instrumentation. + +.. + +.. date: 2023-02-16-23-19-01 +.. gh-issue: 101967 +.. nonce: Kqr1dz +.. section: Core and Builtins + +Fix possible segfault in ``positional_only_passed_as_keyword`` function, +when new list created. + +.. + +.. date: 2023-02-16-16-57-23 +.. gh-issue: 101952 +.. nonce: Zo1dlq +.. section: Core and Builtins + +Fix possible segfault in ``BUILD_SET`` opcode, when new set created. + +.. + +.. date: 2023-02-13-22-21-58 +.. gh-issue: 74895 +.. nonce: esMNtq +.. section: Core and Builtins + +:mod:`socket.getaddrinfo` no longer raises :class:`OverflowError` for +:class:`int` **port** values outside of the C long range. Out of range +values are left up to the underlying string based C library API to report. A +:class:`socket.gaierror` ``SAI_SERVICE`` may occur instead, or no error at +all as not all platform C libraries generate an error. + +.. + +.. date: 2023-02-13-18-21-14 +.. gh-issue: 101799 +.. nonce: wpHbCn +.. section: Core and Builtins + +Add :opcode:`CALL_INTRINSIC_2` and use it instead of +:opcode:`PREP_RERAISE_STAR`. + +.. + +.. date: 2023-02-12-22-40-22 +.. gh-issue: 101857 +.. nonce: _bribG +.. section: Core and Builtins + +Fix xattr support detection on Linux systems by widening the check to linux, +not just glibc. This fixes support for musl. + +.. + +.. date: 2023-02-11-23-14-06 +.. gh-issue: 84783 +.. nonce: _P5sMa +.. section: Core and Builtins + +Make the slice object hashable. + +.. + +.. date: 2023-02-10-15-54-57 +.. gh-issue: 87849 +.. nonce: IUVvPz +.. section: Core and Builtins + +Change the ``SEND`` instruction to leave the receiver on the stack. This +allows the specialized form of ``SEND`` to skip the chain of C calls and +jump directly to the ``RESUME`` in the generator or coroutine. + +.. + +.. date: 2023-02-10-07-21-47 +.. gh-issue: 101765 +.. nonce: MO5LlC +.. section: Core and Builtins + +Fix SystemError / segmentation fault in iter ``__reduce__`` when internal +access of ``builtins.__dict__`` keys mutates the iter object. + +.. + +.. date: 2023-02-10-01-15-57 +.. gh-issue: 101430 +.. nonce: T3Gegb +.. section: Core and Builtins + +Update :mod:`tracemalloc` to handle presize of object properly. Patch by +Dong-hee Na. + +.. + +.. date: 2023-02-08-17-13-31 +.. gh-issue: 101696 +.. nonce: seJhTt +.. section: Core and Builtins + +Invalidate type version tag in ``_PyStaticType_Dealloc`` for static types, +avoiding bug where a false cache hit could crash the interpreter. Patch by +Kumar Aditya. + +.. + +.. date: 2023-02-07-14-56-43 +.. gh-issue: 101632 +.. nonce: Fd1yxk +.. section: Core and Builtins + +Adds a new :opcode:`RETURN_CONST` instruction. + +.. + +.. date: 2023-01-04-12-49-33 +.. gh-issue: 100719 +.. nonce: uRPccL +.. section: Core and Builtins + +Remove gi_code field from generator (and coroutine and async generator) +objects as it is redundant. The frame already includes a reference to the +code object. + +.. + +.. date: 2022-11-02-20-23-47 +.. gh-issue: 98627 +.. nonce: VJkdRM +.. section: Core and Builtins + +When an interpreter is configured to check (and only then), importing an +extension module will now fail when the extension does not support multiple +interpreters (i.e. doesn't implement PEP 489 multi-phase init). This does +not apply to the main interpreter, nor to subinterpreters created with +``Py_NewInterpreter()``. + +.. + +.. date: 2023-03-04-14-46-47 +.. gh-issue: 102302 +.. nonce: -b_s6Z +.. section: Library + +Micro-optimise hashing of :class:`inspect.Parameter`, reducing the time it +takes to hash an instance by around 40%. + +.. + +.. date: 2023-02-28-09-52-25 +.. gh-issue: 101979 +.. nonce: or3hXV +.. section: Library + +Fix a bug where parentheses in the ``metavar`` argument to +:meth:`argparse.ArgumentParser.add_argument` were dropped. Patch by Yeojin +Kim. + +.. + +.. date: 2023-02-26-12-37-17 +.. gh-issue: 91038 +.. nonce: S4rFH_ +.. section: Library + +:meth:`platform.platform` now has boolean default arguments. + +.. + +.. date: 2023-02-23-20-39-52 +.. gh-issue: 81652 +.. nonce: Vxz0Mr +.. section: Library + +Add :data:`mmap.MAP_ALIGNED_SUPER` FreeBSD and :data:`mmap.MAP_CONCEAL` +OpenBSD constants to :mod:`mmap`. Patch by Yeojin Kim. + +.. + +.. date: 2023-02-23-15-06-01 +.. gh-issue: 102179 +.. nonce: P6KQ4c +.. section: Library + +Fix :func:`os.dup2` error message for negative fds. + +.. + +.. date: 2023-02-21-10-05-33 +.. gh-issue: 101961 +.. nonce: 7e56jh +.. section: Library + +For the binary mode, :func:`fileinput.hookcompressed` doesn't set the +``encoding`` value even if the value is ``None``. Patch by Gihwan Kim. + +.. + +.. date: 2023-02-21-07-15-41 +.. gh-issue: 101936 +.. nonce: QVOxHH +.. section: Library + +The default value of ``fp`` becomes :class:`io.BytesIO` if +:exc:`~urllib.error.HTTPError` is initialized without a designated ``fp`` +parameter. Patch by Long Vo. + +.. + +.. date: 2023-02-17-20-24-15 +.. gh-issue: 101566 +.. nonce: FjgWBt +.. section: Library + +In zipfile, sync Path with `zipp 3.14 +`_, including +fix for extractall on the underlying zipfile after being wrapped in +``Path``. + +.. + +.. date: 2023-02-17-19-00-58 +.. gh-issue: 97930 +.. nonce: C_nQjb +.. section: Library + +Apply changes from `importlib_resources 5.12 +`_, +including fix for ``MultiplexedPath`` to support directories in multiple +namespaces (python/importlib_resources#265). + +.. + +.. date: 2023-02-17-18-44-27 +.. gh-issue: 101997 +.. nonce: A6_blD +.. section: Library + +Upgrade pip wheel bundled with ensurepip (pip 23.0.1) + +.. + +.. date: 2023-02-15-01-54-06 +.. gh-issue: 99108 +.. nonce: rjTSic +.. section: Library + +The built-in extension modules for :mod:`hashlib` SHA2 algorithms, used when +OpenSSL does not provide them, now live in a single internal ``_sha2`` +module instead of separate ``_sha256`` and ``_sha512`` modules. + +.. + +.. date: 2023-02-14-09-08-48 +.. gh-issue: 101892 +.. nonce: FMos8l +.. section: Library + +Callable iterators no longer raise :class:`SystemError` when the callable +object exhausts the iterator but forgets to either return a sentinel value +or raise :class:`StopIteration`. + +.. + +.. date: 2023-02-13-12-55-48 +.. gh-issue: 87634 +.. nonce: q-SBhJ +.. section: Library + +Remove locking behavior from :func:`functools.cached_property`. + +.. + +.. date: 2023-02-11-13-23-29 +.. gh-issue: 97786 +.. nonce: QjvQ1B +.. section: Library + +Fix potential undefined behaviour in corner cases of floating-point-to-time +conversions. + +.. + +.. date: 2023-02-10-16-02-29 +.. gh-issue: 101517 +.. nonce: r7S2u8 +.. section: Library + +Fixed bug where :mod:`bdb` looks up the source line with :mod:`linecache` +with a ``lineno=None``, which causes it to fail with an unhandled exception. + +.. + +.. date: 2023-02-10-11-59-13 +.. gh-issue: 101773 +.. nonce: J_kI7y +.. section: Library + +Optimize :class:`fractions.Fraction` for small components. The private +argument ``_normalize`` of the :class:`fractions.Fraction` constructor has +been removed. + +.. + +.. date: 2023-02-08-18-20-58 +.. gh-issue: 101693 +.. nonce: 4_LPXj +.. section: Library + +In :meth:`sqlite3.Cursor.execute`, :exc:`DeprecationWarning` is now emitted +when :ref:`named placeholders ` are used together with +parameters supplied as a :term:`sequence` instead of as a :class:`dict`. +Starting from Python 3.14, using named placeholders with parameters supplied +as a sequence will raise a :exc:`~sqlite3.ProgrammingError`. Patch by Erlend +E. Aasland. + +.. + +.. date: 2023-02-07-22-21-46 +.. gh-issue: 101446 +.. nonce: -c0FdK +.. section: Library + +Change repr of :class:`collections.OrderedDict` to use regular dictionary +formating instead of pairs of keys and values. + +.. + +.. date: 2023-02-07-22-20-32 +.. gh-issue: 101362 +.. nonce: Jlk6mt +.. section: Library + +Speed up :class:`pathlib.PurePath` construction by handling arguments more +uniformly. When a :class:`pathlib.Path` argument is supplied, we use its +string representation rather than joining its parts with +:func:`os.path.join`. + +.. + +.. date: 2023-02-07-21-16-41 +.. gh-issue: 101362 +.. nonce: KMQllM +.. section: Library + +Speed up :class:`pathlib.PurePath` construction by calling +:func:`os.path.join` only when two or more arguments are given. + +.. + +.. date: 2023-02-07-20-46-08 +.. gh-issue: 101362 +.. nonce: 2ckZ6R +.. section: Library + +Speed up :class:`pathlib.Path` construction by running the path flavour +compatibility check only when pathlib is imported. + +.. + +.. date: 2023-02-05-21-40-15 +.. gh-issue: 85984 +.. nonce: Kfzbb2 +.. section: Library + +Refactored the implementation of :func:`pty.fork` to use +:func:`os.login_tty`. + +A :exc:`DeprecationWarning` is now raised by ``pty.master_open()`` and +``pty.slave_open()``. They were undocumented and deprecated long long ago in +the docstring in favor of :func:`pty.openpty`. + +.. + +.. date: 2023-02-04-16-35-46 +.. gh-issue: 101561 +.. nonce: Xo6pIZ +.. section: Library + +Add a new decorator :func:`typing.override`. See :pep:`698` for details. +Patch by Steven Troxler. + +.. + +.. date: 2023-02-01-10-42-16 +.. gh-issue: 63301 +.. nonce: XNxSFh +.. section: Library + +Set exit code when :mod:`tabnanny` CLI exits on error. + +.. + +.. date: 2023-01-27-02-53-50 +.. gh-issue: 101360 +.. nonce: bPB7SL +.. section: Library + +Fix anchor matching in :meth:`pathlib.PureWindowsPath.match`. Path and +pattern anchors are now matched with :mod:`fnmatch`, just like other path +parts. This allows patterns such as ``"*:/Users/*"`` to be matched. + +.. + +.. date: 2023-01-25-00-14-52 +.. gh-issue: 101277 +.. nonce: FceHX7 +.. section: Library + +Remove global state from :mod:`itertools` module (:pep:`687`). Patches by +Erlend E. Aasland. + +.. + +.. date: 2023-01-06-21-14-41 +.. gh-issue: 100809 +.. nonce: I697UT +.. section: Library + +Fix handling of drive-relative paths (like 'C:' and 'C:foo') in +:meth:`pathlib.Path.absolute`. This method now uses the OS API to retrieve +the correct current working directory for the drive. + +.. + +.. date: 2023-01-02-22-41-44 +.. gh-issue: 99138 +.. nonce: 17hp9U +.. section: Library + +Apply :pep:`687` to :mod:`zoneinfo`. Patch by Erlend E. Aasland. + +.. + +.. date: 2022-10-22-09-26-43 +.. gh-issue: 96764 +.. nonce: Dh9Y5L +.. section: Library + +:func:`asyncio.wait_for` now uses :func:`asyncio.timeout` as its underlying +implementation. Patch by Kumar Aditya. + +.. + +.. date: 2022-09-05-12-17-34 +.. gh-issue: 88233 +.. nonce: gff9qJ +.. section: Library + +Correctly preserve "extra" fields in ``zipfile`` regardless of their +ordering relative to a zip64 "extra." + +.. + +.. bpo: 23224 +.. date: 2018-06-20-09-12-21 +.. nonce: zxCQ13 +.. section: Library + +Fix segfaults when creating :class:`lzma.LZMADecompressor` and +:class:`bz2.BZ2Decompressor` objects without calling ``__init__()``, and fix +leakage of locks and internal buffers when calling the ``__init__()`` +methods of :class:`lzma.LZMADecompressor`, :class:`lzma.LZMACompressor`, +:class:`bz2.BZ2Compressor`, and :class:`bz2.BZ2Decompressor` objects +multiple times. + +.. + +.. date: 2023-02-19-10-33-01 +.. gh-issue: 85417 +.. nonce: kYO8u3 +.. section: Documentation + +Update :mod:`cmath` documentation to clarify behaviour on branch cuts. + +.. + +.. date: 2023-02-07-21-43-24 +.. gh-issue: 97725 +.. nonce: cuY7Cd +.. section: Documentation + +Fix :meth:`asyncio.Task.print_stack` description for ``file=None``. Patch by +Oleg Iarygin. + +.. + +.. date: 2023-02-18-10-51-02 +.. gh-issue: 102019 +.. nonce: 0797SJ +.. section: Tests + +Fix deadlock on shutdown if ``test_current_{exception,frames}`` fails. Patch +by Jacob Bower. + +.. + +.. date: 2023-02-11-22-36-10 +.. gh-issue: 85984 +.. nonce: EVXjT9 +.. section: Tests + +Utilize new "winsize" functions from termios in pty tests. + +.. + +.. date: 2023-02-11-20-28-08 +.. gh-issue: 89792 +.. nonce: S-Y5BZ +.. section: Tests + +``test_tools`` now copies up to 10x less source data to a temporary +directory during the ``freeze`` test by ignoring git metadata and other +artifacts. It also limits its python build parallelism based on +os.cpu_count instead of hard coding it as 8 cores. + +.. + +.. date: 2023-01-12-00-49-16 +.. gh-issue: 99942 +.. nonce: DUR8b4 +.. section: Build + +On Android, in a static build, python-config in embed mode no longer +incorrectly reports a library to link to. + +.. + +.. date: 2022-12-20-01-06-17 +.. gh-issue: 99942 +.. nonce: lbmzYj +.. section: Build + +On Android, python.pc now correctly reports the library to link to, the same +as python-config.sh. + +.. + +.. date: 2022-12-18-08-33-28 +.. gh-issue: 100221 +.. nonce: K94Ct3 +.. section: Build + +Fix creating install directories in ``make sharedinstall`` if they exist +outside ``DESTDIR`` already. + +.. + +.. date: 2022-09-14-10-38-15 +.. gh-issue: 96821 +.. nonce: Zk2a9c +.. section: Build + +Explicitly mark C extension modules that need defined signed integer +overflow, and add a configure option :option:`--with-strict-overflow`. Patch +by Matthias Görgens and Shantanu Jain. + +.. + +.. date: 2023-03-01-01-36-39 +.. gh-issue: 102344 +.. nonce: Dgfux4 +.. section: Windows + +Implement ``winreg.QueryValue`` using ``QueryValueEx`` and +``winreg.SetValue`` using ``SetValueEx``. Patch by Max Bachmann. + +.. + +.. date: 2023-02-15-11-08-10 +.. gh-issue: 101881 +.. nonce: fScr3m +.. section: Windows + +Handle read and write operations on non-blocking pipes properly on Windows. + +.. + +.. date: 2023-02-13-18-05-49 +.. gh-issue: 101881 +.. nonce: _TnHzN +.. section: Windows + +Add support for the os.get_blocking() and os.set_blocking() functions on +Windows. + +.. + +.. date: 2023-02-13-16-32-50 +.. gh-issue: 101849 +.. nonce: 7lm_53 +.. section: Windows + +Ensures installer will correctly upgrade existing ``py.exe`` launcher +installs. + +.. + +.. date: 2023-02-10-14-26-05 +.. gh-issue: 101763 +.. nonce: RPaj7r +.. section: Windows + +Updates copy of libffi bundled with Windows installs to 3.4.4. + +.. + +.. date: 2023-02-09-22-09-27 +.. gh-issue: 101759 +.. nonce: zFlqSH +.. section: Windows + +Update Windows installer to SQLite 3.40.1. + +.. + +.. date: 2023-02-07-18-22-54 +.. gh-issue: 101614 +.. nonce: NjVP0n +.. section: Windows + +Correctly handle extensions built against debug binaries that reference +``python3_d.dll``. + +.. + +.. date: 2023-01-25-11-33-54 +.. gh-issue: 101196 +.. nonce: wAX_2g +.. section: Windows + +The functions ``os.path.isdir``, ``os.path.isfile``, ``os.path.islink`` and +``os.path.exists`` are now 13% to 28% faster on Windows, by making fewer +Win32 API calls. + +.. + +.. date: 2023-02-09-22-07-17 +.. gh-issue: 101759 +.. nonce: B0JP2H +.. section: macOS + +Update macOS installer to SQLite 3.40.1. + +.. + +.. date: 2023-02-14-15-53-01 +.. gh-issue: 101907 +.. nonce: HgF1N2 +.. section: C API + +Removes use of non-standard C++ extension in public header files. + +.. + +.. date: 2023-02-09-10-38-20 +.. gh-issue: 99293 +.. nonce: mFqfpp +.. section: C API + +Document that the Py_TPFLAGS_VALID_VERSION_TAG is an internal feature, +should not be used, and will be removed. + +.. + +.. date: 2023-02-06-16-14-30 +.. gh-issue: 101578 +.. nonce: PW5fA9 +.. section: C API + +Add :c:func:`PyErr_GetRaisedException` and +:c:func:`PyErr_SetRaisedException` for saving and restoring the current +exception. These functions return and accept a single exception object, +rather than the triple arguments of the now-deprecated :c:func:`PyErr_Fetch` +and :c:func:`PyErr_Restore`. This is less error prone and a bit more +efficient. + +Add :c:func:`PyException_GetArgs` and :c:func:`PyException_SetArgs` as +convenience functions for retrieving and modifying the +:attr:`~BaseException.args` passed to the exception's constructor. + +.. + +.. date: 2022-04-21-17-25-22 +.. gh-issue: 91744 +.. nonce: FgvaMi +.. section: C API + +Introduced the *Unstable C API tier*, marking APi that is allowed to change +in minor releases without a deprecation period. See :pep:`689` for details. diff --git a/Misc/NEWS.d/next/Build/2022-09-14-10-38-15.gh-issue-96821.Zk2a9c.rst b/Misc/NEWS.d/next/Build/2022-09-14-10-38-15.gh-issue-96821.Zk2a9c.rst deleted file mode 100644 index 865cfde8b06359..00000000000000 --- a/Misc/NEWS.d/next/Build/2022-09-14-10-38-15.gh-issue-96821.Zk2a9c.rst +++ /dev/null @@ -1,3 +0,0 @@ -Explicitly mark C extension modules that need defined signed integer overflow, -and add a configure option :option:`--with-strict-overflow`. -Patch by Matthias Görgens and Shantanu Jain. diff --git a/Misc/NEWS.d/next/Build/2022-12-18-08-33-28.gh-issue-100221.K94Ct3.rst b/Misc/NEWS.d/next/Build/2022-12-18-08-33-28.gh-issue-100221.K94Ct3.rst deleted file mode 100644 index 27c948330cfc17..00000000000000 --- a/Misc/NEWS.d/next/Build/2022-12-18-08-33-28.gh-issue-100221.K94Ct3.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix creating install directories in ``make sharedinstall`` if they exist -outside ``DESTDIR`` already. diff --git a/Misc/NEWS.d/next/Build/2022-12-20-01-06-17.gh-issue-99942.lbmzYj.rst b/Misc/NEWS.d/next/Build/2022-12-20-01-06-17.gh-issue-99942.lbmzYj.rst deleted file mode 100644 index 63a640a9cdf733..00000000000000 --- a/Misc/NEWS.d/next/Build/2022-12-20-01-06-17.gh-issue-99942.lbmzYj.rst +++ /dev/null @@ -1,2 +0,0 @@ -On Android, python.pc now correctly reports the library to link to, the same -as python-config.sh. diff --git a/Misc/NEWS.d/next/Build/2023-01-12-00-49-16.gh-issue-99942.DUR8b4.rst b/Misc/NEWS.d/next/Build/2023-01-12-00-49-16.gh-issue-99942.DUR8b4.rst deleted file mode 100644 index 5b692c3cc458c5..00000000000000 --- a/Misc/NEWS.d/next/Build/2023-01-12-00-49-16.gh-issue-99942.DUR8b4.rst +++ /dev/null @@ -1,2 +0,0 @@ -On Android, in a static build, python-config in embed mode no longer -incorrectly reports a library to link to. diff --git a/Misc/NEWS.d/next/C API/2022-04-21-17-25-22.gh-issue-91744.FgvaMi.rst b/Misc/NEWS.d/next/C API/2022-04-21-17-25-22.gh-issue-91744.FgvaMi.rst deleted file mode 100644 index 20db25ddd0c1f6..00000000000000 --- a/Misc/NEWS.d/next/C API/2022-04-21-17-25-22.gh-issue-91744.FgvaMi.rst +++ /dev/null @@ -1,3 +0,0 @@ -Introduced the *Unstable C API tier*, marking APi that is allowed to change -in minor releases without a deprecation period. -See :pep:`689` for details. diff --git a/Misc/NEWS.d/next/C API/2023-02-06-16-14-30.gh-issue-101578.PW5fA9.rst b/Misc/NEWS.d/next/C API/2023-02-06-16-14-30.gh-issue-101578.PW5fA9.rst deleted file mode 100644 index 27294a9e5179c4..00000000000000 --- a/Misc/NEWS.d/next/C API/2023-02-06-16-14-30.gh-issue-101578.PW5fA9.rst +++ /dev/null @@ -1,10 +0,0 @@ -Add :c:func:`PyErr_GetRaisedException` and :c:func:`PyErr_SetRaisedException` -for saving and restoring the current exception. -These functions return and accept a single exception object, -rather than the triple arguments of the now-deprecated -:c:func:`PyErr_Fetch` and :c:func:`PyErr_Restore`. -This is less error prone and a bit more efficient. - -Add :c:func:`PyException_GetArgs` and :c:func:`PyException_SetArgs` -as convenience functions for retrieving and modifying -the :attr:`~BaseException.args` passed to the exception's constructor. diff --git a/Misc/NEWS.d/next/C API/2023-02-09-10-38-20.gh-issue-99293.mFqfpp.rst b/Misc/NEWS.d/next/C API/2023-02-09-10-38-20.gh-issue-99293.mFqfpp.rst deleted file mode 100644 index 8c0f05543747dc..00000000000000 --- a/Misc/NEWS.d/next/C API/2023-02-09-10-38-20.gh-issue-99293.mFqfpp.rst +++ /dev/null @@ -1,2 +0,0 @@ -Document that the Py_TPFLAGS_VALID_VERSION_TAG is an internal feature, -should not be used, and will be removed. diff --git a/Misc/NEWS.d/next/C API/2023-02-14-15-53-01.gh-issue-101907.HgF1N2.rst b/Misc/NEWS.d/next/C API/2023-02-14-15-53-01.gh-issue-101907.HgF1N2.rst deleted file mode 100644 index cfc0d72cdbca00..00000000000000 --- a/Misc/NEWS.d/next/C API/2023-02-14-15-53-01.gh-issue-101907.HgF1N2.rst +++ /dev/null @@ -1 +0,0 @@ -Removes use of non-standard C++ extension in public header files. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-02-20-23-47.gh-issue-98627.VJkdRM.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-02-20-23-47.gh-issue-98627.VJkdRM.rst deleted file mode 100644 index 3d2d6f6eb0c41f..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2022-11-02-20-23-47.gh-issue-98627.VJkdRM.rst +++ /dev/null @@ -1,5 +0,0 @@ -When an interpreter is configured to check (and only then), importing an -extension module will now fail when the extension does not support multiple -interpreters (i.e. doesn't implement PEP 489 multi-phase init). This does -not apply to the main interpreter, nor to subinterpreters created with -``Py_NewInterpreter()``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-01-04-12-49-33.gh-issue-100719.uRPccL.rst b/Misc/NEWS.d/next/Core and Builtins/2023-01-04-12-49-33.gh-issue-100719.uRPccL.rst deleted file mode 100644 index 2addef27b8ea4d..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-01-04-12-49-33.gh-issue-100719.uRPccL.rst +++ /dev/null @@ -1,3 +0,0 @@ -Remove gi_code field from generator (and coroutine and async generator) -objects as it is redundant. The frame already includes a reference to the -code object. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-07-14-56-43.gh-issue-101632.Fd1yxk.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-07-14-56-43.gh-issue-101632.Fd1yxk.rst deleted file mode 100644 index 136909ca699903..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-07-14-56-43.gh-issue-101632.Fd1yxk.rst +++ /dev/null @@ -1 +0,0 @@ -Adds a new :opcode:`RETURN_CONST` instruction. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-08-17-13-31.gh-issue-101696.seJhTt.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-08-17-13-31.gh-issue-101696.seJhTt.rst deleted file mode 100644 index ff2bbb4b564252..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-08-17-13-31.gh-issue-101696.seJhTt.rst +++ /dev/null @@ -1 +0,0 @@ -Invalidate type version tag in ``_PyStaticType_Dealloc`` for static types, avoiding bug where a false cache hit could crash the interpreter. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-10-01-15-57.gh-issue-101430.T3Gegb.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-10-01-15-57.gh-issue-101430.T3Gegb.rst deleted file mode 100644 index e617d85242144e..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-10-01-15-57.gh-issue-101430.T3Gegb.rst +++ /dev/null @@ -1,2 +0,0 @@ -Update :mod:`tracemalloc` to handle presize of object properly. Patch by -Dong-hee Na. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-10-07-21-47.gh-issue-101765.MO5LlC.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-10-07-21-47.gh-issue-101765.MO5LlC.rst deleted file mode 100644 index cc99779a944ec6..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-10-07-21-47.gh-issue-101765.MO5LlC.rst +++ /dev/null @@ -1 +0,0 @@ -Fix SystemError / segmentation fault in iter ``__reduce__`` when internal access of ``builtins.__dict__`` keys mutates the iter object. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-10-15-54-57.gh-issue-87849.IUVvPz.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-10-15-54-57.gh-issue-87849.IUVvPz.rst deleted file mode 100644 index da5f3ff79fd575..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-10-15-54-57.gh-issue-87849.IUVvPz.rst +++ /dev/null @@ -1,3 +0,0 @@ -Change the ``SEND`` instruction to leave the receiver on the stack. This -allows the specialized form of ``SEND`` to skip the chain of C calls and jump -directly to the ``RESUME`` in the generator or coroutine. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-11-23-14-06.gh-issue-84783._P5sMa.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-11-23-14-06.gh-issue-84783._P5sMa.rst deleted file mode 100644 index e1c851a0825a7f..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-11-23-14-06.gh-issue-84783._P5sMa.rst +++ /dev/null @@ -1 +0,0 @@ -Make the slice object hashable. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-12-22-40-22.gh-issue-101857._bribG.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-12-22-40-22.gh-issue-101857._bribG.rst deleted file mode 100644 index 832cc300fa9433..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-12-22-40-22.gh-issue-101857._bribG.rst +++ /dev/null @@ -1 +0,0 @@ -Fix xattr support detection on Linux systems by widening the check to linux, not just glibc. This fixes support for musl. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-13-18-21-14.gh-issue-101799.wpHbCn.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-13-18-21-14.gh-issue-101799.wpHbCn.rst deleted file mode 100644 index 3233a573be7acd..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-13-18-21-14.gh-issue-101799.wpHbCn.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add :opcode:`CALL_INTRINSIC_2` and use it instead of -:opcode:`PREP_RERAISE_STAR`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-13-22-21-58.gh-issue-74895.esMNtq.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-13-22-21-58.gh-issue-74895.esMNtq.rst deleted file mode 100644 index adbbb601634a60..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-13-22-21-58.gh-issue-74895.esMNtq.rst +++ /dev/null @@ -1,5 +0,0 @@ -:mod:`socket.getaddrinfo` no longer raises :class:`OverflowError` for -:class:`int` **port** values outside of the C long range. Out of range values -are left up to the underlying string based C library API to report. A -:class:`socket.gaierror` ``SAI_SERVICE`` may occur instead, or no error at all -as not all platform C libraries generate an error. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-16-16-57-23.gh-issue-101952.Zo1dlq.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-16-16-57-23.gh-issue-101952.Zo1dlq.rst deleted file mode 100644 index 3902c988c8bf9f..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-16-16-57-23.gh-issue-101952.Zo1dlq.rst +++ /dev/null @@ -1 +0,0 @@ -Fix possible segfault in ``BUILD_SET`` opcode, when new set created. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-16-23-19-01.gh-issue-101967.Kqr1dz.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-16-23-19-01.gh-issue-101967.Kqr1dz.rst deleted file mode 100644 index 6e681f910f5359..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-16-23-19-01.gh-issue-101967.Kqr1dz.rst +++ /dev/null @@ -1 +0,0 @@ -Fix possible segfault in ``positional_only_passed_as_keyword`` function, when new list created. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-17-10-12-13.gh-issue-100982.mJGJQw.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-17-10-12-13.gh-issue-100982.mJGJQw.rst deleted file mode 100644 index 53bbc860c53f37..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-17-10-12-13.gh-issue-100982.mJGJQw.rst +++ /dev/null @@ -1,2 +0,0 @@ -Restrict the scope of the :opcode:`FOR_ITER_RANGE` instruction to the scope of the -original :opcode:`FOR_ITER` instruction, to allow instrumentation. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-20-15-18-33.gh-issue-102056.uHKuwH.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-20-15-18-33.gh-issue-102056.uHKuwH.rst deleted file mode 100644 index 78cd525b365fe5..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-20-15-18-33.gh-issue-102056.uHKuwH.rst +++ /dev/null @@ -1 +0,0 @@ -Fix error handling bugs in interpreter's exception printing code, which could cause a crash on infinite recursion. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-22-15-15-32.gh-issue-102027.Km4G-d.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-22-15-15-32.gh-issue-102027.Km4G-d.rst deleted file mode 100644 index 514a8ef26594dc..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-22-15-15-32.gh-issue-102027.Km4G-d.rst +++ /dev/null @@ -1,2 +0,0 @@ -Use ``GetCurrentProcessId`` on Windows when ``getpid`` is unavailable. Patch by -Max Bachmann. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-24-17-59-39.gh-issue-102126.HTT8Vc.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-24-17-59-39.gh-issue-102126.HTT8Vc.rst deleted file mode 100644 index 68c43688c3df03..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-24-17-59-39.gh-issue-102126.HTT8Vc.rst +++ /dev/null @@ -1 +0,0 @@ -Fix deadlock at shutdown when clearing thread states if any finalizer tries to acquire the runtime head lock. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-26-23-10-32.gh-issue-102250.7MUKoC.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-26-23-10-32.gh-issue-102250.7MUKoC.rst deleted file mode 100644 index 17ab0cd4367991..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-26-23-10-32.gh-issue-102250.7MUKoC.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed a segfault occurring when the interpreter calls a ``__bool__`` method that raises. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-28-21-17-03.gh-issue-102336.-wL3Tm.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-28-21-17-03.gh-issue-102336.-wL3Tm.rst deleted file mode 100644 index 0c3e4bd4b86094..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-28-21-17-03.gh-issue-102336.-wL3Tm.rst +++ /dev/null @@ -1 +0,0 @@ -Cleanup Windows 7 specific special handling. Patch by Max Bachmann. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-04-20-56-12.gh-issue-102356.07KvUd.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-04-20-56-12.gh-issue-102356.07KvUd.rst deleted file mode 100644 index c03fd5266bc301..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-03-04-20-56-12.gh-issue-102356.07KvUd.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a bug that caused a crash when deallocating deeply nested filter -objects. Patch by Marta Gómez Macías. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-06-13-05-33.gh-issue-102416.dz6K5f.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-06-13-05-33.gh-issue-102416.dz6K5f.rst deleted file mode 100644 index 9ffc67cfb7ed56..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-03-06-13-05-33.gh-issue-102416.dz6K5f.rst +++ /dev/null @@ -1 +0,0 @@ -Do not memoize incorrectly automatically generated loop rules in the parser. Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-07-16-56-28.gh-issue-102493.gTXrcD.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-07-16-56-28.gh-issue-102493.gTXrcD.rst deleted file mode 100644 index 4c4e88ca4e7c3c..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-03-07-16-56-28.gh-issue-102493.gTXrcD.rst +++ /dev/null @@ -1 +0,0 @@ -Fix regression in semantics of normalisation in ``PyErr_SetObject``. diff --git a/Misc/NEWS.d/next/Documentation/2023-02-07-21-43-24.gh-issue-97725.cuY7Cd.rst b/Misc/NEWS.d/next/Documentation/2023-02-07-21-43-24.gh-issue-97725.cuY7Cd.rst deleted file mode 100644 index fd9ea049c23968..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2023-02-07-21-43-24.gh-issue-97725.cuY7Cd.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :meth:`asyncio.Task.print_stack` description for ``file=None``. -Patch by Oleg Iarygin. diff --git a/Misc/NEWS.d/next/Documentation/2023-02-19-10-33-01.gh-issue-85417.kYO8u3.rst b/Misc/NEWS.d/next/Documentation/2023-02-19-10-33-01.gh-issue-85417.kYO8u3.rst deleted file mode 100644 index a5532df14795d2..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2023-02-19-10-33-01.gh-issue-85417.kYO8u3.rst +++ /dev/null @@ -1 +0,0 @@ -Update :mod:`cmath` documentation to clarify behaviour on branch cuts. diff --git a/Misc/NEWS.d/next/Library/2018-06-20-09-12-21.bpo-23224.zxCQ13.rst b/Misc/NEWS.d/next/Library/2018-06-20-09-12-21.bpo-23224.zxCQ13.rst deleted file mode 100644 index 8909753c7f9ee6..00000000000000 --- a/Misc/NEWS.d/next/Library/2018-06-20-09-12-21.bpo-23224.zxCQ13.rst +++ /dev/null @@ -1,6 +0,0 @@ -Fix segfaults when creating :class:`lzma.LZMADecompressor` and -:class:`bz2.BZ2Decompressor` objects without calling ``__init__()``, and fix -leakage of locks and internal buffers when calling the ``__init__()`` -methods of :class:`lzma.LZMADecompressor`, :class:`lzma.LZMACompressor`, -:class:`bz2.BZ2Compressor`, and :class:`bz2.BZ2Decompressor` objects -multiple times. diff --git a/Misc/NEWS.d/next/Library/2022-09-05-12-17-34.gh-issue-88233.gff9qJ.rst b/Misc/NEWS.d/next/Library/2022-09-05-12-17-34.gh-issue-88233.gff9qJ.rst deleted file mode 100644 index 806f7011edc398..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-09-05-12-17-34.gh-issue-88233.gff9qJ.rst +++ /dev/null @@ -1,2 +0,0 @@ -Correctly preserve "extra" fields in ``zipfile`` regardless of their -ordering relative to a zip64 "extra." diff --git a/Misc/NEWS.d/next/Library/2022-10-22-09-26-43.gh-issue-96764.Dh9Y5L.rst b/Misc/NEWS.d/next/Library/2022-10-22-09-26-43.gh-issue-96764.Dh9Y5L.rst deleted file mode 100644 index a0174291cbc311..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-22-09-26-43.gh-issue-96764.Dh9Y5L.rst +++ /dev/null @@ -1 +0,0 @@ -:func:`asyncio.wait_for` now uses :func:`asyncio.timeout` as its underlying implementation. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Library/2023-01-02-22-41-44.gh-issue-99138.17hp9U.rst b/Misc/NEWS.d/next/Library/2023-01-02-22-41-44.gh-issue-99138.17hp9U.rst deleted file mode 100644 index 3dd4646f40e1e5..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-01-02-22-41-44.gh-issue-99138.17hp9U.rst +++ /dev/null @@ -1 +0,0 @@ -Apply :pep:`687` to :mod:`zoneinfo`. Patch by Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2023-01-06-21-14-41.gh-issue-100809.I697UT.rst b/Misc/NEWS.d/next/Library/2023-01-06-21-14-41.gh-issue-100809.I697UT.rst deleted file mode 100644 index 54082de88ccf4a..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-01-06-21-14-41.gh-issue-100809.I697UT.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix handling of drive-relative paths (like 'C:' and 'C:foo') in -:meth:`pathlib.Path.absolute`. This method now uses the OS API -to retrieve the correct current working directory for the drive. diff --git a/Misc/NEWS.d/next/Library/2023-01-25-00-14-52.gh-issue-101277.FceHX7.rst b/Misc/NEWS.d/next/Library/2023-01-25-00-14-52.gh-issue-101277.FceHX7.rst deleted file mode 100644 index e09c0e09fb388f..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-01-25-00-14-52.gh-issue-101277.FceHX7.rst +++ /dev/null @@ -1,2 +0,0 @@ -Remove global state from :mod:`itertools` module (:pep:`687`). Patches by -Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2023-01-27-02-53-50.gh-issue-101360.bPB7SL.rst b/Misc/NEWS.d/next/Library/2023-01-27-02-53-50.gh-issue-101360.bPB7SL.rst deleted file mode 100644 index 4cfb136c5db853..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-01-27-02-53-50.gh-issue-101360.bPB7SL.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix anchor matching in :meth:`pathlib.PureWindowsPath.match`. Path and -pattern anchors are now matched with :mod:`fnmatch`, just like other path -parts. This allows patterns such as ``"*:/Users/*"`` to be matched. diff --git a/Misc/NEWS.d/next/Library/2023-02-01-10-42-16.gh-issue-63301.XNxSFh.rst b/Misc/NEWS.d/next/Library/2023-02-01-10-42-16.gh-issue-63301.XNxSFh.rst deleted file mode 100644 index e00e71fb8554f3..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-01-10-42-16.gh-issue-63301.XNxSFh.rst +++ /dev/null @@ -1 +0,0 @@ -Set exit code when :mod:`tabnanny` CLI exits on error. diff --git a/Misc/NEWS.d/next/Library/2023-02-04-16-35-46.gh-issue-101561.Xo6pIZ.rst b/Misc/NEWS.d/next/Library/2023-02-04-16-35-46.gh-issue-101561.Xo6pIZ.rst deleted file mode 100644 index 2f6a4153062e5a..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-04-16-35-46.gh-issue-101561.Xo6pIZ.rst +++ /dev/null @@ -1 +0,0 @@ -Add a new decorator :func:`typing.override`. See :pep:`698` for details. Patch by Steven Troxler. diff --git a/Misc/NEWS.d/next/Library/2023-02-05-21-40-15.gh-issue-85984.Kfzbb2.rst b/Misc/NEWS.d/next/Library/2023-02-05-21-40-15.gh-issue-85984.Kfzbb2.rst deleted file mode 100644 index c91829f2c739af..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-05-21-40-15.gh-issue-85984.Kfzbb2.rst +++ /dev/null @@ -1,4 +0,0 @@ -Refactored the implementation of :func:`pty.fork` to use :func:`os.login_tty`. - -A :exc:`DeprecationWarning` is now raised by ``pty.master_open()`` and ``pty.slave_open()``. They were -undocumented and deprecated long long ago in the docstring in favor of :func:`pty.openpty`. diff --git a/Misc/NEWS.d/next/Library/2023-02-07-20-46-08.gh-issue-101362.2ckZ6R.rst b/Misc/NEWS.d/next/Library/2023-02-07-20-46-08.gh-issue-101362.2ckZ6R.rst deleted file mode 100644 index 8421466cdbb3c9..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-07-20-46-08.gh-issue-101362.2ckZ6R.rst +++ /dev/null @@ -1,2 +0,0 @@ -Speed up :class:`pathlib.Path` construction by running the path flavour -compatibility check only when pathlib is imported. diff --git a/Misc/NEWS.d/next/Library/2023-02-07-21-16-41.gh-issue-101362.KMQllM.rst b/Misc/NEWS.d/next/Library/2023-02-07-21-16-41.gh-issue-101362.KMQllM.rst deleted file mode 100644 index af4ee9ad904868..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-07-21-16-41.gh-issue-101362.KMQllM.rst +++ /dev/null @@ -1,2 +0,0 @@ -Speed up :class:`pathlib.PurePath` construction by calling -:func:`os.path.join` only when two or more arguments are given. diff --git a/Misc/NEWS.d/next/Library/2023-02-07-22-20-32.gh-issue-101362.Jlk6mt.rst b/Misc/NEWS.d/next/Library/2023-02-07-22-20-32.gh-issue-101362.Jlk6mt.rst deleted file mode 100644 index c05f92ae699de9..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-07-22-20-32.gh-issue-101362.Jlk6mt.rst +++ /dev/null @@ -1,4 +0,0 @@ -Speed up :class:`pathlib.PurePath` construction by handling arguments more -uniformly. When a :class:`pathlib.Path` argument is supplied, -we use its string representation rather than joining its parts -with :func:`os.path.join`. diff --git a/Misc/NEWS.d/next/Library/2023-02-07-22-21-46.gh-issue-101446.-c0FdK.rst b/Misc/NEWS.d/next/Library/2023-02-07-22-21-46.gh-issue-101446.-c0FdK.rst deleted file mode 100644 index ddf897b71bb1d1..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-07-22-21-46.gh-issue-101446.-c0FdK.rst +++ /dev/null @@ -1,2 +0,0 @@ -Change repr of :class:`collections.OrderedDict` to use regular dictionary -formating instead of pairs of keys and values. diff --git a/Misc/NEWS.d/next/Library/2023-02-08-18-20-58.gh-issue-101693.4_LPXj.rst b/Misc/NEWS.d/next/Library/2023-02-08-18-20-58.gh-issue-101693.4_LPXj.rst deleted file mode 100644 index e436054b15b657..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-08-18-20-58.gh-issue-101693.4_LPXj.rst +++ /dev/null @@ -1,6 +0,0 @@ -In :meth:`sqlite3.Cursor.execute`, :exc:`DeprecationWarning` is now emitted -when :ref:`named placeholders ` are used together with -parameters supplied as a :term:`sequence` instead of as a :class:`dict`. -Starting from Python 3.14, using named placeholders with parameters supplied -as a sequence will raise a :exc:`~sqlite3.ProgrammingError`. -Patch by Erlend E. Aasland. diff --git a/Misc/NEWS.d/next/Library/2023-02-10-11-59-13.gh-issue-101773.J_kI7y.rst b/Misc/NEWS.d/next/Library/2023-02-10-11-59-13.gh-issue-101773.J_kI7y.rst deleted file mode 100644 index b577d93d28c2ae..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-10-11-59-13.gh-issue-101773.J_kI7y.rst +++ /dev/null @@ -1,2 +0,0 @@ -Optimize :class:`fractions.Fraction` for small components. The private argument -``_normalize`` of the :class:`fractions.Fraction` constructor has been removed. diff --git a/Misc/NEWS.d/next/Library/2023-02-10-16-02-29.gh-issue-101517.r7S2u8.rst b/Misc/NEWS.d/next/Library/2023-02-10-16-02-29.gh-issue-101517.r7S2u8.rst deleted file mode 100644 index a5f6bdfa5ac2f0..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-10-16-02-29.gh-issue-101517.r7S2u8.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed bug where :mod:`bdb` looks up the source line with :mod:`linecache` with a ``lineno=None``, which causes it to fail with an unhandled exception. diff --git a/Misc/NEWS.d/next/Library/2023-02-11-13-23-29.gh-issue-97786.QjvQ1B.rst b/Misc/NEWS.d/next/Library/2023-02-11-13-23-29.gh-issue-97786.QjvQ1B.rst deleted file mode 100644 index df194b67590d67..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-11-13-23-29.gh-issue-97786.QjvQ1B.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix potential undefined behaviour in corner cases of floating-point-to-time -conversions. diff --git a/Misc/NEWS.d/next/Library/2023-02-13-12-55-48.gh-issue-87634.q-SBhJ.rst b/Misc/NEWS.d/next/Library/2023-02-13-12-55-48.gh-issue-87634.q-SBhJ.rst deleted file mode 100644 index a17927500bd9a5..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-13-12-55-48.gh-issue-87634.q-SBhJ.rst +++ /dev/null @@ -1 +0,0 @@ -Remove locking behavior from :func:`functools.cached_property`. diff --git a/Misc/NEWS.d/next/Library/2023-02-14-09-08-48.gh-issue-101892.FMos8l.rst b/Misc/NEWS.d/next/Library/2023-02-14-09-08-48.gh-issue-101892.FMos8l.rst deleted file mode 100644 index d586779b3a8a36..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-14-09-08-48.gh-issue-101892.FMos8l.rst +++ /dev/null @@ -1,3 +0,0 @@ -Callable iterators no longer raise :class:`SystemError` when the -callable object exhausts the iterator but forgets to either return a -sentinel value or raise :class:`StopIteration`. diff --git a/Misc/NEWS.d/next/Library/2023-02-15-01-54-06.gh-issue-99108.rjTSic.rst b/Misc/NEWS.d/next/Library/2023-02-15-01-54-06.gh-issue-99108.rjTSic.rst deleted file mode 100644 index 1612c89c0ea6be..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-15-01-54-06.gh-issue-99108.rjTSic.rst +++ /dev/null @@ -1,3 +0,0 @@ -The built-in extension modules for :mod:`hashlib` SHA2 algorithms, used when -OpenSSL does not provide them, now live in a single internal ``_sha2`` module -instead of separate ``_sha256`` and ``_sha512`` modules. diff --git a/Misc/NEWS.d/next/Library/2023-02-17-18-44-27.gh-issue-101997.A6_blD.rst b/Misc/NEWS.d/next/Library/2023-02-17-18-44-27.gh-issue-101997.A6_blD.rst deleted file mode 100644 index f9dfd46d1ed430..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-17-18-44-27.gh-issue-101997.A6_blD.rst +++ /dev/null @@ -1 +0,0 @@ -Upgrade pip wheel bundled with ensurepip (pip 23.0.1) diff --git a/Misc/NEWS.d/next/Library/2023-02-17-19-00-58.gh-issue-97930.C_nQjb.rst b/Misc/NEWS.d/next/Library/2023-02-17-19-00-58.gh-issue-97930.C_nQjb.rst deleted file mode 100644 index 967e13f752bcd1..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-17-19-00-58.gh-issue-97930.C_nQjb.rst +++ /dev/null @@ -1,4 +0,0 @@ -Apply changes from `importlib_resources 5.12 -`_, -including fix for ``MultiplexedPath`` to support directories in multiple -namespaces (python/importlib_resources#265). diff --git a/Misc/NEWS.d/next/Library/2023-02-17-20-24-15.gh-issue-101566.FjgWBt.rst b/Misc/NEWS.d/next/Library/2023-02-17-20-24-15.gh-issue-101566.FjgWBt.rst deleted file mode 100644 index 5fc1a0288a82dc..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-17-20-24-15.gh-issue-101566.FjgWBt.rst +++ /dev/null @@ -1,4 +0,0 @@ -In zipfile, sync Path with `zipp 3.14 -`_, including -fix for extractall on the underlying zipfile after being wrapped in -``Path``. diff --git a/Misc/NEWS.d/next/Library/2023-02-21-07-15-41.gh-issue-101936.QVOxHH.rst b/Misc/NEWS.d/next/Library/2023-02-21-07-15-41.gh-issue-101936.QVOxHH.rst deleted file mode 100644 index 55841da44b1146..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-21-07-15-41.gh-issue-101936.QVOxHH.rst +++ /dev/null @@ -1,2 +0,0 @@ -The default value of ``fp`` becomes :class:`io.BytesIO` if :exc:`~urllib.error.HTTPError` -is initialized without a designated ``fp`` parameter. Patch by Long Vo. diff --git a/Misc/NEWS.d/next/Library/2023-02-21-10-05-33.gh-issue-101961.7e56jh.rst b/Misc/NEWS.d/next/Library/2023-02-21-10-05-33.gh-issue-101961.7e56jh.rst deleted file mode 100644 index a3d4119e7cbdce..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-21-10-05-33.gh-issue-101961.7e56jh.rst +++ /dev/null @@ -1,2 +0,0 @@ -For the binary mode, :func:`fileinput.hookcompressed` doesn't set the ``encoding`` value -even if the value is ``None``. Patch by Gihwan Kim. diff --git a/Misc/NEWS.d/next/Library/2023-02-23-15-06-01.gh-issue-102179.P6KQ4c.rst b/Misc/NEWS.d/next/Library/2023-02-23-15-06-01.gh-issue-102179.P6KQ4c.rst deleted file mode 100644 index f77493e267ac7e..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-23-15-06-01.gh-issue-102179.P6KQ4c.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :func:`os.dup2` error message for negative fds. diff --git a/Misc/NEWS.d/next/Library/2023-02-23-20-39-52.gh-issue-81652.Vxz0Mr.rst b/Misc/NEWS.d/next/Library/2023-02-23-20-39-52.gh-issue-81652.Vxz0Mr.rst deleted file mode 100644 index 48acce1d863ea6..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-23-20-39-52.gh-issue-81652.Vxz0Mr.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add :data:`mmap.MAP_ALIGNED_SUPER` FreeBSD and :data:`mmap.MAP_CONCEAL` -OpenBSD constants to :mod:`mmap`. Patch by Yeojin Kim. diff --git a/Misc/NEWS.d/next/Library/2023-02-26-12-37-17.gh-issue-91038.S4rFH_.rst b/Misc/NEWS.d/next/Library/2023-02-26-12-37-17.gh-issue-91038.S4rFH_.rst deleted file mode 100644 index 2667ff120fd402..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-26-12-37-17.gh-issue-91038.S4rFH_.rst +++ /dev/null @@ -1 +0,0 @@ -:meth:`platform.platform` now has boolean default arguments. diff --git a/Misc/NEWS.d/next/Library/2023-02-28-09-52-25.gh-issue-101979.or3hXV.rst b/Misc/NEWS.d/next/Library/2023-02-28-09-52-25.gh-issue-101979.or3hXV.rst deleted file mode 100644 index 1efe72439b3a4a..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-28-09-52-25.gh-issue-101979.or3hXV.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a bug where parentheses in the ``metavar`` argument to :meth:`argparse.ArgumentParser.add_argument` were -dropped. Patch by Yeojin Kim. diff --git a/Misc/NEWS.d/next/Library/2023-03-04-14-46-47.gh-issue-102302.-b_s6Z.rst b/Misc/NEWS.d/next/Library/2023-03-04-14-46-47.gh-issue-102302.-b_s6Z.rst deleted file mode 100644 index aaf4e62069ca24..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-04-14-46-47.gh-issue-102302.-b_s6Z.rst +++ /dev/null @@ -1 +0,0 @@ -Micro-optimise hashing of :class:`inspect.Parameter`, reducing the time it takes to hash an instance by around 40%. diff --git a/Misc/NEWS.d/next/Security/2023-01-24-16-12-00.gh-issue-101283.9tqu39.rst b/Misc/NEWS.d/next/Security/2023-01-24-16-12-00.gh-issue-101283.9tqu39.rst deleted file mode 100644 index 0efdfa10234185..00000000000000 --- a/Misc/NEWS.d/next/Security/2023-01-24-16-12-00.gh-issue-101283.9tqu39.rst +++ /dev/null @@ -1,3 +0,0 @@ -:class:`subprocess.Popen` now uses a safer approach to find -``cmd.exe`` when launching with ``shell=True``. Patch by Eryk Sun, -based on a patch by Oleg Iarygin. diff --git a/Misc/NEWS.d/next/Security/2023-02-08-12-57-35.gh-issue-99108.6tnmhA.rst b/Misc/NEWS.d/next/Security/2023-02-08-12-57-35.gh-issue-99108.6tnmhA.rst deleted file mode 100644 index 6a7a309dad5d8f..00000000000000 --- a/Misc/NEWS.d/next/Security/2023-02-08-12-57-35.gh-issue-99108.6tnmhA.rst +++ /dev/null @@ -1,4 +0,0 @@ -Replace the builtin :mod:`hashlib` implementations of SHA2-384 and SHA2-512 -originally from LibTomCrypt with formally verified, side-channel resistant -code from the `HACL* `_ project. -The builtins remain a fallback only used when OpenSSL does not provide them. diff --git a/Misc/NEWS.d/next/Security/2023-02-08-22-03-04.gh-issue-101727.9P5eZz.rst b/Misc/NEWS.d/next/Security/2023-02-08-22-03-04.gh-issue-101727.9P5eZz.rst deleted file mode 100644 index 43acc82063fd7a..00000000000000 --- a/Misc/NEWS.d/next/Security/2023-02-08-22-03-04.gh-issue-101727.9P5eZz.rst +++ /dev/null @@ -1,4 +0,0 @@ -Updated the OpenSSL version used in Windows and macOS binary release builds -to 1.1.1t to address CVE-2023-0286, CVE-2022-4303, and CVE-2022-4303 per -`the OpenSSL 2023-02-07 security advisory -`_. diff --git a/Misc/NEWS.d/next/Security/2023-02-17-10-42-48.gh-issue-99108.MKA8-f.rst b/Misc/NEWS.d/next/Security/2023-02-17-10-42-48.gh-issue-99108.MKA8-f.rst deleted file mode 100644 index 723d8a43a09f9e..00000000000000 --- a/Misc/NEWS.d/next/Security/2023-02-17-10-42-48.gh-issue-99108.MKA8-f.rst +++ /dev/null @@ -1,2 +0,0 @@ -Replace builtin hashlib implementations of MD5 and SHA1 with verified ones -from the HACL* project. diff --git a/Misc/NEWS.d/next/Tests/2023-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst b/Misc/NEWS.d/next/Tests/2023-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst deleted file mode 100644 index 9de278919ef2f8..00000000000000 --- a/Misc/NEWS.d/next/Tests/2023-02-11-20-28-08.gh-issue-89792.S-Y5BZ.rst +++ /dev/null @@ -1,4 +0,0 @@ -``test_tools`` now copies up to 10x less source data to a temporary directory -during the ``freeze`` test by ignoring git metadata and other artifacts. It -also limits its python build parallelism based on os.cpu_count instead of hard -coding it as 8 cores. diff --git a/Misc/NEWS.d/next/Tests/2023-02-11-22-36-10.gh-issue-85984.EVXjT9.rst b/Misc/NEWS.d/next/Tests/2023-02-11-22-36-10.gh-issue-85984.EVXjT9.rst deleted file mode 100644 index 402f99ea6c6ebf..00000000000000 --- a/Misc/NEWS.d/next/Tests/2023-02-11-22-36-10.gh-issue-85984.EVXjT9.rst +++ /dev/null @@ -1 +0,0 @@ -Utilize new "winsize" functions from termios in pty tests. diff --git a/Misc/NEWS.d/next/Tests/2023-02-18-10-51-02.gh-issue-102019.0797SJ.rst b/Misc/NEWS.d/next/Tests/2023-02-18-10-51-02.gh-issue-102019.0797SJ.rst deleted file mode 100644 index 63e36046d26dfe..00000000000000 --- a/Misc/NEWS.d/next/Tests/2023-02-18-10-51-02.gh-issue-102019.0797SJ.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix deadlock on shutdown if ``test_current_{exception,frames}`` fails. Patch -by Jacob Bower. diff --git a/Misc/NEWS.d/next/Windows/2023-01-25-11-33-54.gh-issue-101196.wAX_2g.rst b/Misc/NEWS.d/next/Windows/2023-01-25-11-33-54.gh-issue-101196.wAX_2g.rst deleted file mode 100644 index c61e9b90fb5373..00000000000000 --- a/Misc/NEWS.d/next/Windows/2023-01-25-11-33-54.gh-issue-101196.wAX_2g.rst +++ /dev/null @@ -1,3 +0,0 @@ -The functions ``os.path.isdir``, ``os.path.isfile``, ``os.path.islink`` and -``os.path.exists`` are now 13% to 28% faster on Windows, by making fewer Win32 -API calls. diff --git a/Misc/NEWS.d/next/Windows/2023-02-07-18-22-54.gh-issue-101614.NjVP0n.rst b/Misc/NEWS.d/next/Windows/2023-02-07-18-22-54.gh-issue-101614.NjVP0n.rst deleted file mode 100644 index 8ed0995d78925b..00000000000000 --- a/Misc/NEWS.d/next/Windows/2023-02-07-18-22-54.gh-issue-101614.NjVP0n.rst +++ /dev/null @@ -1 +0,0 @@ -Correctly handle extensions built against debug binaries that reference ``python3_d.dll``. diff --git a/Misc/NEWS.d/next/Windows/2023-02-09-22-09-27.gh-issue-101759.zFlqSH.rst b/Misc/NEWS.d/next/Windows/2023-02-09-22-09-27.gh-issue-101759.zFlqSH.rst deleted file mode 100644 index 62bcac34397d2e..00000000000000 --- a/Misc/NEWS.d/next/Windows/2023-02-09-22-09-27.gh-issue-101759.zFlqSH.rst +++ /dev/null @@ -1 +0,0 @@ -Update Windows installer to SQLite 3.40.1. diff --git a/Misc/NEWS.d/next/Windows/2023-02-10-14-26-05.gh-issue-101763.RPaj7r.rst b/Misc/NEWS.d/next/Windows/2023-02-10-14-26-05.gh-issue-101763.RPaj7r.rst deleted file mode 100644 index e7e5a73afeb532..00000000000000 --- a/Misc/NEWS.d/next/Windows/2023-02-10-14-26-05.gh-issue-101763.RPaj7r.rst +++ /dev/null @@ -1 +0,0 @@ -Updates copy of libffi bundled with Windows installs to 3.4.4. diff --git a/Misc/NEWS.d/next/Windows/2023-02-13-16-32-50.gh-issue-101849.7lm_53.rst b/Misc/NEWS.d/next/Windows/2023-02-13-16-32-50.gh-issue-101849.7lm_53.rst deleted file mode 100644 index 861d4de9f9a650..00000000000000 --- a/Misc/NEWS.d/next/Windows/2023-02-13-16-32-50.gh-issue-101849.7lm_53.rst +++ /dev/null @@ -1 +0,0 @@ -Ensures installer will correctly upgrade existing ``py.exe`` launcher installs. diff --git a/Misc/NEWS.d/next/Windows/2023-02-13-18-05-49.gh-issue-101881._TnHzN.rst b/Misc/NEWS.d/next/Windows/2023-02-13-18-05-49.gh-issue-101881._TnHzN.rst deleted file mode 100644 index ba58dd4f5cb450..00000000000000 --- a/Misc/NEWS.d/next/Windows/2023-02-13-18-05-49.gh-issue-101881._TnHzN.rst +++ /dev/null @@ -1 +0,0 @@ -Add support for the os.get_blocking() and os.set_blocking() functions on Windows. diff --git a/Misc/NEWS.d/next/Windows/2023-02-15-11-08-10.gh-issue-101881.fScr3m.rst b/Misc/NEWS.d/next/Windows/2023-02-15-11-08-10.gh-issue-101881.fScr3m.rst deleted file mode 100644 index 099b2c1c07a665..00000000000000 --- a/Misc/NEWS.d/next/Windows/2023-02-15-11-08-10.gh-issue-101881.fScr3m.rst +++ /dev/null @@ -1 +0,0 @@ -Handle read and write operations on non-blocking pipes properly on Windows. diff --git a/Misc/NEWS.d/next/Windows/2023-03-01-01-36-39.gh-issue-102344.Dgfux4.rst b/Misc/NEWS.d/next/Windows/2023-03-01-01-36-39.gh-issue-102344.Dgfux4.rst deleted file mode 100644 index 4804212be8182c..00000000000000 --- a/Misc/NEWS.d/next/Windows/2023-03-01-01-36-39.gh-issue-102344.Dgfux4.rst +++ /dev/null @@ -1,2 +0,0 @@ -Implement ``winreg.QueryValue`` using ``QueryValueEx`` and -``winreg.SetValue`` using ``SetValueEx``. Patch by Max Bachmann. diff --git a/Misc/NEWS.d/next/macOS/2023-02-09-22-07-17.gh-issue-101759.B0JP2H.rst b/Misc/NEWS.d/next/macOS/2023-02-09-22-07-17.gh-issue-101759.B0JP2H.rst deleted file mode 100644 index fc53d08bffc4fd..00000000000000 --- a/Misc/NEWS.d/next/macOS/2023-02-09-22-07-17.gh-issue-101759.B0JP2H.rst +++ /dev/null @@ -1 +0,0 @@ -Update macOS installer to SQLite 3.40.1. diff --git a/README.rst b/README.rst index b1756e20c141ab..6923b692f6c971 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.12.0 alpha 5 +This is Python version 3.12.0 alpha 6 ===================================== .. image:: https://github.com/python/cpython/workflows/Tests/badge.svg From 1e703a473343ed198c9a06a876b25d7d69d4bbd0 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Tue, 7 Mar 2023 17:10:58 -0700 Subject: [PATCH 14/38] gh-102381: don't call watcher callback with dead object (#102382) Co-authored-by: T. Wouters --- Doc/c-api/code.rst | 17 ++++++++-- Doc/c-api/dict.rst | 21 +++++++++--- Doc/c-api/function.rst | 17 ++++++++-- Include/cpython/code.h | 13 +++++--- Include/cpython/dictobject.h | 21 +++++++----- Include/cpython/funcobject.h | 16 ++++----- Include/internal/pycore_dict.h | 1 + Lib/test/test_capi/test_watchers.py | 52 +++++++++++++++++++++++++++-- Modules/_testcapi/watchers.c | 13 +++++++- Objects/codeobject.c | 38 ++++++++++++++++++++- Objects/dictobject.c | 34 +++++++++++++++++-- Objects/funcobject.c | 38 ++++++++++++++++++++- 12 files changed, 243 insertions(+), 38 deletions(-) diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 062ef3a1fea93c..a99de9904c0740 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -172,6 +172,11 @@ bound into a function. before the destruction of *co* takes place, so the prior state of *co* can be inspected. + If *event* is ``PY_CODE_EVENT_DESTROY``, taking a reference in the callback + to the about-to-be-destroyed code object will resurrect it and prevent it + from being freed at this time. When the resurrected object is destroyed + later, any watcher callbacks active at that time will be called again. + Users of this API should not rely on internal runtime implementation details. Such details may include, but are not limited to, the exact order and timing of creation and destruction of code objects. While @@ -179,9 +184,15 @@ bound into a function. (including whether a callback is invoked or not), it does not change the semantics of the Python code being executed. - If the callback returns with an exception set, it must return ``-1``; this - exception will be printed as an unraisable exception using - :c:func:`PyErr_WriteUnraisable`. Otherwise it should return ``0``. + If the callback sets an exception, it must return ``-1``; this exception will + be printed as an unraisable exception using :c:func:`PyErr_WriteUnraisable`. + Otherwise it should return ``0``. + + There may already be a pending exception set on entry to the callback. In + this case, the callback should return ``0`` with the same exception still + set. This means the callback may not call any other API that can set an + exception unless it saves and clears the exception state first, and restores + it before returning. .. versionadded:: 3.12 diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 34106ee6b1f30d..b9f84cea785644 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -298,13 +298,26 @@ Dictionary Objects dictionary. The callback may inspect but must not modify *dict*; doing so could have - unpredictable effects, including infinite recursion. + unpredictable effects, including infinite recursion. Do not trigger Python + code execution in the callback, as it could modify the dict as a side effect. + + If *event* is ``PyDict_EVENT_DEALLOCATED``, taking a new reference in the + callback to the about-to-be-destroyed dictionary will resurrect it and + prevent it from being freed at this time. When the resurrected object is + destroyed later, any watcher callbacks active at that time will be called + again. Callbacks occur before the notified modification to *dict* takes place, so the prior state of *dict* can be inspected. - If the callback returns with an exception set, it must return ``-1``; this - exception will be printed as an unraisable exception using - :c:func:`PyErr_WriteUnraisable`. Otherwise it should return ``0``. + If the callback sets an exception, it must return ``-1``; this exception will + be printed as an unraisable exception using :c:func:`PyErr_WriteUnraisable`. + Otherwise it should return ``0``. + + There may already be a pending exception set on entry to the callback. In + this case, the callback should return ``0`` with the same exception still + set. This means the callback may not call any other API that can set an + exception unless it saves and clears the exception state first, and restores + it before returning. .. versionadded:: 3.12 diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst index bc7569d0add97d..947ed70404081b 100644 --- a/Doc/c-api/function.rst +++ b/Doc/c-api/function.rst @@ -173,8 +173,19 @@ There are a few functions specific to Python functions. runtime behavior depending on optimization decisions, it does not change the semantics of the Python code being executed. - If the callback returns with an exception set, it must return ``-1``; this - exception will be printed as an unraisable exception using - :c:func:`PyErr_WriteUnraisable`. Otherwise it should return ``0``. + If *event* is ``PyFunction_EVENT_DESTROY``, Taking a reference in the + callback to the about-to-be-destroyed function will resurrect it, preventing + it from being freed at this time. When the resurrected object is destroyed + later, any watcher callbacks active at that time will be called again. + + If the callback sets an exception, it must return ``-1``; this exception will + be printed as an unraisable exception using :c:func:`PyErr_WriteUnraisable`. + Otherwise it should return ``0``. + + There may already be a pending exception set on entry to the callback. In + this case, the callback should return ``0`` with the same exception still + set. This means the callback may not call any other API that can set an + exception unless it saves and clears the exception state first, and restores + it before returning. .. versionadded:: 3.12 diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 0e4bd8a58c165b..abcf1250603dfe 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -224,9 +224,14 @@ PyAPI_FUNC(int) PyCode_Addr2Line(PyCodeObject *, int); PyAPI_FUNC(int) PyCode_Addr2Location(PyCodeObject *, int, int *, int *, int *, int *); -typedef enum PyCodeEvent { - PY_CODE_EVENT_CREATE, - PY_CODE_EVENT_DESTROY +#define PY_FOREACH_CODE_EVENT(V) \ + V(CREATE) \ + V(DESTROY) + +typedef enum { + #define PY_DEF_EVENT(op) PY_CODE_EVENT_##op, + PY_FOREACH_CODE_EVENT(PY_DEF_EVENT) + #undef PY_DEF_EVENT } PyCodeEvent; @@ -236,7 +241,7 @@ typedef enum PyCodeEvent { * The callback is invoked with a borrowed reference to co, after it is * created and before it is destroyed. * - * If the callback returns with an exception set, it must return -1. Otherwise + * If the callback sets an exception, it must return -1. Otherwise * it should return 0. */ typedef int (*PyCode_WatchCallback)( diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index 5001f35654475e..ddada922020aa4 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -16,11 +16,11 @@ typedef struct { /* Dictionary version: globally unique, value change each time the dictionary is modified */ -#ifdef Py_BUILD_CORE +#ifdef Py_BUILD_CORE uint64_t ma_version_tag; #else Py_DEPRECATED(3.12) uint64_t ma_version_tag; -#endif +#endif PyDictKeysObject *ma_keys; @@ -90,13 +90,18 @@ PyAPI_FUNC(PyObject *) _PyDictView_Intersect(PyObject* self, PyObject *other); /* Dictionary watchers */ +#define PY_FOREACH_DICT_EVENT(V) \ + V(ADDED) \ + V(MODIFIED) \ + V(DELETED) \ + V(CLONED) \ + V(CLEARED) \ + V(DEALLOCATED) + typedef enum { - PyDict_EVENT_ADDED, - PyDict_EVENT_MODIFIED, - PyDict_EVENT_DELETED, - PyDict_EVENT_CLONED, - PyDict_EVENT_CLEARED, - PyDict_EVENT_DEALLOCATED, + #define PY_DEF_EVENT(EVENT) PyDict_EVENT_##EVENT, + PY_FOREACH_DICT_EVENT(PY_DEF_EVENT) + #undef PY_DEF_EVENT } PyDict_WatchEvent; // Callback to be invoked when a watched dict is cleared, dealloced, or modified. diff --git a/Include/cpython/funcobject.h b/Include/cpython/funcobject.h index 5979febc2e3421..c716330cc3fbab 100644 --- a/Include/cpython/funcobject.h +++ b/Include/cpython/funcobject.h @@ -131,17 +131,17 @@ PyAPI_DATA(PyTypeObject) PyStaticMethod_Type; PyAPI_FUNC(PyObject *) PyClassMethod_New(PyObject *); PyAPI_FUNC(PyObject *) PyStaticMethod_New(PyObject *); -#define FOREACH_FUNC_EVENT(V) \ - V(CREATE) \ - V(DESTROY) \ - V(MODIFY_CODE) \ - V(MODIFY_DEFAULTS) \ +#define PY_FOREACH_FUNC_EVENT(V) \ + V(CREATE) \ + V(DESTROY) \ + V(MODIFY_CODE) \ + V(MODIFY_DEFAULTS) \ V(MODIFY_KWDEFAULTS) typedef enum { - #define DEF_EVENT(EVENT) PyFunction_EVENT_##EVENT, - FOREACH_FUNC_EVENT(DEF_EVENT) - #undef DEF_EVENT + #define PY_DEF_EVENT(EVENT) PyFunction_EVENT_##EVENT, + PY_FOREACH_FUNC_EVENT(PY_DEF_EVENT) + #undef PY_DEF_EVENT } PyFunction_WatchEvent; /* diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index c74a3437713039..1af5e59a677a9a 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -164,6 +164,7 @@ _PyDict_NotifyEvent(PyDict_WatchEvent event, PyObject *key, PyObject *value) { + assert(Py_REFCNT((PyObject*)mp) > 0); int watcher_bits = mp->ma_version_tag & DICT_VERSION_MASK; if (watcher_bits) { _PyDict_SendEvent(watcher_bits, event, mp, key, value); diff --git a/Lib/test/test_capi/test_watchers.py b/Lib/test/test_capi/test_watchers.py index 1922614ef60558..93f6ef752d0663 100644 --- a/Lib/test/test_capi/test_watchers.py +++ b/Lib/test/test_capi/test_watchers.py @@ -109,10 +109,21 @@ def test_error(self): self.watch(wid, d) with catch_unraisable_exception() as cm: d["foo"] = "bar" - self.assertIs(cm.unraisable.object, d) + self.assertIn( + "PyDict_EVENT_ADDED watcher callback for 0); PyInterpreterState *interp = _PyInterpreterState_GET(); assert(interp->_initialized); uint8_t bits = interp->active_code_watchers; @@ -25,7 +40,21 @@ notify_code_watchers(PyCodeEvent event, PyCodeObject *co) // callback must be non-null if the watcher bit is set assert(cb != NULL); if (cb(event, co) < 0) { - PyErr_WriteUnraisable((PyObject *) co); + // Don't risk resurrecting the object if an unraisablehook keeps + // a reference; pass a string as context. + PyObject *context = NULL; + PyObject *repr = code_repr(co); + if (repr) { + context = PyUnicode_FromFormat( + "%s watcher callback for %U", + code_event_name(event), repr); + Py_DECREF(repr); + } + if (context == NULL) { + context = Py_NewRef(Py_None); + } + PyErr_WriteUnraisable(context); + Py_DECREF(context); } } i++; @@ -1667,7 +1696,14 @@ code_new_impl(PyTypeObject *type, int argcount, int posonlyargcount, static void code_dealloc(PyCodeObject *co) { + assert(Py_REFCNT(co) == 0); + Py_SET_REFCNT(co, 1); notify_code_watchers(PY_CODE_EVENT_DESTROY, co); + if (Py_REFCNT(co) > 1) { + Py_SET_REFCNT(co, Py_REFCNT(co) - 1); + return; + } + Py_SET_REFCNT(co, 0); if (co->co_extra != NULL) { PyInterpreterState *interp = _PyInterpreterState_GET(); diff --git a/Objects/dictobject.c b/Objects/dictobject.c index fc658ca2f4b7f8..e3795e75e3da5e 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -2308,7 +2308,14 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) static void dict_dealloc(PyDictObject *mp) { + assert(Py_REFCNT(mp) == 0); + Py_SET_REFCNT(mp, 1); _PyDict_NotifyEvent(PyDict_EVENT_DEALLOCATED, mp, NULL, NULL); + if (Py_REFCNT(mp) > 1) { + Py_SET_REFCNT(mp, Py_REFCNT(mp) - 1); + return; + } + Py_SET_REFCNT(mp, 0); PyDictValues *values = mp->ma_values; PyDictKeysObject *keys = mp->ma_keys; Py_ssize_t i, n; @@ -5732,6 +5739,18 @@ PyDict_ClearWatcher(int watcher_id) return 0; } +static const char * +dict_event_name(PyDict_WatchEvent event) { + switch (event) { + #define CASE(op) \ + case PyDict_EVENT_##op: \ + return "PyDict_EVENT_" #op; + PY_FOREACH_DICT_EVENT(CASE) + #undef CASE + } + Py_UNREACHABLE(); +} + void _PyDict_SendEvent(int watcher_bits, PyDict_WatchEvent event, @@ -5744,9 +5763,18 @@ _PyDict_SendEvent(int watcher_bits, if (watcher_bits & 1) { PyDict_WatchCallback cb = interp->dict_state.watchers[i]; if (cb && (cb(event, (PyObject*)mp, key, value) < 0)) { - // some dict modification paths (e.g. PyDict_Clear) can't raise, so we - // can't propagate exceptions from dict watchers. - PyErr_WriteUnraisable((PyObject *)mp); + // We don't want to resurrect the dict by potentially having an + // unraisablehook keep a reference to it, so we don't pass the + // dict as context, just an informative string message. Dict + // repr can call arbitrary code, so we invent a simpler version. + PyObject *context = PyUnicode_FromFormat( + "%s watcher callback for ", + dict_event_name(event), mp); + if (context == NULL) { + context = Py_NewRef(Py_None); + } + PyErr_WriteUnraisable(context); + Py_DECREF(context); } } watcher_bits >>= 1; diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 91a6b3dd40a232..99048ea41c6c80 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -8,6 +8,20 @@ #include "pycore_pyerrors.h" // _PyErr_Occurred() #include "structmember.h" // PyMemberDef +static PyObject* func_repr(PyFunctionObject *op); + +static const char * +func_event_name(PyFunction_WatchEvent event) { + switch (event) { + #define CASE(op) \ + case PyFunction_EVENT_##op: \ + return "PyFunction_EVENT_" #op; + PY_FOREACH_FUNC_EVENT(CASE) + #undef CASE + } + Py_UNREACHABLE(); +} + static void notify_func_watchers(PyInterpreterState *interp, PyFunction_WatchEvent event, PyFunctionObject *func, PyObject *new_value) @@ -21,7 +35,21 @@ notify_func_watchers(PyInterpreterState *interp, PyFunction_WatchEvent event, // callback must be non-null if the watcher bit is set assert(cb != NULL); if (cb(event, func, new_value) < 0) { - PyErr_WriteUnraisable((PyObject *) func); + // Don't risk resurrecting the func if an unraisablehook keeps a + // reference; pass a string as context. + PyObject *context = NULL; + PyObject *repr = func_repr(func); + if (repr != NULL) { + context = PyUnicode_FromFormat( + "%s watcher callback for %U", + func_event_name(event), repr); + Py_DECREF(repr); + } + if (context == NULL) { + context = Py_NewRef(Py_None); + } + PyErr_WriteUnraisable(context); + Py_DECREF(context); } } i++; @@ -33,6 +61,7 @@ static inline void handle_func_event(PyFunction_WatchEvent event, PyFunctionObject *func, PyObject *new_value) { + assert(Py_REFCNT(func) > 0); PyInterpreterState *interp = _PyInterpreterState_GET(); assert(interp->_initialized); if (interp->active_func_watchers) { @@ -766,7 +795,14 @@ func_clear(PyFunctionObject *op) static void func_dealloc(PyFunctionObject *op) { + assert(Py_REFCNT(op) == 0); + Py_SET_REFCNT(op, 1); handle_func_event(PyFunction_EVENT_DESTROY, op, NULL); + if (Py_REFCNT(op) > 1) { + Py_SET_REFCNT(op, Py_REFCNT(op) - 1); + return; + } + Py_SET_REFCNT(op, 0); _PyObject_GC_UNTRACK(op); if (op->func_weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject *) op); From e499680100a9d4fa4b673cbcfebc32212b4f848a Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Wed, 8 Mar 2023 09:09:50 +0530 Subject: [PATCH 15/38] fix typo in async generator code field name `ag_code` (#102448) --- Objects/genobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/genobject.c b/Objects/genobject.c index 4ab6581e12ab3a..be08a59ece6b7e 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -1549,7 +1549,7 @@ ag_getframe(PyAsyncGenObject *ag, void *Py_UNUSED(ignored)) static PyObject * ag_getcode(PyGenObject *gen, void *Py_UNUSED(ignored)) { - return _gen_getcode(gen, "ag__code"); + return _gen_getcode(gen, "ag_code"); } static PyGetSetDef async_gen_getsetlist[] = { From 02b9a921cb3e6308ad300ee9c398b24998220931 Mon Sep 17 00:00:00 2001 From: Thomas Wouters Date: Wed, 8 Mar 2023 05:04:38 +0100 Subject: [PATCH 16/38] Post 3.12.0a6 --- Include/patchlevel.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index fd16421eace94a..049cdfa30897ca 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -23,7 +23,7 @@ #define PY_RELEASE_SERIAL 6 /* Version as a string */ -#define PY_VERSION "3.12.0a6" +#define PY_VERSION "3.12.0a6+" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. From 061325e0d2bbec6ff89d03f527c91dc7bfa14003 Mon Sep 17 00:00:00 2001 From: Marcin Wieczorek Date: Wed, 8 Mar 2023 08:25:28 +0100 Subject: [PATCH 17/38] Fix style in argparse.rst (#101733) --- Doc/library/argparse.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 34b4c61649b99f..ee68ac58d3de75 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -34,9 +34,9 @@ around an instance of :class:`argparse.ArgumentParser`. It is a container for argument specifications and has options that apply to the parser as whole:: parser = argparse.ArgumentParser( - prog = 'ProgramName', - description = 'What the program does', - epilog = 'Text at the bottom of help') + prog='ProgramName', + description='What the program does', + epilog='Text at the bottom of help') The :meth:`ArgumentParser.add_argument` method attaches individual argument specifications to the parser. It supports positional arguments, options that From 1a84cc007e207f2dd61f86a7fc3d86632fdce72f Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Wed, 8 Mar 2023 13:29:39 +0530 Subject: [PATCH 18/38] GH-102397: Fix segfault from race condition in signal handling (#102399) Co-authored-by: Gregory P. Smith --- Lib/test/test_signal.py | 15 +++++++++++++++ ...2023-03-04-06-48-34.gh-issue-102397.ACJaOf.rst | 2 ++ Modules/signalmodule.c | 4 ++++ 3 files changed, 21 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-04-06-48-34.gh-issue-102397.ACJaOf.rst diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index 2562a57ea421ff..25afd6aabe0751 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -1406,6 +1406,21 @@ def handler(a, b): signal.raise_signal(signal.SIGINT) self.assertTrue(is_ok) + def test__thread_interrupt_main(self): + # See https://github.com/python/cpython/issues/102397 + code = """if 1: + import _thread + class Foo(): + def __del__(self): + _thread.interrupt_main() + + x = Foo() + """ + + rc, out, err = assert_python_ok('-c', code) + self.assertIn(b'OSError: Signal 2 ignored due to race condition', err) + + class PidfdSignalTest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-04-06-48-34.gh-issue-102397.ACJaOf.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-04-06-48-34.gh-issue-102397.ACJaOf.rst new file mode 100644 index 00000000000000..db0b3f32c2ec0b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-03-04-06-48-34.gh-issue-102397.ACJaOf.rst @@ -0,0 +1,2 @@ +Fix segfault from race condition in signal handling during garbage collection. +Patch by Kumar Aditya. diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index cd26eca351c0ed..0e472e1ee4f9dd 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -148,6 +148,10 @@ get_signal_state(PyObject *module) static inline int compare_handler(PyObject *func, PyObject *dfl_ign_handler) { + // See https://github.com/python/cpython/pull/102399 + if (func == NULL || dfl_ign_handler == NULL) { + return 0; + } assert(PyLong_CheckExact(dfl_ign_handler)); if (!PyLong_CheckExact(func)) { return 0; From 1f557f94c2ee98c2a43bd090a7bf3f39a22ed874 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Wed, 8 Mar 2023 11:19:05 +0300 Subject: [PATCH 19/38] gh-101100: Fix sphinx warnings in `zipapp` and `zipfile` modules (#102526) --- Doc/library/zipapp.rst | 2 +- Doc/library/zipfile.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/zipapp.rst b/Doc/library/zipapp.rst index fb40a2b3e964e4..981020b13cd988 100644 --- a/Doc/library/zipapp.rst +++ b/Doc/library/zipapp.rst @@ -215,7 +215,7 @@ using the :func:`create_archive` function:: >>> import zipapp >>> zipapp.create_archive('old_archive.pyz', 'new_archive.pyz', '/usr/bin/python3') -To update the file in place, do the replacement in memory using a :class:`BytesIO` +To update the file in place, do the replacement in memory using a :class:`~io.BytesIO` object, and then overwrite the source afterwards. Note that there is a risk when overwriting a file in place that an error will result in the loss of the original file. This code does not protect against such errors, but diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 0195abc3a992c1..e2a085d6e98e67 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -288,7 +288,7 @@ ZipFile Objects (``ZipExtFile``) is read-only and provides the following methods: :meth:`~io.BufferedIOBase.read`, :meth:`~io.IOBase.readline`, :meth:`~io.IOBase.readlines`, :meth:`~io.IOBase.seek`, - :meth:`~io.IOBase.tell`, :meth:`__iter__`, :meth:`~iterator.__next__`. + :meth:`~io.IOBase.tell`, :meth:`~container.__iter__`, :meth:`~iterator.__next__`. These objects can operate independently of the ZipFile. With ``mode='w'``, a writable file handle is returned, which supports the From 7d801f245e2021d19daff105ce722f22aa844391 Mon Sep 17 00:00:00 2001 From: sblondon Date: Wed, 8 Mar 2023 11:24:39 +0100 Subject: [PATCH 20/38] Remove or update bitbucket links (GH-101963) Since Mercurial removal from bitbucket.org, some links are broken. They are replaced by github.com or webarchive.org links if available. Otherwise, they are removed. Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> --- Doc/distributing/index.rst | 3 +-- Doc/installing/index.rst | 3 +-- Doc/library/venv.rst | 2 +- Lib/test/test_pathlib.py | 2 +- PC/launcher.c | 3 +-- PC/launcher2.c | 3 +-- 6 files changed, 6 insertions(+), 10 deletions(-) diff --git a/Doc/distributing/index.rst b/Doc/distributing/index.rst index 2ae2726d4e4b92..21389adedf9c15 100644 --- a/Doc/distributing/index.rst +++ b/Doc/distributing/index.rst @@ -39,8 +39,7 @@ Key terms developers and documentation authors responsible for the maintenance and evolution of the standard packaging tools and the associated metadata and file format standards. They maintain a variety of tools, documentation - and issue trackers on both `GitHub `__ and - `Bitbucket `__. + and issue trackers on `GitHub `__. * ``distutils`` is the original build and distribution system first added to the Python standard library in 1998. While direct use of ``distutils`` is being phased out, it still laid the foundation for the current packaging diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst index e158bf1c4c0c7f..5aec5178d48f3d 100644 --- a/Doc/installing/index.rst +++ b/Doc/installing/index.rst @@ -52,8 +52,7 @@ Key terms developers and documentation authors responsible for the maintenance and evolution of the standard packaging tools and the associated metadata and file format standards. They maintain a variety of tools, documentation, - and issue trackers on both `GitHub `__ and - `Bitbucket `__. + and issue trackers on `GitHub `__. * ``distutils`` is the original build and distribution system first added to the Python standard library in 1998. While direct use of ``distutils`` is being phased out, it still laid the foundation for the current packaging diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 2a41096de006b8..8eb0b35eaa12df 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -478,7 +478,7 @@ subclass which installs setuptools and pip into a created virtual environment:: :param context: The information for the virtual environment creation request being processed. """ - url = 'https://bitbucket.org/pypa/setuptools/downloads/ez_setup.py' + url = "https://bootstrap.pypa.io/ez_setup.py" self.install_script(context, 'setuptools', url) # clear up the setuptools archive which gets downloaded pred = lambda o: o.startswith('setuptools-') and o.endswith('.tar.gz') diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index df9c1f6ba65deb..f8e2f44d27fc1e 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -1938,7 +1938,7 @@ def test_resolve_common(self): @os_helper.skip_unless_symlink def test_resolve_dot(self): - # See https://bitbucket.org/pitrou/pathlib/issue/9/pathresolve-fails-on-complex-symlinks + # See http://web.archive.org/web/20200623062557/https://bitbucket.org/pitrou/pathlib/issues/9/ p = self.cls(BASE) self.dirlink('.', join('0')) self.dirlink(os.path.join('0', '0'), join('1')) diff --git a/PC/launcher.c b/PC/launcher.c index da566a180168c5..0776e57249c427 100644 --- a/PC/launcher.c +++ b/PC/launcher.c @@ -770,8 +770,7 @@ run_child(wchar_t * cmdline) window, or fetching a message). As this launcher doesn't do this directly, that cursor remains even after the child process does these things. We avoid that by doing a simple post+get message. - See http://bugs.python.org/issue17290 and - https://bitbucket.org/vinay.sajip/pylauncher/issue/20/busy-cursor-for-a-long-time-when-running + See http://bugs.python.org/issue17290 */ MSG msg; diff --git a/PC/launcher2.c b/PC/launcher2.c index beeb2ae46b83f0..932665387f1966 100644 --- a/PC/launcher2.c +++ b/PC/launcher2.c @@ -2473,8 +2473,7 @@ launchEnvironment(const SearchInfo *search, const EnvironmentInfo *launch, wchar window, or fetching a message). As this launcher doesn't do this directly, that cursor remains even after the child process does these things. We avoid that by doing a simple post+get message. - See http://bugs.python.org/issue17290 and - https://bitbucket.org/vinay.sajip/pylauncher/issue/20/busy-cursor-for-a-long-time-when-running + See http://bugs.python.org/issue17290 */ MSG msg; From 401d7a7f009ca2e282b1a0d1b880dc602afd39dc Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Wed, 8 Mar 2023 11:45:38 +0000 Subject: [PATCH 21/38] gh-102515: Remove unused imports in the `Lib/` directory (#102516) --- Lib/_pylong.py | 1 - Lib/concurrent/futures/process.py | 2 ++ Lib/dataclasses.py | 1 - Lib/importlib/_abc.py | 1 - Lib/sysconfig.py | 2 +- Lib/test/_test_venv_multiprocessing.py | 1 - Lib/test/fork_wait.py | 2 +- Lib/test/test__xxinterpchannels.py | 1 - Lib/test/test__xxsubinterpreters.py | 2 -- Lib/test/test_asyncgen.py | 1 - Lib/test/test_asyncio/test_ssl.py | 1 - Lib/test/test_asyncio/test_subprocess.py | 1 - Lib/test/test_ctypes/test_callbacks.py | 1 - Lib/test/test_dataclasses.py | 1 - Lib/test/test_enum.py | 2 -- Lib/test/test_grammar.py | 2 -- Lib/test/test_hashlib.py | 1 - Lib/test/test_httplib.py | 2 -- Lib/test/test_imaplib.py | 2 -- Lib/test/test_isinstance.py | 1 - Lib/test/test_minidom.py | 1 - Lib/test/test_peg_generator/test_c_parser.py | 1 - Lib/test/test_shlex.py | 1 - Lib/test/test_sys_setprofile.py | 1 - Lib/test/test_sys_settrace.py | 1 - Lib/test/test_tkinter/test_widgets.py | 3 +-- Lib/test/test_tools/test_sundry.py | 1 - Lib/test/test_traceback.py | 1 - Lib/test/test_ttk/test_widgets.py | 4 +--- Lib/test/test_unicodedata.py | 3 +-- Lib/test/test_unittest/test_loader.py | 1 - Lib/test/test_wmi.py | 1 - Lib/test/test_zipfile64.py | 2 +- 33 files changed, 8 insertions(+), 41 deletions(-) diff --git a/Lib/_pylong.py b/Lib/_pylong.py index d14c1d93836327..936346e187ff69 100644 --- a/Lib/_pylong.py +++ b/Lib/_pylong.py @@ -12,7 +12,6 @@ tricky or non-obvious code is not worth it. For people looking for maximum performance, they should use something like gmpy2.""" -import sys import re import decimal diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index bee162430a6f8e..3a8637b6fa1b47 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -49,6 +49,8 @@ from concurrent.futures import _base import queue import multiprocessing as mp +# This import is required to load the multiprocessing.connection submodule +# so that it can be accessed later as `mp.connection` import multiprocessing.connection from multiprocessing.queues import Queue import threading diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 5c0257eba186d1..8bc8594d674bc0 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -4,7 +4,6 @@ import types import inspect import keyword -import builtins import functools import itertools import abc diff --git a/Lib/importlib/_abc.py b/Lib/importlib/_abc.py index 08320563896521..693b466112638f 100644 --- a/Lib/importlib/_abc.py +++ b/Lib/importlib/_abc.py @@ -1,7 +1,6 @@ """Subset of importlib.abc used to reduce importlib.util imports.""" from . import _bootstrap import abc -import warnings class Loader(metaclass=abc.ABCMeta): diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index c61100a6da8503..122d441bd19f5e 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -3,7 +3,7 @@ import os import sys import threading -from os.path import pardir, realpath +from os.path import realpath __all__ = [ 'get_config_h_filename', diff --git a/Lib/test/_test_venv_multiprocessing.py b/Lib/test/_test_venv_multiprocessing.py index 044a0c6cd3f5ca..ad985dd8d56bb4 100644 --- a/Lib/test/_test_venv_multiprocessing.py +++ b/Lib/test/_test_venv_multiprocessing.py @@ -1,7 +1,6 @@ import multiprocessing import random import sys -import time def fill_queue(queue, code): queue.put(code) diff --git a/Lib/test/fork_wait.py b/Lib/test/fork_wait.py index c26c7aaaeb4306..8c32895f5e09e5 100644 --- a/Lib/test/fork_wait.py +++ b/Lib/test/fork_wait.py @@ -9,7 +9,7 @@ active threads survive in the child after a fork(); this is an error. """ -import os, sys, time, unittest +import os, time, unittest import threading from test import support from test.support import threading_helper diff --git a/Lib/test/test__xxinterpchannels.py b/Lib/test/test__xxinterpchannels.py index 03bb5c80b8dac9..69bda89a1688f5 100644 --- a/Lib/test/test__xxinterpchannels.py +++ b/Lib/test/test__xxinterpchannels.py @@ -1,6 +1,5 @@ from collections import namedtuple import contextlib -import os import sys from textwrap import dedent import threading diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index 687fcf3b770522..965967e3f2734b 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -1,4 +1,3 @@ -from collections import namedtuple import contextlib import itertools import os @@ -6,7 +5,6 @@ import sys from textwrap import dedent import threading -import time import unittest import _testcapi diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 0421efdbf9dac9..09e4010b0e53d6 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -2,7 +2,6 @@ import types import unittest import contextlib -import warnings from test.support.import_helper import import_module from test.support import gc_collect, requires_working_socket diff --git a/Lib/test/test_asyncio/test_ssl.py b/Lib/test/test_asyncio/test_ssl.py index aaf3c37101f52a..e9cc735613fb8e 100644 --- a/Lib/test/test_asyncio/test_ssl.py +++ b/Lib/test/test_asyncio/test_ssl.py @@ -1,5 +1,4 @@ import asyncio -import asyncio.sslproto import contextlib import gc import logging diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index f1ad10a9903fe8..eba6e2d1f28f3e 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -1,5 +1,4 @@ import os -import shutil import signal import sys import unittest diff --git a/Lib/test/test_ctypes/test_callbacks.py b/Lib/test/test_ctypes/test_callbacks.py index e8fa3e6f7aca51..b185e388ab1527 100644 --- a/Lib/test/test_ctypes/test_callbacks.py +++ b/Lib/test/test_ctypes/test_callbacks.py @@ -1,4 +1,3 @@ -import sys import functools import unittest from test import support diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 81a36aa241acf7..589f229f462359 100644 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -2261,7 +2261,6 @@ def test_base_has_init(self): class B: def __init__(self): self.z = 100 - pass # Make sure that declaring this class doesn't raise an error. # The issue is that we can't override __init__ in our class, diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 0a2e0c14d268af..2b14590b2c21af 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -20,7 +20,6 @@ from test import support from test.support import ALWAYS_EQ from test.support import threading_helper -from textwrap import dedent from datetime import timedelta python_version = sys.version_info[:2] @@ -1187,7 +1186,6 @@ class MyEnum(HexInt, enum.Enum): # class SillyInt(HexInt): __qualname__ = 'SillyInt' - pass class MyOtherEnum(SillyInt, enum.Enum): __qualname__ = 'MyOtherEnum' D = 4 diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 5b946020994e31..ced9000f75f2e5 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -3,7 +3,6 @@ from test.support import check_syntax_error from test.support import import_helper -from test.support.warnings_helper import check_syntax_warning import inspect import unittest import sys @@ -15,7 +14,6 @@ # with import machinery import test.ann_module as ann_module import typing -from collections import ChainMap from test import ann_module2 import test diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index 5ead8857943592..08cb5eb0c2bbab 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -22,7 +22,6 @@ from test.support import os_helper from test.support import requires_resource from test.support import threading_helper -from test.support import warnings_helper from http.client import HTTPException diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 620a5b19109a8c..9ff6afcbadec54 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -8,7 +8,6 @@ import re import socket import threading -import warnings import unittest from unittest import mock @@ -17,7 +16,6 @@ from test import support from test.support import os_helper from test.support import socket_helper -from test.support import warnings_helper support.requires_working_socket(module=True) diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 7626d9572e1e96..60f5b671b1da48 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -13,7 +13,6 @@ from test.support import verbose, run_with_tz, run_with_locale, cpython_only from test.support import hashlib_helper from test.support import threading_helper -from test.support import warnings_helper import unittest from unittest import mock from datetime import datetime, timezone, timedelta @@ -751,7 +750,6 @@ class NonUTF8Server(SimpleIMAPHandler): typ, data = client.login('user', 'pass') self.assertEqual(typ, 'OK') client.enable('UTF8=ACCEPT') - pass @threading_helper.reap_threads def test_enable_UTF8_True_append(self): diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py index 2fcf6ebbee7e34..bf9332e40aeaf2 100644 --- a/Lib/test/test_isinstance.py +++ b/Lib/test/test_isinstance.py @@ -3,7 +3,6 @@ # testing of error conditions uncovered when using extension types. import unittest -import sys import typing from test import support diff --git a/Lib/test/test_minidom.py b/Lib/test/test_minidom.py index 2ca3908bd1caac..699265ccadc7f9 100644 --- a/Lib/test/test_minidom.py +++ b/Lib/test/test_minidom.py @@ -6,7 +6,6 @@ from test import support import unittest -import pyexpat import xml.dom.minidom from xml.dom.minidom import parse, Attr, Node, Document, parseString diff --git a/Lib/test/test_peg_generator/test_c_parser.py b/Lib/test/test_peg_generator/test_c_parser.py index 1b3fcbb92f8292..d34ffef0dbc5ec 100644 --- a/Lib/test/test_peg_generator/test_c_parser.py +++ b/Lib/test/test_peg_generator/test_c_parser.py @@ -24,7 +24,6 @@ generate_parser_c_extension, generate_c_parser_source, ) - from pegen.ast_dump import ast_dump TEST_TEMPLATE = """ diff --git a/Lib/test/test_shlex.py b/Lib/test/test_shlex.py index 92598dbbd5f293..797c91ee7effdf 100644 --- a/Lib/test/test_shlex.py +++ b/Lib/test/test_shlex.py @@ -3,7 +3,6 @@ import shlex import string import unittest -from unittest import mock # The original test data set was from shellwords, by Hartmut Goebel. diff --git a/Lib/test/test_sys_setprofile.py b/Lib/test/test_sys_setprofile.py index acae433cd0a549..49e076c77d167a 100644 --- a/Lib/test/test_sys_setprofile.py +++ b/Lib/test/test_sys_setprofile.py @@ -2,7 +2,6 @@ import pprint import sys import unittest -from test import support class TestGetProfile(unittest.TestCase): diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index a251b2272e95eb..4907c930e143d5 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -2,7 +2,6 @@ from test import support import unittest -from unittest.mock import MagicMock import sys import difflib import gc diff --git a/Lib/test/test_tkinter/test_widgets.py b/Lib/test/test_tkinter/test_widgets.py index 6fde93cbecc73f..64c9472706549b 100644 --- a/Lib/test/test_tkinter/test_widgets.py +++ b/Lib/test/test_tkinter/test_widgets.py @@ -9,8 +9,7 @@ AbstractDefaultRootTest) from test.test_tkinter.widget_tests import ( add_standard_options, - AbstractWidgetTest, StandardOptionsTests, IntegerSizeTests, PixelSizeTests, - setUpModule) + AbstractWidgetTest, StandardOptionsTests, IntegerSizeTests, PixelSizeTests) requires('gui') diff --git a/Lib/test/test_tools/test_sundry.py b/Lib/test/test_tools/test_sundry.py index 81f06763980a32..6a3dc12781b2b6 100644 --- a/Lib/test/test_tools/test_sundry.py +++ b/Lib/test/test_tools/test_sundry.py @@ -6,7 +6,6 @@ """ import os -import sys import unittest from test.support import import_helper diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 95b1bae4f60850..92c5a000585855 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -6,7 +6,6 @@ import sys import types import inspect -import importlib import builtins import unittest import re diff --git a/Lib/test/test_ttk/test_widgets.py b/Lib/test/test_ttk/test_widgets.py index 6f47ccb8e8b3de..79d65b496abdc6 100644 --- a/Lib/test/test_ttk/test_widgets.py +++ b/Lib/test/test_ttk/test_widgets.py @@ -8,8 +8,7 @@ from test.test_tkinter.support import (AbstractTkTest, tcl_version, get_tk_patchlevel, simulate_mouse_click, AbstractDefaultRootTest) from test.test_tkinter.widget_tests import (add_standard_options, - AbstractWidgetTest, StandardOptionsTests, IntegerSizeTests, PixelSizeTests, - setUpModule) + AbstractWidgetTest, StandardOptionsTests, IntegerSizeTests, PixelSizeTests) requires('gui') @@ -50,7 +49,6 @@ def test_configure_style(self): widget2 = self.create(class_='Foo') self.assertEqual(widget2['class'], 'Foo') # XXX - pass class WidgetTest(AbstractTkTest, unittest.TestCase): diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index 74503c89e559a0..3dc0790ca15b41 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -12,8 +12,7 @@ import unicodedata import unittest from test.support import (open_urlresource, requires_resource, script_helper, - cpython_only, check_disallow_instantiation, - ResourceDenied) + cpython_only, check_disallow_instantiation) class UnicodeMethodsTest(unittest.TestCase): diff --git a/Lib/test/test_unittest/test_loader.py b/Lib/test/test_unittest/test_loader.py index bbdfb247ebaada..a203145a791b1a 100644 --- a/Lib/test/test_unittest/test_loader.py +++ b/Lib/test/test_unittest/test_loader.py @@ -1,7 +1,6 @@ import functools import sys import types -import warnings import unittest diff --git a/Lib/test/test_wmi.py b/Lib/test/test_wmi.py index 3f579595290524..3445702846d8a0 100644 --- a/Lib/test/test_wmi.py +++ b/Lib/test/test_wmi.py @@ -1,7 +1,6 @@ # Test the internal _wmi module on Windows # This is used by the platform module, and potentially others -import sys import unittest from test.support import import_helper, requires_resource diff --git a/Lib/test/test_zipfile64.py b/Lib/test/test_zipfile64.py index be654a8478b04b..2e1affe0252858 100644 --- a/Lib/test/test_zipfile64.py +++ b/Lib/test/test_zipfile64.py @@ -11,7 +11,7 @@ 'test requires loads of disk-space bytes and a long time to run' ) -import zipfile, os, unittest +import zipfile, unittest import time import sys From b097925858c6975c73e989226cf278cc382c0416 Mon Sep 17 00:00:00 2001 From: JosephSBoyle <48555120+JosephSBoyle@users.noreply.github.com> Date: Wed, 8 Mar 2023 13:58:14 +0000 Subject: [PATCH 22/38] gh-102507 Remove invisible pagebreak characters (#102531) Co-authored-by: AlexWaygood --- Lib/email/__init__.py | 1 - Lib/email/base64mime.py | 4 ---- Lib/email/charset.py | 5 ----- Lib/email/encoders.py | 4 ---- Lib/email/feedparser.py | 2 -- Lib/email/generator.py | 5 +---- Lib/email/header.py | 5 ----- Lib/email/iterators.py | 3 --- Lib/email/mime/base.py | 1 - Lib/email/mime/message.py | 1 - Lib/email/mime/multipart.py | 1 - Lib/email/mime/nonmultipart.py | 1 - Lib/email/mime/text.py | 1 - Lib/email/parser.py | 3 +-- Modules/_io/bufferedio.c | 3 --- Tools/i18n/pygettext.py | 11 ++++------- 16 files changed, 6 insertions(+), 45 deletions(-) diff --git a/Lib/email/__init__.py b/Lib/email/__init__.py index fae872439edc66..9fa47783004185 100644 --- a/Lib/email/__init__.py +++ b/Lib/email/__init__.py @@ -25,7 +25,6 @@ ] - # Some convenience routines. Don't import Parser and Message as side-effects # of importing email since those cascadingly import most of the rest of the # email package. diff --git a/Lib/email/base64mime.py b/Lib/email/base64mime.py index a7cc37365c6f9a..4cdf22666e3016 100644 --- a/Lib/email/base64mime.py +++ b/Lib/email/base64mime.py @@ -45,7 +45,6 @@ MISC_LEN = 7 - # Helpers def header_length(bytearray): """Return the length of s when it is encoded with base64.""" @@ -57,7 +56,6 @@ def header_length(bytearray): return n - def header_encode(header_bytes, charset='iso-8859-1'): """Encode a single header line with Base64 encoding in a given charset. @@ -72,7 +70,6 @@ def header_encode(header_bytes, charset='iso-8859-1'): return '=?%s?b?%s?=' % (charset, encoded) - def body_encode(s, maxlinelen=76, eol=NL): r"""Encode a string with base64. @@ -98,7 +95,6 @@ def body_encode(s, maxlinelen=76, eol=NL): return EMPTYSTRING.join(encvec) - def decode(string): """Decode a raw base64 string, returning a bytes object. diff --git a/Lib/email/charset.py b/Lib/email/charset.py index 791b6584b24757..9af269442fb8af 100644 --- a/Lib/email/charset.py +++ b/Lib/email/charset.py @@ -18,7 +18,6 @@ from email.encoders import encode_7or8bit - # Flags for types of header encodings QP = 1 # Quoted-Printable BASE64 = 2 # Base64 @@ -32,7 +31,6 @@ EMPTYSTRING = '' - # Defaults CHARSETS = { # input header enc body enc output conv @@ -104,7 +102,6 @@ } - # Convenience functions for extending the above mappings def add_charset(charset, header_enc=None, body_enc=None, output_charset=None): """Add character set properties to the global registry. @@ -153,7 +150,6 @@ def add_codec(charset, codecname): CODEC_MAP[charset] = codecname - # Convenience function for encoding strings, taking into account # that they might be unknown-8bit (ie: have surrogate-escaped bytes) def _encode(string, codec): @@ -163,7 +159,6 @@ def _encode(string, codec): return string.encode(codec) - class Charset: """Map character sets to their email properties. diff --git a/Lib/email/encoders.py b/Lib/email/encoders.py index 0a66acb6240bd7..17bd1ab7b19f32 100644 --- a/Lib/email/encoders.py +++ b/Lib/email/encoders.py @@ -16,7 +16,6 @@ from quopri import encodestring as _encodestring - def _qencode(s): enc = _encodestring(s, quotetabs=True) # Must encode spaces, which quopri.encodestring() doesn't do @@ -34,7 +33,6 @@ def encode_base64(msg): msg['Content-Transfer-Encoding'] = 'base64' - def encode_quopri(msg): """Encode the message's payload in quoted-printable. @@ -46,7 +44,6 @@ def encode_quopri(msg): msg['Content-Transfer-Encoding'] = 'quoted-printable' - def encode_7or8bit(msg): """Set the Content-Transfer-Encoding header to 7bit or 8bit.""" orig = msg.get_payload(decode=True) @@ -64,6 +61,5 @@ def encode_7or8bit(msg): msg['Content-Transfer-Encoding'] = '7bit' - def encode_noop(msg): """Do nothing.""" diff --git a/Lib/email/feedparser.py b/Lib/email/feedparser.py index 97d3f5144d606f..6bc4e0c4e59895 100644 --- a/Lib/email/feedparser.py +++ b/Lib/email/feedparser.py @@ -41,7 +41,6 @@ NeedMoreData = object() - class BufferedSubFile(object): """A file-ish object that can have new data loaded into it. @@ -132,7 +131,6 @@ def __next__(self): return line - class FeedParser: """A feed-style parser of email.""" diff --git a/Lib/email/generator.py b/Lib/email/generator.py index 885e6ba98540a7..7ccbe10eb76856 100644 --- a/Lib/email/generator.py +++ b/Lib/email/generator.py @@ -22,7 +22,6 @@ fcre = re.compile(r'^From ', re.MULTILINE) - class Generator: """Generates output from a Message object tree. @@ -392,7 +391,7 @@ def _make_boundary(cls, text=None): def _compile_re(cls, s, flags): return re.compile(s, flags) - + class BytesGenerator(Generator): """Generates a bytes version of a Message object tree. @@ -443,7 +442,6 @@ def _compile_re(cls, s, flags): return re.compile(s.encode('ascii'), flags) - _FMT = '[Non-text (%(type)s) part of message omitted, filename %(filename)s]' class DecodedGenerator(Generator): @@ -503,7 +501,6 @@ def _dispatch(self, msg): }, file=self) - # Helper used by Generator._make_boundary _width = len(repr(sys.maxsize-1)) _fmt = '%%0%dd' % _width diff --git a/Lib/email/header.py b/Lib/email/header.py index 4ab0032bc66123..984851a7d9a679 100644 --- a/Lib/email/header.py +++ b/Lib/email/header.py @@ -52,12 +52,10 @@ _embedded_header = re.compile(r'\n[^ \t]+:') - # Helpers _max_append = email.quoprimime._max_append - def decode_header(header): """Decode a message header value without converting charset. @@ -152,7 +150,6 @@ def decode_header(header): return collapsed - def make_header(decoded_seq, maxlinelen=None, header_name=None, continuation_ws=' '): """Create a Header from a sequence of pairs as returned by decode_header() @@ -175,7 +172,6 @@ def make_header(decoded_seq, maxlinelen=None, header_name=None, return h - class Header: def __init__(self, s=None, charset=None, maxlinelen=None, header_name=None, @@ -409,7 +405,6 @@ def _normalize(self): self._chunks = chunks - class _ValueFormatter: def __init__(self, headerlen, maxlen, continuation_ws, splitchars): self._maxlen = maxlen diff --git a/Lib/email/iterators.py b/Lib/email/iterators.py index b5502ee975266b..3410935e38f476 100644 --- a/Lib/email/iterators.py +++ b/Lib/email/iterators.py @@ -15,7 +15,6 @@ from io import StringIO - # This function will become a method of the Message class def walk(self): """Walk over the message tree, yielding each subpart. @@ -29,7 +28,6 @@ def walk(self): yield from subpart.walk() - # These two functions are imported into the Iterators.py interface module. def body_line_iterator(msg, decode=False): """Iterate over the parts, returning string payloads line-by-line. @@ -55,7 +53,6 @@ def typed_subpart_iterator(msg, maintype='text', subtype=None): yield subpart - def _structure(msg, fp=None, level=0, include_default=False): """A handy debugging aid""" if fp is None: diff --git a/Lib/email/mime/base.py b/Lib/email/mime/base.py index 1a3f9b51f6c045..f601f621cec393 100644 --- a/Lib/email/mime/base.py +++ b/Lib/email/mime/base.py @@ -11,7 +11,6 @@ from email import message - class MIMEBase(message.Message): """Base class for MIME specializations.""" diff --git a/Lib/email/mime/message.py b/Lib/email/mime/message.py index 07e4f2d1196151..61836b5a7861fc 100644 --- a/Lib/email/mime/message.py +++ b/Lib/email/mime/message.py @@ -10,7 +10,6 @@ from email.mime.nonmultipart import MIMENonMultipart - class MIMEMessage(MIMENonMultipart): """Class representing message/* MIME documents.""" diff --git a/Lib/email/mime/multipart.py b/Lib/email/mime/multipart.py index 2d3f288810dd91..94d81c771a474e 100644 --- a/Lib/email/mime/multipart.py +++ b/Lib/email/mime/multipart.py @@ -9,7 +9,6 @@ from email.mime.base import MIMEBase - class MIMEMultipart(MIMEBase): """Base class for MIME multipart/* type messages.""" diff --git a/Lib/email/mime/nonmultipart.py b/Lib/email/mime/nonmultipart.py index e1f51968b59eb1..a41386eb148c0c 100644 --- a/Lib/email/mime/nonmultipart.py +++ b/Lib/email/mime/nonmultipart.py @@ -10,7 +10,6 @@ from email.mime.base import MIMEBase - class MIMENonMultipart(MIMEBase): """Base class for MIME non-multipart type messages.""" diff --git a/Lib/email/mime/text.py b/Lib/email/mime/text.py index 35b442383002b2..dfe53c426b2ac4 100644 --- a/Lib/email/mime/text.py +++ b/Lib/email/mime/text.py @@ -10,7 +10,6 @@ from email.mime.nonmultipart import MIMENonMultipart - class MIMEText(MIMENonMultipart): """Class for generating text/* type MIME documents.""" diff --git a/Lib/email/parser.py b/Lib/email/parser.py index e94d455baa5262..06d99b17f2f9c4 100644 --- a/Lib/email/parser.py +++ b/Lib/email/parser.py @@ -64,7 +64,6 @@ def parsestr(self, text, headersonly=False): return self.parse(StringIO(text), headersonly=headersonly) - class HeaderParser(Parser): def parse(self, fp, headersonly=True): return Parser.parse(self, fp, True) @@ -72,7 +71,7 @@ def parse(self, fp, headersonly=True): def parsestr(self, text, headersonly=True): return Parser.parsestr(self, text, True) - + class BytesParser: def __init__(self, *args, **kw): diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 960026707fc5ed..2c71768be97870 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -1742,7 +1742,6 @@ _bufferedreader_peek_unlocked(buffered *self) self->pos = 0; return PyBytes_FromStringAndSize(self->buffer, r); } - /* @@ -2052,7 +2051,6 @@ _io_BufferedWriter_write_impl(buffered *self, Py_buffer *buffer) LEAVE_BUFFERED(self) return res; } - /* @@ -2266,7 +2264,6 @@ bufferedrwpair_closed_get(rwpair *self, void *context) } return PyObject_GetAttr((PyObject *) self->writer, &_Py_ID(closed)); } - /* diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py index 7ada79105db1ca..3a0b27ba420e7a 100755 --- a/Tools/i18n/pygettext.py +++ b/Tools/i18n/pygettext.py @@ -174,7 +174,6 @@ EMPTYSTRING = '' - # The normal pot-file header. msgmerge and Emacs's po-mode work better if it's # there. pot_header = _('''\ @@ -196,7 +195,7 @@ ''') - + def usage(code, msg=''): print(__doc__ % globals(), file=sys.stderr) if msg: @@ -204,7 +203,6 @@ def usage(code, msg=''): sys.exit(code) - def make_escapes(pass_nonascii): global escapes, escape if pass_nonascii: @@ -258,7 +256,7 @@ def normalize(s, encoding): s = '""\n"' + lineterm.join(lines) + '"' return s - + def containsAny(str, set): """Check whether 'str' contains ANY of the chars in 'set'""" return 1 in [c in str for c in set] @@ -307,7 +305,7 @@ def getFilesForName(name): return [] - + class TokenEater: def __init__(self, options): self.__options = options @@ -515,7 +513,6 @@ def write(self, fp): print('msgstr ""\n', file=fp) - def main(): global default_keywords try: @@ -675,7 +672,7 @@ class Options: if closep: fp.close() - + if __name__ == '__main__': main() # some more test strings From 11a2c6ce516b24b2435cb627742a6c4df92d411c Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 8 Mar 2023 17:03:18 +0000 Subject: [PATCH 23/38] gh-102192: Replace PyErr_Fetch/Restore etc by more efficient alternatives (in Objects/) (#102218) --- Objects/dictobject.c | 2 +- Objects/exceptions.c | 39 +++++++++---------------- Objects/frameobject.c | 5 ++-- Objects/genobject.c | 64 +++++++++++------------------------------ Objects/object.c | 25 +++++++--------- Objects/odictobject.c | 5 ++-- Objects/typeobject.c | 10 +++---- Objects/weakrefobject.c | 7 ++--- 8 files changed, 51 insertions(+), 106 deletions(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index e3795e75e3da5e..75c92172a91778 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -119,7 +119,7 @@ As a consequence of this, split keys have a maximum size of 16. #include "pycore_dict.h" // PyDictKeysObject #include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() #include "pycore_object.h" // _PyObject_GC_TRACK() -#include "pycore_pyerrors.h" // _PyErr_Fetch() +#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyThreadState_GET() #include "stringlib/eq.h" // unicode_eq() diff --git a/Objects/exceptions.c b/Objects/exceptions.c index a473cbdfeda7fc..c6fb6a3f19b2d0 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -3781,16 +3781,13 @@ PyObject * _PyErr_TrySetFromCause(const char *format, ...) { PyObject* msg_prefix; - PyObject *exc, *val, *tb; - PyTypeObject *caught_type; PyObject *instance_args; Py_ssize_t num_args, caught_type_size, base_exc_size; - PyObject *new_exc, *new_val, *new_tb; va_list vargs; int same_basic_size; - PyErr_Fetch(&exc, &val, &tb); - caught_type = (PyTypeObject *)exc; + PyObject *exc = PyErr_GetRaisedException(); + PyTypeObject *caught_type = Py_TYPE(exc); /* Ensure type info indicates no extra state is stored at the C level * and that the type can be reinstantiated using PyErr_Format */ @@ -3810,31 +3807,30 @@ _PyErr_TrySetFromCause(const char *format, ...) * more state than just the exception type. Accordingly, we just * leave it alone. */ - PyErr_Restore(exc, val, tb); + PyErr_SetRaisedException(exc); return NULL; } /* Check the args are empty or contain a single string */ - PyErr_NormalizeException(&exc, &val, &tb); - instance_args = ((PyBaseExceptionObject *)val)->args; + instance_args = ((PyBaseExceptionObject *)exc)->args; num_args = PyTuple_GET_SIZE(instance_args); if (num_args > 1 || (num_args == 1 && !PyUnicode_CheckExact(PyTuple_GET_ITEM(instance_args, 0)))) { /* More than 1 arg, or the one arg we do have isn't a string */ - PyErr_Restore(exc, val, tb); + PyErr_SetRaisedException(exc); return NULL; } /* Ensure the instance dict is also empty */ - if (!_PyObject_IsInstanceDictEmpty(val)) { + if (!_PyObject_IsInstanceDictEmpty(exc)) { /* While we could potentially copy a non-empty instance dictionary * to the replacement exception, for now we take the more * conservative path of leaving exceptions with attributes set * alone. */ - PyErr_Restore(exc, val, tb); + PyErr_SetRaisedException(exc); return NULL; } @@ -3847,28 +3843,19 @@ _PyErr_TrySetFromCause(const char *format, ...) * types as well, but that's quite a bit trickier due to the extra * state potentially stored on OSError instances. */ - /* Ensure the traceback is set correctly on the existing exception */ - if (tb != NULL) { - PyException_SetTraceback(val, tb); - Py_DECREF(tb); - } - va_start(vargs, format); msg_prefix = PyUnicode_FromFormatV(format, vargs); va_end(vargs); if (msg_prefix == NULL) { Py_DECREF(exc); - Py_DECREF(val); return NULL; } - PyErr_Format(exc, "%U (%s: %S)", - msg_prefix, Py_TYPE(val)->tp_name, val); - Py_DECREF(exc); + PyErr_Format((PyObject*)Py_TYPE(exc), "%U (%s: %S)", + msg_prefix, Py_TYPE(exc)->tp_name, exc); Py_DECREF(msg_prefix); - PyErr_Fetch(&new_exc, &new_val, &new_tb); - PyErr_NormalizeException(&new_exc, &new_val, &new_tb); - PyException_SetCause(new_val, val); - PyErr_Restore(new_exc, new_val, new_tb); - return new_val; + PyObject *new_exc = PyErr_GetRaisedException(); + PyException_SetCause(new_exc, exc); + PyErr_SetRaisedException(new_exc); + return new_exc; } diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 34143c9a40b293..133c991bf701c4 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1308,7 +1308,6 @@ _PyFrame_LocalsToFast(_PyInterpreterFrame *frame, int clear) /* Merge locals into fast locals */ PyObject *locals; PyObject **fast; - PyObject *error_type, *error_value, *error_traceback; PyCodeObject *co; locals = frame->f_locals; if (locals == NULL) { @@ -1317,7 +1316,7 @@ _PyFrame_LocalsToFast(_PyInterpreterFrame *frame, int clear) fast = _PyFrame_GetLocalsArray(frame); co = frame->f_code; - PyErr_Fetch(&error_type, &error_value, &error_traceback); + PyObject *exc = PyErr_GetRaisedException(); for (int i = 0; i < co->co_nlocalsplus; i++) { _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); @@ -1374,7 +1373,7 @@ _PyFrame_LocalsToFast(_PyInterpreterFrame *frame, int clear) } Py_XDECREF(value); } - PyErr_Restore(error_type, error_value, error_traceback); + PyErr_SetRaisedException(exc); } void diff --git a/Objects/genobject.c b/Objects/genobject.c index be08a59ece6b7e..61463774310f88 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -69,8 +69,6 @@ void _PyGen_Finalize(PyObject *self) { PyGenObject *gen = (PyGenObject *)self; - PyObject *res = NULL; - PyObject *error_type, *error_value, *error_traceback; if (gen->gi_frame_state >= FRAME_COMPLETED) { /* Generator isn't paused, so no need to close */ @@ -82,23 +80,22 @@ _PyGen_Finalize(PyObject *self) PyObject *finalizer = agen->ag_origin_or_finalizer; if (finalizer && !agen->ag_closed) { /* Save the current exception, if any. */ - PyErr_Fetch(&error_type, &error_value, &error_traceback); - - res = PyObject_CallOneArg(finalizer, self); + PyObject *exc = PyErr_GetRaisedException(); + PyObject *res = PyObject_CallOneArg(finalizer, self); if (res == NULL) { PyErr_WriteUnraisable(self); } else { Py_DECREF(res); } /* Restore the saved exception. */ - PyErr_Restore(error_type, error_value, error_traceback); + PyErr_SetRaisedException(exc); return; } } /* Save the current exception, if any. */ - PyErr_Fetch(&error_type, &error_value, &error_traceback); + PyObject *exc = PyErr_GetRaisedException(); /* If `gen` is a coroutine, and if it was never awaited on, issue a RuntimeWarning. */ @@ -109,20 +106,19 @@ _PyGen_Finalize(PyObject *self) _PyErr_WarnUnawaitedCoroutine((PyObject *)gen); } else { - res = gen_close(gen, NULL); - } - - if (res == NULL) { - if (PyErr_Occurred()) { - PyErr_WriteUnraisable(self); + PyObject *res = gen_close(gen, NULL); + if (res == NULL) { + if (PyErr_Occurred()) { + PyErr_WriteUnraisable(self); + } + } + else { + Py_DECREF(res); } - } - else { - Py_DECREF(res); } /* Restore the saved exception. */ - PyErr_Restore(error_type, error_value, error_traceback); + PyErr_SetRaisedException(exc); } static void @@ -648,39 +644,11 @@ _PyGen_SetStopIterationValue(PyObject *value) int _PyGen_FetchStopIterationValue(PyObject **pvalue) { - PyObject *et, *ev, *tb; PyObject *value = NULL; - if (PyErr_ExceptionMatches(PyExc_StopIteration)) { - PyErr_Fetch(&et, &ev, &tb); - if (ev) { - /* exception will usually be normalised already */ - if (PyObject_TypeCheck(ev, (PyTypeObject *) et)) { - value = Py_NewRef(((PyStopIterationObject *)ev)->value); - Py_DECREF(ev); - } else if (et == PyExc_StopIteration && !PyTuple_Check(ev)) { - /* Avoid normalisation and take ev as value. - * - * Normalization is required if the value is a tuple, in - * that case the value of StopIteration would be set to - * the first element of the tuple. - * - * (See _PyErr_CreateException code for details.) - */ - value = ev; - } else { - /* normalisation required */ - PyErr_NormalizeException(&et, &ev, &tb); - if (!PyObject_TypeCheck(ev, (PyTypeObject *)PyExc_StopIteration)) { - PyErr_Restore(et, ev, tb); - return -1; - } - value = Py_NewRef(((PyStopIterationObject *)ev)->value); - Py_DECREF(ev); - } - } - Py_XDECREF(et); - Py_XDECREF(tb); + PyObject *exc = PyErr_GetRaisedException(); + value = Py_NewRef(((PyStopIterationObject *)exc)->value); + Py_DECREF(exc); } else if (PyErr_Occurred()) { return -1; } diff --git a/Objects/object.c b/Objects/object.c index 446c7b1f5f0302..5db2b6af21ef13 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -370,13 +370,12 @@ _PyObject_Dump(PyObject* op) fflush(stderr); PyGILState_STATE gil = PyGILState_Ensure(); - PyObject *error_type, *error_value, *error_traceback; - PyErr_Fetch(&error_type, &error_value, &error_traceback); + PyObject *exc = PyErr_GetRaisedException(); (void)PyObject_Print(op, stderr, 0); fflush(stderr); - PyErr_Restore(error_type, error_value, error_traceback); + PyErr_SetRaisedException(exc); PyGILState_Release(gil); fprintf(stderr, "\n"); @@ -860,25 +859,22 @@ set_attribute_error_context(PyObject* v, PyObject* name) return 0; } // Intercept AttributeError exceptions and augment them to offer suggestions later. - PyObject *type, *value, *traceback; - PyErr_Fetch(&type, &value, &traceback); - PyErr_NormalizeException(&type, &value, &traceback); - // Check if the normalized exception is indeed an AttributeError - if (!PyErr_GivenExceptionMatches(value, PyExc_AttributeError)) { + PyObject *exc = PyErr_GetRaisedException(); + if (!PyErr_GivenExceptionMatches(exc, PyExc_AttributeError)) { goto restore; } - PyAttributeErrorObject* the_exc = (PyAttributeErrorObject*) value; + PyAttributeErrorObject* the_exc = (PyAttributeErrorObject*) exc; // Check if this exception was already augmented if (the_exc->name || the_exc->obj) { goto restore; } // Augment the exception with the name and object - if (PyObject_SetAttr(value, &_Py_ID(name), name) || - PyObject_SetAttr(value, &_Py_ID(obj), v)) { + if (PyObject_SetAttr(exc, &_Py_ID(name), name) || + PyObject_SetAttr(exc, &_Py_ID(obj), v)) { return 1; } restore: - PyErr_Restore(type, value, traceback); + PyErr_SetRaisedException(exc); return 0; } @@ -2190,9 +2186,8 @@ Py_ReprLeave(PyObject *obj) PyObject *dict; PyObject *list; Py_ssize_t i; - PyObject *error_type, *error_value, *error_traceback; - PyErr_Fetch(&error_type, &error_value, &error_traceback); + PyObject *exc = PyErr_GetRaisedException(); dict = PyThreadState_GetDict(); if (dict == NULL) @@ -2213,7 +2208,7 @@ Py_ReprLeave(PyObject *obj) finally: /* ignore exceptions because there is no way to report them. */ - PyErr_Restore(error_type, error_value, error_traceback); + PyErr_SetRaisedException(exc); } /* Trashcan support. */ diff --git a/Objects/odictobject.c b/Objects/odictobject.c index 215a8af54fb266..39b0f684510578 100644 --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -1556,10 +1556,9 @@ _PyODict_SetItem_KnownHash(PyObject *od, PyObject *key, PyObject *value, res = _odict_add_new_node((PyODictObject *)od, key, hash); if (res < 0) { /* Revert setting the value on the dict */ - PyObject *exc, *val, *tb; - PyErr_Fetch(&exc, &val, &tb); + PyObject *exc = PyErr_GetRaisedException(); (void) _PyDict_DelItem_KnownHash(od, key, hash); - _PyErr_ChainExceptions(exc, val, tb); + _PyErr_ChainExceptions1(exc); } } return res; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 981930f58417d8..f486b83fd69e64 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4397,10 +4397,9 @@ static void type_dealloc_common(PyTypeObject *type) { if (type->tp_bases != NULL) { - PyObject *tp, *val, *tb; - PyErr_Fetch(&tp, &val, &tb); + PyObject *exc = PyErr_GetRaisedException(); remove_all_subclasses(type, type->tp_bases); - PyErr_Restore(tp, val, tb); + PyErr_SetRaisedException(exc); } } @@ -8445,10 +8444,9 @@ slot_tp_finalize(PyObject *self) { int unbound; PyObject *del, *res; - PyObject *error_type, *error_value, *error_traceback; /* Save the current exception, if any. */ - PyErr_Fetch(&error_type, &error_value, &error_traceback); + PyObject *exc = PyErr_GetRaisedException(); /* Execute __del__ method, if any. */ del = lookup_maybe_method(self, &_Py_ID(__del__), &unbound); @@ -8462,7 +8460,7 @@ slot_tp_finalize(PyObject *self) } /* Restore the saved exception. */ - PyErr_Restore(error_type, error_value, error_traceback); + PyErr_SetRaisedException(exc); } static PyObject * diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index bd7720e2753307..5a3e49a6fe45e3 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -959,9 +959,8 @@ PyObject_ClearWeakRefs(PyObject *object) if (*list != NULL) { PyWeakReference *current = *list; Py_ssize_t count = _PyWeakref_GetWeakrefCount(current); - PyObject *err_type, *err_value, *err_tb; + PyObject *exc = PyErr_GetRaisedException(); - PyErr_Fetch(&err_type, &err_value, &err_tb); if (count == 1) { PyObject *callback = current->wr_callback; @@ -980,7 +979,7 @@ PyObject_ClearWeakRefs(PyObject *object) tuple = PyTuple_New(count * 2); if (tuple == NULL) { - _PyErr_ChainExceptions(err_type, err_value, err_tb); + _PyErr_ChainExceptions1(exc); return; } @@ -1010,7 +1009,7 @@ PyObject_ClearWeakRefs(PyObject *object) Py_DECREF(tuple); } assert(!PyErr_Occurred()); - PyErr_Restore(err_type, err_value, err_tb); + PyErr_SetRaisedException(exc); } } From cbb0aa71d040022db61390380b8aebc7c04f3275 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 8 Mar 2023 12:03:50 -0700 Subject: [PATCH 24/38] gh-102304: Consolidate Direct Usage of _Py_RefTotal (gh-102514) This simplifies further changes to _Py_RefTotal (e.g. make it atomic or move it to PyInterpreterState). https://github.com/python/cpython/issues/102304 --- Include/cpython/object.h | 1 + Include/internal/pycore_object.h | 23 ++++++++-- Include/object.h | 25 ++++++++-- Modules/_testcapimodule.c | 7 +-- Objects/bytesobject.c | 9 ++-- Objects/dictobject.c | 10 ++-- Objects/object.c | 79 +++++++++++++++++++++++++++----- Objects/structseq.c | 2 +- Objects/tupleobject.c | 9 ++-- Objects/unicodeobject.c | 7 +-- 10 files changed, 127 insertions(+), 45 deletions(-) diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 3f26f2487d70cc..7b687d311359c3 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -3,6 +3,7 @@ #endif PyAPI_FUNC(void) _Py_NewReference(PyObject *op); +PyAPI_FUNC(void) _Py_NewReferenceNoTotal(PyObject *op); #ifdef Py_TRACE_REFS /* Py_TRACE_REFS is such major surgery that we call external routines. */ diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 8796dfe2f6b8cf..e15685f174ebcf 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -37,11 +37,23 @@ PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc( #define _Py_FatalRefcountError(message) \ _Py_FatalRefcountErrorFunc(__func__, (message)) + +#ifdef Py_REF_DEBUG +/* The symbol is only exposed in the API for the sake of extensions + built against the pre-3.12 stable ABI. */ +PyAPI_DATA(Py_ssize_t) _Py_RefTotal; + +extern void _Py_AddRefTotal(Py_ssize_t); +extern void _Py_IncRefTotal(void); +extern void _Py_DecRefTotal(void); +# define _Py_DEC_REFTOTAL() _Py_RefTotal-- +#endif + // Increment reference count by n static inline void _Py_RefcntAdd(PyObject* op, Py_ssize_t n) { #ifdef Py_REF_DEBUG - _Py_RefTotal += n; + _Py_AddRefTotal(n); #endif op->ob_refcnt += n; } @@ -52,7 +64,7 @@ _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) { _Py_DECREF_STAT_INC(); #ifdef Py_REF_DEBUG - _Py_RefTotal--; + _Py_DEC_REFTOTAL(); #endif if (--op->ob_refcnt != 0) { assert(op->ob_refcnt > 0); @@ -70,7 +82,7 @@ _Py_DECREF_NO_DEALLOC(PyObject *op) { _Py_DECREF_STAT_INC(); #ifdef Py_REF_DEBUG - _Py_RefTotal--; + _Py_DEC_REFTOTAL(); #endif op->ob_refcnt--; #ifdef Py_DEBUG @@ -80,6 +92,11 @@ _Py_DECREF_NO_DEALLOC(PyObject *op) #endif } +#ifdef Py_REF_DEBUG +# undef _Py_DEC_REFTOTAL +#endif + + PyAPI_FUNC(int) _PyType_CheckConsistency(PyTypeObject *type); PyAPI_FUNC(int) _PyDict_CheckConsistency(PyObject *mp, int check_content); diff --git a/Include/object.h b/Include/object.h index 3774f126730005..844b9c4a51c3e4 100644 --- a/Include/object.h +++ b/Include/object.h @@ -490,7 +490,21 @@ you can count such references to the type object.) */ #ifdef Py_REF_DEBUG -PyAPI_DATA(Py_ssize_t) _Py_RefTotal; +# if defined(Py_LIMITED_API) && Py_LIMITED_API+0 < 0x030A0000 +extern Py_ssize_t _Py_RefTotal; +# define _Py_INC_REFTOTAL() _Py_RefTotal++ +# define _Py_DEC_REFTOTAL() _Py_RefTotal-- +# elif defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +extern void _Py_IncRefTotal(void); +extern void _Py_DecRefTotal(void); +# define _Py_INC_REFTOTAL() _Py_IncRefTotal() +# define _Py_DEC_REFTOTAL() _Py_DecRefTotal() +# elif !defined(Py_LIMITED_API) || Py_LIMITED_API+0 > 0x030C0000 +extern void _Py_IncRefTotal_DO_NOT_USE_THIS(void); +extern void _Py_DecRefTotal_DO_NOT_USE_THIS(void); +# define _Py_INC_REFTOTAL() _Py_IncRefTotal_DO_NOT_USE_THIS() +# define _Py_DEC_REFTOTAL() _Py_DecRefTotal_DO_NOT_USE_THIS() +# endif PyAPI_FUNC(void) _Py_NegativeRefcount(const char *filename, int lineno, PyObject *op); #endif /* Py_REF_DEBUG */ @@ -519,8 +533,8 @@ static inline void Py_INCREF(PyObject *op) // Non-limited C API and limited C API for Python 3.9 and older access // directly PyObject.ob_refcnt. #ifdef Py_REF_DEBUG - _Py_RefTotal++; -#endif + _Py_INC_REFTOTAL(); +#endif // Py_REF_DEBUG op->ob_refcnt++; #endif } @@ -539,7 +553,7 @@ static inline void Py_DECREF(PyObject *op) { static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) { _Py_DECREF_STAT_INC(); - _Py_RefTotal--; + _Py_DEC_REFTOTAL(); if (--op->ob_refcnt != 0) { if (op->ob_refcnt < 0) { _Py_NegativeRefcount(filename, lineno, op); @@ -564,6 +578,9 @@ static inline void Py_DECREF(PyObject *op) #define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op)) #endif +#undef _Py_INC_REFTOTAL +#undef _Py_DEC_REFTOTAL + /* Safely decref `op` and set `op` to NULL, especially useful in tp_clear * and tp_dealloc implementations. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 10e507d6b481de..ea67017a1ba3b1 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1654,15 +1654,10 @@ slot_tp_del(PyObject *self) */ { Py_ssize_t refcnt = Py_REFCNT(self); - _Py_NewReference(self); + _Py_NewReferenceNoTotal(self); Py_SET_REFCNT(self, refcnt); } assert(!PyType_IS_GC(Py_TYPE(self)) || PyObject_GC_IsTracked(self)); - /* If Py_REF_DEBUG macro is defined, _Py_NewReference() increased - _Py_RefTotal, so we need to undo that. */ -#ifdef Py_REF_DEBUG - _Py_RefTotal--; -#endif } static PyObject * diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 657443f31fa709..687a654bdae137 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3060,21 +3060,20 @@ _PyBytes_Resize(PyObject **pv, Py_ssize_t newsize) Py_DECREF(v); return 0; } - /* XXX UNREF/NEWREF interface should be more symmetrical */ -#ifdef Py_REF_DEBUG - _Py_RefTotal--; -#endif #ifdef Py_TRACE_REFS _Py_ForgetReference(v); #endif *pv = (PyObject *) PyObject_Realloc(v, PyBytesObject_SIZE + newsize); if (*pv == NULL) { +#ifdef Py_REF_DEBUG + _Py_DecRefTotal(); +#endif PyObject_Free(v); PyErr_NoMemory(); return -1; } - _Py_NewReference(*pv); + _Py_NewReferenceNoTotal(*pv); sv = (PyBytesObject *) *pv; Py_SET_SIZE(sv, newsize); sv->ob_sval[newsize] = '\0'; diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 75c92172a91778..a60f275742a534 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -303,7 +303,7 @@ static inline void dictkeys_incref(PyDictKeysObject *dk) { #ifdef Py_REF_DEBUG - _Py_RefTotal++; + _Py_IncRefTotal(); #endif dk->dk_refcnt++; } @@ -313,7 +313,7 @@ dictkeys_decref(PyDictKeysObject *dk) { assert(dk->dk_refcnt > 0); #ifdef Py_REF_DEBUG - _Py_RefTotal--; + _Py_DecRefTotal(); #endif if (--dk->dk_refcnt == 0) { free_keys_object(dk); @@ -633,7 +633,7 @@ new_keys_object(uint8_t log2_size, bool unicode) } } #ifdef Py_REF_DEBUG - _Py_RefTotal++; + _Py_IncRefTotal(); #endif dk->dk_refcnt = 1; dk->dk_log2_size = log2_size; @@ -821,7 +821,7 @@ clone_combined_dict_keys(PyDictObject *orig) we have it now; calling dictkeys_incref would be an error as keys->dk_refcnt is already set to 1 (after memcpy). */ #ifdef Py_REF_DEBUG - _Py_RefTotal++; + _Py_IncRefTotal(); #endif return keys; } @@ -1520,7 +1520,7 @@ dictresize(PyDictObject *mp, uint8_t log2_newsize, int unicode) // We can not use free_keys_object here because key's reference // are moved already. #ifdef Py_REF_DEBUG - _Py_RefTotal--; + _Py_DecRefTotal(); #endif if (oldkeys == Py_EMPTY_KEYS) { oldkeys->dk_refcnt--; diff --git a/Objects/object.c b/Objects/object.c index 5db2b6af21ef13..38da4d497a96e7 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -56,6 +56,24 @@ _PyObject_CheckConsistency(PyObject *op, int check_content) #ifdef Py_REF_DEBUG Py_ssize_t _Py_RefTotal; +static inline void +reftotal_increment(void) +{ + _Py_RefTotal++; +} + +static inline void +reftotal_decrement(void) +{ + _Py_RefTotal--; +} + +void +_Py_AddRefTotal(Py_ssize_t n) +{ + _Py_RefTotal += n; +} + Py_ssize_t _Py_GetRefTotal(void) { @@ -121,6 +139,32 @@ _Py_NegativeRefcount(const char *filename, int lineno, PyObject *op) filename, lineno, __func__); } +/* This is exposed strictly for use in Py_INCREF(). */ +PyAPI_FUNC(void) +_Py_IncRefTotal_DO_NOT_USE_THIS(void) +{ + reftotal_increment(); +} + +/* This is exposed strictly for use in Py_DECREF(). */ +PyAPI_FUNC(void) +_Py_DecRefTotal_DO_NOT_USE_THIS(void) +{ + reftotal_decrement(); +} + +void +_Py_IncRefTotal(void) +{ + reftotal_increment(); +} + +void +_Py_DecRefTotal(void) +{ + reftotal_decrement(); +} + #endif /* Py_REF_DEBUG */ void @@ -138,12 +182,18 @@ Py_DecRef(PyObject *o) void _Py_IncRef(PyObject *o) { +#ifdef Py_REF_DEBUG + reftotal_increment(); +#endif Py_INCREF(o); } void _Py_DecRef(PyObject *o) { +#ifdef Py_REF_DEBUG + reftotal_decrement(); +#endif Py_DECREF(o); } @@ -238,17 +288,12 @@ PyObject_CallFinalizerFromDealloc(PyObject *self) /* tp_finalize resurrected it! Make it look like the original Py_DECREF * never happened. */ Py_ssize_t refcnt = Py_REFCNT(self); - _Py_NewReference(self); + _Py_NewReferenceNoTotal(self); Py_SET_REFCNT(self, refcnt); _PyObject_ASSERT(self, (!_PyType_IS_GC(Py_TYPE(self)) || _PyObject_GC_IS_TRACKED(self))); - /* If Py_REF_DEBUG macro is defined, _Py_NewReference() increased - _Py_RefTotal, so we need to undo that. */ -#ifdef Py_REF_DEBUG - _Py_RefTotal--; -#endif return -1; } @@ -2010,21 +2055,33 @@ _PyTypes_FiniTypes(PyInterpreterState *interp) } -void -_Py_NewReference(PyObject *op) +static inline void +new_reference(PyObject *op) { if (_PyRuntime.tracemalloc.config.tracing) { _PyTraceMalloc_NewReference(op); } -#ifdef Py_REF_DEBUG - _Py_RefTotal++; -#endif Py_SET_REFCNT(op, 1); #ifdef Py_TRACE_REFS _Py_AddToAllObjects(op, 1); #endif } +void +_Py_NewReference(PyObject *op) +{ +#ifdef Py_REF_DEBUG + reftotal_increment(); +#endif + new_reference(op); +} + +void +_Py_NewReferenceNoTotal(PyObject *op) +{ + new_reference(op); +} + #ifdef Py_TRACE_REFS void diff --git a/Objects/structseq.c b/Objects/structseq.c index 100ccfef0a23c4..c20962ecd82563 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -592,7 +592,7 @@ _PyStructSequence_FiniType(PyTypeObject *type) // Don't use Py_DECREF(): static type must not be deallocated Py_SET_REFCNT(type, 0); #ifdef Py_REF_DEBUG - _Py_RefTotal--; + _Py_DecRefTotal(); #endif // Make sure that _PyStructSequence_InitType() will initialize diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 6ee93ab5adc281..59c0251639d3dd 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -930,10 +930,6 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) return *pv == NULL ? -1 : 0; } - /* XXX UNREF/NEWREF interface should be more symmetrical */ -#ifdef Py_REF_DEBUG - _Py_RefTotal--; -#endif if (_PyObject_GC_IS_TRACKED(v)) { _PyObject_GC_UNTRACK(v); } @@ -947,10 +943,13 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) sv = PyObject_GC_Resize(PyTupleObject, v, newsize); if (sv == NULL) { *pv = NULL; +#ifdef Py_REF_DEBUG + _Py_DecRefTotal(); +#endif PyObject_GC_Del(v); return -1; } - _Py_NewReference((PyObject *) sv); + _Py_NewReferenceNoTotal((PyObject *) sv); /* Zero out items added by growing */ if (newsize > oldsize) memset(&sv->ob_item[oldsize], 0, diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 1ba30421c66dba..2d50f9c340f2f3 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -947,21 +947,18 @@ resize_compact(PyObject *unicode, Py_ssize_t length) _PyUnicode_UTF8(unicode) = NULL; _PyUnicode_UTF8_LENGTH(unicode) = 0; } -#ifdef Py_REF_DEBUG - _Py_RefTotal--; -#endif #ifdef Py_TRACE_REFS _Py_ForgetReference(unicode); #endif new_unicode = (PyObject *)PyObject_Realloc(unicode, new_size); if (new_unicode == NULL) { - _Py_NewReference(unicode); + _Py_NewReferenceNoTotal(unicode); PyErr_NoMemory(); return NULL; } unicode = new_unicode; - _Py_NewReference(unicode); + _Py_NewReferenceNoTotal(unicode); _PyUnicode_LENGTH(unicode) = length; #ifdef Py_DEBUG From 66ff374d4f353ae427c148d2a1d141d223303a82 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 8 Mar 2023 15:56:36 -0700 Subject: [PATCH 25/38] gh-100227: Move func_state.next_version to PyInterpreterState (gh-102334) https://github.com/python/cpython/issues/100227 --- Include/internal/pycore_function.h | 2 +- Include/internal/pycore_interp.h | 1 + Include/internal/pycore_runtime.h | 2 -- Include/internal/pycore_runtime_init.h | 6 +++--- Objects/funcobject.c | 5 +++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Include/internal/pycore_function.h b/Include/internal/pycore_function.h index 5cedb33d7e3afd..11988149843fef 100644 --- a/Include/internal/pycore_function.h +++ b/Include/internal/pycore_function.h @@ -10,7 +10,7 @@ extern "C" { #define FUNC_MAX_WATCHERS 8 -struct _py_func_runtime_state { +struct _py_func_state { uint32_t next_version; }; diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 7ef9c40153e421..9efed0a1cf90c2 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -141,6 +141,7 @@ struct _is { struct _Py_float_state float_state; struct _Py_long_state long_state; struct _dtoa_state dtoa; + struct _py_func_state func_state; /* Using a cache is very effective since typically only a single slice is created and then deleted again. */ PySliceObject *slice_cache; diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 2350eaab5976ca..e0e3d4ace0cfde 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -13,7 +13,6 @@ extern "C" { #include "pycore_dict_state.h" // struct _Py_dict_runtime_state #include "pycore_floatobject.h" // struct _Py_float_runtime_state #include "pycore_faulthandler.h" // struct _faulthandler_runtime_state -#include "pycore_function.h" // struct _func_runtime_state #include "pycore_global_objects.h" // struct _Py_global_objects #include "pycore_import.h" // struct _import_runtime_state #include "pycore_interp.h" // PyInterpreterState @@ -155,7 +154,6 @@ typedef struct pyruntimestate { struct _Py_float_runtime_state float_state; struct _Py_unicode_runtime_state unicode_state; struct _Py_dict_runtime_state dict_state; - struct _py_func_runtime_state func_state; struct { /* Used to set PyTypeObject.tp_version_tag */ diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index a2cc7c87c2f3e2..aeabcfd2b9056c 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -68,9 +68,6 @@ extern PyTypeObject _PyExc_MemoryError; .dict_state = { \ .next_keys_version = 2, \ }, \ - .func_state = { \ - .next_version = 1, \ - }, \ .types = { \ .next_version_tag = 1, \ }, \ @@ -116,6 +113,9 @@ extern PyTypeObject _PyExc_MemoryError; }, \ }, \ .dtoa = _dtoa_state_INIT(&(INTERP)), \ + .func_state = { \ + .next_version = 1, \ + }, \ .static_objects = { \ .singletons = { \ ._not_used = 1, \ diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 99048ea41c6c80..ce5d7bda32c032 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -227,10 +227,11 @@ uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func) if (func->vectorcall != _PyFunction_Vectorcall) { return 0; } - if (_PyRuntime.func_state.next_version == 0) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (interp->func_state.next_version == 0) { return 0; } - uint32_t v = _PyRuntime.func_state.next_version++; + uint32_t v = interp->func_state.next_version++; func->func_version = v; return v; } From 5e5acd291f4387876afc641163e9f8ae5c65086c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 8 Mar 2023 18:04:16 -0700 Subject: [PATCH 26/38] gh-100227: Move next_keys_version to PyInterpreterState (gh-102335) https://github.com/python/cpython/issues/100227 --- Include/internal/pycore_dict.h | 3 ++- Include/internal/pycore_dict_state.h | 4 +++- Include/internal/pycore_runtime_init.h | 6 +++--- Objects/dictobject.c | 7 ++++--- Python/specialize.c | 16 +++++++++++----- 5 files changed, 23 insertions(+), 13 deletions(-) diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 1af5e59a677a9a..12c3c708e29b95 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -37,7 +37,8 @@ extern PyObject *_PyDict_FromKeys(PyObject *, PyObject *, PyObject *); /* Gets a version number unique to the current state of the keys of dict, if possible. * Returns the version number, or zero if it was not possible to get a version number. */ -extern uint32_t _PyDictKeys_GetVersionForCurrentState(PyDictKeysObject *dictkeys); +extern uint32_t _PyDictKeys_GetVersionForCurrentState( + PyInterpreterState *interp, PyDictKeysObject *dictkeys); extern size_t _PyDict_KeysSize(PyDictKeysObject *keys); diff --git a/Include/internal/pycore_dict_state.h b/Include/internal/pycore_dict_state.h index 77375ea8beb877..c5142ee6763a86 100644 --- a/Include/internal/pycore_dict_state.h +++ b/Include/internal/pycore_dict_state.h @@ -14,7 +14,6 @@ struct _Py_dict_runtime_state { * It is incremented each time that a dictionary is created and each * time that a dictionary is modified. */ uint64_t global_version; - uint32_t next_keys_version; }; @@ -30,6 +29,8 @@ struct _Py_dict_runtime_state { #define DICT_MAX_WATCHERS 8 struct _Py_dict_state { + uint32_t next_keys_version; + #if PyDict_MAXFREELIST > 0 /* Dictionary reuse scheme to save calls to malloc and free */ PyDictObject *free_list[PyDict_MAXFREELIST]; @@ -37,6 +38,7 @@ struct _Py_dict_state { int numfree; int keys_numfree; #endif + PyDict_WatchCallback watchers[DICT_MAX_WATCHERS]; }; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index aeabcfd2b9056c..efc82b43a61dab 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -65,9 +65,6 @@ extern PyTypeObject _PyExc_MemoryError; .float_format = _py_float_format_unknown, \ .double_format = _py_float_format_unknown, \ }, \ - .dict_state = { \ - .next_keys_version = 2, \ - }, \ .types = { \ .next_version_tag = 1, \ }, \ @@ -113,6 +110,9 @@ extern PyTypeObject _PyExc_MemoryError; }, \ }, \ .dtoa = _dtoa_state_INIT(&(INTERP)), \ + .dict_state = { \ + .next_keys_version = 2, \ + }, \ .func_state = { \ .next_version = 1, \ }, \ diff --git a/Objects/dictobject.c b/Objects/dictobject.c index a60f275742a534..b58e93f4320d42 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5655,15 +5655,16 @@ _PyDictKeys_DecRef(PyDictKeysObject *keys) dictkeys_decref(keys); } -uint32_t _PyDictKeys_GetVersionForCurrentState(PyDictKeysObject *dictkeys) +uint32_t _PyDictKeys_GetVersionForCurrentState(PyInterpreterState *interp, + PyDictKeysObject *dictkeys) { if (dictkeys->dk_version != 0) { return dictkeys->dk_version; } - if (_PyRuntime.dict_state.next_keys_version == 0) { + if (interp->dict_state.next_keys_version == 0) { return 0; } - uint32_t v = _PyRuntime.dict_state.next_keys_version++; + uint32_t v = interp->dict_state.next_keys_version++; dictkeys->dk_version = v; return v; } diff --git a/Python/specialize.c b/Python/specialize.c index 3405d2b0ab0680..0a7af8991eec83 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -512,7 +512,8 @@ specialize_module_load_attr( SPEC_FAIL_OUT_OF_RANGE); return -1; } - uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(dict->ma_keys); + uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState( + _PyInterpreterState_GET(), dict->ma_keys); if (keys_version == 0) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); return -1; @@ -1063,7 +1064,8 @@ PyObject *descr, DescriptorClassification kind) SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_SHADOWED); return 0; } - uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(keys); + uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState( + _PyInterpreterState_GET(), keys); if (keys_version == 0) { SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); return 0; @@ -1134,12 +1136,14 @@ _Py_Specialize_LoadGlobal( SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_EXPECTED_ERROR); goto fail; } + PyInterpreterState *interp = _PyInterpreterState_GET(); if (index != DKIX_EMPTY) { if (index != (uint16_t)index) { SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE); goto fail; } - uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(globals_keys); + uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState( + interp, globals_keys); if (keys_version == 0) { SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS); goto fail; @@ -1167,12 +1171,14 @@ _Py_Specialize_LoadGlobal( SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE); goto fail; } - uint32_t globals_version = _PyDictKeys_GetVersionForCurrentState(globals_keys); + uint32_t globals_version = _PyDictKeys_GetVersionForCurrentState( + interp, globals_keys); if (globals_version == 0) { SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS); goto fail; } - uint32_t builtins_version = _PyDictKeys_GetVersionForCurrentState(builtin_keys); + uint32_t builtins_version = _PyDictKeys_GetVersionForCurrentState( + interp, builtin_keys); if (builtins_version == 0) { SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS); goto fail; From 58d761e5b5f253892a697941c71bac0159a1d74e Mon Sep 17 00:00:00 2001 From: "T. Wouters" Date: Wed, 8 Mar 2023 18:39:33 -0800 Subject: [PATCH 27/38] GH-84783: Document GH-101264 (Make the slice object hashable) in What's New. (#102548) --- Doc/whatsnew/3.12.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index c0c021c679147f..d982cb62ec2f4e 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -189,6 +189,8 @@ Other Language Changes part of comprehensions (like ``a``) is still disallowed, as per :pep:`572`. (Contributed by Nikita Sobolev in :gh:`100581`.) +* :class:`slice` objects are now hashable, allowing them to be used as dict keys and + set items. (Contributed by Furkan Onder in :gh:`101264`.) New Modules =========== From b45d14b88611fefc6f054226d3e1117082d322c8 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 9 Mar 2023 08:16:30 -0700 Subject: [PATCH 28/38] gh-100227: Move dict_state.global_version to PyInterpreterState (gh-102338) https://github.com/python/cpython/issues/100227 --- Include/internal/pycore_dict.h | 11 +- Include/internal/pycore_dict_state.h | 12 +- Include/internal/pycore_runtime.h | 2 - Objects/dictobject.c | 236 +++++++++++++++++---------- Python/bytecodes.c | 4 +- Python/generated_cases.c.h | 4 +- 6 files changed, 160 insertions(+), 109 deletions(-) diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 12c3c708e29b95..6253e0841ad349 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -149,8 +149,8 @@ static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) { #define DICT_VERSION_INCREMENT (1 << DICT_MAX_WATCHERS) #define DICT_VERSION_MASK (DICT_VERSION_INCREMENT - 1) -#define DICT_NEXT_VERSION() \ - (_PyRuntime.dict_state.global_version += DICT_VERSION_INCREMENT) +#define DICT_NEXT_VERSION(INTERP) \ + ((INTERP)->dict_state.global_version += DICT_VERSION_INCREMENT) void _PyDict_SendEvent(int watcher_bits, @@ -160,7 +160,8 @@ _PyDict_SendEvent(int watcher_bits, PyObject *value); static inline uint64_t -_PyDict_NotifyEvent(PyDict_WatchEvent event, +_PyDict_NotifyEvent(PyInterpreterState *interp, + PyDict_WatchEvent event, PyDictObject *mp, PyObject *key, PyObject *value) @@ -169,9 +170,9 @@ _PyDict_NotifyEvent(PyDict_WatchEvent event, int watcher_bits = mp->ma_version_tag & DICT_VERSION_MASK; if (watcher_bits) { _PyDict_SendEvent(watcher_bits, event, mp, key, value); - return DICT_NEXT_VERSION() | watcher_bits; + return DICT_NEXT_VERSION(interp) | watcher_bits; } - return DICT_NEXT_VERSION(); + return DICT_NEXT_VERSION(interp); } extern PyObject *_PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values); diff --git a/Include/internal/pycore_dict_state.h b/Include/internal/pycore_dict_state.h index c5142ee6763a86..d608088ed2d7ce 100644 --- a/Include/internal/pycore_dict_state.h +++ b/Include/internal/pycore_dict_state.h @@ -9,14 +9,6 @@ extern "C" { #endif -struct _Py_dict_runtime_state { - /*Global counter used to set ma_version_tag field of dictionary. - * It is incremented each time that a dictionary is created and each - * time that a dictionary is modified. */ - uint64_t global_version; -}; - - #ifndef WITH_FREELISTS // without freelists # define PyDict_MAXFREELIST 0 @@ -29,6 +21,10 @@ struct _Py_dict_runtime_state { #define DICT_MAX_WATCHERS 8 struct _Py_dict_state { + /*Global counter used to set ma_version_tag field of dictionary. + * It is incremented each time that a dictionary is created and each + * time that a dictionary is modified. */ + uint64_t global_version; uint32_t next_keys_version; #if PyDict_MAXFREELIST > 0 diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index e0e3d4ace0cfde..520109ca440444 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -10,7 +10,6 @@ extern "C" { #include "pycore_atomic.h" /* _Py_atomic_address */ #include "pycore_ceval_state.h" // struct _ceval_runtime_state -#include "pycore_dict_state.h" // struct _Py_dict_runtime_state #include "pycore_floatobject.h" // struct _Py_float_runtime_state #include "pycore_faulthandler.h" // struct _faulthandler_runtime_state #include "pycore_global_objects.h" // struct _Py_global_objects @@ -153,7 +152,6 @@ typedef struct pyruntimestate { struct _Py_float_runtime_state float_state; struct _Py_unicode_runtime_state unicode_state; - struct _Py_dict_runtime_state dict_state; struct { /* Used to set PyTypeObject.tp_version_tag */ diff --git a/Objects/dictobject.c b/Objects/dictobject.c index b58e93f4320d42..227e438a8dfffc 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -232,7 +232,8 @@ equally good collision statistics, needed less code & used less memory. */ -static int dictresize(PyDictObject *mp, uint8_t log_newsize, int unicode); +static int dictresize(PyInterpreterState *interp, PyDictObject *mp, + uint8_t log_newsize, int unicode); static PyObject* dict_iter(PyDictObject *dict); @@ -241,9 +242,8 @@ static PyObject* dict_iter(PyDictObject *dict); #if PyDict_MAXFREELIST > 0 static struct _Py_dict_state * -get_dict_state(void) +get_dict_state(PyInterpreterState *interp) { - PyInterpreterState *interp = _PyInterpreterState_GET(); return &interp->dict_state; } #endif @@ -289,7 +289,8 @@ void _PyDict_DebugMallocStats(FILE *out) { #if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(); + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct _Py_dict_state *state = get_dict_state(interp); _PyDebugAllocatorStats(out, "free PyDictObject", state->numfree, sizeof(PyDictObject)); #endif @@ -297,7 +298,7 @@ _PyDict_DebugMallocStats(FILE *out) #define DK_MASK(dk) (DK_SIZE(dk)-1) -static void free_keys_object(PyDictKeysObject *keys); +static void free_keys_object(PyInterpreterState *interp, PyDictKeysObject *keys); static inline void dictkeys_incref(PyDictKeysObject *dk) @@ -309,14 +310,14 @@ dictkeys_incref(PyDictKeysObject *dk) } static inline void -dictkeys_decref(PyDictKeysObject *dk) +dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk) { assert(dk->dk_refcnt > 0); #ifdef Py_REF_DEBUG _Py_DecRefTotal(); #endif if (--dk->dk_refcnt == 0) { - free_keys_object(dk); + free_keys_object(interp, dk); } } @@ -586,7 +587,7 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) static PyDictKeysObject* -new_keys_object(uint8_t log2_size, bool unicode) +new_keys_object(PyInterpreterState *interp, uint8_t log2_size, bool unicode) { PyDictKeysObject *dk; Py_ssize_t usable; @@ -612,7 +613,7 @@ new_keys_object(uint8_t log2_size, bool unicode) } #if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(); + struct _Py_dict_state *state = get_dict_state(interp); #ifdef Py_DEBUG // new_keys_object() must not be called after _PyDict_Fini() assert(state->keys_numfree != -1); @@ -648,7 +649,7 @@ new_keys_object(uint8_t log2_size, bool unicode) } static void -free_keys_object(PyDictKeysObject *keys) +free_keys_object(PyInterpreterState *interp, PyDictKeysObject *keys) { assert(keys != Py_EMPTY_KEYS); if (DK_IS_UNICODE(keys)) { @@ -668,7 +669,7 @@ free_keys_object(PyDictKeysObject *keys) } } #if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(); + struct _Py_dict_state *state = get_dict_state(interp); #ifdef Py_DEBUG // free_keys_object() must not be called after _PyDict_Fini() assert(state->keys_numfree != -1); @@ -709,12 +710,14 @@ free_values(PyDictValues *values) /* Consumes a reference to the keys object */ static PyObject * -new_dict(PyDictKeysObject *keys, PyDictValues *values, Py_ssize_t used, int free_values_on_failure) +new_dict(PyInterpreterState *interp, + PyDictKeysObject *keys, PyDictValues *values, + Py_ssize_t used, int free_values_on_failure) { PyDictObject *mp; assert(keys != NULL); #if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(); + struct _Py_dict_state *state = get_dict_state(interp); #ifdef Py_DEBUG // new_dict() must not be called after _PyDict_Fini() assert(state->numfree != -1); @@ -731,7 +734,7 @@ new_dict(PyDictKeysObject *keys, PyDictValues *values, Py_ssize_t used, int free { mp = PyObject_GC_New(PyDictObject, &PyDict_Type); if (mp == NULL) { - dictkeys_decref(keys); + dictkeys_decref(interp, keys); if (free_values_on_failure) { free_values(values); } @@ -741,7 +744,7 @@ new_dict(PyDictKeysObject *keys, PyDictValues *values, Py_ssize_t used, int free mp->ma_keys = keys; mp->ma_values = values; mp->ma_used = used; - mp->ma_version_tag = DICT_NEXT_VERSION(); + mp->ma_version_tag = DICT_NEXT_VERSION(interp); ASSERT_CONSISTENT(mp); return (PyObject *)mp; } @@ -754,19 +757,19 @@ shared_keys_usable_size(PyDictKeysObject *keys) /* Consumes a reference to the keys object */ static PyObject * -new_dict_with_shared_keys(PyDictKeysObject *keys) +new_dict_with_shared_keys(PyInterpreterState *interp, PyDictKeysObject *keys) { size_t size = shared_keys_usable_size(keys); PyDictValues *values = new_values(size); if (values == NULL) { - dictkeys_decref(keys); + dictkeys_decref(interp, keys); return PyErr_NoMemory(); } ((char *)values)[-2] = 0; for (size_t i = 0; i < size; i++) { values->values[i] = NULL; } - return new_dict(keys, values, 0, 1); + return new_dict(interp, keys, values, 0, 1); } @@ -829,8 +832,9 @@ clone_combined_dict_keys(PyDictObject *orig) PyObject * PyDict_New(void) { + PyInterpreterState *interp = _PyInterpreterState_GET(); dictkeys_incref(Py_EMPTY_KEYS); - return new_dict(Py_EMPTY_KEYS, NULL, 0, 0); + return new_dict(interp, Py_EMPTY_KEYS, NULL, 0, 0); } /* Search index of hash table from offset of entry table */ @@ -1170,9 +1174,9 @@ find_empty_slot(PyDictKeysObject *keys, Py_hash_t hash) } static int -insertion_resize(PyDictObject *mp, int unicode) +insertion_resize(PyInterpreterState *interp, PyDictObject *mp, int unicode) { - return dictresize(mp, calculate_log2_keysize(GROWTH_RATE(mp)), unicode); + return dictresize(interp, mp, calculate_log2_keysize(GROWTH_RATE(mp)), unicode); } static Py_ssize_t @@ -1214,12 +1218,13 @@ Returns -1 if an error occurred, or 0 on success. Consumes key and value references. */ static int -insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) +insertdict(PyInterpreterState *interp, PyDictObject *mp, + PyObject *key, Py_hash_t hash, PyObject *value) { PyObject *old_value; if (DK_IS_UNICODE(mp->ma_keys) && !PyUnicode_CheckExact(key)) { - if (insertion_resize(mp, 0) < 0) + if (insertion_resize(interp, mp, 0) < 0) goto Fail; assert(mp->ma_keys->dk_kind == DICT_KEYS_GENERAL); } @@ -1231,13 +1236,14 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) MAINTAIN_TRACKING(mp, key, value); if (ix == DKIX_EMPTY) { - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_ADDED, mp, key, value); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_ADDED, mp, key, value); /* Insert into new slot. */ mp->ma_keys->dk_version = 0; assert(old_value == NULL); if (mp->ma_keys->dk_usable <= 0) { /* Need to resize. */ - if (insertion_resize(mp, 1) < 0) + if (insertion_resize(interp, mp, 1) < 0) goto Fail; } @@ -1275,7 +1281,8 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) } if (old_value != value) { - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, mp, key, value); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_MODIFIED, mp, key, value); if (_PyDict_HasSplitTable(mp)) { mp->ma_values->values[ix] = value; if (old_value == NULL) { @@ -1308,21 +1315,23 @@ insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) // Same to insertdict but specialized for ma_keys = Py_EMPTY_KEYS. // Consumes key and value references. static int -insert_to_emptydict(PyDictObject *mp, PyObject *key, Py_hash_t hash, - PyObject *value) +insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, + PyObject *key, Py_hash_t hash, PyObject *value) { assert(mp->ma_keys == Py_EMPTY_KEYS); - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_ADDED, mp, key, value); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_ADDED, mp, key, value); int unicode = PyUnicode_CheckExact(key); - PyDictKeysObject *newkeys = new_keys_object(PyDict_LOG_MINSIZE, unicode); + PyDictKeysObject *newkeys = new_keys_object( + interp, PyDict_LOG_MINSIZE, unicode); if (newkeys == NULL) { Py_DECREF(key); Py_DECREF(value); return -1; } - dictkeys_decref(Py_EMPTY_KEYS); + dictkeys_decref(interp, Py_EMPTY_KEYS); mp->ma_keys = newkeys; mp->ma_values = NULL; @@ -1397,7 +1406,8 @@ This function supports: - Generic -> Generic */ static int -dictresize(PyDictObject *mp, uint8_t log2_newsize, int unicode) +dictresize(PyInterpreterState *interp, PyDictObject *mp, + uint8_t log2_newsize, int unicode) { PyDictKeysObject *oldkeys; PyDictValues *oldvalues; @@ -1421,7 +1431,7 @@ dictresize(PyDictObject *mp, uint8_t log2_newsize, int unicode) */ /* Allocate a new table. */ - mp->ma_keys = new_keys_object(log2_newsize, unicode); + mp->ma_keys = new_keys_object(interp, log2_newsize, unicode); if (mp->ma_keys == NULL) { mp->ma_keys = oldkeys; return -1; @@ -1462,7 +1472,7 @@ dictresize(PyDictObject *mp, uint8_t log2_newsize, int unicode) } build_indices_unicode(mp->ma_keys, newentries, numentries); } - dictkeys_decref(oldkeys); + dictkeys_decref(interp, oldkeys); mp->ma_values = NULL; free_values(oldvalues); } @@ -1530,7 +1540,7 @@ dictresize(PyDictObject *mp, uint8_t log2_newsize, int unicode) assert(oldkeys->dk_kind != DICT_KEYS_SPLIT); assert(oldkeys->dk_refcnt == 1); #if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(); + struct _Py_dict_state *state = get_dict_state(interp); #ifdef Py_DEBUG // dictresize() must not be called after _PyDict_Fini() assert(state->keys_numfree != -1); @@ -1557,7 +1567,7 @@ dictresize(PyDictObject *mp, uint8_t log2_newsize, int unicode) } static PyObject * -dict_new_presized(Py_ssize_t minused, bool unicode) +dict_new_presized(PyInterpreterState *interp, Py_ssize_t minused, bool unicode) { const uint8_t log2_max_presize = 17; const Py_ssize_t max_presize = ((Py_ssize_t)1) << log2_max_presize; @@ -1578,16 +1588,17 @@ dict_new_presized(Py_ssize_t minused, bool unicode) log2_newsize = estimate_log2_keysize(minused); } - new_keys = new_keys_object(log2_newsize, unicode); + new_keys = new_keys_object(interp, log2_newsize, unicode); if (new_keys == NULL) return NULL; - return new_dict(new_keys, NULL, 0, 0); + return new_dict(interp, new_keys, NULL, 0, 0); } PyObject * _PyDict_NewPresized(Py_ssize_t minused) { - return dict_new_presized(minused, false); + PyInterpreterState *interp = _PyInterpreterState_GET(); + return dict_new_presized(interp, minused, false); } PyObject * @@ -1597,6 +1608,7 @@ _PyDict_FromItems(PyObject *const *keys, Py_ssize_t keys_offset, { bool unicode = true; PyObject *const *ks = keys; + PyInterpreterState *interp = _PyInterpreterState_GET(); for (Py_ssize_t i = 0; i < length; i++) { if (!PyUnicode_CheckExact(*ks)) { @@ -1606,7 +1618,7 @@ _PyDict_FromItems(PyObject *const *keys, Py_ssize_t keys_offset, ks += keys_offset; } - PyObject *dict = dict_new_presized(length, unicode); + PyObject *dict = dict_new_presized(interp, length, unicode); if (dict == NULL) { return NULL; } @@ -1834,11 +1846,12 @@ _PyDict_SetItem_Take2(PyDictObject *mp, PyObject *key, PyObject *value) return -1; } } + PyInterpreterState *interp = _PyInterpreterState_GET(); if (mp->ma_keys == Py_EMPTY_KEYS) { - return insert_to_emptydict(mp, key, hash, value); + return insert_to_emptydict(interp, mp, key, hash, value); } /* insertdict() handles any resizing that might be necessary */ - return insertdict(mp, key, hash, value); + return insertdict(interp, mp, key, hash, value); } /* CAUTION: PyDict_SetItem() must guarantee that it won't resize the @@ -1875,11 +1888,12 @@ _PyDict_SetItem_KnownHash(PyObject *op, PyObject *key, PyObject *value, assert(hash != -1); mp = (PyDictObject *)op; + PyInterpreterState *interp = _PyInterpreterState_GET(); if (mp->ma_keys == Py_EMPTY_KEYS) { - return insert_to_emptydict(mp, Py_NewRef(key), hash, Py_NewRef(value)); + return insert_to_emptydict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); } /* insertdict() handles any resizing that might be necessary */ - return insertdict(mp, Py_NewRef(key), hash, Py_NewRef(value)); + return insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value)); } static void @@ -1977,7 +1991,9 @@ _PyDict_DelItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) return -1; } - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_DELETED, mp, key, NULL); + PyInterpreterState *interp = _PyInterpreterState_GET(); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_DELETED, mp, key, NULL); return delitem_common(mp, hash, ix, old_value, new_version); } @@ -2020,7 +2036,9 @@ _PyDict_DelItemIf(PyObject *op, PyObject *key, assert(hashpos >= 0); if (res > 0) { - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_DELETED, mp, key, NULL); + PyInterpreterState *interp = _PyInterpreterState_GET(); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_DELETED, mp, key, NULL); return delitem_common(mp, hashpos, ix, old_value, new_version); } else { return 0; @@ -2045,7 +2063,9 @@ PyDict_Clear(PyObject *op) return; } /* Empty the dict... */ - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_CLEARED, mp, NULL, NULL); + PyInterpreterState *interp = _PyInterpreterState_GET(); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_CLEARED, mp, NULL, NULL); dictkeys_incref(Py_EMPTY_KEYS); mp->ma_keys = Py_EMPTY_KEYS; mp->ma_values = NULL; @@ -2057,11 +2077,11 @@ PyDict_Clear(PyObject *op) for (i = 0; i < n; i++) Py_CLEAR(oldvalues->values[i]); free_values(oldvalues); - dictkeys_decref(oldkeys); + dictkeys_decref(interp, oldkeys); } else { assert(oldkeys->dk_refcnt == 1); - dictkeys_decref(oldkeys); + dictkeys_decref(interp, oldkeys); } ASSERT_CONSISTENT(mp); } @@ -2165,6 +2185,7 @@ _PyDict_Pop_KnownHash(PyObject *dict, PyObject *key, Py_hash_t hash, PyObject *d Py_ssize_t ix; PyObject *old_value; PyDictObject *mp; + PyInterpreterState *interp = _PyInterpreterState_GET(); assert(PyDict_Check(dict)); mp = (PyDictObject *)dict; @@ -2187,7 +2208,8 @@ _PyDict_Pop_KnownHash(PyObject *dict, PyObject *key, Py_hash_t hash, PyObject *d return NULL; } assert(old_value != NULL); - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_DELETED, mp, key, NULL); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_DELETED, mp, key, NULL); delitem_common(mp, hash, ix, Py_NewRef(old_value), new_version); ASSERT_CONSISTENT(mp); @@ -2222,6 +2244,7 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) PyObject *key; PyObject *d; int status; + PyInterpreterState *interp = _PyInterpreterState_GET(); d = _PyObject_CallNoArgs(cls); if (d == NULL) @@ -2236,13 +2259,16 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) Py_hash_t hash; int unicode = DK_IS_UNICODE(((PyDictObject*)iterable)->ma_keys); - if (dictresize(mp, estimate_log2_keysize(PyDict_GET_SIZE(iterable)), unicode)) { + if (dictresize(interp, mp, + estimate_log2_keysize(PyDict_GET_SIZE(iterable)), + unicode)) { Py_DECREF(d); return NULL; } while (_PyDict_Next(iterable, &pos, &key, &oldvalue, &hash)) { - if (insertdict(mp, Py_NewRef(key), hash, Py_NewRef(value))) { + if (insertdict(interp, mp, + Py_NewRef(key), hash, Py_NewRef(value))) { Py_DECREF(d); return NULL; } @@ -2255,13 +2281,14 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) PyObject *key; Py_hash_t hash; - if (dictresize(mp, estimate_log2_keysize(PySet_GET_SIZE(iterable)), 0)) { + if (dictresize(interp, mp, + estimate_log2_keysize(PySet_GET_SIZE(iterable)), 0)) { Py_DECREF(d); return NULL; } while (_PySet_NextEntry(iterable, &pos, &key, &hash)) { - if (insertdict(mp, Py_NewRef(key), hash, Py_NewRef(value))) { + if (insertdict(interp, mp, Py_NewRef(key), hash, Py_NewRef(value))) { Py_DECREF(d); return NULL; } @@ -2308,9 +2335,10 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) static void dict_dealloc(PyDictObject *mp) { + PyInterpreterState *interp = _PyInterpreterState_GET(); assert(Py_REFCNT(mp) == 0); Py_SET_REFCNT(mp, 1); - _PyDict_NotifyEvent(PyDict_EVENT_DEALLOCATED, mp, NULL, NULL); + _PyDict_NotifyEvent(interp, PyDict_EVENT_DEALLOCATED, mp, NULL, NULL); if (Py_REFCNT(mp) > 1) { Py_SET_REFCNT(mp, Py_REFCNT(mp) - 1); return; @@ -2328,14 +2356,14 @@ dict_dealloc(PyDictObject *mp) Py_XDECREF(values->values[i]); } free_values(values); - dictkeys_decref(keys); + dictkeys_decref(interp, keys); } else if (keys != NULL) { assert(keys->dk_refcnt == 1 || keys == Py_EMPTY_KEYS); - dictkeys_decref(keys); + dictkeys_decref(interp, keys); } #if PyDict_MAXFREELIST > 0 - struct _Py_dict_state *state = get_dict_state(); + struct _Py_dict_state *state = get_dict_state(interp); #ifdef Py_DEBUG // new_dict() must not be called after _PyDict_Fini() assert(state->numfree != -1); @@ -2765,7 +2793,7 @@ PyDict_MergeFromSeq2(PyObject *d, PyObject *seq2, int override) } static int -dict_merge(PyObject *a, PyObject *b, int override) +dict_merge(PyInterpreterState *interp, PyObject *a, PyObject *b, int override) { PyDictObject *mp, *other; @@ -2799,13 +2827,14 @@ dict_merge(PyObject *a, PyObject *b, int override) other->ma_used == okeys->dk_nentries && (DK_LOG_SIZE(okeys) == PyDict_LOG_MINSIZE || USABLE_FRACTION(DK_SIZE(okeys)/2) < other->ma_used)) { - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_CLONED, mp, b, NULL); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_CLONED, mp, b, NULL); PyDictKeysObject *keys = clone_combined_dict_keys(other); if (keys == NULL) { return -1; } - dictkeys_decref(mp->ma_keys); + dictkeys_decref(interp, mp->ma_keys); mp->ma_keys = keys; if (mp->ma_values != NULL) { free_values(mp->ma_values); @@ -2830,7 +2859,9 @@ dict_merge(PyObject *a, PyObject *b, int override) */ if (USABLE_FRACTION(DK_SIZE(mp->ma_keys)) < other->ma_used) { int unicode = DK_IS_UNICODE(other->ma_keys); - if (dictresize(mp, estimate_log2_keysize(mp->ma_used + other->ma_used), unicode)) { + if (dictresize(interp, mp, + estimate_log2_keysize(mp->ma_used + other->ma_used), + unicode)) { return -1; } } @@ -2845,12 +2876,14 @@ dict_merge(PyObject *a, PyObject *b, int override) Py_INCREF(key); Py_INCREF(value); if (override == 1) { - err = insertdict(mp, Py_NewRef(key), hash, Py_NewRef(value)); + err = insertdict(interp, mp, + Py_NewRef(key), hash, Py_NewRef(value)); } else { err = _PyDict_Contains_KnownHash(a, key, hash); if (err == 0) { - err = insertdict(mp, Py_NewRef(key), hash, Py_NewRef(value)); + err = insertdict(interp, mp, + Py_NewRef(key), hash, Py_NewRef(value)); } else if (err > 0) { if (override != 0) { @@ -2936,20 +2969,23 @@ dict_merge(PyObject *a, PyObject *b, int override) int PyDict_Update(PyObject *a, PyObject *b) { - return dict_merge(a, b, 1); + PyInterpreterState *interp = _PyInterpreterState_GET(); + return dict_merge(interp, a, b, 1); } int PyDict_Merge(PyObject *a, PyObject *b, int override) { + PyInterpreterState *interp = _PyInterpreterState_GET(); /* XXX Deprecate override not in (0, 1). */ - return dict_merge(a, b, override != 0); + return dict_merge(interp, a, b, override != 0); } int _PyDict_MergeEx(PyObject *a, PyObject *b, int override) { - return dict_merge(a, b, override); + PyInterpreterState *interp = _PyInterpreterState_GET(); + return dict_merge(interp, a, b, override); } static PyObject * @@ -2963,6 +2999,7 @@ PyDict_Copy(PyObject *o) { PyObject *copy; PyDictObject *mp; + PyInterpreterState *interp = _PyInterpreterState_GET(); if (o == NULL || !PyDict_Check(o)) { PyErr_BadInternalCall(); @@ -2991,7 +3028,7 @@ PyDict_Copy(PyObject *o) split_copy->ma_values = newvalues; split_copy->ma_keys = mp->ma_keys; split_copy->ma_used = mp->ma_used; - split_copy->ma_version_tag = DICT_NEXT_VERSION(); + split_copy->ma_version_tag = DICT_NEXT_VERSION(interp); dictkeys_incref(mp->ma_keys); for (size_t i = 0; i < size; i++) { PyObject *value = mp->ma_values->values[i]; @@ -3024,7 +3061,7 @@ PyDict_Copy(PyObject *o) if (keys == NULL) { return NULL; } - PyDictObject *new = (PyDictObject *)new_dict(keys, NULL, 0, 0); + PyDictObject *new = (PyDictObject *)new_dict(interp, keys, NULL, 0, 0); if (new == NULL) { /* In case of an error, `new_dict()` takes care of cleaning up `keys`. */ @@ -3044,7 +3081,7 @@ PyDict_Copy(PyObject *o) copy = PyDict_New(); if (copy == NULL) return NULL; - if (dict_merge(copy, o, 1) == 0) + if (dict_merge(interp, copy, o, 1) == 0) return copy; Py_DECREF(copy); return NULL; @@ -3244,6 +3281,7 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) PyDictObject *mp = (PyDictObject *)d; PyObject *value; Py_hash_t hash; + PyInterpreterState *interp = _PyInterpreterState_GET(); if (!PyDict_Check(d)) { PyErr_BadInternalCall(); @@ -3257,7 +3295,7 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) } if (mp->ma_keys == Py_EMPTY_KEYS) { - if (insert_to_emptydict(mp, Py_NewRef(key), hash, + if (insert_to_emptydict(interp, mp, Py_NewRef(key), hash, Py_NewRef(defaultobj)) < 0) { return NULL; } @@ -3265,7 +3303,7 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) } if (!PyUnicode_CheckExact(key) && DK_IS_UNICODE(mp->ma_keys)) { - if (insertion_resize(mp, 0) < 0) { + if (insertion_resize(interp, mp, 0) < 0) { return NULL; } } @@ -3275,11 +3313,12 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) return NULL; if (ix == DKIX_EMPTY) { - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_ADDED, mp, key, defaultobj); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_ADDED, mp, key, defaultobj); mp->ma_keys->dk_version = 0; value = defaultobj; if (mp->ma_keys->dk_usable <= 0) { - if (insertion_resize(mp, 1) < 0) { + if (insertion_resize(interp, mp, 1) < 0) { return NULL; } } @@ -3314,7 +3353,8 @@ PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj) assert(mp->ma_keys->dk_usable >= 0); } else if (value == NULL) { - uint64_t new_version = _PyDict_NotifyEvent(PyDict_EVENT_ADDED, mp, key, defaultobj); + uint64_t new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_ADDED, mp, key, defaultobj); value = defaultobj; assert(_PyDict_HasSplitTable(mp)); assert(mp->ma_values->values[ix] == NULL); @@ -3395,6 +3435,7 @@ dict_popitem_impl(PyDictObject *self) Py_ssize_t i, j; PyObject *res; uint64_t new_version; + PyInterpreterState *interp = _PyInterpreterState_GET(); /* Allocate the result tuple before checking the size. Believe it * or not, this allocation could trigger a garbage collection which @@ -3415,7 +3456,7 @@ dict_popitem_impl(PyDictObject *self) } /* Convert split table to combined table */ if (self->ma_keys->dk_kind == DICT_KEYS_SPLIT) { - if (dictresize(self, DK_LOG_SIZE(self->ma_keys), 1)) { + if (dictresize(interp, self, DK_LOG_SIZE(self->ma_keys), 1)) { Py_DECREF(res); return NULL; } @@ -3434,7 +3475,8 @@ dict_popitem_impl(PyDictObject *self) assert(i >= 0); key = ep0[i].me_key; - new_version = _PyDict_NotifyEvent(PyDict_EVENT_DELETED, self, key, NULL); + new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_DELETED, self, key, NULL); hash = unicode_get_hash(key); value = ep0[i].me_value; ep0[i].me_key = NULL; @@ -3449,7 +3491,8 @@ dict_popitem_impl(PyDictObject *self) assert(i >= 0); key = ep0[i].me_key; - new_version = _PyDict_NotifyEvent(PyDict_EVENT_DELETED, self, key, NULL); + new_version = _PyDict_NotifyEvent( + interp, PyDict_EVENT_DELETED, self, key, NULL); hash = ep0[i].me_hash; value = ep0[i].me_value; ep0[i].me_key = NULL; @@ -3708,7 +3751,8 @@ dict_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PyDictObject *d = (PyDictObject *)self; d->ma_used = 0; - d->ma_version_tag = DICT_NEXT_VERSION(); + d->ma_version_tag = DICT_NEXT_VERSION( + _PyInterpreterState_GET()); dictkeys_incref(Py_EMPTY_KEYS); d->ma_keys = Py_EMPTY_KEYS; d->ma_values = NULL; @@ -5262,7 +5306,9 @@ dictvalues_reversed(_PyDictViewObject *dv, PyObject *Py_UNUSED(ignored)) PyDictKeysObject * _PyDict_NewKeysForClass(void) { - PyDictKeysObject *keys = new_keys_object(NEXT_LOG2_SHARED_KEYS_MAX_SIZE, 1); + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyDictKeysObject *keys = new_keys_object( + interp, NEXT_LOG2_SHARED_KEYS_MAX_SIZE, 1); if (keys == NULL) { PyErr_Clear(); } @@ -5306,6 +5352,7 @@ init_inline_values(PyObject *obj, PyTypeObject *tp) int _PyObject_InitializeDict(PyObject *obj) { + PyInterpreterState *interp = _PyInterpreterState_GET(); PyTypeObject *tp = Py_TYPE(obj); if (tp->tp_dictoffset == 0) { return 0; @@ -5317,7 +5364,7 @@ _PyObject_InitializeDict(PyObject *obj) PyObject *dict; if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) { dictkeys_incref(CACHED_KEYS(tp)); - dict = new_dict_with_shared_keys(CACHED_KEYS(tp)); + dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp)); } else { dict = PyDict_New(); @@ -5331,7 +5378,8 @@ _PyObject_InitializeDict(PyObject *obj) } static PyObject * -make_dict_from_instance_attributes(PyDictKeysObject *keys, PyDictValues *values) +make_dict_from_instance_attributes(PyInterpreterState *interp, + PyDictKeysObject *keys, PyDictValues *values) { dictkeys_incref(keys); Py_ssize_t used = 0; @@ -5344,7 +5392,7 @@ make_dict_from_instance_attributes(PyDictKeysObject *keys, PyDictValues *values) track += _PyObject_GC_MAY_BE_TRACKED(val); } } - PyObject *res = new_dict(keys, values, used, 0); + PyObject *res = new_dict(interp, keys, values, used, 0); if (track && res) { _PyObject_GC_TRACK(res); } @@ -5354,15 +5402,17 @@ make_dict_from_instance_attributes(PyDictKeysObject *keys, PyDictValues *values) PyObject * _PyObject_MakeDictFromInstanceAttributes(PyObject *obj, PyDictValues *values) { + PyInterpreterState *interp = _PyInterpreterState_GET(); PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); OBJECT_STAT_INC(dict_materialized_on_request); - return make_dict_from_instance_attributes(keys, values); + return make_dict_from_instance_attributes(interp, keys, values); } int _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, PyObject *name, PyObject *value) { + PyInterpreterState *interp = _PyInterpreterState_GET(); PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); assert(keys != NULL); assert(values != NULL); @@ -5385,7 +5435,8 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, OBJECT_STAT_INC(dict_materialized_str_subclass); } #endif - PyObject *dict = make_dict_from_instance_attributes(keys, values); + PyObject *dict = make_dict_from_instance_attributes( + interp, keys, values); if (dict == NULL) { return -1; } @@ -5564,13 +5615,15 @@ PyObject * PyObject_GenericGetDict(PyObject *obj, void *context) { PyObject *dict; + PyInterpreterState *interp = _PyInterpreterState_GET(); PyTypeObject *tp = Py_TYPE(obj); if (_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT)) { PyDictOrValues *dorv_ptr = _PyObject_DictOrValuesPointer(obj); if (_PyDictOrValues_IsValues(*dorv_ptr)) { PyDictValues *values = _PyDictOrValues_GetValues(*dorv_ptr); OBJECT_STAT_INC(dict_materialized_on_request); - dict = make_dict_from_instance_attributes(CACHED_KEYS(tp), values); + dict = make_dict_from_instance_attributes( + interp, CACHED_KEYS(tp), values); if (dict != NULL) { dorv_ptr->dict = dict; } @@ -5579,7 +5632,7 @@ PyObject_GenericGetDict(PyObject *obj, void *context) dict = _PyDictOrValues_GetDict(*dorv_ptr); if (dict == NULL) { dictkeys_incref(CACHED_KEYS(tp)); - dict = new_dict_with_shared_keys(CACHED_KEYS(tp)); + dict = new_dict_with_shared_keys(interp, CACHED_KEYS(tp)); dorv_ptr->dict = dict; } } @@ -5596,7 +5649,8 @@ PyObject_GenericGetDict(PyObject *obj, void *context) PyTypeObject *tp = Py_TYPE(obj); if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) { dictkeys_incref(CACHED_KEYS(tp)); - *dictptr = dict = new_dict_with_shared_keys(CACHED_KEYS(tp)); + *dictptr = dict = new_dict_with_shared_keys( + interp, CACHED_KEYS(tp)); } else { *dictptr = dict = PyDict_New(); @@ -5613,6 +5667,7 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, PyObject *dict; int res; PyDictKeysObject *cached; + PyInterpreterState *interp = _PyInterpreterState_GET(); assert(dictptr != NULL); if ((tp->tp_flags & Py_TPFLAGS_HEAPTYPE) && (cached = CACHED_KEYS(tp))) { @@ -5620,7 +5675,7 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, dict = *dictptr; if (dict == NULL) { dictkeys_incref(cached); - dict = new_dict_with_shared_keys(cached); + dict = new_dict_with_shared_keys(interp, cached); if (dict == NULL) return -1; *dictptr = dict; @@ -5652,7 +5707,8 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, void _PyDictKeys_DecRef(PyDictKeysObject *keys) { - dictkeys_decref(keys); + PyInterpreterState *interp = _PyInterpreterState_GET(); + dictkeys_decref(interp, keys); } uint32_t _PyDictKeys_GetVersionForCurrentState(PyInterpreterState *interp, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 63dbecad3b45fc..45d50726674321 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1679,7 +1679,7 @@ dummy_func( DEOPT_IF(ep->me_key != name, STORE_ATTR); old_value = ep->me_value; DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, value); + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } else { @@ -1687,7 +1687,7 @@ dummy_func( DEOPT_IF(ep->me_key != name, STORE_ATTR); old_value = ep->me_value; DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, value); + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } Py_DECREF(old_value); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 82e18505b0d430..51357cda4c8949 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2114,7 +2114,7 @@ DEOPT_IF(ep->me_key != name, STORE_ATTR); old_value = ep->me_value; DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, value); + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } else { @@ -2122,7 +2122,7 @@ DEOPT_IF(ep->me_key != name, STORE_ATTR); old_value = ep->me_value; DEOPT_IF(old_value == NULL, STORE_ATTR); - new_version = _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, value); + new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } Py_DECREF(old_value); From cf6e7c5e551b3513817d6a77ba88253dc8473298 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 9 Mar 2023 09:46:21 -0700 Subject: [PATCH 29/38] gh-100227: Isolate the Import State to Each Interpreter (gh-101941) Specific changes: * move the import lock to PyInterpreterState * move the "find_and_load" diagnostic state to PyInterpreterState Note that the import lock exists to keep multiple imports of the same module in the same interpreter (but in different threads) from stomping on each other. Independently, we use a distinct global lock to protect globally shared import state, especially related to loaded extension modules. For now we can rely on the GIL as that lock but with a per-interpreter GIL we'll need a new global lock. The remaining state in _PyRuntimeState.imports will (probably) continue being global. https://github.com/python/cpython/issues/100227 --- Include/cpython/import.h | 4 +- Include/internal/pycore_import.h | 34 +++++---- Include/internal/pycore_runtime_init.h | 10 --- Modules/posixmodule.c | 13 ++-- Python/import.c | 98 ++++++++++++++------------ 5 files changed, 85 insertions(+), 74 deletions(-) diff --git a/Include/cpython/import.h b/Include/cpython/import.h index a58801b47f1bec..2bca4ade4c4f2c 100644 --- a/Include/cpython/import.h +++ b/Include/cpython/import.h @@ -10,8 +10,8 @@ PyAPI_FUNC(PyObject *) _PyImport_GetModuleId(_Py_Identifier *name); PyAPI_FUNC(int) _PyImport_SetModule(PyObject *name, PyObject *module); PyAPI_FUNC(int) _PyImport_SetModuleString(const char *name, PyObject* module); -PyAPI_FUNC(void) _PyImport_AcquireLock(void); -PyAPI_FUNC(int) _PyImport_ReleaseLock(void); +PyAPI_FUNC(void) _PyImport_AcquireLock(PyInterpreterState *interp); +PyAPI_FUNC(int) _PyImport_ReleaseLock(PyInterpreterState *interp); PyAPI_FUNC(int) _PyImport_FixupBuiltin( PyObject *mod, diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index b7ffe01c0c0e20..69ed6273b7e609 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -21,17 +21,6 @@ struct _import_runtime_state { This is initialized lazily in _PyImport_FixupExtensionObject(). Modules are added there and looked up in _imp.find_extension(). */ PyObject *extensions; - /* The global import lock. */ - struct { - PyThread_type_lock mutex; - unsigned long thread; - int level; - } lock; - struct { - int import_level; - _PyTime_t accumulated; - int header; - } find_and_load; /* Package context -- the full module name for package imports */ const char * pkgcontext; }; @@ -69,6 +58,18 @@ struct _import_state { int dlopenflags; #endif PyObject *import_func; + /* The global import lock. */ + struct { + PyThread_type_lock mutex; + unsigned long thread; + int level; + } lock; + /* diagnostic info in PyImport_ImportModuleLevelObject() */ + struct { + int import_level; + _PyTime_t accumulated; + int header; + } find_and_load; }; #ifdef HAVE_DLOPEN @@ -86,8 +87,15 @@ struct _import_state { #define IMPORTS_INIT \ { \ - .override_frozen_modules = 0, \ DLOPENFLAGS_INIT \ + .lock = { \ + .mutex = NULL, \ + .thread = PYTHREAD_INVALID_THREAD_ID, \ + .level = 0, \ + }, \ + .find_and_load = { \ + .header = 1, \ + }, \ } extern void _PyImport_ClearCore(PyInterpreterState *interp); @@ -138,7 +146,7 @@ extern void _PyImport_FiniExternal(PyInterpreterState *interp); #ifdef HAVE_FORK -extern PyStatus _PyImport_ReInitLock(void); +extern PyStatus _PyImport_ReInitLock(PyInterpreterState *interp); #endif diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index efc82b43a61dab..bdecac944dfd3a 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -40,16 +40,6 @@ extern PyTypeObject _PyExc_MemoryError; in accordance with the specification. */ \ .autoTSSkey = Py_tss_NEEDS_INIT, \ .parser = _parser_runtime_state_INIT, \ - .imports = { \ - .lock = { \ - .mutex = NULL, \ - .thread = PYTHREAD_INVALID_THREAD_ID, \ - .level = 0, \ - }, \ - .find_and_load = { \ - .header = 1, \ - }, \ - }, \ .ceval = { \ .perf = _PyEval_RUNTIME_PERF_INIT, \ }, \ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 7beb2cee64a05c..a3d86cbe7a57fb 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -567,18 +567,21 @@ run_at_forkers(PyObject *lst, int reverse) void PyOS_BeforeFork(void) { - run_at_forkers(_PyInterpreterState_GET()->before_forkers, 1); + PyInterpreterState *interp = _PyInterpreterState_GET(); + run_at_forkers(interp->before_forkers, 1); - _PyImport_AcquireLock(); + _PyImport_AcquireLock(interp); } void PyOS_AfterFork_Parent(void) { - if (_PyImport_ReleaseLock() <= 0) + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (_PyImport_ReleaseLock(interp) <= 0) { Py_FatalError("failed releasing import lock after fork"); + } - run_at_forkers(_PyInterpreterState_GET()->after_forkers_parent, 0); + run_at_forkers(interp->after_forkers_parent, 0); } void @@ -604,7 +607,7 @@ PyOS_AfterFork_Child(void) goto fatal_error; } - status = _PyImport_ReInitLock(); + status = _PyImport_ReInitLock(tstate->interp); if (_PyStatus_EXCEPTION(status)) { goto fatal_error; } diff --git a/Python/import.c b/Python/import.c index 57d4eea148810f..1bf4199e125aa5 100644 --- a/Python/import.c +++ b/Python/import.c @@ -56,11 +56,6 @@ static struct _inittab *inittab_copy = NULL; #define LAST_MODULE_INDEX _PyRuntime.imports.last_module_index #define EXTENSIONS _PyRuntime.imports.extensions -#define import_lock _PyRuntime.imports.lock.mutex -#define import_lock_thread _PyRuntime.imports.lock.thread -#define import_lock_level _PyRuntime.imports.lock.level - -#define FIND_AND_LOAD _PyRuntime.imports.find_and_load #define PKGCONTEXT (_PyRuntime.imports.pkgcontext) @@ -85,6 +80,16 @@ static struct _inittab *inittab_copy = NULL; #define IMPORT_FUNC(interp) \ (interp)->imports.import_func +#define IMPORT_LOCK(interp) \ + (interp)->imports.lock.mutex +#define IMPORT_LOCK_THREAD(interp) \ + (interp)->imports.lock.thread +#define IMPORT_LOCK_LEVEL(interp) \ + (interp)->imports.lock.level + +#define FIND_AND_LOAD(interp) \ + (interp)->imports.find_and_load + /*******************/ /* the import lock */ @@ -95,45 +100,45 @@ static struct _inittab *inittab_copy = NULL; These calls are serialized by the global interpreter lock. */ void -_PyImport_AcquireLock(void) +_PyImport_AcquireLock(PyInterpreterState *interp) { unsigned long me = PyThread_get_thread_ident(); if (me == PYTHREAD_INVALID_THREAD_ID) return; /* Too bad */ - if (import_lock == NULL) { - import_lock = PyThread_allocate_lock(); - if (import_lock == NULL) + if (IMPORT_LOCK(interp) == NULL) { + IMPORT_LOCK(interp) = PyThread_allocate_lock(); + if (IMPORT_LOCK(interp) == NULL) return; /* Nothing much we can do. */ } - if (import_lock_thread == me) { - import_lock_level++; + if (IMPORT_LOCK_THREAD(interp) == me) { + IMPORT_LOCK_LEVEL(interp)++; return; } - if (import_lock_thread != PYTHREAD_INVALID_THREAD_ID || - !PyThread_acquire_lock(import_lock, 0)) + if (IMPORT_LOCK_THREAD(interp) != PYTHREAD_INVALID_THREAD_ID || + !PyThread_acquire_lock(IMPORT_LOCK(interp), 0)) { PyThreadState *tstate = PyEval_SaveThread(); - PyThread_acquire_lock(import_lock, WAIT_LOCK); + PyThread_acquire_lock(IMPORT_LOCK(interp), WAIT_LOCK); PyEval_RestoreThread(tstate); } - assert(import_lock_level == 0); - import_lock_thread = me; - import_lock_level = 1; + assert(IMPORT_LOCK_LEVEL(interp) == 0); + IMPORT_LOCK_THREAD(interp) = me; + IMPORT_LOCK_LEVEL(interp) = 1; } int -_PyImport_ReleaseLock(void) +_PyImport_ReleaseLock(PyInterpreterState *interp) { unsigned long me = PyThread_get_thread_ident(); - if (me == PYTHREAD_INVALID_THREAD_ID || import_lock == NULL) + if (me == PYTHREAD_INVALID_THREAD_ID || IMPORT_LOCK(interp) == NULL) return 0; /* Too bad */ - if (import_lock_thread != me) + if (IMPORT_LOCK_THREAD(interp) != me) return -1; - import_lock_level--; - assert(import_lock_level >= 0); - if (import_lock_level == 0) { - import_lock_thread = PYTHREAD_INVALID_THREAD_ID; - PyThread_release_lock(import_lock); + IMPORT_LOCK_LEVEL(interp)--; + assert(IMPORT_LOCK_LEVEL(interp) >= 0); + if (IMPORT_LOCK_LEVEL(interp) == 0) { + IMPORT_LOCK_THREAD(interp) = PYTHREAD_INVALID_THREAD_ID; + PyThread_release_lock(IMPORT_LOCK(interp)); } return 1; } @@ -144,23 +149,23 @@ _PyImport_ReleaseLock(void) We now acquire the import lock around fork() calls but on some platforms (Solaris 9 and earlier? see isue7242) that still left us with problems. */ PyStatus -_PyImport_ReInitLock(void) +_PyImport_ReInitLock(PyInterpreterState *interp) { - if (import_lock != NULL) { - if (_PyThread_at_fork_reinit(&import_lock) < 0) { + if (IMPORT_LOCK(interp) != NULL) { + if (_PyThread_at_fork_reinit(&IMPORT_LOCK(interp)) < 0) { return _PyStatus_ERR("failed to create a new lock"); } } - if (import_lock_level > 1) { + if (IMPORT_LOCK_LEVEL(interp) > 1) { /* Forked as a side effect of import */ unsigned long me = PyThread_get_thread_ident(); - PyThread_acquire_lock(import_lock, WAIT_LOCK); - import_lock_thread = me; - import_lock_level--; + PyThread_acquire_lock(IMPORT_LOCK(interp), WAIT_LOCK); + IMPORT_LOCK_THREAD(interp) = me; + IMPORT_LOCK_LEVEL(interp)--; } else { - import_lock_thread = PYTHREAD_INVALID_THREAD_ID; - import_lock_level = 0; + IMPORT_LOCK_THREAD(interp) = PYTHREAD_INVALID_THREAD_ID; + IMPORT_LOCK_LEVEL(interp) = 0; } return _PyStatus_OK(); } @@ -2506,8 +2511,8 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) PyObject *mod = NULL; PyInterpreterState *interp = tstate->interp; int import_time = _PyInterpreterState_GetConfig(interp)->import_time; -#define import_level FIND_AND_LOAD.import_level -#define accumulated FIND_AND_LOAD.accumulated +#define import_level FIND_AND_LOAD(interp).import_level +#define accumulated FIND_AND_LOAD(interp).accumulated _PyTime_t t1 = 0, accumulated_copy = accumulated; @@ -2528,7 +2533,7 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) * _PyDict_GetItemIdWithError(). */ if (import_time) { -#define header FIND_AND_LOAD.header +#define header FIND_AND_LOAD(interp).header if (header) { fputs("import time: self [us] | cumulative | imported package\n", stderr); @@ -2867,10 +2872,6 @@ _PyImport_Fini(void) { /* Destroy the database used by _PyImport_{Fixup,Find}Extension */ _extensions_cache_clear_all(); - if (import_lock != NULL) { - PyThread_free_lock(import_lock); - import_lock = NULL; - } /* Use the same memory allocator as _PyImport_Init(). */ PyMemAllocatorEx old_alloc; @@ -2959,6 +2960,11 @@ _PyImport_FiniCore(PyInterpreterState *interp) PyErr_WriteUnraisable(NULL); } + if (IMPORT_LOCK(interp) != NULL) { + PyThread_free_lock(IMPORT_LOCK(interp)); + IMPORT_LOCK(interp) = NULL; + } + _PyImport_ClearCore(interp); } @@ -3090,7 +3096,9 @@ static PyObject * _imp_lock_held_impl(PyObject *module) /*[clinic end generated code: output=8b89384b5e1963fc input=9b088f9b217d9bdf]*/ { - return PyBool_FromLong(import_lock_thread != PYTHREAD_INVALID_THREAD_ID); + PyInterpreterState *interp = _PyInterpreterState_GET(); + return PyBool_FromLong( + IMPORT_LOCK_THREAD(interp) != PYTHREAD_INVALID_THREAD_ID); } /*[clinic input] @@ -3106,7 +3114,8 @@ static PyObject * _imp_acquire_lock_impl(PyObject *module) /*[clinic end generated code: output=1aff58cb0ee1b026 input=4a2d4381866d5fdc]*/ { - _PyImport_AcquireLock(); + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyImport_AcquireLock(interp); Py_RETURN_NONE; } @@ -3122,7 +3131,8 @@ static PyObject * _imp_release_lock_impl(PyObject *module) /*[clinic end generated code: output=7faab6d0be178b0a input=934fb11516dd778b]*/ { - if (_PyImport_ReleaseLock() < 0) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (_PyImport_ReleaseLock(interp) < 0) { PyErr_SetString(PyExc_RuntimeError, "not holding the import lock"); return NULL; From ca066bdbed85094a9c4d9930823ce3587807db48 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 9 Mar 2023 09:50:33 -0700 Subject: [PATCH 30/38] gh-81057: Vendor a Subset of distutils for the c-analyzer Tool (gh-102505) distutils was removed in November. However, the c-analyzer relies on it. To solve that here, we vendor the parts the tool needs so it can be run against 3.12+. (Also see gh-92584.) Note that we may end up removing this code later in favor of a solution in common with the peg_generator tool (which also relies on distutils). At the least, the copy here makes sure the c-analyzer tool works on 3.12+ in the meantime. --- Tools/c-analyzer/distutils/README | 2 + Tools/c-analyzer/distutils/__init__.py | 0 Tools/c-analyzer/distutils/_msvccompiler.py | 203 ++++++++ Tools/c-analyzer/distutils/bcppcompiler.py | 109 ++++ Tools/c-analyzer/distutils/ccompiler.py | 470 ++++++++++++++++++ Tools/c-analyzer/distutils/cygwinccompiler.py | 286 +++++++++++ Tools/c-analyzer/distutils/debug.py | 5 + Tools/c-analyzer/distutils/dep_util.py | 29 ++ Tools/c-analyzer/distutils/errors.py | 48 ++ Tools/c-analyzer/distutils/log.py | 63 +++ Tools/c-analyzer/distutils/msvc9compiler.py | 438 ++++++++++++++++ Tools/c-analyzer/distutils/msvccompiler.py | 327 ++++++++++++ Tools/c-analyzer/distutils/spawn.py | 48 ++ Tools/c-analyzer/distutils/unixccompiler.py | 102 ++++ Tools/c-analyzer/distutils/util.py | 171 +++++++ 15 files changed, 2301 insertions(+) create mode 100644 Tools/c-analyzer/distutils/README create mode 100644 Tools/c-analyzer/distutils/__init__.py create mode 100644 Tools/c-analyzer/distutils/_msvccompiler.py create mode 100644 Tools/c-analyzer/distutils/bcppcompiler.py create mode 100644 Tools/c-analyzer/distutils/ccompiler.py create mode 100644 Tools/c-analyzer/distutils/cygwinccompiler.py create mode 100644 Tools/c-analyzer/distutils/debug.py create mode 100644 Tools/c-analyzer/distutils/dep_util.py create mode 100644 Tools/c-analyzer/distutils/errors.py create mode 100644 Tools/c-analyzer/distutils/log.py create mode 100644 Tools/c-analyzer/distutils/msvc9compiler.py create mode 100644 Tools/c-analyzer/distutils/msvccompiler.py create mode 100644 Tools/c-analyzer/distutils/spawn.py create mode 100644 Tools/c-analyzer/distutils/unixccompiler.py create mode 100644 Tools/c-analyzer/distutils/util.py diff --git a/Tools/c-analyzer/distutils/README b/Tools/c-analyzer/distutils/README new file mode 100644 index 00000000000000..b260b8e06fac06 --- /dev/null +++ b/Tools/c-analyzer/distutils/README @@ -0,0 +1,2 @@ +This is a partial copy of distutils as it was removed in 0faa0ba240e. +It only includes the parts needed by the C parser. diff --git a/Tools/c-analyzer/distutils/__init__.py b/Tools/c-analyzer/distutils/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/Tools/c-analyzer/distutils/_msvccompiler.py b/Tools/c-analyzer/distutils/_msvccompiler.py new file mode 100644 index 00000000000000..1e67870d13323d --- /dev/null +++ b/Tools/c-analyzer/distutils/_msvccompiler.py @@ -0,0 +1,203 @@ +"""distutils._msvccompiler + +Contains MSVCCompiler, an implementation of the abstract CCompiler class +for Microsoft Visual Studio 2015. + +The module is compatible with VS 2015 and later. You can find legacy support +for older versions in distutils.msvc9compiler and distutils.msvccompiler. +""" + +# Written by Perry Stoll +# hacked by Robin Becker and Thomas Heller to do a better job of +# finding DevStudio (through the registry) +# ported to VS 2005 and VS 2008 by Christian Heimes +# ported to VS 2015 by Steve Dower + +import os +import subprocess +import winreg + +from distutils.errors import DistutilsPlatformError +from distutils.ccompiler import CCompiler +from distutils import log + +from itertools import count + +def _find_vc2015(): + try: + key = winreg.OpenKeyEx( + winreg.HKEY_LOCAL_MACHINE, + r"Software\Microsoft\VisualStudio\SxS\VC7", + access=winreg.KEY_READ | winreg.KEY_WOW64_32KEY + ) + except OSError: + log.debug("Visual C++ is not registered") + return None, None + + best_version = 0 + best_dir = None + with key: + for i in count(): + try: + v, vc_dir, vt = winreg.EnumValue(key, i) + except OSError: + break + if v and vt == winreg.REG_SZ and os.path.isdir(vc_dir): + try: + version = int(float(v)) + except (ValueError, TypeError): + continue + if version >= 14 and version > best_version: + best_version, best_dir = version, vc_dir + return best_version, best_dir + +def _find_vc2017(): + """Returns "15, path" based on the result of invoking vswhere.exe + If no install is found, returns "None, None" + + The version is returned to avoid unnecessarily changing the function + result. It may be ignored when the path is not None. + + If vswhere.exe is not available, by definition, VS 2017 is not + installed. + """ + root = os.environ.get("ProgramFiles(x86)") or os.environ.get("ProgramFiles") + if not root: + return None, None + + try: + path = subprocess.check_output([ + os.path.join(root, "Microsoft Visual Studio", "Installer", "vswhere.exe"), + "-latest", + "-prerelease", + "-requires", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", + "-property", "installationPath", + "-products", "*", + ], encoding="mbcs", errors="strict").strip() + except (subprocess.CalledProcessError, OSError, UnicodeDecodeError): + return None, None + + path = os.path.join(path, "VC", "Auxiliary", "Build") + if os.path.isdir(path): + return 15, path + + return None, None + +PLAT_SPEC_TO_RUNTIME = { + 'x86' : 'x86', + 'x86_amd64' : 'x64', + 'x86_arm' : 'arm', + 'x86_arm64' : 'arm64' +} + +def _find_vcvarsall(plat_spec): + # bpo-38597: Removed vcruntime return value + _, best_dir = _find_vc2017() + + if not best_dir: + best_version, best_dir = _find_vc2015() + + if not best_dir: + log.debug("No suitable Visual C++ version found") + return None, None + + vcvarsall = os.path.join(best_dir, "vcvarsall.bat") + if not os.path.isfile(vcvarsall): + log.debug("%s cannot be found", vcvarsall) + return None, None + + return vcvarsall, None + +def _get_vc_env(plat_spec): + if os.getenv("DISTUTILS_USE_SDK"): + return { + key.lower(): value + for key, value in os.environ.items() + } + + vcvarsall, _ = _find_vcvarsall(plat_spec) + if not vcvarsall: + raise DistutilsPlatformError("Unable to find vcvarsall.bat") + + try: + out = subprocess.check_output( + 'cmd /u /c "{}" {} && set'.format(vcvarsall, plat_spec), + stderr=subprocess.STDOUT, + ).decode('utf-16le', errors='replace') + except subprocess.CalledProcessError as exc: + log.error(exc.output) + raise DistutilsPlatformError("Error executing {}" + .format(exc.cmd)) + + env = { + key.lower(): value + for key, _, value in + (line.partition('=') for line in out.splitlines()) + if key and value + } + + return env + +def _find_exe(exe, paths=None): + """Return path to an MSVC executable program. + + Tries to find the program in several places: first, one of the + MSVC program search paths from the registry; next, the directories + in the PATH environment variable. If any of those work, return an + absolute path that is known to exist. If none of them work, just + return the original program name, 'exe'. + """ + if not paths: + paths = os.getenv('path').split(os.pathsep) + for p in paths: + fn = os.path.join(os.path.abspath(p), exe) + if os.path.isfile(fn): + return fn + return exe + +# A map keyed by get_platform() return values to values accepted by +# 'vcvarsall.bat'. Always cross-compile from x86 to work with the +# lighter-weight MSVC installs that do not include native 64-bit tools. +PLAT_TO_VCVARS = { + 'win32' : 'x86', + 'win-amd64' : 'x86_amd64', + 'win-arm32' : 'x86_arm', + 'win-arm64' : 'x86_arm64' +} + +class MSVCCompiler(CCompiler) : + """Concrete class that implements an interface to Microsoft Visual C++, + as defined by the CCompiler abstract class.""" + + compiler_type = 'msvc' + + # Just set this so CCompiler's constructor doesn't barf. We currently + # don't use the 'set_executables()' bureaucracy provided by CCompiler, + # as it really isn't necessary for this sort of single-compiler class. + # Would be nice to have a consistent interface with UnixCCompiler, + # though, so it's worth thinking about. + executables = {} + + # Private class data (need to distinguish C from C++ source for compiler) + _c_extensions = ['.c'] + _cpp_extensions = ['.cc', '.cpp', '.cxx'] + _rc_extensions = ['.rc'] + _mc_extensions = ['.mc'] + + # Needed for the filename generation methods provided by the + # base class, CCompiler. + src_extensions = (_c_extensions + _cpp_extensions + + _rc_extensions + _mc_extensions) + res_extension = '.res' + obj_extension = '.obj' + static_lib_extension = '.lib' + shared_lib_extension = '.dll' + static_lib_format = shared_lib_format = '%s%s' + exe_extension = '.exe' + + + def __init__(self, verbose=0, dry_run=0, force=0): + CCompiler.__init__ (self, verbose, dry_run, force) + # target platform (.plat_name is consistent with 'bdist') + self.plat_name = None + self.initialized = False diff --git a/Tools/c-analyzer/distutils/bcppcompiler.py b/Tools/c-analyzer/distutils/bcppcompiler.py new file mode 100644 index 00000000000000..4575b665c10c0e --- /dev/null +++ b/Tools/c-analyzer/distutils/bcppcompiler.py @@ -0,0 +1,109 @@ +"""distutils.bcppcompiler + +Contains BorlandCCompiler, an implementation of the abstract CCompiler class +for the Borland C++ compiler. +""" + +# This implementation by Lyle Johnson, based on the original msvccompiler.py +# module and using the directions originally published by Gordon Williams. + +# XXX looks like there's a LOT of overlap between these two classes: +# someone should sit down and factor out the common code as +# WindowsCCompiler! --GPW + + +import os +from distutils.errors import DistutilsExecError, CompileError +from distutils.ccompiler import \ + CCompiler, gen_preprocess_options +from distutils.dep_util import newer + +class BCPPCompiler(CCompiler) : + """Concrete class that implements an interface to the Borland C/C++ + compiler, as defined by the CCompiler abstract class. + """ + + compiler_type = 'bcpp' + + # Just set this so CCompiler's constructor doesn't barf. We currently + # don't use the 'set_executables()' bureaucracy provided by CCompiler, + # as it really isn't necessary for this sort of single-compiler class. + # Would be nice to have a consistent interface with UnixCCompiler, + # though, so it's worth thinking about. + executables = {} + + # Private class data (need to distinguish C from C++ source for compiler) + _c_extensions = ['.c'] + _cpp_extensions = ['.cc', '.cpp', '.cxx'] + + # Needed for the filename generation methods provided by the + # base class, CCompiler. + src_extensions = _c_extensions + _cpp_extensions + obj_extension = '.obj' + static_lib_extension = '.lib' + shared_lib_extension = '.dll' + static_lib_format = shared_lib_format = '%s%s' + exe_extension = '.exe' + + + def __init__ (self, + verbose=0, + dry_run=0, + force=0): + + CCompiler.__init__ (self, verbose, dry_run, force) + + # These executables are assumed to all be in the path. + # Borland doesn't seem to use any special registry settings to + # indicate their installation locations. + + self.cc = "bcc32.exe" + self.linker = "ilink32.exe" + self.lib = "tlib.exe" + + self.preprocess_options = None + self.compile_options = ['/tWM', '/O2', '/q', '/g0'] + self.compile_options_debug = ['/tWM', '/Od', '/q', '/g0'] + + self.ldflags_shared = ['/Tpd', '/Gn', '/q', '/x'] + self.ldflags_shared_debug = ['/Tpd', '/Gn', '/q', '/x'] + self.ldflags_static = [] + self.ldflags_exe = ['/Gn', '/q', '/x'] + self.ldflags_exe_debug = ['/Gn', '/q', '/x','/r'] + + + # -- Worker methods ------------------------------------------------ + + def preprocess (self, + source, + output_file=None, + macros=None, + include_dirs=None, + extra_preargs=None, + extra_postargs=None): + + (_, macros, include_dirs) = \ + self._fix_compile_args(None, macros, include_dirs) + pp_opts = gen_preprocess_options(macros, include_dirs) + pp_args = ['cpp32.exe'] + pp_opts + if output_file is not None: + pp_args.append('-o' + output_file) + if extra_preargs: + pp_args[:0] = extra_preargs + if extra_postargs: + pp_args.extend(extra_postargs) + pp_args.append(source) + + # We need to preprocess: either we're being forced to, or the + # source file is newer than the target (or the target doesn't + # exist). + if self.force or output_file is None or newer(source, output_file): + if output_file: + self.mkpath(os.path.dirname(output_file)) + try: + self.spawn(pp_args) + except DistutilsExecError as msg: + print(msg) + raise CompileError(msg) + + # preprocess() diff --git a/Tools/c-analyzer/distutils/ccompiler.py b/Tools/c-analyzer/distutils/ccompiler.py new file mode 100644 index 00000000000000..13e43103b94f5e --- /dev/null +++ b/Tools/c-analyzer/distutils/ccompiler.py @@ -0,0 +1,470 @@ +"""distutils.ccompiler + +Contains CCompiler, an abstract base class that defines the interface +for the Distutils compiler abstraction model.""" + +import sys, os, re +from distutils.errors import ( + DistutilsModuleError, DistutilsPlatformError, +) +from distutils.util import split_quoted + +class CCompiler: + """Abstract base class to define the interface that must be implemented + by real compiler classes. Also has some utility methods used by + several compiler classes. + + The basic idea behind a compiler abstraction class is that each + instance can be used for all the compile/link steps in building a + single project. Thus, attributes common to all of those compile and + link steps -- include directories, macros to define, libraries to link + against, etc. -- are attributes of the compiler instance. To allow for + variability in how individual files are treated, most of those + attributes may be varied on a per-compilation or per-link basis. + """ + + # 'compiler_type' is a class attribute that identifies this class. It + # keeps code that wants to know what kind of compiler it's dealing with + # from having to import all possible compiler classes just to do an + # 'isinstance'. In concrete CCompiler subclasses, 'compiler_type' + # should really, really be one of the keys of the 'compiler_class' + # dictionary (see below -- used by the 'new_compiler()' factory + # function) -- authors of new compiler interface classes are + # responsible for updating 'compiler_class'! + compiler_type = None + + # XXX things not handled by this compiler abstraction model: + # * client can't provide additional options for a compiler, + # e.g. warning, optimization, debugging flags. Perhaps this + # should be the domain of concrete compiler abstraction classes + # (UnixCCompiler, MSVCCompiler, etc.) -- or perhaps the base + # class should have methods for the common ones. + # * can't completely override the include or library searchg + # path, ie. no "cc -I -Idir1 -Idir2" or "cc -L -Ldir1 -Ldir2". + # I'm not sure how widely supported this is even by Unix + # compilers, much less on other platforms. And I'm even less + # sure how useful it is; maybe for cross-compiling, but + # support for that is a ways off. (And anyways, cross + # compilers probably have a dedicated binary with the + # right paths compiled in. I hope.) + # * can't do really freaky things with the library list/library + # dirs, e.g. "-Ldir1 -lfoo -Ldir2 -lfoo" to link against + # different versions of libfoo.a in different locations. I + # think this is useless without the ability to null out the + # library search path anyways. + + + # Subclasses that rely on the standard filename generation methods + # implemented below should override these; see the comment near + # those methods ('object_filenames()' et. al.) for details: + src_extensions = None # list of strings + obj_extension = None # string + static_lib_extension = None + shared_lib_extension = None # string + static_lib_format = None # format string + shared_lib_format = None # prob. same as static_lib_format + exe_extension = None # string + + # Default language settings. language_map is used to detect a source + # file or Extension target language, checking source filenames. + # language_order is used to detect the language precedence, when deciding + # what language to use when mixing source types. For example, if some + # extension has two files with ".c" extension, and one with ".cpp", it + # is still linked as c++. + language_map = {".c" : "c", + ".cc" : "c++", + ".cpp" : "c++", + ".cxx" : "c++", + ".m" : "objc", + } + language_order = ["c++", "objc", "c"] + + def __init__(self, verbose=0, dry_run=0, force=0): + self.dry_run = dry_run + self.force = force + self.verbose = verbose + + # 'output_dir': a common output directory for object, library, + # shared object, and shared library files + self.output_dir = None + + # 'macros': a list of macro definitions (or undefinitions). A + # macro definition is a 2-tuple (name, value), where the value is + # either a string or None (no explicit value). A macro + # undefinition is a 1-tuple (name,). + self.macros = [] + + # 'include_dirs': a list of directories to search for include files + self.include_dirs = [] + + # 'libraries': a list of libraries to include in any link + # (library names, not filenames: eg. "foo" not "libfoo.a") + self.libraries = [] + + # 'library_dirs': a list of directories to search for libraries + self.library_dirs = [] + + # 'runtime_library_dirs': a list of directories to search for + # shared libraries/objects at runtime + self.runtime_library_dirs = [] + + # 'objects': a list of object files (or similar, such as explicitly + # named library files) to include on any link + self.objects = [] + + for key in self.executables.keys(): + self.set_executable(key, self.executables[key]) + + def set_executables(self, **kwargs): + """Define the executables (and options for them) that will be run + to perform the various stages of compilation. The exact set of + executables that may be specified here depends on the compiler + class (via the 'executables' class attribute), but most will have: + compiler the C/C++ compiler + linker_so linker used to create shared objects and libraries + linker_exe linker used to create binary executables + archiver static library creator + + On platforms with a command-line (Unix, DOS/Windows), each of these + is a string that will be split into executable name and (optional) + list of arguments. (Splitting the string is done similarly to how + Unix shells operate: words are delimited by spaces, but quotes and + backslashes can override this. See + 'distutils.util.split_quoted()'.) + """ + + # Note that some CCompiler implementation classes will define class + # attributes 'cpp', 'cc', etc. with hard-coded executable names; + # this is appropriate when a compiler class is for exactly one + # compiler/OS combination (eg. MSVCCompiler). Other compiler + # classes (UnixCCompiler, in particular) are driven by information + # discovered at run-time, since there are many different ways to do + # basically the same things with Unix C compilers. + + for key in kwargs: + if key not in self.executables: + raise ValueError("unknown executable '%s' for class %s" % + (key, self.__class__.__name__)) + self.set_executable(key, kwargs[key]) + + def set_executable(self, key, value): + if isinstance(value, str): + setattr(self, key, split_quoted(value)) + else: + setattr(self, key, value) + + def _find_macro(self, name): + i = 0 + for defn in self.macros: + if defn[0] == name: + return i + i += 1 + return None + + def _check_macro_definitions(self, definitions): + """Ensures that every element of 'definitions' is a valid macro + definition, ie. either (name,value) 2-tuple or a (name,) tuple. Do + nothing if all definitions are OK, raise TypeError otherwise. + """ + for defn in definitions: + if not (isinstance(defn, tuple) and + (len(defn) in (1, 2) and + (isinstance (defn[1], str) or defn[1] is None)) and + isinstance (defn[0], str)): + raise TypeError(("invalid macro definition '%s': " % defn) + \ + "must be tuple (string,), (string, string), or " + \ + "(string, None)") + + + # -- Bookkeeping methods ------------------------------------------- + + def define_macro(self, name, value=None): + """Define a preprocessor macro for all compilations driven by this + compiler object. The optional parameter 'value' should be a + string; if it is not supplied, then the macro will be defined + without an explicit value and the exact outcome depends on the + compiler used (XXX true? does ANSI say anything about this?) + """ + # Delete from the list of macro definitions/undefinitions if + # already there (so that this one will take precedence). + i = self._find_macro (name) + if i is not None: + del self.macros[i] + + self.macros.append((name, value)) + + def undefine_macro(self, name): + """Undefine a preprocessor macro for all compilations driven by + this compiler object. If the same macro is defined by + 'define_macro()' and undefined by 'undefine_macro()' the last call + takes precedence (including multiple redefinitions or + undefinitions). If the macro is redefined/undefined on a + per-compilation basis (ie. in the call to 'compile()'), then that + takes precedence. + """ + # Delete from the list of macro definitions/undefinitions if + # already there (so that this one will take precedence). + i = self._find_macro (name) + if i is not None: + del self.macros[i] + + undefn = (name,) + self.macros.append(undefn) + + def add_include_dir(self, dir): + """Add 'dir' to the list of directories that will be searched for + header files. The compiler is instructed to search directories in + the order in which they are supplied by successive calls to + 'add_include_dir()'. + """ + self.include_dirs.append(dir) + + def set_include_dirs(self, dirs): + """Set the list of directories that will be searched to 'dirs' (a + list of strings). Overrides any preceding calls to + 'add_include_dir()'; subsequence calls to 'add_include_dir()' add + to the list passed to 'set_include_dirs()'. This does not affect + any list of standard include directories that the compiler may + search by default. + """ + self.include_dirs = dirs[:] + + + # -- Private utility methods -------------------------------------- + # (here for the convenience of subclasses) + + # Helper method to prep compiler in subclass compile() methods + + def _fix_compile_args(self, output_dir, macros, include_dirs): + """Typecheck and fix-up some of the arguments to the 'compile()' + method, and return fixed-up values. Specifically: if 'output_dir' + is None, replaces it with 'self.output_dir'; ensures that 'macros' + is a list, and augments it with 'self.macros'; ensures that + 'include_dirs' is a list, and augments it with 'self.include_dirs'. + Guarantees that the returned values are of the correct type, + i.e. for 'output_dir' either string or None, and for 'macros' and + 'include_dirs' either list or None. + """ + if output_dir is None: + output_dir = self.output_dir + elif not isinstance(output_dir, str): + raise TypeError("'output_dir' must be a string or None") + + if macros is None: + macros = self.macros + elif isinstance(macros, list): + macros = macros + (self.macros or []) + else: + raise TypeError("'macros' (if supplied) must be a list of tuples") + + if include_dirs is None: + include_dirs = self.include_dirs + elif isinstance(include_dirs, (list, tuple)): + include_dirs = list(include_dirs) + (self.include_dirs or []) + else: + raise TypeError( + "'include_dirs' (if supplied) must be a list of strings") + + return output_dir, macros, include_dirs + + + # -- Worker methods ------------------------------------------------ + # (must be implemented by subclasses) + + def preprocess(self, source, output_file=None, macros=None, + include_dirs=None, extra_preargs=None, extra_postargs=None): + """Preprocess a single C/C++ source file, named in 'source'. + Output will be written to file named 'output_file', or stdout if + 'output_file' not supplied. 'macros' is a list of macro + definitions as for 'compile()', which will augment the macros set + with 'define_macro()' and 'undefine_macro()'. 'include_dirs' is a + list of directory names that will be added to the default list. + + Raises PreprocessError on failure. + """ + pass + + + # -- Miscellaneous methods ----------------------------------------- + # These are all used by the 'gen_lib_options() function; there is + # no appropriate default implementation so subclasses should + # implement all of these. + +# def library_dir_option(self, dir): +# """Return the compiler option to add 'dir' to the list of +# directories searched for libraries. +# """ +# raise NotImplementedError +# +# def runtime_library_dir_option(self, dir): +# """Return the compiler option to add 'dir' to the list of +# directories searched for runtime libraries. +# """ +# raise NotImplementedError +# +# def library_option(self, lib): +# """Return the compiler option to add 'lib' to the list of libraries +# linked into the shared library or executable. +# """ +# raise NotImplementedError +# +# def find_library_file (self, dirs, lib, debug=0): +# """Search the specified list of directories for a static or shared +# library file 'lib' and return the full path to that file. If +# 'debug' true, look for a debugging version (if that makes sense on +# the current platform). Return None if 'lib' wasn't found in any of +# the specified directories. +# """ +# raise NotImplementedError + + + # -- Utility methods ----------------------------------------------- + + def spawn(self, cmd): + raise NotImplementedError + + +# Map a sys.platform/os.name ('posix', 'nt') to the default compiler +# type for that platform. Keys are interpreted as re match +# patterns. Order is important; platform mappings are preferred over +# OS names. +_default_compilers = ( + + # Platform string mappings + + # on a cygwin built python we can use gcc like an ordinary UNIXish + # compiler + ('cygwin.*', 'unix'), + + # OS name mappings + ('posix', 'unix'), + ('nt', 'msvc'), + + ) + +def get_default_compiler(osname=None, platform=None): + """Determine the default compiler to use for the given platform. + + osname should be one of the standard Python OS names (i.e. the + ones returned by os.name) and platform the common value + returned by sys.platform for the platform in question. + + The default values are os.name and sys.platform in case the + parameters are not given. + """ + if osname is None: + osname = os.name + if platform is None: + platform = sys.platform + for pattern, compiler in _default_compilers: + if re.match(pattern, platform) is not None or \ + re.match(pattern, osname) is not None: + return compiler + # Default to Unix compiler + return 'unix' + +# Map compiler types to (module_name, class_name) pairs -- ie. where to +# find the code that implements an interface to this compiler. (The module +# is assumed to be in the 'distutils' package.) +compiler_class = { 'unix': ('unixccompiler', 'UnixCCompiler', + "standard UNIX-style compiler"), + 'msvc': ('_msvccompiler', 'MSVCCompiler', + "Microsoft Visual C++"), + 'cygwin': ('cygwinccompiler', 'CygwinCCompiler', + "Cygwin port of GNU C Compiler for Win32"), + 'mingw32': ('cygwinccompiler', 'Mingw32CCompiler', + "Mingw32 port of GNU C Compiler for Win32"), + 'bcpp': ('bcppcompiler', 'BCPPCompiler', + "Borland C++ Compiler"), + } + + +def new_compiler(plat=None, compiler=None, verbose=0, dry_run=0, force=0): + """Generate an instance of some CCompiler subclass for the supplied + platform/compiler combination. 'plat' defaults to 'os.name' + (eg. 'posix', 'nt'), and 'compiler' defaults to the default compiler + for that platform. Currently only 'posix' and 'nt' are supported, and + the default compilers are "traditional Unix interface" (UnixCCompiler + class) and Visual C++ (MSVCCompiler class). Note that it's perfectly + possible to ask for a Unix compiler object under Windows, and a + Microsoft compiler object under Unix -- if you supply a value for + 'compiler', 'plat' is ignored. + """ + if plat is None: + plat = os.name + + try: + if compiler is None: + compiler = get_default_compiler(plat) + + (module_name, class_name, long_description) = compiler_class[compiler] + except KeyError: + msg = "don't know how to compile C/C++ code on platform '%s'" % plat + if compiler is not None: + msg = msg + " with '%s' compiler" % compiler + raise DistutilsPlatformError(msg) + + try: + module_name = "distutils." + module_name + __import__ (module_name) + module = sys.modules[module_name] + klass = vars(module)[class_name] + except ImportError: + raise + raise DistutilsModuleError( + "can't compile C/C++ code: unable to load module '%s'" % \ + module_name) + except KeyError: + raise DistutilsModuleError( + "can't compile C/C++ code: unable to find class '%s' " + "in module '%s'" % (class_name, module_name)) + + # XXX The None is necessary to preserve backwards compatibility + # with classes that expect verbose to be the first positional + # argument. + return klass(None, dry_run, force) + + +def gen_preprocess_options(macros, include_dirs): + """Generate C pre-processor options (-D, -U, -I) as used by at least + two types of compilers: the typical Unix compiler and Visual C++. + 'macros' is the usual thing, a list of 1- or 2-tuples, where (name,) + means undefine (-U) macro 'name', and (name,value) means define (-D) + macro 'name' to 'value'. 'include_dirs' is just a list of directory + names to be added to the header file search path (-I). Returns a list + of command-line options suitable for either Unix compilers or Visual + C++. + """ + # XXX it would be nice (mainly aesthetic, and so we don't generate + # stupid-looking command lines) to go over 'macros' and eliminate + # redundant definitions/undefinitions (ie. ensure that only the + # latest mention of a particular macro winds up on the command + # line). I don't think it's essential, though, since most (all?) + # Unix C compilers only pay attention to the latest -D or -U + # mention of a macro on their command line. Similar situation for + # 'include_dirs'. I'm punting on both for now. Anyways, weeding out + # redundancies like this should probably be the province of + # CCompiler, since the data structures used are inherited from it + # and therefore common to all CCompiler classes. + pp_opts = [] + for macro in macros: + if not (isinstance(macro, tuple) and 1 <= len(macro) <= 2): + raise TypeError( + "bad macro definition '%s': " + "each element of 'macros' list must be a 1- or 2-tuple" + % macro) + + if len(macro) == 1: # undefine this macro + pp_opts.append("-U%s" % macro[0]) + elif len(macro) == 2: + if macro[1] is None: # define with no explicit value + pp_opts.append("-D%s" % macro[0]) + else: + # XXX *don't* need to be clever about quoting the + # macro value here, because we're going to avoid the + # shell at all costs when we spawn the command! + pp_opts.append("-D%s=%s" % macro) + + for dir in include_dirs: + pp_opts.append("-I%s" % dir) + return pp_opts diff --git a/Tools/c-analyzer/distutils/cygwinccompiler.py b/Tools/c-analyzer/distutils/cygwinccompiler.py new file mode 100644 index 00000000000000..a3505f31f1f0a2 --- /dev/null +++ b/Tools/c-analyzer/distutils/cygwinccompiler.py @@ -0,0 +1,286 @@ +"""distutils.cygwinccompiler + +Provides the CygwinCCompiler class, a subclass of UnixCCompiler that +handles the Cygwin port of the GNU C compiler to Windows. It also contains +the Mingw32CCompiler class which handles the mingw32 port of GCC (same as +cygwin in no-cygwin mode). +""" + +# problems: +# +# * if you use a msvc compiled python version (1.5.2) +# 1. you have to insert a __GNUC__ section in its config.h +# 2. you have to generate an import library for its dll +# - create a def-file for python??.dll +# - create an import library using +# dlltool --dllname python15.dll --def python15.def \ +# --output-lib libpython15.a +# +# see also http://starship.python.net/crew/kernr/mingw32/Notes.html +# +# * We put export_symbols in a def-file, and don't use +# --export-all-symbols because it doesn't worked reliable in some +# tested configurations. And because other windows compilers also +# need their symbols specified this no serious problem. +# +# tested configurations: +# +# * cygwin gcc 2.91.57/ld 2.9.4/dllwrap 0.2.4 works +# (after patching python's config.h and for C++ some other include files) +# see also http://starship.python.net/crew/kernr/mingw32/Notes.html +# * mingw32 gcc 2.95.2/ld 2.9.4/dllwrap 0.2.4 works +# (ld doesn't support -shared, so we use dllwrap) +# * cygwin gcc 2.95.2/ld 2.10.90/dllwrap 2.10.90 works now +# - its dllwrap doesn't work, there is a bug in binutils 2.10.90 +# see also http://sources.redhat.com/ml/cygwin/2000-06/msg01274.html +# - using gcc -mdll instead dllwrap doesn't work without -static because +# it tries to link against dlls instead their import libraries. (If +# it finds the dll first.) +# By specifying -static we force ld to link against the import libraries, +# this is windows standard and there are normally not the necessary symbols +# in the dlls. +# *** only the version of June 2000 shows these problems +# * cygwin gcc 3.2/ld 2.13.90 works +# (ld supports -shared) +# * mingw gcc 3.2/ld 2.13 works +# (ld supports -shared) + +import os +import sys +from subprocess import Popen, PIPE, check_output +import re + +from distutils.unixccompiler import UnixCCompiler +from distutils.errors import CCompilerError +from distutils.version import LooseVersion +from distutils.spawn import find_executable + +def get_msvcr(): + """Include the appropriate MSVC runtime library if Python was built + with MSVC 7.0 or later. + """ + msc_pos = sys.version.find('MSC v.') + if msc_pos != -1: + msc_ver = sys.version[msc_pos+6:msc_pos+10] + if msc_ver == '1300': + # MSVC 7.0 + return ['msvcr70'] + elif msc_ver == '1310': + # MSVC 7.1 + return ['msvcr71'] + elif msc_ver == '1400': + # VS2005 / MSVC 8.0 + return ['msvcr80'] + elif msc_ver == '1500': + # VS2008 / MSVC 9.0 + return ['msvcr90'] + elif msc_ver == '1600': + # VS2010 / MSVC 10.0 + return ['msvcr100'] + else: + raise ValueError("Unknown MS Compiler version %s " % msc_ver) + + +class CygwinCCompiler(UnixCCompiler): + """ Handles the Cygwin port of the GNU C compiler to Windows. + """ + compiler_type = 'cygwin' + obj_extension = ".o" + static_lib_extension = ".a" + shared_lib_extension = ".dll" + static_lib_format = "lib%s%s" + shared_lib_format = "%s%s" + exe_extension = ".exe" + + def __init__(self, verbose=0, dry_run=0, force=0): + + UnixCCompiler.__init__(self, verbose, dry_run, force) + + status, details = check_config_h() + self.debug_print("Python's GCC status: %s (details: %s)" % + (status, details)) + if status is not CONFIG_H_OK: + self.warn( + "Python's pyconfig.h doesn't seem to support your compiler. " + "Reason: %s. " + "Compiling may fail because of undefined preprocessor macros." + % details) + + self.gcc_version, self.ld_version, self.dllwrap_version = \ + get_versions() + self.debug_print(self.compiler_type + ": gcc %s, ld %s, dllwrap %s\n" % + (self.gcc_version, + self.ld_version, + self.dllwrap_version) ) + + # ld_version >= "2.10.90" and < "2.13" should also be able to use + # gcc -mdll instead of dllwrap + # Older dllwraps had own version numbers, newer ones use the + # same as the rest of binutils ( also ld ) + # dllwrap 2.10.90 is buggy + if self.ld_version >= "2.10.90": + self.linker_dll = "gcc" + else: + self.linker_dll = "dllwrap" + + # ld_version >= "2.13" support -shared so use it instead of + # -mdll -static + if self.ld_version >= "2.13": + shared_option = "-shared" + else: + shared_option = "-mdll -static" + + # Hard-code GCC because that's what this is all about. + # XXX optimization, warnings etc. should be customizable. + self.set_executables(compiler='gcc -mcygwin -O -Wall', + compiler_so='gcc -mcygwin -mdll -O -Wall', + compiler_cxx='g++ -mcygwin -O -Wall', + linker_exe='gcc -mcygwin', + linker_so=('%s -mcygwin %s' % + (self.linker_dll, shared_option))) + + # cygwin and mingw32 need different sets of libraries + if self.gcc_version == "2.91.57": + # cygwin shouldn't need msvcrt, but without the dlls will crash + # (gcc version 2.91.57) -- perhaps something about initialization + self.dll_libraries=["msvcrt"] + self.warn( + "Consider upgrading to a newer version of gcc") + else: + # Include the appropriate MSVC runtime library if Python was built + # with MSVC 7.0 or later. + self.dll_libraries = get_msvcr() + + +# the same as cygwin plus some additional parameters +class Mingw32CCompiler(CygwinCCompiler): + """ Handles the Mingw32 port of the GNU C compiler to Windows. + """ + compiler_type = 'mingw32' + + def __init__(self, verbose=0, dry_run=0, force=0): + + CygwinCCompiler.__init__ (self, verbose, dry_run, force) + + # ld_version >= "2.13" support -shared so use it instead of + # -mdll -static + if self.ld_version >= "2.13": + shared_option = "-shared" + else: + shared_option = "-mdll -static" + + # A real mingw32 doesn't need to specify a different entry point, + # but cygwin 2.91.57 in no-cygwin-mode needs it. + if self.gcc_version <= "2.91.57": + entry_point = '--entry _DllMain@12' + else: + entry_point = '' + + if is_cygwingcc(): + raise CCompilerError( + 'Cygwin gcc cannot be used with --compiler=mingw32') + + self.set_executables(compiler='gcc -O -Wall', + compiler_so='gcc -mdll -O -Wall', + compiler_cxx='g++ -O -Wall', + linker_exe='gcc', + linker_so='%s %s %s' + % (self.linker_dll, shared_option, + entry_point)) + # Maybe we should also append -mthreads, but then the finished + # dlls need another dll (mingwm10.dll see Mingw32 docs) + # (-mthreads: Support thread-safe exception handling on `Mingw32') + + # no additional libraries needed + self.dll_libraries=[] + + # Include the appropriate MSVC runtime library if Python was built + # with MSVC 7.0 or later. + self.dll_libraries = get_msvcr() + +# Because these compilers aren't configured in Python's pyconfig.h file by +# default, we should at least warn the user if he is using an unmodified +# version. + +CONFIG_H_OK = "ok" +CONFIG_H_NOTOK = "not ok" +CONFIG_H_UNCERTAIN = "uncertain" + +def check_config_h(): + """Check if the current Python installation appears amenable to building + extensions with GCC. + + Returns a tuple (status, details), where 'status' is one of the following + constants: + + - CONFIG_H_OK: all is well, go ahead and compile + - CONFIG_H_NOTOK: doesn't look good + - CONFIG_H_UNCERTAIN: not sure -- unable to read pyconfig.h + + 'details' is a human-readable string explaining the situation. + + Note there are two ways to conclude "OK": either 'sys.version' contains + the string "GCC" (implying that this Python was built with GCC), or the + installed "pyconfig.h" contains the string "__GNUC__". + """ + + # XXX since this function also checks sys.version, it's not strictly a + # "pyconfig.h" check -- should probably be renamed... + + import sysconfig + + # if sys.version contains GCC then python was compiled with GCC, and the + # pyconfig.h file should be OK + if "GCC" in sys.version: + return CONFIG_H_OK, "sys.version mentions 'GCC'" + + # let's see if __GNUC__ is mentioned in python.h + fn = sysconfig.get_config_h_filename() + try: + config_h = open(fn) + try: + if "__GNUC__" in config_h.read(): + return CONFIG_H_OK, "'%s' mentions '__GNUC__'" % fn + else: + return CONFIG_H_NOTOK, "'%s' does not mention '__GNUC__'" % fn + finally: + config_h.close() + except OSError as exc: + return (CONFIG_H_UNCERTAIN, + "couldn't read '%s': %s" % (fn, exc.strerror)) + +RE_VERSION = re.compile(br'(\d+\.\d+(\.\d+)*)') + +def _find_exe_version(cmd): + """Find the version of an executable by running `cmd` in the shell. + + If the command is not found, or the output does not match + `RE_VERSION`, returns None. + """ + executable = cmd.split()[0] + if find_executable(executable) is None: + return None + out = Popen(cmd, shell=True, stdout=PIPE).stdout + try: + out_string = out.read() + finally: + out.close() + result = RE_VERSION.search(out_string) + if result is None: + return None + # LooseVersion works with strings + # so we need to decode our bytes + return LooseVersion(result.group(1).decode()) + +def get_versions(): + """ Try to find out the versions of gcc, ld and dllwrap. + + If not possible it returns None for it. + """ + commands = ['gcc -dumpversion', 'ld -v', 'dllwrap --version'] + return tuple([_find_exe_version(cmd) for cmd in commands]) + +def is_cygwingcc(): + '''Try to determine if the gcc that would be used is from cygwin.''' + out_string = check_output(['gcc', '-dumpmachine']) + return out_string.strip().endswith(b'cygwin') diff --git a/Tools/c-analyzer/distutils/debug.py b/Tools/c-analyzer/distutils/debug.py new file mode 100644 index 00000000000000..daf1660f0d8211 --- /dev/null +++ b/Tools/c-analyzer/distutils/debug.py @@ -0,0 +1,5 @@ +import os + +# If DISTUTILS_DEBUG is anything other than the empty string, we run in +# debug mode. +DEBUG = os.environ.get('DISTUTILS_DEBUG') diff --git a/Tools/c-analyzer/distutils/dep_util.py b/Tools/c-analyzer/distutils/dep_util.py new file mode 100644 index 00000000000000..318c830f2eab3e --- /dev/null +++ b/Tools/c-analyzer/distutils/dep_util.py @@ -0,0 +1,29 @@ +"""distutils.dep_util + +Utility functions for simple, timestamp-based dependency of files +and groups of files; also, function based entirely on such +timestamp dependency analysis.""" + +import os +from distutils.errors import DistutilsFileError + + +def newer (source, target): + """Return true if 'source' exists and is more recently modified than + 'target', or if 'source' exists and 'target' doesn't. Return false if + both exist and 'target' is the same age or younger than 'source'. + Raise DistutilsFileError if 'source' does not exist. + """ + if not os.path.exists(source): + raise DistutilsFileError("file '%s' does not exist" % + os.path.abspath(source)) + if not os.path.exists(target): + return 1 + + from stat import ST_MTIME + mtime1 = os.stat(source)[ST_MTIME] + mtime2 = os.stat(target)[ST_MTIME] + + return mtime1 > mtime2 + +# newer () diff --git a/Tools/c-analyzer/distutils/errors.py b/Tools/c-analyzer/distutils/errors.py new file mode 100644 index 00000000000000..10f8e316f675f0 --- /dev/null +++ b/Tools/c-analyzer/distutils/errors.py @@ -0,0 +1,48 @@ +"""distutils.errors + +Provides exceptions used by the Distutils modules. Note that Distutils +modules may raise standard exceptions; in particular, SystemExit is +usually raised for errors that are obviously the end-user's fault +(eg. bad command-line arguments). + +This module is safe to use in "from ... import *" mode; it only exports +symbols whose names start with "Distutils" and end with "Error".""" + +class DistutilsError (Exception): + """The root of all Distutils evil.""" + pass + +class DistutilsModuleError (DistutilsError): + """Unable to load an expected module, or to find an expected class + within some module (in particular, command modules and classes).""" + pass + +class DistutilsFileError (DistutilsError): + """Any problems in the filesystem: expected file not found, etc. + Typically this is for problems that we detect before OSError + could be raised.""" + pass + +class DistutilsPlatformError (DistutilsError): + """We don't know how to do something on the current platform (but + we do know how to do it on some platform) -- eg. trying to compile + C files on a platform not supported by a CCompiler subclass.""" + pass + +class DistutilsExecError (DistutilsError): + """Any problems executing an external program (such as the C + compiler, when compiling C files).""" + pass + +# Exception classes used by the CCompiler implementation classes +class CCompilerError (Exception): + """Some compile/link operation failed.""" + +class PreprocessError (CCompilerError): + """Failure to preprocess one or more C/C++ files.""" + +class CompileError (CCompilerError): + """Failure to compile one or more C/C++ source files.""" + +class UnknownFileError (CCompilerError): + """Attempt to process an unknown file type.""" diff --git a/Tools/c-analyzer/distutils/log.py b/Tools/c-analyzer/distutils/log.py new file mode 100644 index 00000000000000..26ecf22ae19bef --- /dev/null +++ b/Tools/c-analyzer/distutils/log.py @@ -0,0 +1,63 @@ +"""A simple log mechanism styled after PEP 282.""" + +# The class here is styled after PEP 282 so that it could later be +# replaced with a standard Python logging implementation. + +DEBUG = 1 +INFO = 2 +WARN = 3 +ERROR = 4 +FATAL = 5 + +import sys + +class Log: + + def __init__(self, threshold=WARN): + self.threshold = threshold + + def _log(self, level, msg, args): + if level not in (DEBUG, INFO, WARN, ERROR, FATAL): + raise ValueError('%s wrong log level' % str(level)) + + if level >= self.threshold: + if args: + msg = msg % args + if level in (WARN, ERROR, FATAL): + stream = sys.stderr + else: + stream = sys.stdout + try: + stream.write('%s\n' % msg) + except UnicodeEncodeError: + # emulate backslashreplace error handler + encoding = stream.encoding + msg = msg.encode(encoding, "backslashreplace").decode(encoding) + stream.write('%s\n' % msg) + stream.flush() + + def log(self, level, msg, *args): + self._log(level, msg, args) + + def debug(self, msg, *args): + self._log(DEBUG, msg, args) + + def info(self, msg, *args): + self._log(INFO, msg, args) + + def warn(self, msg, *args): + self._log(WARN, msg, args) + + def error(self, msg, *args): + self._log(ERROR, msg, args) + + def fatal(self, msg, *args): + self._log(FATAL, msg, args) + +_global_log = Log() +log = _global_log.log +debug = _global_log.debug +info = _global_log.info +warn = _global_log.warn +error = _global_log.error +fatal = _global_log.fatal diff --git a/Tools/c-analyzer/distutils/msvc9compiler.py b/Tools/c-analyzer/distutils/msvc9compiler.py new file mode 100644 index 00000000000000..38fff9b2d53120 --- /dev/null +++ b/Tools/c-analyzer/distutils/msvc9compiler.py @@ -0,0 +1,438 @@ +"""distutils.msvc9compiler + +Contains MSVCCompiler, an implementation of the abstract CCompiler class +for the Microsoft Visual Studio 2008. + +The module is compatible with VS 2005 and VS 2008. You can find legacy support +for older versions of VS in distutils.msvccompiler. +""" + +# Written by Perry Stoll +# hacked by Robin Becker and Thomas Heller to do a better job of +# finding DevStudio (through the registry) +# ported to VS2005 and VS 2008 by Christian Heimes + +import os +import subprocess +import sys +import re + +from distutils.errors import DistutilsPlatformError +from distutils.ccompiler import CCompiler +from distutils import log + +import winreg + +RegOpenKeyEx = winreg.OpenKeyEx +RegEnumKey = winreg.EnumKey +RegEnumValue = winreg.EnumValue +RegError = winreg.error + +HKEYS = (winreg.HKEY_USERS, + winreg.HKEY_CURRENT_USER, + winreg.HKEY_LOCAL_MACHINE, + winreg.HKEY_CLASSES_ROOT) + +NATIVE_WIN64 = (sys.platform == 'win32' and sys.maxsize > 2**32) +if NATIVE_WIN64: + # Visual C++ is a 32-bit application, so we need to look in + # the corresponding registry branch, if we're running a + # 64-bit Python on Win64 + VS_BASE = r"Software\Wow6432Node\Microsoft\VisualStudio\%0.1f" + WINSDK_BASE = r"Software\Wow6432Node\Microsoft\Microsoft SDKs\Windows" + NET_BASE = r"Software\Wow6432Node\Microsoft\.NETFramework" +else: + VS_BASE = r"Software\Microsoft\VisualStudio\%0.1f" + WINSDK_BASE = r"Software\Microsoft\Microsoft SDKs\Windows" + NET_BASE = r"Software\Microsoft\.NETFramework" + +# A map keyed by get_platform() return values to values accepted by +# 'vcvarsall.bat'. Note a cross-compile may combine these (eg, 'x86_amd64' is +# the param to cross-compile on x86 targeting amd64.) +PLAT_TO_VCVARS = { + 'win32' : 'x86', + 'win-amd64' : 'amd64', +} + +class Reg: + """Helper class to read values from the registry + """ + + def get_value(cls, path, key): + for base in HKEYS: + d = cls.read_values(base, path) + if d and key in d: + return d[key] + raise KeyError(key) + get_value = classmethod(get_value) + + def read_keys(cls, base, key): + """Return list of registry keys.""" + try: + handle = RegOpenKeyEx(base, key) + except RegError: + return None + L = [] + i = 0 + while True: + try: + k = RegEnumKey(handle, i) + except RegError: + break + L.append(k) + i += 1 + return L + read_keys = classmethod(read_keys) + + def read_values(cls, base, key): + """Return dict of registry keys and values. + + All names are converted to lowercase. + """ + try: + handle = RegOpenKeyEx(base, key) + except RegError: + return None + d = {} + i = 0 + while True: + try: + name, value, type = RegEnumValue(handle, i) + except RegError: + break + name = name.lower() + d[cls.convert_mbcs(name)] = cls.convert_mbcs(value) + i += 1 + return d + read_values = classmethod(read_values) + + def convert_mbcs(s): + dec = getattr(s, "decode", None) + if dec is not None: + try: + s = dec("mbcs") + except UnicodeError: + pass + return s + convert_mbcs = staticmethod(convert_mbcs) + +class MacroExpander: + + def __init__(self, version): + self.macros = {} + self.vsbase = VS_BASE % version + self.load_macros(version) + + def set_macro(self, macro, path, key): + self.macros["$(%s)" % macro] = Reg.get_value(path, key) + + def load_macros(self, version): + self.set_macro("VCInstallDir", self.vsbase + r"\Setup\VC", "productdir") + self.set_macro("VSInstallDir", self.vsbase + r"\Setup\VS", "productdir") + self.set_macro("FrameworkDir", NET_BASE, "installroot") + try: + if version >= 8.0: + self.set_macro("FrameworkSDKDir", NET_BASE, + "sdkinstallrootv2.0") + else: + raise KeyError("sdkinstallrootv2.0") + except KeyError: + raise DistutilsPlatformError( + """Python was built with Visual Studio 2008; +extensions must be built with a compiler than can generate compatible binaries. +Visual Studio 2008 was not found on this system. If you have Cygwin installed, +you can try compiling with MingW32, by passing "-c mingw32" to setup.py.""") + + if version >= 9.0: + self.set_macro("FrameworkVersion", self.vsbase, "clr version") + self.set_macro("WindowsSdkDir", WINSDK_BASE, "currentinstallfolder") + else: + p = r"Software\Microsoft\NET Framework Setup\Product" + for base in HKEYS: + try: + h = RegOpenKeyEx(base, p) + except RegError: + continue + key = RegEnumKey(h, 0) + d = Reg.get_value(base, r"%s\%s" % (p, key)) + self.macros["$(FrameworkVersion)"] = d["version"] + + def sub(self, s): + for k, v in self.macros.items(): + s = s.replace(k, v) + return s + +def get_build_version(): + """Return the version of MSVC that was used to build Python. + + For Python 2.3 and up, the version number is included in + sys.version. For earlier versions, assume the compiler is MSVC 6. + """ + prefix = "MSC v." + i = sys.version.find(prefix) + if i == -1: + return 6 + i = i + len(prefix) + s, rest = sys.version[i:].split(" ", 1) + majorVersion = int(s[:-2]) - 6 + if majorVersion >= 13: + # v13 was skipped and should be v14 + majorVersion += 1 + minorVersion = int(s[2:3]) / 10.0 + # I don't think paths are affected by minor version in version 6 + if majorVersion == 6: + minorVersion = 0 + if majorVersion >= 6: + return majorVersion + minorVersion + # else we don't know what version of the compiler this is + return None + +def normalize_and_reduce_paths(paths): + """Return a list of normalized paths with duplicates removed. + + The current order of paths is maintained. + """ + # Paths are normalized so things like: /a and /a/ aren't both preserved. + reduced_paths = [] + for p in paths: + np = os.path.normpath(p) + # XXX(nnorwitz): O(n**2), if reduced_paths gets long perhaps use a set. + if np not in reduced_paths: + reduced_paths.append(np) + return reduced_paths + +def removeDuplicates(variable): + """Remove duplicate values of an environment variable. + """ + oldList = variable.split(os.pathsep) + newList = [] + for i in oldList: + if i not in newList: + newList.append(i) + newVariable = os.pathsep.join(newList) + return newVariable + +def find_vcvarsall(version): + """Find the vcvarsall.bat file + + At first it tries to find the productdir of VS 2008 in the registry. If + that fails it falls back to the VS90COMNTOOLS env var. + """ + vsbase = VS_BASE % version + try: + productdir = Reg.get_value(r"%s\Setup\VC" % vsbase, + "productdir") + except KeyError: + log.debug("Unable to find productdir in registry") + productdir = None + + if not productdir or not os.path.isdir(productdir): + toolskey = "VS%0.f0COMNTOOLS" % version + toolsdir = os.environ.get(toolskey, None) + + if toolsdir and os.path.isdir(toolsdir): + productdir = os.path.join(toolsdir, os.pardir, os.pardir, "VC") + productdir = os.path.abspath(productdir) + if not os.path.isdir(productdir): + log.debug("%s is not a valid directory" % productdir) + return None + else: + log.debug("Env var %s is not set or invalid" % toolskey) + if not productdir: + log.debug("No productdir found") + return None + vcvarsall = os.path.join(productdir, "vcvarsall.bat") + if os.path.isfile(vcvarsall): + return vcvarsall + log.debug("Unable to find vcvarsall.bat") + return None + +def query_vcvarsall(version, arch="x86"): + """Launch vcvarsall.bat and read the settings from its environment + """ + vcvarsall = find_vcvarsall(version) + interesting = {"include", "lib", "libpath", "path"} + result = {} + + if vcvarsall is None: + raise DistutilsPlatformError("Unable to find vcvarsall.bat") + log.debug("Calling 'vcvarsall.bat %s' (version=%s)", arch, version) + popen = subprocess.Popen('"%s" %s & set' % (vcvarsall, arch), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + try: + stdout, stderr = popen.communicate() + if popen.wait() != 0: + raise DistutilsPlatformError(stderr.decode("mbcs")) + + stdout = stdout.decode("mbcs") + for line in stdout.split("\n"): + line = Reg.convert_mbcs(line) + if '=' not in line: + continue + line = line.strip() + key, value = line.split('=', 1) + key = key.lower() + if key in interesting: + if value.endswith(os.pathsep): + value = value[:-1] + result[key] = removeDuplicates(value) + + finally: + popen.stdout.close() + popen.stderr.close() + + if len(result) != len(interesting): + raise ValueError(str(list(result.keys()))) + + return result + +# More globals +VERSION = get_build_version() +if VERSION < 8.0: + raise DistutilsPlatformError("VC %0.1f is not supported by this module" % VERSION) +# MACROS = MacroExpander(VERSION) + +class MSVCCompiler(CCompiler) : + """Concrete class that implements an interface to Microsoft Visual C++, + as defined by the CCompiler abstract class.""" + + compiler_type = 'msvc' + + # Just set this so CCompiler's constructor doesn't barf. We currently + # don't use the 'set_executables()' bureaucracy provided by CCompiler, + # as it really isn't necessary for this sort of single-compiler class. + # Would be nice to have a consistent interface with UnixCCompiler, + # though, so it's worth thinking about. + executables = {} + + # Private class data (need to distinguish C from C++ source for compiler) + _c_extensions = ['.c'] + _cpp_extensions = ['.cc', '.cpp', '.cxx'] + _rc_extensions = ['.rc'] + _mc_extensions = ['.mc'] + + # Needed for the filename generation methods provided by the + # base class, CCompiler. + src_extensions = (_c_extensions + _cpp_extensions + + _rc_extensions + _mc_extensions) + res_extension = '.res' + obj_extension = '.obj' + static_lib_extension = '.lib' + shared_lib_extension = '.dll' + static_lib_format = shared_lib_format = '%s%s' + exe_extension = '.exe' + + def __init__(self, verbose=0, dry_run=0, force=0): + CCompiler.__init__ (self, verbose, dry_run, force) + self.__version = VERSION + self.__root = r"Software\Microsoft\VisualStudio" + # self.__macros = MACROS + self.__paths = [] + # target platform (.plat_name is consistent with 'bdist') + self.plat_name = None + self.__arch = None # deprecated name + self.initialized = False + + # -- Worker methods ------------------------------------------------ + + def manifest_setup_ldargs(self, output_filename, build_temp, ld_args): + # If we need a manifest at all, an embedded manifest is recommended. + # See MSDN article titled + # "How to: Embed a Manifest Inside a C/C++ Application" + # (currently at http://msdn2.microsoft.com/en-us/library/ms235591(VS.80).aspx) + # Ask the linker to generate the manifest in the temp dir, so + # we can check it, and possibly embed it, later. + temp_manifest = os.path.join( + build_temp, + os.path.basename(output_filename) + ".manifest") + ld_args.append('/MANIFESTFILE:' + temp_manifest) + + def manifest_get_embed_info(self, target_desc, ld_args): + # If a manifest should be embedded, return a tuple of + # (manifest_filename, resource_id). Returns None if no manifest + # should be embedded. See http://bugs.python.org/issue7833 for why + # we want to avoid any manifest for extension modules if we can. + for arg in ld_args: + if arg.startswith("/MANIFESTFILE:"): + temp_manifest = arg.split(":", 1)[1] + break + else: + # no /MANIFESTFILE so nothing to do. + return None + if target_desc == CCompiler.EXECUTABLE: + # by default, executables always get the manifest with the + # CRT referenced. + mfid = 1 + else: + # Extension modules try and avoid any manifest if possible. + mfid = 2 + temp_manifest = self._remove_visual_c_ref(temp_manifest) + if temp_manifest is None: + return None + return temp_manifest, mfid + + def _remove_visual_c_ref(self, manifest_file): + try: + # Remove references to the Visual C runtime, so they will + # fall through to the Visual C dependency of Python.exe. + # This way, when installed for a restricted user (e.g. + # runtimes are not in WinSxS folder, but in Python's own + # folder), the runtimes do not need to be in every folder + # with .pyd's. + # Returns either the filename of the modified manifest or + # None if no manifest should be embedded. + manifest_f = open(manifest_file) + try: + manifest_buf = manifest_f.read() + finally: + manifest_f.close() + pattern = re.compile( + r"""|)""", + re.DOTALL) + manifest_buf = re.sub(pattern, "", manifest_buf) + pattern = r"\s*" + manifest_buf = re.sub(pattern, "", manifest_buf) + # Now see if any other assemblies are referenced - if not, we + # don't want a manifest embedded. + pattern = re.compile( + r"""|)""", re.DOTALL) + if re.search(pattern, manifest_buf) is None: + return None + + manifest_f = open(manifest_file, 'w') + try: + manifest_f.write(manifest_buf) + return manifest_file + finally: + manifest_f.close() + except OSError: + pass + + # -- Miscellaneous methods ----------------------------------------- + + # Helper methods for using the MSVC registry settings + + def find_exe(self, exe): + """Return path to an MSVC executable program. + + Tries to find the program in several places: first, one of the + MSVC program search paths from the registry; next, the directories + in the PATH environment variable. If any of those work, return an + absolute path that is known to exist. If none of them work, just + return the original program name, 'exe'. + """ + for p in self.__paths: + fn = os.path.join(os.path.abspath(p), exe) + if os.path.isfile(fn): + return fn + + # didn't find it; try existing path + for p in os.environ['Path'].split(';'): + fn = os.path.join(os.path.abspath(p),exe) + if os.path.isfile(fn): + return fn + + return exe diff --git a/Tools/c-analyzer/distutils/msvccompiler.py b/Tools/c-analyzer/distutils/msvccompiler.py new file mode 100644 index 00000000000000..c0864b1f95d4e2 --- /dev/null +++ b/Tools/c-analyzer/distutils/msvccompiler.py @@ -0,0 +1,327 @@ +"""distutils.msvccompiler + +Contains MSVCCompiler, an implementation of the abstract CCompiler class +for the Microsoft Visual Studio. +""" + +# Written by Perry Stoll +# hacked by Robin Becker and Thomas Heller to do a better job of +# finding DevStudio (through the registry) + +import sys, os +from distutils.errors import DistutilsPlatformError +from distutils.ccompiler import CCompiler +from distutils import log + +_can_read_reg = False +try: + import winreg + + _can_read_reg = True + hkey_mod = winreg + + RegOpenKeyEx = winreg.OpenKeyEx + RegEnumKey = winreg.EnumKey + RegEnumValue = winreg.EnumValue + RegError = winreg.error + +except ImportError: + try: + import win32api + import win32con + _can_read_reg = True + hkey_mod = win32con + + RegOpenKeyEx = win32api.RegOpenKeyEx + RegEnumKey = win32api.RegEnumKey + RegEnumValue = win32api.RegEnumValue + RegError = win32api.error + except ImportError: + log.info("Warning: Can't read registry to find the " + "necessary compiler setting\n" + "Make sure that Python modules winreg, " + "win32api or win32con are installed.") + +if _can_read_reg: + HKEYS = (hkey_mod.HKEY_USERS, + hkey_mod.HKEY_CURRENT_USER, + hkey_mod.HKEY_LOCAL_MACHINE, + hkey_mod.HKEY_CLASSES_ROOT) + +def read_keys(base, key): + """Return list of registry keys.""" + try: + handle = RegOpenKeyEx(base, key) + except RegError: + return None + L = [] + i = 0 + while True: + try: + k = RegEnumKey(handle, i) + except RegError: + break + L.append(k) + i += 1 + return L + +def read_values(base, key): + """Return dict of registry keys and values. + + All names are converted to lowercase. + """ + try: + handle = RegOpenKeyEx(base, key) + except RegError: + return None + d = {} + i = 0 + while True: + try: + name, value, type = RegEnumValue(handle, i) + except RegError: + break + name = name.lower() + d[convert_mbcs(name)] = convert_mbcs(value) + i += 1 + return d + +def convert_mbcs(s): + dec = getattr(s, "decode", None) + if dec is not None: + try: + s = dec("mbcs") + except UnicodeError: + pass + return s + +class MacroExpander: + def __init__(self, version): + self.macros = {} + self.load_macros(version) + + def set_macro(self, macro, path, key): + for base in HKEYS: + d = read_values(base, path) + if d: + self.macros["$(%s)" % macro] = d[key] + break + + def load_macros(self, version): + vsbase = r"Software\Microsoft\VisualStudio\%0.1f" % version + self.set_macro("VCInstallDir", vsbase + r"\Setup\VC", "productdir") + self.set_macro("VSInstallDir", vsbase + r"\Setup\VS", "productdir") + net = r"Software\Microsoft\.NETFramework" + self.set_macro("FrameworkDir", net, "installroot") + try: + if version > 7.0: + self.set_macro("FrameworkSDKDir", net, "sdkinstallrootv1.1") + else: + self.set_macro("FrameworkSDKDir", net, "sdkinstallroot") + except KeyError as exc: # + raise DistutilsPlatformError( + """Python was built with Visual Studio 2003; +extensions must be built with a compiler than can generate compatible binaries. +Visual Studio 2003 was not found on this system. If you have Cygwin installed, +you can try compiling with MingW32, by passing "-c mingw32" to setup.py.""") + + p = r"Software\Microsoft\NET Framework Setup\Product" + for base in HKEYS: + try: + h = RegOpenKeyEx(base, p) + except RegError: + continue + key = RegEnumKey(h, 0) + d = read_values(base, r"%s\%s" % (p, key)) + self.macros["$(FrameworkVersion)"] = d["version"] + + def sub(self, s): + for k, v in self.macros.items(): + s = s.replace(k, v) + return s + +def get_build_version(): + """Return the version of MSVC that was used to build Python. + + For Python 2.3 and up, the version number is included in + sys.version. For earlier versions, assume the compiler is MSVC 6. + """ + prefix = "MSC v." + i = sys.version.find(prefix) + if i == -1: + return 6 + i = i + len(prefix) + s, rest = sys.version[i:].split(" ", 1) + majorVersion = int(s[:-2]) - 6 + if majorVersion >= 13: + # v13 was skipped and should be v14 + majorVersion += 1 + minorVersion = int(s[2:3]) / 10.0 + # I don't think paths are affected by minor version in version 6 + if majorVersion == 6: + minorVersion = 0 + if majorVersion >= 6: + return majorVersion + minorVersion + # else we don't know what version of the compiler this is + return None + +def get_build_architecture(): + """Return the processor architecture. + + Possible results are "Intel" or "AMD64". + """ + + prefix = " bit (" + i = sys.version.find(prefix) + if i == -1: + return "Intel" + j = sys.version.find(")", i) + return sys.version[i+len(prefix):j] + +def normalize_and_reduce_paths(paths): + """Return a list of normalized paths with duplicates removed. + + The current order of paths is maintained. + """ + # Paths are normalized so things like: /a and /a/ aren't both preserved. + reduced_paths = [] + for p in paths: + np = os.path.normpath(p) + # XXX(nnorwitz): O(n**2), if reduced_paths gets long perhaps use a set. + if np not in reduced_paths: + reduced_paths.append(np) + return reduced_paths + + +class MSVCCompiler(CCompiler) : + """Concrete class that implements an interface to Microsoft Visual C++, + as defined by the CCompiler abstract class.""" + + compiler_type = 'msvc' + + # Just set this so CCompiler's constructor doesn't barf. We currently + # don't use the 'set_executables()' bureaucracy provided by CCompiler, + # as it really isn't necessary for this sort of single-compiler class. + # Would be nice to have a consistent interface with UnixCCompiler, + # though, so it's worth thinking about. + executables = {} + + # Private class data (need to distinguish C from C++ source for compiler) + _c_extensions = ['.c'] + _cpp_extensions = ['.cc', '.cpp', '.cxx'] + _rc_extensions = ['.rc'] + _mc_extensions = ['.mc'] + + # Needed for the filename generation methods provided by the + # base class, CCompiler. + src_extensions = (_c_extensions + _cpp_extensions + + _rc_extensions + _mc_extensions) + res_extension = '.res' + obj_extension = '.obj' + static_lib_extension = '.lib' + shared_lib_extension = '.dll' + static_lib_format = shared_lib_format = '%s%s' + exe_extension = '.exe' + + def __init__(self, verbose=0, dry_run=0, force=0): + CCompiler.__init__ (self, verbose, dry_run, force) + self.__version = get_build_version() + self.__arch = get_build_architecture() + if self.__arch == "Intel": + # x86 + if self.__version >= 7: + self.__root = r"Software\Microsoft\VisualStudio" + self.__macros = MacroExpander(self.__version) + else: + self.__root = r"Software\Microsoft\Devstudio" + self.__product = "Visual Studio version %s" % self.__version + else: + # Win64. Assume this was built with the platform SDK + self.__product = "Microsoft SDK compiler %s" % (self.__version + 6) + + self.initialized = False + + + # -- Miscellaneous methods ----------------------------------------- + + # Helper methods for using the MSVC registry settings + + def find_exe(self, exe): + """Return path to an MSVC executable program. + + Tries to find the program in several places: first, one of the + MSVC program search paths from the registry; next, the directories + in the PATH environment variable. If any of those work, return an + absolute path that is known to exist. If none of them work, just + return the original program name, 'exe'. + """ + for p in self.__paths: + fn = os.path.join(os.path.abspath(p), exe) + if os.path.isfile(fn): + return fn + + # didn't find it; try existing path + for p in os.environ['Path'].split(';'): + fn = os.path.join(os.path.abspath(p),exe) + if os.path.isfile(fn): + return fn + + return exe + + def get_msvc_paths(self, path, platform='x86'): + """Get a list of devstudio directories (include, lib or path). + + Return a list of strings. The list will be empty if unable to + access the registry or appropriate registry keys not found. + """ + if not _can_read_reg: + return [] + + path = path + " dirs" + if self.__version >= 7: + key = (r"%s\%0.1f\VC\VC_OBJECTS_PLATFORM_INFO\Win32\Directories" + % (self.__root, self.__version)) + else: + key = (r"%s\6.0\Build System\Components\Platforms" + r"\Win32 (%s)\Directories" % (self.__root, platform)) + + for base in HKEYS: + d = read_values(base, key) + if d: + if self.__version >= 7: + return self.__macros.sub(d[path]).split(";") + else: + return d[path].split(";") + # MSVC 6 seems to create the registry entries we need only when + # the GUI is run. + if self.__version == 6: + for base in HKEYS: + if read_values(base, r"%s\6.0" % self.__root) is not None: + self.warn("It seems you have Visual Studio 6 installed, " + "but the expected registry settings are not present.\n" + "You must at least run the Visual Studio GUI once " + "so that these entries are created.") + break + return [] + + def set_path_env_var(self, name): + """Set environment variable 'name' to an MSVC path type value. + + This is equivalent to a SET command prior to execution of spawned + commands. + """ + + if name == "lib": + p = self.get_msvc_paths("library") + else: + p = self.get_msvc_paths(name) + if p: + os.environ[name] = ';'.join(p) + + +if get_build_version() >= 8.0: + log.debug("Importing new compiler from distutils.msvc9compiler") + OldMSVCCompiler = MSVCCompiler + from distutils.msvc9compiler import MSVCCompiler + # get_build_architecture not really relevant now we support cross-compile + from distutils.msvc9compiler import MacroExpander diff --git a/Tools/c-analyzer/distutils/spawn.py b/Tools/c-analyzer/distutils/spawn.py new file mode 100644 index 00000000000000..eb9794964d7acb --- /dev/null +++ b/Tools/c-analyzer/distutils/spawn.py @@ -0,0 +1,48 @@ +"""distutils.spawn + +Provides the 'spawn()' function, a front-end to various platform- +specific functions for launching another program in a sub-process. +Also provides the 'find_executable()' to search the path for a given +executable name. +""" + +import sys +import os +import os.path + + +def find_executable(executable, path=None): + """Tries to find 'executable' in the directories listed in 'path'. + + A string listing directories separated by 'os.pathsep'; defaults to + os.environ['PATH']. Returns the complete filename or None if not found. + """ + _, ext = os.path.splitext(executable) + if (sys.platform == 'win32') and (ext != '.exe'): + executable = executable + '.exe' + + if os.path.isfile(executable): + return executable + + if path is None: + path = os.environ.get('PATH', None) + if path is None: + try: + path = os.confstr("CS_PATH") + except (AttributeError, ValueError): + # os.confstr() or CS_PATH is not available + path = os.defpath + # bpo-35755: Don't use os.defpath if the PATH environment variable is + # set to an empty string + + # PATH='' doesn't match, whereas PATH=':' looks in the current directory + if not path: + return None + + paths = path.split(os.pathsep) + for p in paths: + f = os.path.join(p, executable) + if os.path.isfile(f): + # the file exists, we have a shot at spawn working + return f + return None diff --git a/Tools/c-analyzer/distutils/unixccompiler.py b/Tools/c-analyzer/distutils/unixccompiler.py new file mode 100644 index 00000000000000..1a3a8e5255070d --- /dev/null +++ b/Tools/c-analyzer/distutils/unixccompiler.py @@ -0,0 +1,102 @@ +"""distutils.unixccompiler + +Contains the UnixCCompiler class, a subclass of CCompiler that handles +the "typical" Unix-style command-line C compiler: + * macros defined with -Dname[=value] + * macros undefined with -Uname + * include search directories specified with -Idir + * libraries specified with -lllib + * library search directories specified with -Ldir + * compile handled by 'cc' (or similar) executable with -c option: + compiles .c to .o + * link static library handled by 'ar' command (possibly with 'ranlib') + * link shared library handled by 'cc -shared' +""" + +import os, sys, re + +from distutils.dep_util import newer +from distutils.ccompiler import CCompiler, gen_preprocess_options +from distutils.errors import DistutilsExecError, CompileError + +# XXX Things not currently handled: +# * optimization/debug/warning flags; we just use whatever's in Python's +# Makefile and live with it. Is this adequate? If not, we might +# have to have a bunch of subclasses GNUCCompiler, SGICCompiler, +# SunCCompiler, and I suspect down that road lies madness. +# * even if we don't know a warning flag from an optimization flag, +# we need some way for outsiders to feed preprocessor/compiler/linker +# flags in to us -- eg. a sysadmin might want to mandate certain flags +# via a site config file, or a user might want to set something for +# compiling this module distribution only via the setup.py command +# line, whatever. As long as these options come from something on the +# current system, they can be as system-dependent as they like, and we +# should just happily stuff them into the preprocessor/compiler/linker +# options and carry on. + + +class UnixCCompiler(CCompiler): + + compiler_type = 'unix' + + # These are used by CCompiler in two places: the constructor sets + # instance attributes 'preprocessor', 'compiler', etc. from them, and + # 'set_executable()' allows any of these to be set. The defaults here + # are pretty generic; they will probably have to be set by an outsider + # (eg. using information discovered by the sysconfig about building + # Python extensions). + executables = {'preprocessor' : None, + 'compiler' : ["cc"], + 'compiler_so' : ["cc"], + 'compiler_cxx' : ["cc"], + 'linker_so' : ["cc", "-shared"], + 'linker_exe' : ["cc"], + 'archiver' : ["ar", "-cr"], + 'ranlib' : None, + } + + if sys.platform[:6] == "darwin": + executables['ranlib'] = ["ranlib"] + + # Needed for the filename generation methods provided by the base + # class, CCompiler. NB. whoever instantiates/uses a particular + # UnixCCompiler instance should set 'shared_lib_ext' -- we set a + # reasonable common default here, but it's not necessarily used on all + # Unices! + + src_extensions = [".c",".C",".cc",".cxx",".cpp",".m"] + obj_extension = ".o" + static_lib_extension = ".a" + shared_lib_extension = ".so" + dylib_lib_extension = ".dylib" + xcode_stub_lib_extension = ".tbd" + static_lib_format = shared_lib_format = dylib_lib_format = "lib%s%s" + xcode_stub_lib_format = dylib_lib_format + if sys.platform == "cygwin": + exe_extension = ".exe" + + def preprocess(self, source, output_file=None, macros=None, + include_dirs=None, extra_preargs=None, extra_postargs=None): + fixed_args = self._fix_compile_args(None, macros, include_dirs) + ignore, macros, include_dirs = fixed_args + pp_opts = gen_preprocess_options(macros, include_dirs) + pp_args = self.preprocessor + pp_opts + if output_file: + pp_args.extend(['-o', output_file]) + if extra_preargs: + pp_args[:0] = extra_preargs + if extra_postargs: + pp_args.extend(extra_postargs) + pp_args.append(source) + + # We need to preprocess: either we're being forced to, or we're + # generating output to stdout, or there's a target output file and + # the source file is newer than the target (or the target doesn't + # exist). + if self.force or output_file is None or newer(source, output_file): + if output_file: + self.mkpath(os.path.dirname(output_file)) + try: + self.spawn(pp_args) + except DistutilsExecError as msg: + raise CompileError(msg) diff --git a/Tools/c-analyzer/distutils/util.py b/Tools/c-analyzer/distutils/util.py new file mode 100644 index 00000000000000..89ca094336fdb8 --- /dev/null +++ b/Tools/c-analyzer/distutils/util.py @@ -0,0 +1,171 @@ +"""distutils.util + +Miscellaneous utility functions -- anything that doesn't fit into +one of the other *util.py modules. +""" + +import os +import re +import string +import sys +from distutils.errors import DistutilsPlatformError + +def get_host_platform(): + """Return a string that identifies the current platform. This is used mainly to + distinguish platform-specific build directories and platform-specific built + distributions. Typically includes the OS name and version and the + architecture (as supplied by 'os.uname()'), although the exact information + included depends on the OS; eg. on Linux, the kernel version isn't + particularly important. + + Examples of returned values: + linux-i586 + linux-alpha (?) + solaris-2.6-sun4u + + Windows will return one of: + win-amd64 (64bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc) + win32 (all others - specifically, sys.platform is returned) + + For other non-POSIX platforms, currently just returns 'sys.platform'. + + """ + if os.name == 'nt': + if 'amd64' in sys.version.lower(): + return 'win-amd64' + if '(arm)' in sys.version.lower(): + return 'win-arm32' + if '(arm64)' in sys.version.lower(): + return 'win-arm64' + return sys.platform + + # Set for cross builds explicitly + if "_PYTHON_HOST_PLATFORM" in os.environ: + return os.environ["_PYTHON_HOST_PLATFORM"] + + if os.name != "posix" or not hasattr(os, 'uname'): + # XXX what about the architecture? NT is Intel or Alpha, + # Mac OS is M68k or PPC, etc. + return sys.platform + + # Try to distinguish various flavours of Unix + + (osname, host, release, version, machine) = os.uname() + + # Convert the OS name to lowercase, remove '/' characters, and translate + # spaces (for "Power Macintosh") + osname = osname.lower().replace('/', '') + machine = machine.replace(' ', '_') + machine = machine.replace('/', '-') + + if osname[:5] == "linux": + # At least on Linux/Intel, 'machine' is the processor -- + # i386, etc. + # XXX what about Alpha, SPARC, etc? + return "%s-%s" % (osname, machine) + elif osname[:5] == "sunos": + if release[0] >= "5": # SunOS 5 == Solaris 2 + osname = "solaris" + release = "%d.%s" % (int(release[0]) - 3, release[2:]) + # We can't use "platform.architecture()[0]" because a + # bootstrap problem. We use a dict to get an error + # if some suspicious happens. + bitness = {2147483647:"32bit", 9223372036854775807:"64bit"} + machine += ".%s" % bitness[sys.maxsize] + # fall through to standard osname-release-machine representation + elif osname[:3] == "aix": + from _aix_support import aix_platform + return aix_platform() + elif osname[:6] == "cygwin": + osname = "cygwin" + rel_re = re.compile (r'[\d.]+', re.ASCII) + m = rel_re.match(release) + if m: + release = m.group() + elif osname[:6] == "darwin": + import _osx_support, sysconfig + osname, release, machine = _osx_support.get_platform_osx( + sysconfig.get_config_vars(), + osname, release, machine) + + return "%s-%s-%s" % (osname, release, machine) + +def get_platform(): + if os.name == 'nt': + TARGET_TO_PLAT = { + 'x86' : 'win32', + 'x64' : 'win-amd64', + 'arm' : 'win-arm32', + } + return TARGET_TO_PLAT.get(os.environ.get('VSCMD_ARG_TGT_ARCH')) or get_host_platform() + else: + return get_host_platform() + + +# Needed by 'split_quoted()' +_wordchars_re = _squote_re = _dquote_re = None +def _init_regex(): + global _wordchars_re, _squote_re, _dquote_re + _wordchars_re = re.compile(r'[^\\\'\"%s ]*' % string.whitespace) + _squote_re = re.compile(r"'(?:[^'\\]|\\.)*'") + _dquote_re = re.compile(r'"(?:[^"\\]|\\.)*"') + +def split_quoted (s): + """Split a string up according to Unix shell-like rules for quotes and + backslashes. In short: words are delimited by spaces, as long as those + spaces are not escaped by a backslash, or inside a quoted string. + Single and double quotes are equivalent, and the quote characters can + be backslash-escaped. The backslash is stripped from any two-character + escape sequence, leaving only the escaped character. The quote + characters are stripped from any quoted string. Returns a list of + words. + """ + + # This is a nice algorithm for splitting up a single string, since it + # doesn't require character-by-character examination. It was a little + # bit of a brain-bender to get it working right, though... + if _wordchars_re is None: _init_regex() + + s = s.strip() + words = [] + pos = 0 + + while s: + m = _wordchars_re.match(s, pos) + end = m.end() + if end == len(s): + words.append(s[:end]) + break + + if s[end] in string.whitespace: # unescaped, unquoted whitespace: now + words.append(s[:end]) # we definitely have a word delimiter + s = s[end:].lstrip() + pos = 0 + + elif s[end] == '\\': # preserve whatever is being escaped; + # will become part of the current word + s = s[:end] + s[end+1:] + pos = end+1 + + else: + if s[end] == "'": # slurp singly-quoted string + m = _squote_re.match(s, end) + elif s[end] == '"': # slurp doubly-quoted string + m = _dquote_re.match(s, end) + else: + raise RuntimeError("this can't happen (bad char '%c')" % s[end]) + + if m is None: + raise ValueError("bad string (mismatched %s quotes?)" % s[end]) + + (beg, end) = m.span() + s = s[:beg] + s[beg+1:end-1] + s[end:] + pos = m.end() - 2 + + if pos >= len(s): + words.append(s) + break + + return words + +# split_quoted () From c6858d1e7f4cd3184d5ddea4025ad5dfc7596546 Mon Sep 17 00:00:00 2001 From: Max Bachmann Date: Thu, 9 Mar 2023 22:09:12 +0100 Subject: [PATCH 31/38] gh-102255: Improve build support for Windows API partitions (GH-102256) Add `MS_WINDOWS_DESKTOP`, `MS_WINDOWS_APPS`, `MS_WINDOWS_SYSTEM` and `MS_WINDOWS_GAMES` preprocessor definitions to allow switching off functionality missing from particular API partitions ("partitions" are used in Windows to identify overlapping subsets of APIs). CPython only officially supports `MS_WINDOWS_DESKTOP` and `MS_WINDOWS_SYSTEM` (APPS is included by normal desktop builds, but APPS without DESKTOP is not covered). Other configurations are a convenience for people building their own runtimes. `MS_WINDOWS_GAMES` is for the Xbox subset of the Windows API, which is also available on client OS, but is restricted compared to `MS_WINDOWS_DESKTOP`. These restrictions may change over time, as they relate to the build headers rather than the OS support, and so we assume that Xbox builds will use the latest available version of the GDK. --- Include/internal/pycore_fileutils.h | 8 + Lib/test/test_os.py | 8 +- ...-02-26-11-43-56.gh-issue-102255.cRnI5x.rst | 1 + Modules/_io/_iomodule.c | 6 +- Modules/_io/_iomodule.h | 6 +- Modules/_io/clinic/winconsoleio.c.h | 42 ++-- Modules/_io/fileio.c | 2 + Modules/_io/winconsoleio.c | 6 +- Modules/_localemodule.c | 8 +- Modules/_multiprocessing/multiprocessing.h | 4 +- Modules/_pickle.c | 6 + Modules/_randommodule.c | 6 +- Modules/_ssl.c | 4 + Modules/_winapi.c | 13 ++ Modules/clinic/posixmodule.c.h | 10 +- Modules/errnomodule.c | 2 + Modules/faulthandler.c | 2 +- Modules/getpath.c | 2 + Modules/mmapmodule.c | 8 +- Modules/posixmodule.c | 58 +++-- Modules/selectmodule.c | 6 +- Modules/socketmodule.c | 60 +++-- Modules/timemodule.c | 8 +- Objects/obmalloc.c | 1 - PC/clinic/msvcrtmodule.c.h | 42 +++- PC/clinic/winreg.c.h | 218 +++++++++++++++++- PC/config.c | 8 + PC/config_minimal.c | 6 + PC/msvcrtmodule.c | 22 ++ PC/pyconfig.h | 24 +- PC/winreg.c | 25 ++ Parser/myreadline.c | 8 +- Python/dynload_win.c | 8 + Python/fileutils.c | 89 ++++++- Python/pylifecycle.c | 2 +- Python/sysmodule.c | 4 + 36 files changed, 633 insertions(+), 100 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-26-11-43-56.gh-issue-102255.cRnI5x.rst diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index f8e2bf22590888..445ac0a3d955d9 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -251,6 +251,14 @@ extern int _Py_add_relfile(wchar_t *dirname, extern size_t _Py_find_basename(const wchar_t *filename); PyAPI_FUNC(wchar_t *) _Py_normpath(wchar_t *path, Py_ssize_t size); +// The Windows Games API family does not provide these functions +// so provide our own implementations. Remove them in case they get added +// to the Games API family +#if defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) +#include + +extern HRESULT PathCchSkipRoot(const wchar_t *pszPath, const wchar_t **ppszRootEnd); +#endif /* defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) */ // Macros to protect CRT calls against instant termination when passed an // invalid parameter (bpo-23524). IPH stands for Invalid Parameter Handler. diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 792794ca109489..ba6feb69ea1712 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -3084,11 +3084,13 @@ def test_device_encoding(self): class PidTests(unittest.TestCase): @unittest.skipUnless(hasattr(os, 'getppid'), "test needs os.getppid") def test_getppid(self): - p = subprocess.Popen([sys.executable, '-c', + p = subprocess.Popen([sys._base_executable, '-c', 'import os; print(os.getppid())'], - stdout=subprocess.PIPE) - stdout, _ = p.communicate() + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, error = p.communicate() # We are the parent of our subprocess + self.assertEqual(error, b'') self.assertEqual(int(stdout), os.getpid()) def check_waitpid(self, code, exitcode, callback=None): diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-26-11-43-56.gh-issue-102255.cRnI5x.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-26-11-43-56.gh-issue-102255.cRnI5x.rst new file mode 100644 index 00000000000000..daabc3c15f6ee2 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-02-26-11-43-56.gh-issue-102255.cRnI5x.rst @@ -0,0 +1 @@ +Improve build support for the Xbox. Patch by Max Bachmann. diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index d8d836b8382eb1..1506755427fc0d 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -317,7 +317,7 @@ _io_open_impl(PyObject *module, PyObject *file, const char *mode, _PyIO_State *state = get_io_state(module); { PyObject *RawIO_class = (PyObject *)state->PyFileIO_Type; -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO const PyConfig *config = _Py_GetConfig(); if (!config->legacy_windows_stdio && _PyIO_get_console_type(path_or_fd) != '\0') { RawIO_class = (PyObject *)&PyWindowsConsoleIO_Type; @@ -660,7 +660,7 @@ static PyTypeObject* static_types[] = { // PyRawIOBase_Type(PyIOBase_Type) subclasses &_PyBytesIOBuffer_Type, -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO &PyWindowsConsoleIO_Type, #endif }; @@ -718,7 +718,7 @@ PyInit__io(void) } // Set type base classes -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO PyWindowsConsoleIO_Type.tp_base = &PyRawIOBase_Type; #endif diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h index 02daef9e85677e..d7224e56f9a722 100644 --- a/Modules/_io/_iomodule.h +++ b/Modules/_io/_iomodule.h @@ -26,9 +26,9 @@ extern PyType_Spec fileio_spec; extern PyType_Spec stringio_spec; extern PyType_Spec textiowrapper_spec; -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO extern PyTypeObject PyWindowsConsoleIO_Type; -#endif /* MS_WINDOWS */ +#endif /* HAVE_WINDOWS_CONSOLE_IO */ /* These functions are used as METH_NOARGS methods, are normally called * with args=NULL, and return a new reference. @@ -178,7 +178,7 @@ find_io_state_by_def(PyTypeObject *type) extern _PyIO_State *_PyIO_get_module_state(void); -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO extern char _PyIO_get_console_type(PyObject *); #endif diff --git a/Modules/_io/clinic/winconsoleio.c.h b/Modules/_io/clinic/winconsoleio.c.h index df834dbde40f5b..4c5cd0892c4a6d 100644 --- a/Modules/_io/clinic/winconsoleio.c.h +++ b/Modules/_io/clinic/winconsoleio.c.h @@ -8,7 +8,7 @@ preserve #endif -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_close__doc__, "close($self, /)\n" @@ -31,9 +31,9 @@ _io__WindowsConsoleIO_close(winconsoleio *self, PyObject *Py_UNUSED(ignored)) return _io__WindowsConsoleIO_close_impl(self); } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO___init____doc__, "_WindowsConsoleIO(file, mode=\'r\', closefd=True, opener=None)\n" @@ -131,9 +131,9 @@ _io__WindowsConsoleIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) return return_value; } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_fileno__doc__, "fileno($self, /)\n" @@ -153,9 +153,9 @@ _io__WindowsConsoleIO_fileno(winconsoleio *self, PyObject *Py_UNUSED(ignored)) return _io__WindowsConsoleIO_fileno_impl(self); } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_readable__doc__, "readable($self, /)\n" @@ -175,9 +175,9 @@ _io__WindowsConsoleIO_readable(winconsoleio *self, PyObject *Py_UNUSED(ignored)) return _io__WindowsConsoleIO_readable_impl(self); } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_writable__doc__, "writable($self, /)\n" @@ -197,9 +197,9 @@ _io__WindowsConsoleIO_writable(winconsoleio *self, PyObject *Py_UNUSED(ignored)) return _io__WindowsConsoleIO_writable_impl(self); } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_readinto__doc__, "readinto($self, buffer, /)\n" @@ -239,9 +239,9 @@ _io__WindowsConsoleIO_readinto(winconsoleio *self, PyObject *arg) return return_value; } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_readall__doc__, "readall($self, /)\n" @@ -263,9 +263,9 @@ _io__WindowsConsoleIO_readall(winconsoleio *self, PyObject *Py_UNUSED(ignored)) return _io__WindowsConsoleIO_readall_impl(self); } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_read__doc__, "read($self, size=-1, /)\n" @@ -305,9 +305,9 @@ _io__WindowsConsoleIO_read(winconsoleio *self, PyObject *const *args, Py_ssize_t return return_value; } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_write__doc__, "write($self, b, /)\n" @@ -348,9 +348,9 @@ _io__WindowsConsoleIO_write(winconsoleio *self, PyObject *arg) return return_value; } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ -#if defined(MS_WINDOWS) +#if defined(HAVE_WINDOWS_CONSOLE_IO) PyDoc_STRVAR(_io__WindowsConsoleIO_isatty__doc__, "isatty($self, /)\n" @@ -370,7 +370,7 @@ _io__WindowsConsoleIO_isatty(winconsoleio *self, PyObject *Py_UNUSED(ignored)) return _io__WindowsConsoleIO_isatty_impl(self); } -#endif /* defined(MS_WINDOWS) */ +#endif /* defined(HAVE_WINDOWS_CONSOLE_IO) */ #ifndef _IO__WINDOWSCONSOLEIO_CLOSE_METHODDEF #define _IO__WINDOWSCONSOLEIO_CLOSE_METHODDEF @@ -407,4 +407,4 @@ _io__WindowsConsoleIO_isatty(winconsoleio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #define _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #endif /* !defined(_IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF) */ -/*[clinic end generated code: output=4920e9068e0cf08a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=163e934aa9b0ef16 input=a9049054013a1b77]*/ diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 35a498ce5a8354..1118d86e6c9a10 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -37,7 +37,9 @@ #ifdef MS_WINDOWS /* can simulate truncate with Win32 API functions; see file_truncate */ #define HAVE_FTRUNCATE +#ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN +#endif #include #endif diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index de07b50f5ce4cb..f836e230243020 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -11,7 +11,7 @@ #include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH #include "pycore_object.h" // _PyObject_GC_UNTRACK() -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO #include "structmember.h" // PyMemberDef #ifdef HAVE_SYS_TYPES_H @@ -22,7 +22,9 @@ #endif #include /* For offsetof */ +#ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN +#endif #include #include @@ -1177,4 +1179,4 @@ PyTypeObject PyWindowsConsoleIO_Type = { 0, /* tp_finalize */ }; -#endif /* MS_WINDOWS */ +#endif /* HAVE_WINDOWS_CONSOLE_IO */ diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index 23c38e14d997d1..96675cdfb661ad 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -35,7 +35,9 @@ This software comes with no warranty. Use at your own risk. #endif #if defined(MS_WINDOWS) +#ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN +#endif #include #endif @@ -457,12 +459,12 @@ _locale__getdefaultlocale_impl(PyObject *module) PyOS_snprintf(encoding, sizeof(encoding), "cp%u", GetACP()); - if (GetLocaleInfo(LOCALE_USER_DEFAULT, + if (GetLocaleInfoA(LOCALE_USER_DEFAULT, LOCALE_SISO639LANGNAME, locale, sizeof(locale))) { Py_ssize_t i = strlen(locale); locale[i++] = '_'; - if (GetLocaleInfo(LOCALE_USER_DEFAULT, + if (GetLocaleInfoA(LOCALE_USER_DEFAULT, LOCALE_SISO3166CTRYNAME, locale+i, (int)(sizeof(locale)-i))) return Py_BuildValue("ss", locale, encoding); @@ -474,7 +476,7 @@ _locale__getdefaultlocale_impl(PyObject *module) locale[0] = '0'; locale[1] = 'x'; - if (GetLocaleInfo(LOCALE_USER_DEFAULT, LOCALE_IDEFAULTLANGUAGE, + if (GetLocaleInfoA(LOCALE_USER_DEFAULT, LOCALE_IDEFAULTLANGUAGE, locale+2, sizeof(locale)-2)) { return Py_BuildValue("ss", locale, encoding); } diff --git a/Modules/_multiprocessing/multiprocessing.h b/Modules/_multiprocessing/multiprocessing.h index b595e5a8dd18de..dfc2a8e0799a60 100644 --- a/Modules/_multiprocessing/multiprocessing.h +++ b/Modules/_multiprocessing/multiprocessing.h @@ -12,7 +12,9 @@ */ #ifdef MS_WINDOWS -# define WIN32_LEAN_AND_MEAN +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif # include # include # include /* getpid() */ diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 1b34977806b661..a26732af8ba2a1 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -42,6 +42,12 @@ enum { #define FLOAT FLOAT_ #define INT INT_ #define LONG LONG_ + +/* This can already be defined on Windows to set the character set + the Windows header files treat as default */ +#ifdef UNICODE +#undef UNICODE +#endif #endif /* Pickle opcodes. These must be kept updated with pickle.py. diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 68060c07033d34..6e22053239305a 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -77,6 +77,10 @@ # include // getpid() #endif +#ifdef MS_WINDOWS +# include +#endif + /* Period parameters -- These are all magic. Don't change. */ #define N 624 #define M 397 @@ -259,7 +263,7 @@ random_seed_time_pid(RandomObject *self) key[0] = (uint32_t)(now & 0xffffffffU); key[1] = (uint32_t)(now >> 32); -#ifdef MS_WINDOWS_NON_DESKTOP +#if defined(MS_WINDOWS) && !defined(MS_WINDOWS_DESKTOP) && !defined(MS_WINDOWS_SYSTEM) key[2] = (uint32_t)GetCurrentProcessId(); #elif defined(HAVE_GETPID) key[2] = (uint32_t)getpid(); diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 8f03a846aed089..28112317bc289e 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -28,6 +28,10 @@ /* Include symbols from _socket module */ #include "socketmodule.h" +#ifdef MS_WINDOWS +# include +#endif + #include "_ssl.h" /* Redefined below for Windows debug builds after important #includes */ diff --git a/Modules/_winapi.c b/Modules/_winapi.c index eefc2571ee8287..83cde7501176b6 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -39,8 +39,11 @@ #include "structmember.h" // PyMemberDef +#ifndef WINDOWS_LEAN_AND_MEAN #define WINDOWS_LEAN_AND_MEAN +#endif #include "windows.h" +#include #include #include "winreparse.h" @@ -63,6 +66,14 @@ #define T_HANDLE T_POINTER +// winbase.h limits the STARTF_* flags to the desktop API as of 10.0.19041. +#ifndef STARTF_USESHOWWINDOW +#define STARTF_USESHOWWINDOW 0x00000001 +#endif +#ifndef STARTF_USESTDHANDLES +#define STARTF_USESTDHANDLES 0x00000100 +#endif + typedef struct { PyTypeObject *overlapped_type; } WinApiState; @@ -1201,8 +1212,10 @@ _winapi_ExitProcess_impl(PyObject *module, UINT ExitCode) /*[clinic end generated code: output=a387deb651175301 input=4f05466a9406c558]*/ { #if defined(Py_DEBUG) +#ifdef MS_WINDOWS_DESKTOP SetErrorMode(SEM_FAILCRITICALERRORS|SEM_NOALIGNMENTFAULTEXCEPT| SEM_NOGPFAULTERRORBOX|SEM_NOOPENFILEERRORBOX); +#endif _CrtSetReportMode(_CRT_ASSERT, _CRTDBG_MODE_DEBUG); #endif diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index dcd25c28370c93..6565f8df935cb8 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -10988,7 +10988,7 @@ os_getrandom(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #endif /* defined(HAVE_GETRANDOM_SYSCALL) */ -#if defined(MS_WINDOWS) +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM)) PyDoc_STRVAR(os__add_dll_directory__doc__, "_add_dll_directory($module, /, path)\n" @@ -11057,9 +11057,9 @@ os__add_dll_directory(PyObject *module, PyObject *const *args, Py_ssize_t nargs, return return_value; } -#endif /* defined(MS_WINDOWS) */ +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM)) */ -#if defined(MS_WINDOWS) +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM)) PyDoc_STRVAR(os__remove_dll_directory__doc__, "_remove_dll_directory($module, /, cookie)\n" @@ -11120,7 +11120,7 @@ os__remove_dll_directory(PyObject *module, PyObject *const *args, Py_ssize_t nar return return_value; } -#endif /* defined(MS_WINDOWS) */ +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM)) */ #if (defined(WIFEXITED) || defined(MS_WINDOWS)) @@ -11796,4 +11796,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=1b0eb6a76b1a0e28 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9495478e51701b8a input=a9049054013a1b77]*/ diff --git a/Modules/errnomodule.c b/Modules/errnomodule.c index 4de4144520aa48..df4e494ba8a973 100644 --- a/Modules/errnomodule.c +++ b/Modules/errnomodule.c @@ -5,7 +5,9 @@ /* Windows socket errors (WSA*) */ #ifdef MS_WINDOWS +#ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN +#endif #include /* The following constants were added to errno.h in VS2010 but have preferred WSA equivalents. */ diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 5309a3728c5e07..bfe35fed7a450a 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -953,7 +953,7 @@ faulthandler_unregister_py(PyObject *self, PyObject *args) static void faulthandler_suppress_crash_report(void) { -#ifdef MS_WINDOWS +#ifdef MS_WINDOWS_DESKTOP UINT mode; /* Configure Windows to not display the Windows Error Reporting dialog */ diff --git a/Modules/getpath.c b/Modules/getpath.c index c807a3c8e9a2b9..2f20521592ce2e 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -745,10 +745,12 @@ static int library_to_dict(PyObject *dict, const char *key) { #ifdef MS_WINDOWS +#ifdef Py_ENABLE_SHARED extern HMODULE PyWin_DLLhModule; if (PyWin_DLLhModule) { return winmodule_to_dict(dict, key, PyWin_DLLhModule); } +#endif #elif defined(WITH_NEXT_FRAMEWORK) static char modPath[MAXPATHLEN + 1]; static int modPathInitialized = -1; diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 6054c2853d7a78..fe76ca6eafaa88 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -29,6 +29,10 @@ #include "structmember.h" // PyMemberDef #include // offsetof() +// to support MS_WINDOWS_SYSTEM OpenFileMappingA / CreateFileMappingA +// need to be replaced with OpenFileMappingW / CreateFileMappingW +#if !defined(MS_WINDOWS) || defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_GAMES) + #ifndef MS_WINDOWS #define UNIX # ifdef HAVE_FCNTL_H @@ -647,7 +651,7 @@ mmap_flush_method(mmap_object *self, PyObject *args) if (self->access == ACCESS_READ || self->access == ACCESS_COPY) Py_RETURN_NONE; -#ifdef MS_WINDOWS +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM) if (!FlushViewOfFile(self->data+offset, size)) { PyErr_SetFromWindowsErr(GetLastError()); return NULL; @@ -1724,3 +1728,5 @@ PyInit_mmap(void) { return PyModuleDef_Init(&mmapmodule); } + +#endif /* !MS_WINDOWS || MS_WINDOWS_DESKTOP || MS_WINDOWS_GAMES */ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index a3d86cbe7a57fb..0d534f35f6a420 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -26,11 +26,15 @@ #ifdef MS_WINDOWS # include -# include +# if !defined(MS_WINDOWS_GAMES) || defined(MS_WINDOWS_DESKTOP) +# include +# endif # include # include // UNLEN # include "osdefs.h" // SEP -# define HAVE_SYMLINK +# if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) +# define HAVE_SYMLINK +# endif /* MS_WINDOWS_DESKTOP | MS_WINDOWS_SYSTEM */ #endif #include "structmember.h" // PyMemberDef @@ -311,7 +315,7 @@ corresponding Unix manual entries for more information on calls."); # include #endif -#if defined(MS_WINDOWS) +#ifdef HAVE_WINDOWS_CONSOLE_IO # define TERMSIZE_USE_CONIO #elif defined(HAVE_SYS_IOCTL_H) # include @@ -321,7 +325,7 @@ corresponding Unix manual entries for more information on calls."); # if defined(TIOCGWINSZ) # define TERMSIZE_USE_IOCTL # endif -#endif /* MS_WINDOWS */ +#endif /* HAVE_WINDOWS_CONSOLE_IO */ /* Various compilers have only certain posix functions */ /* XXX Gosh I wish these were all moved into pyconfig.h */ @@ -329,21 +333,25 @@ corresponding Unix manual entries for more information on calls."); # define HAVE_OPENDIR 1 # define HAVE_SYSTEM 1 # include -#else -# ifdef _MSC_VER - /* Microsoft compiler */ +#elif defined( _MSC_VER) + /* Microsoft compiler */ +# if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM) # define HAVE_GETPPID 1 +# endif /* MS_WINDOWS_DESKTOP | MS_WINDOWS_APP | MS_WINDOWS_SYSTEM */ +# if defined(MS_WINDOWS_DESKTOP) # define HAVE_GETLOGIN 1 +# endif /* MS_WINDOWS_DESKTOP */ +# if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) # define HAVE_SPAWNV 1 # define HAVE_EXECV 1 # define HAVE_WSPAWNV 1 # define HAVE_WEXECV 1 -# define HAVE_PIPE 1 # define HAVE_SYSTEM 1 # define HAVE_CWAIT 1 -# define HAVE_FSYNC 1 -# define fsync _commit -# endif /* _MSC_VER */ +# endif /* MS_WINDOWS_DESKTOP | MS_WINDOWS_SYSTEM */ +# define HAVE_PIPE 1 +# define HAVE_FSYNC 1 +# define fsync _commit #endif /* ! __WATCOMC__ || __QNX__ */ /*[clinic input] @@ -1536,7 +1544,7 @@ convertenviron(void) #ifdef MS_WINDOWS /* _wenviron must be initialized in this way if the program is started through main() instead of wmain(). */ - _wgetenv(L""); + (void)_wgetenv(L""); e = _wenviron; #elif defined(WITH_NEXT_FRAMEWORK) || (defined(__APPLE__) && defined(Py_ENABLE_SHARED)) /* environ is not accessible as an extern in a shared object on OSX; use @@ -1785,6 +1793,10 @@ attributes_from_dir(LPCWSTR pszFile, BY_HANDLE_FILE_INFORMATION *info, ULONG *re if (n && (pszFile[n - 1] == L'\\' || pszFile[n - 1] == L'/')) { // cannot use PyMem_Malloc here because we do not hold the GIL filename = (LPCWSTR)malloc((n + 1) * sizeof(filename[0])); + if(!filename) { + SetLastError(ERROR_NOT_ENOUGH_MEMORY); + return FALSE; + } wcsncpy_s((LPWSTR)filename, n + 1, pszFile, n); while (--n > 0 && (filename[n] == L'\\' || filename[n] == L'/')) { ((LPWSTR)filename)[n] = L'\0'; @@ -7933,10 +7945,10 @@ static PyObject * os_getpid_impl(PyObject *module) /*[clinic end generated code: output=9ea6fdac01ed2b3c input=5a9a00f0ab68aa00]*/ { -#ifdef MS_WINDOWS_NON_DESKTOP - return PyLong_FromUnsignedLong(GetCurrentProcessId()); -#else +#if !defined(MS_WINDOWS) || defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) return PyLong_FromPid(getpid()); +#else + return PyLong_FromUnsignedLong(GetCurrentProcessId()); #endif } #endif /* defined(HAVE_GETPID) */ @@ -8392,6 +8404,7 @@ os_kill_impl(PyObject *module, pid_t pid, Py_ssize_t signal) DWORD err; HANDLE handle; +#ifdef HAVE_WINDOWS_CONSOLE_IO /* Console processes which share a common console can be sent CTRL+C or CTRL+BREAK events, provided they handle said events. */ if (sig == CTRL_C_EVENT || sig == CTRL_BREAK_EVENT) { @@ -8399,9 +8412,11 @@ os_kill_impl(PyObject *module, pid_t pid, Py_ssize_t signal) err = GetLastError(); PyErr_SetFromWindowsErr(err); } - else + else { Py_RETURN_NONE; + } } +#endif /* HAVE_WINDOWS_CONSOLE_IO */ /* If the signal is outside of what GenerateConsoleCtrlEvent can use, attempt to open and terminate the process. */ @@ -13776,7 +13791,9 @@ os_cpu_count_impl(PyObject *module) { int ncpu = 0; #ifdef MS_WINDOWS +#ifdef MS_WINDOWS_DESKTOP ncpu = GetActiveProcessorCount(ALL_PROCESSOR_GROUPS); +#endif #elif defined(__hpux) ncpu = mpctl(MPC_GETNUMSPUS, NULL, NULL); #elif defined(HAVE_SYSCONF) && defined(_SC_NPROCESSORS_ONLN) @@ -13848,6 +13865,10 @@ os_set_inheritable_impl(PyObject *module, int fd, int inheritable) #ifdef MS_WINDOWS +#ifndef HANDLE_FLAG_INHERIT +#define HANDLE_FLAG_INHERIT 0x00000001 +#endif + /*[clinic input] os.get_handle_inheritable -> bool handle: intptr_t @@ -15023,7 +15044,8 @@ os_getrandom_impl(PyObject *module, Py_ssize_t size, int flags) } #endif /* HAVE_GETRANDOM_SYSCALL */ -#ifdef MS_WINDOWS +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM) + /* bpo-36085: Helper functions for managing DLL search directories * on win32 */ @@ -15114,7 +15136,7 @@ os__remove_dll_directory_impl(PyObject *module, PyObject *cookie) Py_RETURN_NONE; } -#endif +#endif /* MS_WINDOWS_APP || MS_WINDOWS_SYSTEM */ /* Only check if WIFEXITED is available: expect that it comes diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index df4043de08dacc..5a1e40d0b4a482 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -57,8 +57,10 @@ extern void bzero(void *, int); #endif #ifdef MS_WINDOWS -# define WIN32_LEAN_AND_MEAN -# include +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# include #else # define SOCKET int #endif diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 43a0cc0f963f9d..b7927750e334b7 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -270,7 +270,7 @@ shutdown(how) -- shut down traffic in one or both directions\n\ # include -#else +#else /* MS_WINDOWS */ /* MS_WINDOWS includes */ # ifdef HAVE_FCNTL_H @@ -281,7 +281,6 @@ shutdown(how) -- shut down traffic in one or both directions\n\ # include /* Macros based on the IPPROTO enum, see: https://bugs.python.org/issue29515 */ -#ifdef MS_WINDOWS #define IPPROTO_ICMP IPPROTO_ICMP #define IPPROTO_IGMP IPPROTO_IGMP #define IPPROTO_GGP IPPROTO_GGP @@ -312,7 +311,6 @@ shutdown(how) -- shut down traffic in one or both directions\n\ #define IPPROTO_PGM IPPROTO_PGM // WinSock2 only #define IPPROTO_L2TP IPPROTO_L2TP // WinSock2 only #define IPPROTO_SCTP IPPROTO_SCTP // WinSock2 only -#endif /* MS_WINDOWS */ /* Provides the IsWindows7SP1OrGreater() function */ #include @@ -348,13 +346,18 @@ remove_unusable_flags(PyObject *m) { PyObject *dict; OSVERSIONINFOEX info; - DWORDLONG dwlConditionMask; dict = PyModule_GetDict(m); if (dict == NULL) { return -1; } - +#ifndef MS_WINDOWS_DESKTOP + info.dwOSVersionInfoSize = sizeof(info); + if (!GetVersionExW((OSVERSIONINFOW*) &info)) { + PyErr_SetFromWindowsErr(0); + return -1; + } +#else /* set to Windows 10, except BuildNumber. */ memset(&info, 0, sizeof(info)); info.dwOSVersionInfoSize = sizeof(info); @@ -362,19 +365,30 @@ remove_unusable_flags(PyObject *m) info.dwMinorVersion = 0; /* set Condition Mask */ - dwlConditionMask = 0; + DWORDLONG dwlConditionMask = 0; VER_SET_CONDITION(dwlConditionMask, VER_MAJORVERSION, VER_GREATER_EQUAL); VER_SET_CONDITION(dwlConditionMask, VER_MINORVERSION, VER_GREATER_EQUAL); VER_SET_CONDITION(dwlConditionMask, VER_BUILDNUMBER, VER_GREATER_EQUAL); +#endif for (int i=0; i 10 || + (info.dwMajorVersion == 10 && info.dwMinorVersion > 0) || + (info.dwMajorVersion == 10 && info.dwMinorVersion == 0 && + info.dwBuildNumber >= win_runtime_flags[i].build_number); +#endif + if (isSupported) { break; } else { @@ -497,14 +511,14 @@ remove_unusable_flags(PyObject *m) #endif #endif -#ifdef MS_WINDOWS +#ifdef MS_WINDOWS_DESKTOP #define sockaddr_rc SOCKADDR_BTH_REDEF #define USE_BLUETOOTH 1 #define AF_BLUETOOTH AF_BTH #define BTPROTO_RFCOMM BTHPROTO_RFCOMM #define _BT_RC_MEMB(sa, memb) ((sa)->memb) -#endif +#endif /* MS_WINDOWS_DESKTOP */ /* Convert "sock_addr_t *" to "struct sockaddr *". */ #define SAS2SA(x) (&((x)->sa)) @@ -2869,11 +2883,16 @@ sock_accept(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) newfd = ctx.result; #ifdef MS_WINDOWS +#if defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) +#ifndef HANDLE_FLAG_INHERIT +#define HANDLE_FLAG_INHERIT 0x00000001 +#endif if (!SetHandleInformation((HANDLE)newfd, HANDLE_FLAG_INHERIT, 0)) { PyErr_SetFromWindowsErr(0); SOCKETCLOSE(newfd); goto finally; } +#endif #else #if defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) @@ -5434,11 +5453,6 @@ sock_initobj_impl(PySocketSockObject *self, int family, int type, int proto, proto = 0; } #ifdef MS_WINDOWS - /* Windows implementation */ -#ifndef WSA_FLAG_NO_HANDLE_INHERIT -#define WSA_FLAG_NO_HANDLE_INHERIT 0x80 -#endif - Py_BEGIN_ALLOW_THREADS fd = WSASocketW(family, type, proto, NULL, 0, @@ -6150,8 +6164,9 @@ socket_dup(PyObject *self, PyObject *fdobj) #endif fd = PyLong_AsSocket_t(fdobj); - if (fd == (SOCKET_T)(-1) && PyErr_Occurred()) + if (fd == (SOCKET_T)(-1) && PyErr_Occurred()) { return NULL; + } #ifdef MS_WINDOWS if (WSADuplicateSocketW(fd, GetCurrentProcessId(), &info)) @@ -6160,8 +6175,9 @@ socket_dup(PyObject *self, PyObject *fdobj) newfd = WSASocketW(FROM_PROTOCOL_INFO, FROM_PROTOCOL_INFO, FROM_PROTOCOL_INFO, &info, 0, WSA_FLAG_OVERLAPPED); - if (newfd == INVALID_SOCKET) + if (newfd == INVALID_SOCKET) { return set_error(); + } if (!SetHandleInformation((HANDLE)newfd, HANDLE_FLAG_INHERIT, 0)) { closesocket(newfd); @@ -6171,13 +6187,15 @@ socket_dup(PyObject *self, PyObject *fdobj) #else /* On UNIX, dup can be used to duplicate the file descriptor of a socket */ newfd = _Py_dup(fd); - if (newfd == INVALID_SOCKET) + if (newfd == INVALID_SOCKET) { return NULL; + } #endif newfdobj = PyLong_FromSocket_t(newfd); - if (newfdobj == NULL) + if (newfdobj == NULL) { SOCKETCLOSE(newfd); + } return newfdobj; } diff --git a/Modules/timemodule.c b/Modules/timemodule.c index c2bacaae0c0339..c50e689bb6986c 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -30,7 +30,9 @@ # include #else # ifdef MS_WINDOWS -# define WIN32_LEAN_AND_MEAN +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif # include # endif /* MS_WINDOWS */ #endif /* !__WATCOMC__ || __QNX__ */ @@ -1135,7 +1137,9 @@ time_tzset(PyObject *self, PyObject *unused) return NULL; } +#if !defined(MS_WINDOWS) || defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) tzset(); +#endif /* Reset timezone, altzone, daylight and tzname */ if (init_timezone(m) < 0) { @@ -1753,7 +1757,9 @@ init_timezone(PyObject *m) */ #ifdef HAVE_DECL_TZNAME PyObject *otz0, *otz1; +#if !defined(MS_WINDOWS) || defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) tzset(); +#endif PyModule_AddIntConstant(m, "timezone", _Py_timezone); #ifdef HAVE_ALTZONE PyModule_AddIntConstant(m, "altzone", altzone); diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 276c5a276c06e6..5e1bcda1d976bb 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -8,7 +8,6 @@ #include // malloc() #include - #undef uint #define uint pymem_uint diff --git a/PC/clinic/msvcrtmodule.c.h b/PC/clinic/msvcrtmodule.c.h index d808ef0bbd0ffe..b708c6cdde757c 100644 --- a/PC/clinic/msvcrtmodule.c.h +++ b/PC/clinic/msvcrtmodule.c.h @@ -261,6 +261,8 @@ msvcrt_getch(PyObject *module, PyObject *Py_UNUSED(ignored)) return return_value; } +#if defined(MS_WINDOWS_DESKTOP) + PyDoc_STRVAR(msvcrt_getwch__doc__, "getwch($module, /)\n" "--\n" @@ -285,6 +287,8 @@ msvcrt_getwch(PyObject *module, PyObject *Py_UNUSED(ignored)) return return_value; } +#endif /* defined(MS_WINDOWS_DESKTOP) */ + PyDoc_STRVAR(msvcrt_getche__doc__, "getche($module, /)\n" "--\n" @@ -309,6 +313,8 @@ msvcrt_getche(PyObject *module, PyObject *Py_UNUSED(ignored)) return return_value; } +#if defined(MS_WINDOWS_DESKTOP) + PyDoc_STRVAR(msvcrt_getwche__doc__, "getwche($module, /)\n" "--\n" @@ -333,6 +339,8 @@ msvcrt_getwche(PyObject *module, PyObject *Py_UNUSED(ignored)) return return_value; } +#endif /* defined(MS_WINDOWS_DESKTOP) */ + PyDoc_STRVAR(msvcrt_putch__doc__, "putch($module, char, /)\n" "--\n" @@ -367,6 +375,8 @@ msvcrt_putch(PyObject *module, PyObject *arg) return return_value; } +#if defined(MS_WINDOWS_DESKTOP) + PyDoc_STRVAR(msvcrt_putwch__doc__, "putwch($module, unicode_char, /)\n" "--\n" @@ -403,6 +413,8 @@ msvcrt_putwch(PyObject *module, PyObject *arg) return return_value; } +#endif /* defined(MS_WINDOWS_DESKTOP) */ + PyDoc_STRVAR(msvcrt_ungetch__doc__, "ungetch($module, char, /)\n" "--\n" @@ -441,6 +453,8 @@ msvcrt_ungetch(PyObject *module, PyObject *arg) return return_value; } +#if defined(MS_WINDOWS_DESKTOP) + PyDoc_STRVAR(msvcrt_ungetwch__doc__, "ungetwch($module, unicode_char, /)\n" "--\n" @@ -477,6 +491,8 @@ msvcrt_ungetwch(PyObject *module, PyObject *arg) return return_value; } +#endif /* defined(MS_WINDOWS_DESKTOP) */ + #if defined(_DEBUG) PyDoc_STRVAR(msvcrt_CrtSetReportFile__doc__, @@ -610,6 +626,8 @@ msvcrt_set_error_mode(PyObject *module, PyObject *arg) #endif /* defined(_DEBUG) */ +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM)) + PyDoc_STRVAR(msvcrt_GetErrorMode__doc__, "GetErrorMode($module, /)\n" "--\n" @@ -628,6 +646,8 @@ msvcrt_GetErrorMode(PyObject *module, PyObject *Py_UNUSED(ignored)) return msvcrt_GetErrorMode_impl(module); } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM)) */ + PyDoc_STRVAR(msvcrt_SetErrorMode__doc__, "SetErrorMode($module, mode, /)\n" "--\n" @@ -656,6 +676,22 @@ msvcrt_SetErrorMode(PyObject *module, PyObject *arg) return return_value; } +#ifndef MSVCRT_GETWCH_METHODDEF + #define MSVCRT_GETWCH_METHODDEF +#endif /* !defined(MSVCRT_GETWCH_METHODDEF) */ + +#ifndef MSVCRT_GETWCHE_METHODDEF + #define MSVCRT_GETWCHE_METHODDEF +#endif /* !defined(MSVCRT_GETWCHE_METHODDEF) */ + +#ifndef MSVCRT_PUTWCH_METHODDEF + #define MSVCRT_PUTWCH_METHODDEF +#endif /* !defined(MSVCRT_PUTWCH_METHODDEF) */ + +#ifndef MSVCRT_UNGETWCH_METHODDEF + #define MSVCRT_UNGETWCH_METHODDEF +#endif /* !defined(MSVCRT_UNGETWCH_METHODDEF) */ + #ifndef MSVCRT_CRTSETREPORTFILE_METHODDEF #define MSVCRT_CRTSETREPORTFILE_METHODDEF #endif /* !defined(MSVCRT_CRTSETREPORTFILE_METHODDEF) */ @@ -667,4 +703,8 @@ msvcrt_SetErrorMode(PyObject *module, PyObject *arg) #ifndef MSVCRT_SET_ERROR_MODE_METHODDEF #define MSVCRT_SET_ERROR_MODE_METHODDEF #endif /* !defined(MSVCRT_SET_ERROR_MODE_METHODDEF) */ -/*[clinic end generated code: output=204bae9fee7f6124 input=a9049054013a1b77]*/ + +#ifndef MSVCRT_GETERRORMODE_METHODDEF + #define MSVCRT_GETERRORMODE_METHODDEF +#endif /* !defined(MSVCRT_GETERRORMODE_METHODDEF) */ +/*[clinic end generated code: output=2db6197608a6aab3 input=a9049054013a1b77]*/ diff --git a/PC/clinic/winreg.c.h b/PC/clinic/winreg.c.h index 2834d9967a7726..7a9474301da8a1 100644 --- a/PC/clinic/winreg.c.h +++ b/PC/clinic/winreg.c.h @@ -8,6 +8,8 @@ preserve #endif +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_HKEYType_Close__doc__, "Close($self, /)\n" "--\n" @@ -28,6 +30,10 @@ winreg_HKEYType_Close(PyHKEYObject *self, PyObject *Py_UNUSED(ignored)) return winreg_HKEYType_Close_impl(self); } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_HKEYType_Detach__doc__, "Detach($self, /)\n" "--\n" @@ -54,6 +60,10 @@ winreg_HKEYType_Detach(PyHKEYObject *self, PyObject *Py_UNUSED(ignored)) return winreg_HKEYType_Detach_impl(self); } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_HKEYType___enter____doc__, "__enter__($self, /)\n" "--\n" @@ -77,6 +87,10 @@ winreg_HKEYType___enter__(PyHKEYObject *self, PyObject *Py_UNUSED(ignored)) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_HKEYType___exit____doc__, "__exit__($self, /, exc_type, exc_value, traceback)\n" "--\n" @@ -136,6 +150,10 @@ winreg_HKEYType___exit__(PyHKEYObject *self, PyObject *const *args, Py_ssize_t n return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_CloseKey__doc__, "CloseKey($module, hkey, /)\n" "--\n" @@ -151,6 +169,10 @@ PyDoc_STRVAR(winreg_CloseKey__doc__, #define WINREG_CLOSEKEY_METHODDEF \ {"CloseKey", (PyCFunction)winreg_CloseKey, METH_O, winreg_CloseKey__doc__}, +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) + PyDoc_STRVAR(winreg_ConnectRegistry__doc__, "ConnectRegistry($module, computer_name, key, /)\n" "--\n" @@ -213,6 +235,10 @@ winreg_ConnectRegistry(PyObject *module, PyObject *const *args, Py_ssize_t nargs return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_CreateKey__doc__, "CreateKey($module, key, sub_key, /)\n" "--\n" @@ -278,6 +304,10 @@ winreg_CreateKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_CreateKeyEx__doc__, "CreateKeyEx($module, /, key, sub_key, reserved=0,\n" " access=winreg.KEY_WRITE)\n" @@ -398,6 +428,10 @@ winreg_CreateKeyEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_DeleteKey__doc__, "DeleteKey($module, key, sub_key, /)\n" "--\n" @@ -452,6 +486,10 @@ winreg_DeleteKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_DeleteKeyEx__doc__, "DeleteKeyEx($module, /, key, sub_key, access=winreg.KEY_WOW64_64KEY,\n" " reserved=0)\n" @@ -565,6 +603,10 @@ winreg_DeleteKeyEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_DeleteValue__doc__, "DeleteValue($module, key, value, /)\n" "--\n" @@ -617,6 +659,10 @@ winreg_DeleteValue(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_EnumKey__doc__, "EnumKey($module, key, index, /)\n" "--\n" @@ -661,6 +707,10 @@ winreg_EnumKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_EnumValue__doc__, "EnumValue($module, key, index, /)\n" "--\n" @@ -714,6 +764,10 @@ winreg_EnumValue(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_ExpandEnvironmentStrings__doc__, "ExpandEnvironmentStrings($module, string, /)\n" "--\n" @@ -750,6 +804,10 @@ winreg_ExpandEnvironmentStrings(PyObject *module, PyObject *arg) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) + PyDoc_STRVAR(winreg_FlushKey__doc__, "FlushKey($module, key, /)\n" "--\n" @@ -790,6 +848,10 @@ winreg_FlushKey(PyObject *module, PyObject *arg) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) + PyDoc_STRVAR(winreg_LoadKey__doc__, "LoadKey($module, key, sub_key, file_name, /)\n" "--\n" @@ -866,6 +928,10 @@ winreg_LoadKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_OpenKey__doc__, "OpenKey($module, /, key, sub_key, reserved=0, access=winreg.KEY_READ)\n" "--\n" @@ -979,6 +1045,10 @@ winreg_OpenKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_OpenKeyEx__doc__, "OpenKeyEx($module, /, key, sub_key, reserved=0, access=winreg.KEY_READ)\n" "--\n" @@ -1092,6 +1162,10 @@ winreg_OpenKeyEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_QueryInfoKey__doc__, "QueryInfoKey($module, key, /)\n" "--\n" @@ -1128,6 +1202,10 @@ winreg_QueryInfoKey(PyObject *module, PyObject *arg) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_QueryValue__doc__, "QueryValue($module, key, sub_key, /)\n" "--\n" @@ -1189,6 +1267,10 @@ winreg_QueryValue(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_QueryValueEx__doc__, "QueryValueEx($module, key, name, /)\n" "--\n" @@ -1246,6 +1328,10 @@ winreg_QueryValueEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) + PyDoc_STRVAR(winreg_SaveKey__doc__, "SaveKey($module, key, file_name, /)\n" "--\n" @@ -1303,6 +1389,10 @@ winreg_SaveKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_SetValue__doc__, "SetValue($module, key, sub_key, type, value, /)\n" "--\n" @@ -1384,6 +1474,10 @@ winreg_SetValue(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) + PyDoc_STRVAR(winreg_SetValueEx__doc__, "SetValueEx($module, key, value_name, reserved, type, value, /)\n" "--\n" @@ -1478,6 +1572,10 @@ winreg_SetValueEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) + PyDoc_STRVAR(winreg_DisableReflectionKey__doc__, "DisableReflectionKey($module, key, /)\n" "--\n" @@ -1514,6 +1612,10 @@ winreg_DisableReflectionKey(PyObject *module, PyObject *arg) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) + PyDoc_STRVAR(winreg_EnableReflectionKey__doc__, "EnableReflectionKey($module, key, /)\n" "--\n" @@ -1548,6 +1650,10 @@ winreg_EnableReflectionKey(PyObject *module, PyObject *arg) return return_value; } +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) */ + +#if (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) + PyDoc_STRVAR(winreg_QueryReflectionKey__doc__, "QueryReflectionKey($module, key, /)\n" "--\n" @@ -1579,4 +1685,114 @@ winreg_QueryReflectionKey(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=7e817dc5edc914d3 input=a9049054013a1b77]*/ + +#endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) && (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM)) */ + +#ifndef WINREG_HKEYTYPE_CLOSE_METHODDEF + #define WINREG_HKEYTYPE_CLOSE_METHODDEF +#endif /* !defined(WINREG_HKEYTYPE_CLOSE_METHODDEF) */ + +#ifndef WINREG_HKEYTYPE_DETACH_METHODDEF + #define WINREG_HKEYTYPE_DETACH_METHODDEF +#endif /* !defined(WINREG_HKEYTYPE_DETACH_METHODDEF) */ + +#ifndef WINREG_HKEYTYPE___ENTER___METHODDEF + #define WINREG_HKEYTYPE___ENTER___METHODDEF +#endif /* !defined(WINREG_HKEYTYPE___ENTER___METHODDEF) */ + +#ifndef WINREG_HKEYTYPE___EXIT___METHODDEF + #define WINREG_HKEYTYPE___EXIT___METHODDEF +#endif /* !defined(WINREG_HKEYTYPE___EXIT___METHODDEF) */ + +#ifndef WINREG_CLOSEKEY_METHODDEF + #define WINREG_CLOSEKEY_METHODDEF +#endif /* !defined(WINREG_CLOSEKEY_METHODDEF) */ + +#ifndef WINREG_CONNECTREGISTRY_METHODDEF + #define WINREG_CONNECTREGISTRY_METHODDEF +#endif /* !defined(WINREG_CONNECTREGISTRY_METHODDEF) */ + +#ifndef WINREG_CREATEKEY_METHODDEF + #define WINREG_CREATEKEY_METHODDEF +#endif /* !defined(WINREG_CREATEKEY_METHODDEF) */ + +#ifndef WINREG_CREATEKEYEX_METHODDEF + #define WINREG_CREATEKEYEX_METHODDEF +#endif /* !defined(WINREG_CREATEKEYEX_METHODDEF) */ + +#ifndef WINREG_DELETEKEY_METHODDEF + #define WINREG_DELETEKEY_METHODDEF +#endif /* !defined(WINREG_DELETEKEY_METHODDEF) */ + +#ifndef WINREG_DELETEKEYEX_METHODDEF + #define WINREG_DELETEKEYEX_METHODDEF +#endif /* !defined(WINREG_DELETEKEYEX_METHODDEF) */ + +#ifndef WINREG_DELETEVALUE_METHODDEF + #define WINREG_DELETEVALUE_METHODDEF +#endif /* !defined(WINREG_DELETEVALUE_METHODDEF) */ + +#ifndef WINREG_ENUMKEY_METHODDEF + #define WINREG_ENUMKEY_METHODDEF +#endif /* !defined(WINREG_ENUMKEY_METHODDEF) */ + +#ifndef WINREG_ENUMVALUE_METHODDEF + #define WINREG_ENUMVALUE_METHODDEF +#endif /* !defined(WINREG_ENUMVALUE_METHODDEF) */ + +#ifndef WINREG_EXPANDENVIRONMENTSTRINGS_METHODDEF + #define WINREG_EXPANDENVIRONMENTSTRINGS_METHODDEF +#endif /* !defined(WINREG_EXPANDENVIRONMENTSTRINGS_METHODDEF) */ + +#ifndef WINREG_FLUSHKEY_METHODDEF + #define WINREG_FLUSHKEY_METHODDEF +#endif /* !defined(WINREG_FLUSHKEY_METHODDEF) */ + +#ifndef WINREG_LOADKEY_METHODDEF + #define WINREG_LOADKEY_METHODDEF +#endif /* !defined(WINREG_LOADKEY_METHODDEF) */ + +#ifndef WINREG_OPENKEY_METHODDEF + #define WINREG_OPENKEY_METHODDEF +#endif /* !defined(WINREG_OPENKEY_METHODDEF) */ + +#ifndef WINREG_OPENKEYEX_METHODDEF + #define WINREG_OPENKEYEX_METHODDEF +#endif /* !defined(WINREG_OPENKEYEX_METHODDEF) */ + +#ifndef WINREG_QUERYINFOKEY_METHODDEF + #define WINREG_QUERYINFOKEY_METHODDEF +#endif /* !defined(WINREG_QUERYINFOKEY_METHODDEF) */ + +#ifndef WINREG_QUERYVALUE_METHODDEF + #define WINREG_QUERYVALUE_METHODDEF +#endif /* !defined(WINREG_QUERYVALUE_METHODDEF) */ + +#ifndef WINREG_QUERYVALUEEX_METHODDEF + #define WINREG_QUERYVALUEEX_METHODDEF +#endif /* !defined(WINREG_QUERYVALUEEX_METHODDEF) */ + +#ifndef WINREG_SAVEKEY_METHODDEF + #define WINREG_SAVEKEY_METHODDEF +#endif /* !defined(WINREG_SAVEKEY_METHODDEF) */ + +#ifndef WINREG_SETVALUE_METHODDEF + #define WINREG_SETVALUE_METHODDEF +#endif /* !defined(WINREG_SETVALUE_METHODDEF) */ + +#ifndef WINREG_SETVALUEEX_METHODDEF + #define WINREG_SETVALUEEX_METHODDEF +#endif /* !defined(WINREG_SETVALUEEX_METHODDEF) */ + +#ifndef WINREG_DISABLEREFLECTIONKEY_METHODDEF + #define WINREG_DISABLEREFLECTIONKEY_METHODDEF +#endif /* !defined(WINREG_DISABLEREFLECTIONKEY_METHODDEF) */ + +#ifndef WINREG_ENABLEREFLECTIONKEY_METHODDEF + #define WINREG_ENABLEREFLECTIONKEY_METHODDEF +#endif /* !defined(WINREG_ENABLEREFLECTIONKEY_METHODDEF) */ + +#ifndef WINREG_QUERYREFLECTIONKEY_METHODDEF + #define WINREG_QUERYREFLECTIONKEY_METHODDEF +#endif /* !defined(WINREG_QUERYREFLECTIONKEY_METHODDEF) */ +/*[clinic end generated code: output=715db416dc1321ee input=a9049054013a1b77]*/ diff --git a/PC/config.c b/PC/config.c index b1481d79e6508d..9d0fe6f87df69a 100644 --- a/PC/config.c +++ b/PC/config.c @@ -43,10 +43,14 @@ extern PyObject* PyInit__collections(void); extern PyObject* PyInit__heapq(void); extern PyObject* PyInit__bisect(void); extern PyObject* PyInit__symtable(void); +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_GAMES) extern PyObject* PyInit_mmap(void); +#endif extern PyObject* PyInit__csv(void); extern PyObject* PyInit__sre(void); +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES) extern PyObject* PyInit_winreg(void); +#endif extern PyObject* PyInit__struct(void); extern PyObject* PyInit__datetime(void); extern PyObject* PyInit__functools(void); @@ -122,10 +126,14 @@ struct _inittab _PyImport_Inittab[] = { {"itertools", PyInit_itertools}, {"_collections", PyInit__collections}, {"_symtable", PyInit__symtable}, +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_GAMES) {"mmap", PyInit_mmap}, +#endif {"_csv", PyInit__csv}, {"_sre", PyInit__sre}, +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES) {"winreg", PyInit_winreg}, +#endif {"_struct", PyInit__struct}, {"_datetime", PyInit__datetime}, {"_functools", PyInit__functools}, diff --git a/PC/config_minimal.c b/PC/config_minimal.c index 928a4efd32e132..9a66ea1d1cd348 100644 --- a/PC/config_minimal.c +++ b/PC/config_minimal.c @@ -5,8 +5,10 @@ #include "Python.h" +#ifdef Py_ENABLE_SHARED /* Define extern variables omitted from minimal builds */ void *PyWin_DLLhModule = NULL; +#endif extern PyObject* PyInit_faulthandler(void); @@ -14,7 +16,9 @@ extern PyObject* PyInit__tracemalloc(void); extern PyObject* PyInit_gc(void); extern PyObject* PyInit_nt(void); extern PyObject* PyInit__signal(void); +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES) extern PyObject* PyInit_winreg(void); +#endif extern PyObject* PyInit__ast(void); extern PyObject* PyInit__io(void); @@ -35,7 +39,9 @@ struct _inittab _PyImport_Inittab[] = { {"_tokenize", PyInit__tokenize}, {"_tracemalloc", PyInit__tracemalloc}, +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES) {"winreg", PyInit_winreg}, +#endif /* This module "lives in" with marshal.c */ {"marshal", PyMarshal_Init}, diff --git a/PC/msvcrtmodule.c b/PC/msvcrtmodule.c index 988d9c95aaa22e..face4d03af9d4f 100644 --- a/PC/msvcrtmodule.c +++ b/PC/msvcrtmodule.c @@ -253,6 +253,8 @@ msvcrt_getch_impl(PyObject *module) return ch; } +#ifdef MS_WINDOWS_DESKTOP + /*[clinic input] msvcrt.getwch -> wchar_t @@ -271,6 +273,8 @@ msvcrt_getwch_impl(PyObject *module) return ch; } +#endif /* MS_WINDOWS_DESKTOP */ + /*[clinic input] msvcrt.getche -> byte_char @@ -289,6 +293,8 @@ msvcrt_getche_impl(PyObject *module) return ch; } +#ifdef MS_WINDOWS_DESKTOP + /*[clinic input] msvcrt.getwche -> wchar_t @@ -307,6 +313,8 @@ msvcrt_getwche_impl(PyObject *module) return ch; } +#endif /* MS_WINDOWS_DESKTOP */ + /*[clinic input] msvcrt.putch @@ -326,6 +334,8 @@ msvcrt_putch_impl(PyObject *module, char char_value) Py_RETURN_NONE; } +#ifdef MS_WINDOWS_DESKTOP + /*[clinic input] msvcrt.putwch @@ -346,6 +356,8 @@ msvcrt_putwch_impl(PyObject *module, int unicode_char) } +#endif /* MS_WINDOWS_DESKTOP */ + /*[clinic input] msvcrt.ungetch @@ -374,6 +386,8 @@ msvcrt_ungetch_impl(PyObject *module, char char_value) Py_RETURN_NONE; } +#ifdef MS_WINDOWS_DESKTOP + /*[clinic input] msvcrt.ungetwch @@ -398,6 +412,8 @@ msvcrt_ungetwch_impl(PyObject *module, int unicode_char) Py_RETURN_NONE; } +#endif /* MS_WINDOWS_DESKTOP */ + #ifdef _DEBUG /*[clinic input] msvcrt.CrtSetReportFile -> HANDLE @@ -475,6 +491,8 @@ msvcrt_set_error_mode_impl(PyObject *module, int mode) } #endif /* _DEBUG */ +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM) + /*[clinic input] msvcrt.GetErrorMode @@ -494,6 +512,8 @@ msvcrt_GetErrorMode_impl(PyObject *module) return PyLong_FromUnsignedLong(res); } +#endif /* MS_WINDOWS_APP || MS_WINDOWS_SYSTEM */ + /*[clinic input] msvcrt.SetErrorMode @@ -601,10 +621,12 @@ PyInit_msvcrt(void) insertint(d, "LK_NBRLCK", _LK_NBRLCK); insertint(d, "LK_RLCK", _LK_RLCK); insertint(d, "LK_UNLCK", _LK_UNLCK); +#ifdef MS_WINDOWS_DESKTOP insertint(d, "SEM_FAILCRITICALERRORS", SEM_FAILCRITICALERRORS); insertint(d, "SEM_NOALIGNMENTFAULTEXCEPT", SEM_NOALIGNMENTFAULTEXCEPT); insertint(d, "SEM_NOGPFAULTERRORBOX", SEM_NOGPFAULTERRORBOX); insertint(d, "SEM_NOOPENFILEERRORBOX", SEM_NOOPENFILEERRORBOX); +#endif #ifdef _DEBUG insertint(d, "CRT_WARN", _CRT_WARN); insertint(d, "CRT_ERROR", _CRT_ERROR); diff --git a/PC/pyconfig.h b/PC/pyconfig.h index a34d420ab7ecaa..8a3bf8968ce29d 100644 --- a/PC/pyconfig.h +++ b/PC/pyconfig.h @@ -72,9 +72,27 @@ WIN32 is still required for the locale module. #define USE_SOCKET #endif -#if defined(WINAPI_FAMILY) && (WINAPI_FAMILY != WINAPI_FAMILY_DESKTOP_APP) -#define MS_WINDOWS_NON_DESKTOP +#if defined(Py_BUILD_CORE) || defined(Py_BUILD_CORE_BUILTIN) || defined(Py_BUILD_CORE_MODULE) +#include + +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) +#define MS_WINDOWS_DESKTOP +#endif +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP) +#define MS_WINDOWS_APP +#endif +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_SYSTEM) +#define MS_WINDOWS_SYSTEM #endif +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_GAMES) +#define MS_WINDOWS_GAMES +#endif + +/* Define to 1 if you support windows console io */ +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_APP) || defined(MS_WINDOWS_SYSTEM) +#define HAVE_WINDOWS_CONSOLE_IO 1 +#endif +#endif /* Py_BUILD_CORE || Py_BUILD_CORE_BUILTIN || Py_BUILD_CORE_MODULE */ /* Compiler specific defines */ @@ -300,7 +318,7 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ # endif /* Py_BUILD_CORE */ #endif /* MS_COREDLL */ -#if defined(MS_WIN64) +#ifdef MS_WIN64 /* maintain "win32" sys.platform for backward compatibility of Python code, the Win64 API should be close enough to the Win32 API to make this preferable */ diff --git a/PC/winreg.c b/PC/winreg.c index 86efed09855b01..073598a12a68aa 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -18,6 +18,8 @@ #include "structmember.h" // PyMemberDef #include +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES) + static BOOL PyHKEY_AsHKEY(PyObject *ob, HKEY *pRes, BOOL bNoneOK); static BOOL clinic_HKEY_converter(PyObject *ob, void *p); static PyObject *PyHKEY_FromHKEY(HKEY h); @@ -829,6 +831,8 @@ winreg_CloseKey(PyObject *module, PyObject *hkey) Py_RETURN_NONE; } +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) + /*[clinic input] winreg.ConnectRegistry -> HKEY @@ -866,6 +870,8 @@ winreg_ConnectRegistry_impl(PyObject *module, return retKey; } +#endif /* MS_WINDOWS_DESKTOP || MS_WINDOWS_SYSTEM */ + /*[clinic input] winreg.CreateKey -> HKEY @@ -1272,6 +1278,8 @@ winreg_ExpandEnvironmentStrings_impl(PyObject *module, return o; } +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) + /*[clinic input] winreg.FlushKey @@ -1305,6 +1313,9 @@ winreg_FlushKey_impl(PyObject *module, HKEY key) Py_RETURN_NONE; } +#endif /* MS_WINDOWS_DESKTOP || MS_WINDOWS_SYSTEM */ + +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) /*[clinic input] winreg.LoadKey @@ -1354,6 +1365,8 @@ winreg_LoadKey_impl(PyObject *module, HKEY key, const Py_UNICODE *sub_key, Py_RETURN_NONE; } +#endif /* MS_WINDOWS_DESKTOP || MS_WINDOWS_SYSTEM */ + /*[clinic input] winreg.OpenKey -> HKEY @@ -1463,6 +1476,7 @@ winreg_QueryInfoKey_impl(PyObject *module, HKEY key) return ret; } + /*[clinic input] winreg.QueryValue @@ -1634,6 +1648,8 @@ winreg_QueryValueEx_impl(PyObject *module, HKEY key, const Py_UNICODE *name) return result; } +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) + /*[clinic input] winreg.SaveKey @@ -1679,6 +1695,8 @@ winreg_SaveKey_impl(PyObject *module, HKEY key, const Py_UNICODE *file_name) Py_RETURN_NONE; } +#endif /* MS_WINDOWS_DESKTOP || MS_WINDOWS_SYSTEM */ + /*[clinic input] winreg.SetValue @@ -1776,6 +1794,7 @@ winreg_SetValue_impl(PyObject *module, HKEY key, const Py_UNICODE *sub_key, return result; } + /*[clinic input] winreg.SetValueEx @@ -1861,6 +1880,8 @@ winreg_SetValueEx_impl(PyObject *module, HKEY key, return result; } +#if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) + /*[clinic input] winreg.DisableReflectionKey @@ -2009,6 +2030,8 @@ winreg_QueryReflectionKey_impl(PyObject *module, HKEY key) return PyBool_FromLong(result); } +#endif /* MS_WINDOWS_DESKTOP || MS_WINDOWS_SYSTEM */ + static struct PyMethodDef winreg_methods[] = { WINREG_CLOSEKEY_METHODDEF WINREG_CONNECTREGISTRY_METHODDEF @@ -2149,3 +2172,5 @@ PyMODINIT_FUNC PyInit_winreg(void) ADD_INT(REG_RESOURCE_REQUIREMENTS_LIST); return m; } + +#endif /* MS_WINDOWS_DESKTOP || MS_WINDOWS_SYSTEM || MS_WINDOWS_GAMES */ diff --git a/Parser/myreadline.c b/Parser/myreadline.c index d55fcefbb6f206..3f0e29f051a438 100644 --- a/Parser/myreadline.c +++ b/Parser/myreadline.c @@ -13,7 +13,9 @@ #include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH #include "pycore_pystate.h" // _PyThreadState_GET() #ifdef MS_WINDOWS +# ifndef WIN32_LEAN_AND_MEAN # define WIN32_LEAN_AND_MEAN +# endif # include "windows.h" #endif /* MS_WINDOWS */ @@ -108,7 +110,7 @@ my_fgets(PyThreadState* tstate, char *buf, int len, FILE *fp) /* NOTREACHED */ } -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO /* Readline implementation using ReadConsoleW */ extern char _get_console_type(HANDLE handle); @@ -233,7 +235,7 @@ _PyOS_WindowsConsoleReadline(PyThreadState *tstate, HANDLE hStdIn) return buf; } -#endif +#endif /* HAVE_WINDOWS_CONSOLE_IO */ /* Readline implementation using fgets() */ @@ -246,7 +248,7 @@ PyOS_StdioReadline(FILE *sys_stdin, FILE *sys_stdout, const char *prompt) PyThreadState *tstate = _PyOS_ReadlineTState; assert(tstate != NULL); -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); if (!config->legacy_windows_stdio && sys_stdin == stdin) { HANDLE hStdIn, hStdErr; diff --git a/Python/dynload_win.c b/Python/dynload_win.c index 7bd04d573df4ad..acab05e2c6def3 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -163,6 +163,7 @@ static char *GetPythonImport (HINSTANCE hModule) return NULL; } +#ifdef Py_ENABLE_SHARED /* Load python3.dll before loading any extension module that might refer to it. That way, we can be sure that always the python3.dll corresponding to this python DLL is loaded, not a python3.dll that might be on the path @@ -216,6 +217,7 @@ _Py_CheckPython3(void) return hPython3 != NULL; #undef MAXPATHLEN } +#endif /* Py_ENABLE_SHARED */ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, const char *shortname, @@ -224,7 +226,9 @@ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, dl_funcptr p; char funcname[258], *import_python; +#ifdef Py_ENABLE_SHARED _Py_CheckPython3(); +#endif /* Py_ENABLE_SHARED */ wchar_t *wpathname = PyUnicode_AsWideCharString(pathname, NULL); if (wpathname == NULL) @@ -234,10 +238,12 @@ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, { HINSTANCE hDLL = NULL; +#ifdef MS_WINDOWS_DESKTOP unsigned int old_mode; /* Don't display a message box when Python can't load a DLL */ old_mode = SetErrorMode(SEM_FAILCRITICALERRORS); +#endif /* bpo-36085: We use LoadLibraryEx with restricted search paths to avoid DLL preloading attacks and enable use of the @@ -250,8 +256,10 @@ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, Py_END_ALLOW_THREADS PyMem_Free(wpathname); +#ifdef MS_WINDOWS_DESKTOP /* restore old error mode settings */ SetErrorMode(old_mode); +#endif if (hDLL==NULL){ PyObject *message; diff --git a/Python/fileutils.c b/Python/fileutils.c index 93bee9ee007cd4..4ac759c45a3a1e 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -8,7 +8,11 @@ #ifdef MS_WINDOWS # include # include -# include // PathCchCombineEx +# if defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) +# define PATHCCH_ALLOW_LONG_PATHS 0x01 +# else +# include // PathCchCombineEx +# endif extern int winerror_to_errno(int); #endif @@ -77,7 +81,8 @@ _Py_device_encoding(int fd) if (!valid) Py_RETURN_NONE; -#if defined(MS_WINDOWS) +#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO UINT cp; if (fd == 0) cp = GetConsoleCP(); @@ -92,6 +97,9 @@ _Py_device_encoding(int fd) } return PyUnicode_FromFormat("cp%u", (unsigned int)cp); +#else + Py_RETURN_NONE; +#endif /* HAVE_WINDOWS_CONSOLE_IO */ #else if (_PyRuntime.preconfig.utf8_mode) { _Py_DECLARE_STR(utf_8, "utf-8"); @@ -1270,6 +1278,13 @@ _Py_stat(PyObject *path, struct stat *statbuf) #endif } +#ifdef MS_WINDOWS +// For some Windows API partitions, SetHandleInformation() is declared +// but none of the handle flags are defined. +#ifndef HANDLE_FLAG_INHERIT +#define HANDLE_FLAG_INHERIT 0x00000001 +#endif +#endif /* This function MUST be kept async-signal-safe on POSIX when raise=0. */ static int @@ -2096,6 +2111,72 @@ _Py_abspath(const wchar_t *path, wchar_t **abspath_p) #endif } +// The Windows Games API family implements the PathCch* APIs in the Xbox OS, +// but does not expose them yet. Load them dynamically until +// 1) they are officially exposed +// 2) we stop supporting older versions of the GDK which do not expose them +#if defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) +HRESULT +PathCchSkipRoot(const wchar_t *path, const wchar_t **rootEnd) +{ + static int initialized = 0; + typedef HRESULT(__stdcall *PPathCchSkipRoot) (PCWSTR pszPath, + PCWSTR *ppszRootEnd); + static PPathCchSkipRoot _PathCchSkipRoot; + + if (initialized == 0) { + HMODULE pathapi = LoadLibraryExW(L"api-ms-win-core-path-l1-1-0.dll", NULL, + LOAD_LIBRARY_SEARCH_SYSTEM32); + if (pathapi) { + _PathCchSkipRoot = (PPathCchSkipRoot)GetProcAddress( + pathapi, "PathCchSkipRoot"); + } + else { + _PathCchSkipRoot = NULL; + } + initialized = 1; + } + + if (!_PathCchSkipRoot) { + return E_NOINTERFACE; + } + + return _PathCchSkipRoot(path, rootEnd); +} + +static HRESULT +PathCchCombineEx(wchar_t *buffer, size_t bufsize, const wchar_t *dirname, + const wchar_t *relfile, unsigned long flags) +{ + static int initialized = 0; + typedef HRESULT(__stdcall *PPathCchCombineEx) (PWSTR pszPathOut, + size_t cchPathOut, + PCWSTR pszPathIn, + PCWSTR pszMore, + unsigned long dwFlags); + static PPathCchCombineEx _PathCchCombineEx; + + if (initialized == 0) { + HMODULE pathapi = LoadLibraryExW(L"api-ms-win-core-path-l1-1-0.dll", NULL, + LOAD_LIBRARY_SEARCH_SYSTEM32); + if (pathapi) { + _PathCchCombineEx = (PPathCchCombineEx)GetProcAddress( + pathapi, "PathCchCombineEx"); + } + else { + _PathCchCombineEx = NULL; + } + initialized = 1; + } + + if (!_PathCchCombineEx) { + return E_NOINTERFACE; + } + + return _PathCchCombineEx(buffer, bufsize, dirname, relfile, flags); +} + +#endif /* defined(MS_WINDOWS_GAMES) && !defined(MS_WINDOWS_DESKTOP) */ // The caller must ensure "buffer" is big enough. static int @@ -2491,12 +2572,12 @@ _Py_get_blocking(int fd) success = GetNamedPipeHandleStateW(handle, &mode, NULL, NULL, NULL, NULL, 0); Py_END_ALLOW_THREADS - + if (!success) { PyErr_SetFromWindowsErr(0); return -1; } - + return !(mode & PIPE_NOWAIT); } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index e80dd30c89dfd0..82e94090a6027a 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2289,7 +2289,7 @@ create_stdio(const PyConfig *config, PyObject* io, raw = Py_NewRef(buf); } -#ifdef MS_WINDOWS +#ifdef HAVE_WINDOWS_CONSOLE_IO /* Windows console IO is always UTF-8 encoded */ PyTypeObject *winconsoleio_type = (PyTypeObject *)_PyImport_GetModuleAttr( &_Py_ID(_io), &_Py_ID(_WindowsConsoleIO)); diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 207abb964bcac9..764fb70bae6c38 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1490,6 +1490,9 @@ static PyStructSequence_Desc windows_version_desc = { static PyObject * _sys_getwindowsversion_from_kernel32() { +#ifndef MS_WINDOWS_DESKTOP + return NULL; +#else HANDLE hKernel32; wchar_t kernel32_path[MAX_PATH]; LPVOID verblock; @@ -1523,6 +1526,7 @@ _sys_getwindowsversion_from_kernel32() realBuild = HIWORD(ffi->dwProductVersionLS); PyMem_RawFree(verblock); return Py_BuildValue("(kkk)", realMajor, realMinor, realBuild); +#endif /* !MS_WINDOWS_DESKTOP */ } /* Disable deprecation warnings about GetVersionEx as the result is From 71cf7c3dddd9c49ec70c1a95547f2fcd5daa7034 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Thu, 9 Mar 2023 23:06:20 +0000 Subject: [PATCH 32/38] gh-102378: don't bother stripping `/` from __text_signature__ (#102379) --- Lib/inspect.py | 57 ++++++------------- Lib/test/test_inspect.py | 23 +++----- ...-03-03-19-53-08.gh-issue-102378.kRdOZc.rst | 1 + 3 files changed, 25 insertions(+), 56 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-03-03-19-53-08.gh-issue-102378.kRdOZc.rst diff --git a/Lib/inspect.py b/Lib/inspect.py index 166667c62cdccf..edc23b0ffa9201 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2106,26 +2106,21 @@ def _signature_strip_non_python_syntax(signature): Private helper function. Takes a signature in Argument Clinic's extended signature format. - Returns a tuple of three things: - * that signature re-rendered in standard Python syntax, + Returns a tuple of two things: + * that signature re-rendered in standard Python syntax, and * the index of the "self" parameter (generally 0), or None if - the function does not have a "self" parameter, and - * the index of the last "positional only" parameter, - or None if the signature has no positional-only parameters. + the function does not have a "self" parameter. """ if not signature: - return signature, None, None + return signature, None self_parameter = None - last_positional_only = None lines = [l.encode('ascii') for l in signature.split('\n') if l] generator = iter(lines).__next__ token_stream = tokenize.tokenize(generator) - delayed_comma = False - skip_next_comma = False text = [] add = text.append @@ -2142,35 +2137,18 @@ def _signature_strip_non_python_syntax(signature): if type == OP: if string == ',': - if skip_next_comma: - skip_next_comma = False - else: - assert not delayed_comma - delayed_comma = True - current_parameter += 1 - continue - - if string == '/': - assert not skip_next_comma - assert last_positional_only is None - skip_next_comma = True - last_positional_only = current_parameter - 1 - continue + current_parameter += 1 if (type == ERRORTOKEN) and (string == '$'): assert self_parameter is None self_parameter = current_parameter continue - if delayed_comma: - delayed_comma = False - if not ((type == OP) and (string == ')')): - add(', ') add(string) if (string == ','): add(' ') clean_signature = ''.join(text) - return clean_signature, self_parameter, last_positional_only + return clean_signature, self_parameter def _signature_fromstr(cls, obj, s, skip_bound_arg=True): @@ -2179,8 +2157,7 @@ def _signature_fromstr(cls, obj, s, skip_bound_arg=True): """ Parameter = cls._parameter_cls - clean_signature, self_parameter, last_positional_only = \ - _signature_strip_non_python_syntax(s) + clean_signature, self_parameter = _signature_strip_non_python_syntax(s) program = "def foo" + clean_signature + ": pass" @@ -2269,17 +2246,17 @@ def p(name_node, default_node, default=empty): parameters.append(Parameter(name, kind, default=default, annotation=empty)) # non-keyword-only parameters - args = reversed(f.args.args) - defaults = reversed(f.args.defaults) - iter = itertools.zip_longest(args, defaults, fillvalue=None) - if last_positional_only is not None: - kind = Parameter.POSITIONAL_ONLY - else: - kind = Parameter.POSITIONAL_OR_KEYWORD - for i, (name, default) in enumerate(reversed(list(iter))): + total_non_kw_args = len(f.args.posonlyargs) + len(f.args.args) + required_non_kw_args = total_non_kw_args - len(f.args.defaults) + defaults = itertools.chain(itertools.repeat(None, required_non_kw_args), f.args.defaults) + + kind = Parameter.POSITIONAL_ONLY + for (name, default) in zip(f.args.posonlyargs, defaults): + p(name, default) + + kind = Parameter.POSITIONAL_OR_KEYWORD + for (name, default) in zip(f.args.args, defaults): p(name, default) - if i == last_positional_only: - kind = Parameter.POSITIONAL_OR_KEYWORD # *args if f.args.vararg: diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 92aba519d28a08..02f8378d0413ea 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -4230,56 +4230,47 @@ def foo(a): pass class TestSignaturePrivateHelpers(unittest.TestCase): def _strip_non_python_syntax(self, input, - clean_signature, self_parameter, last_positional_only): + clean_signature, self_parameter): computed_clean_signature, \ - computed_self_parameter, \ - computed_last_positional_only = \ + computed_self_parameter = \ inspect._signature_strip_non_python_syntax(input) self.assertEqual(computed_clean_signature, clean_signature) self.assertEqual(computed_self_parameter, self_parameter) - self.assertEqual(computed_last_positional_only, last_positional_only) def test_signature_strip_non_python_syntax(self): self._strip_non_python_syntax( "($module, /, path, mode, *, dir_fd=None, " + "effective_ids=False,\n follow_symlinks=True)", - "(module, path, mode, *, dir_fd=None, " + + "(module, /, path, mode, *, dir_fd=None, " + "effective_ids=False, follow_symlinks=True)", - 0, 0) self._strip_non_python_syntax( "($module, word, salt, /)", - "(module, word, salt)", - 0, - 2) + "(module, word, salt, /)", + 0) self._strip_non_python_syntax( "(x, y=None, z=None, /)", - "(x, y=None, z=None)", - None, - 2) + "(x, y=None, z=None, /)", + None) self._strip_non_python_syntax( "(x, y=None, z=None)", "(x, y=None, z=None)", - None, None) self._strip_non_python_syntax( "(x,\n y=None,\n z = None )", "(x, y=None, z=None)", - None, None) self._strip_non_python_syntax( "", "", - None, None) self._strip_non_python_syntax( - None, None, None, None) diff --git a/Misc/NEWS.d/next/Library/2023-03-03-19-53-08.gh-issue-102378.kRdOZc.rst b/Misc/NEWS.d/next/Library/2023-03-03-19-53-08.gh-issue-102378.kRdOZc.rst new file mode 100644 index 00000000000000..d30f65f30d109a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-03-19-53-08.gh-issue-102378.kRdOZc.rst @@ -0,0 +1 @@ +Private helper method ``inspect._signature_strip_non_python_syntax`` will no longer strip ``/`` from the input string. From 2999e02836f9112de6b17784eaca762fb87e71a9 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Fri, 10 Mar 2023 09:02:32 +0000 Subject: [PATCH 33/38] gh-102192: Replace PyErr_Fetch/Restore etc by more efficient alternatives in `_ctypes` (#102477) --- Modules/_ctypes/_ctypes.c | 11 ++--------- Modules/_ctypes/callproc.c | 36 +++++++++++++++++++----------------- 2 files changed, 21 insertions(+), 26 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 8690f2c1b07852..6f92ca08dd537b 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -2200,7 +2200,6 @@ PyCSimpleType_from_param(PyObject *type, PyObject *value) struct fielddesc *fd; PyObject *as_parameter; int res; - PyObject *exc, *val, *tb; /* If the value is already an instance of the requested type, we can use it as is */ @@ -2234,33 +2233,27 @@ PyCSimpleType_from_param(PyObject *type, PyObject *value) parg->obj = fd->setfunc(&parg->value, value, 0); if (parg->obj) return (PyObject *)parg; - PyErr_Fetch(&exc, &val, &tb); + PyObject *exc = PyErr_GetRaisedException(); Py_DECREF(parg); if (_PyObject_LookupAttr(value, &_Py_ID(_as_parameter_), &as_parameter) < 0) { Py_XDECREF(exc); - Py_XDECREF(val); - Py_XDECREF(tb); return NULL; } if (as_parameter) { if (_Py_EnterRecursiveCall("while processing _as_parameter_")) { Py_DECREF(as_parameter); Py_XDECREF(exc); - Py_XDECREF(val); - Py_XDECREF(tb); return NULL; } value = PyCSimpleType_from_param(type, as_parameter); _Py_LeaveRecursiveCall(); Py_DECREF(as_parameter); Py_XDECREF(exc); - Py_XDECREF(val); - Py_XDECREF(tb); return value; } if (exc) { - PyErr_Restore(exc, val, tb); + PyErr_SetRaisedException(exc); } else { PyErr_SetString(PyExc_TypeError, "wrong type"); diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index f6d98bbeebc24e..4438727332bc11 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1013,41 +1013,43 @@ static PyObject *GetResult(PyObject *restype, void *result, PyObject *checker) void _ctypes_extend_error(PyObject *exc_class, const char *fmt, ...) { va_list vargs; - PyObject *tp, *v, *tb, *s, *cls_str, *msg_str; va_start(vargs, fmt); - s = PyUnicode_FromFormatV(fmt, vargs); + PyObject *s = PyUnicode_FromFormatV(fmt, vargs); va_end(vargs); - if (!s) + if (s == NULL) { return; + } - PyErr_Fetch(&tp, &v, &tb); - PyErr_NormalizeException(&tp, &v, &tb); - if (PyType_Check(tp)) - cls_str = PyType_GetName((PyTypeObject *)tp); - else - cls_str = PyObject_Str(tp); + assert(PyErr_Occurred()); + PyObject *exc = PyErr_GetRaisedException(); + assert(exc != NULL); + PyObject *cls_str = PyType_GetName(Py_TYPE(exc)); if (cls_str) { PyUnicode_AppendAndDel(&s, cls_str); PyUnicode_AppendAndDel(&s, PyUnicode_FromString(": ")); - if (s == NULL) + if (s == NULL) { goto error; - } else + } + } + else { PyErr_Clear(); - msg_str = PyObject_Str(v); - if (msg_str) + } + + PyObject *msg_str = PyObject_Str(exc); + if (msg_str) { PyUnicode_AppendAndDel(&s, msg_str); + } else { PyErr_Clear(); PyUnicode_AppendAndDel(&s, PyUnicode_FromString("???")); } - if (s == NULL) + if (s == NULL) { goto error; + } PyErr_SetObject(exc_class, s); error: - Py_XDECREF(tp); - Py_XDECREF(v); - Py_XDECREF(tb); + Py_XDECREF(exc); Py_XDECREF(s); } From cb35882773a3ffc7fe0671e64848f4c926a2d52f Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 10 Mar 2023 12:21:37 +0000 Subject: [PATCH 34/38] gh-102519: Add os.listdrives, os.listvolumes and os.listmounts on Windows (GH-102544) --- Doc/whatsnew/3.12.rst | 4 + .../pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + .../internal/pycore_runtime_init_generated.h | 1 + .../internal/pycore_unicodeobject_generated.h | 2 + Lib/test/test_os.py | 43 ++++ ...-03-08-23-08-38.gh-issue-102519.wlcsFI.rst | 2 + Modules/clinic/posixmodule.c.h | 128 +++++++++++- Modules/posixmodule.c | 194 ++++++++++++++++++ 9 files changed, 375 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-03-08-23-08-38.gh-issue-102519.wlcsFI.rst diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index d982cb62ec2f4e..48b7aab0595ebb 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -294,6 +294,10 @@ os method to check if the entry is a junction. (Contributed by Charles Machalow in :gh:`99547`.) +* Add :func:`os.listdrives`, :func:`os.listvolumes` and :func:`os.listmounts` + functions on Windows for enumerating drives, volumes and mount points. + (Contributed by Steve Dower in :gh:`102519`.) + os.path ------- diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index dc5cd58d853534..4b12ae523c3260 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1216,6 +1216,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(value)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(values)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(version)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(volume)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(warnings)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(warnoptions)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(wbits)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 8b23aa15479301..17fb9ffbbf9f11 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -702,6 +702,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(value) STRUCT_FOR_ID(values) STRUCT_FOR_ID(version) + STRUCT_FOR_ID(volume) STRUCT_FOR_ID(warnings) STRUCT_FOR_ID(warnoptions) STRUCT_FOR_ID(wbits) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 471efadb13bb4f..b240be57369d9d 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1208,6 +1208,7 @@ extern "C" { INIT_ID(value), \ INIT_ID(values), \ INIT_ID(version), \ + INIT_ID(volume), \ INIT_ID(warnings), \ INIT_ID(warnoptions), \ INIT_ID(wbits), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index b47d240e492ff9..52af37a8e60aa8 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1310,6 +1310,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(version); PyUnicode_InternInPlace(&string); + string = &_Py_ID(volume); + PyUnicode_InternInPlace(&string); string = &_Py_ID(warnings); PyUnicode_InternInPlace(&string); string = &_Py_ID(warnoptions); diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index ba6feb69ea1712..253e2a23238f12 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2648,6 +2648,49 @@ def test_listdir_extended_path(self): [os.fsencode(path) for path in self.created_paths]) +@unittest.skipUnless(os.name == "nt", "NT specific tests") +class Win32ListdriveTests(unittest.TestCase): + """Test listdrive, listmounts and listvolume on Windows.""" + + def setUp(self): + # Get drives and volumes from fsutil + out = subprocess.check_output( + ["fsutil.exe", "volume", "list"], + cwd=os.path.join(os.getenv("SystemRoot", "\\Windows"), "System32"), + encoding="mbcs", + errors="ignore", + ) + lines = out.splitlines() + self.known_volumes = {l for l in lines if l.startswith('\\\\?\\')} + self.known_drives = {l for l in lines if l[1:] == ':\\'} + self.known_mounts = {l for l in lines if l[1:3] == ':\\'} + + def test_listdrives(self): + drives = os.listdrives() + self.assertIsInstance(drives, list) + self.assertSetEqual( + self.known_drives, + self.known_drives & set(drives), + ) + + def test_listvolumes(self): + volumes = os.listvolumes() + self.assertIsInstance(volumes, list) + self.assertSetEqual( + self.known_volumes, + self.known_volumes & set(volumes), + ) + + def test_listmounts(self): + for volume in os.listvolumes(): + mounts = os.listmounts(volume) + self.assertIsInstance(mounts, list) + self.assertSetEqual( + set(mounts), + self.known_mounts & set(mounts), + ) + + @unittest.skipUnless(hasattr(os, 'readlink'), 'needs os.readlink()') class ReadlinkTests(unittest.TestCase): filelink = 'readlinktest' diff --git a/Misc/NEWS.d/next/Library/2023-03-08-23-08-38.gh-issue-102519.wlcsFI.rst b/Misc/NEWS.d/next/Library/2023-03-08-23-08-38.gh-issue-102519.wlcsFI.rst new file mode 100644 index 00000000000000..f47e4f70b1301d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-08-23-08-38.gh-issue-102519.wlcsFI.rst @@ -0,0 +1,2 @@ +Add :func:`os.listdrives`, :func:`os.listvolumes` and :func:`os.listmounts` +functions on Windows for enumerating drives, volumes and mount points diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 6565f8df935cb8..8b0550d832fc0a 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -1601,6 +1601,120 @@ os_listdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * #if defined(MS_WINDOWS) +PyDoc_STRVAR(os_listdrives__doc__, +"listdrives($module, /)\n" +"--\n" +"\n" +"Return a list containing the names of drives in the system.\n" +"\n" +"A drive name typically looks like \'C:\\\\\'."); + +#define OS_LISTDRIVES_METHODDEF \ + {"listdrives", (PyCFunction)os_listdrives, METH_NOARGS, os_listdrives__doc__}, + +static PyObject * +os_listdrives_impl(PyObject *module); + +static PyObject * +os_listdrives(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return os_listdrives_impl(module); +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(os_listvolumes__doc__, +"listvolumes($module, /)\n" +"--\n" +"\n" +"Return a list containing the volumes in the system.\n" +"\n" +"Volumes are typically represented as a GUID path."); + +#define OS_LISTVOLUMES_METHODDEF \ + {"listvolumes", (PyCFunction)os_listvolumes, METH_NOARGS, os_listvolumes__doc__}, + +static PyObject * +os_listvolumes_impl(PyObject *module); + +static PyObject * +os_listvolumes(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return os_listvolumes_impl(module); +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(os_listmounts__doc__, +"listmounts($module, /, volume)\n" +"--\n" +"\n" +"Return a list containing mount points for a particular volume.\n" +"\n" +"\'volume\' should be a GUID path as returned from os.listvolumes."); + +#define OS_LISTMOUNTS_METHODDEF \ + {"listmounts", _PyCFunction_CAST(os_listmounts), METH_FASTCALL|METH_KEYWORDS, os_listmounts__doc__}, + +static PyObject * +os_listmounts_impl(PyObject *module, path_t *volume); + +static PyObject * +os_listmounts(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(volume), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"volume", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "listmounts", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + path_t volume = PATH_T_INITIALIZE("listmounts", "volume", 0, 0); + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!path_converter(args[0], &volume)) { + goto exit; + } + return_value = os_listmounts_impl(module, &volume); + +exit: + /* Cleanup for volume */ + path_cleanup(&volume); + + return return_value; +} + +#endif /* defined(MS_WINDOWS) */ + +#if defined(MS_WINDOWS) + PyDoc_STRVAR(os__getfullpathname__doc__, "_getfullpathname($module, path, /)\n" "--\n" @@ -11253,6 +11367,18 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #define OS_LINK_METHODDEF #endif /* !defined(OS_LINK_METHODDEF) */ +#ifndef OS_LISTDRIVES_METHODDEF + #define OS_LISTDRIVES_METHODDEF +#endif /* !defined(OS_LISTDRIVES_METHODDEF) */ + +#ifndef OS_LISTVOLUMES_METHODDEF + #define OS_LISTVOLUMES_METHODDEF +#endif /* !defined(OS_LISTVOLUMES_METHODDEF) */ + +#ifndef OS_LISTMOUNTS_METHODDEF + #define OS_LISTMOUNTS_METHODDEF +#endif /* !defined(OS_LISTMOUNTS_METHODDEF) */ + #ifndef OS__GETFULLPATHNAME_METHODDEF #define OS__GETFULLPATHNAME_METHODDEF #endif /* !defined(OS__GETFULLPATHNAME_METHODDEF) */ @@ -11796,4 +11922,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=9495478e51701b8a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=47750e0e29c8d707 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 0d534f35f6a420..7d91f7e4bac76b 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -4229,7 +4229,198 @@ os_listdir_impl(PyObject *module, path_t *path) #endif } + #ifdef MS_WINDOWS + +/*[clinic input] +os.listdrives + +Return a list containing the names of drives in the system. + +A drive name typically looks like 'C:\\'. + +[clinic start generated code]*/ + +static PyObject * +os_listdrives_impl(PyObject *module) +/*[clinic end generated code: output=aaece9dacdf682b5 input=1af9ccc9e583798e]*/ +{ + /* Number of possible drives is limited, so 256 should always be enough. + On the day when it is not, listmounts() will have to be used. */ + wchar_t buffer[256]; + DWORD buflen = Py_ARRAY_LENGTH(buffer); + PyObject *result = NULL; + if (PySys_Audit("os.listdrives", NULL) < 0) { + return NULL; + } + + Py_BEGIN_ALLOW_THREADS; + buflen = GetLogicalDriveStringsW(buflen, buffer); + Py_END_ALLOW_THREADS; + + if (!buflen) { + PyErr_SetFromWindowsErr(0); + return NULL; + } else if (buflen >= Py_ARRAY_LENGTH(buffer)) { + PyErr_SetFromWindowsErr(ERROR_MORE_DATA); + return NULL; + } + + /* buflen includes a null terminator, so remove it */ + PyObject *str = PyUnicode_FromWideChar(buffer, buflen - 1); + if (str) { + PyObject *nullchar = PyUnicode_FromStringAndSize("\0", 1); + if (nullchar) { + result = PyUnicode_Split(str, nullchar, -1); + Py_DECREF(nullchar); + } + Py_DECREF(str); + } + return result; +} + +/*[clinic input] +os.listvolumes + +Return a list containing the volumes in the system. + +Volumes are typically represented as a GUID path. + +[clinic start generated code]*/ + +static PyObject * +os_listvolumes_impl(PyObject *module) +/*[clinic end generated code: output=534e10ea2bf9d386 input=f6e4e70371f11e99]*/ +{ + PyObject *result = PyList_New(0); + HANDLE find = INVALID_HANDLE_VALUE; + wchar_t buffer[MAX_PATH + 1]; + if (!result) { + return NULL; + } + if (PySys_Audit("os.listvolumes", NULL) < 0) { + Py_DECREF(result); + return NULL; + } + + int err = 0; + Py_BEGIN_ALLOW_THREADS; + find = FindFirstVolumeW(buffer, Py_ARRAY_LENGTH(buffer)); + if (find == INVALID_HANDLE_VALUE) { + err = GetLastError(); + } + Py_END_ALLOW_THREADS; + + while (!err) { + PyObject *s = PyUnicode_FromWideChar(buffer, -1); + if (!s || PyList_Append(result, s) < 0) { + Py_XDECREF(s); + Py_CLEAR(result); + break; + } + Py_DECREF(s); + + Py_BEGIN_ALLOW_THREADS; + if (!FindNextVolumeW(find, buffer, Py_ARRAY_LENGTH(buffer))) { + err = GetLastError(); + } + Py_END_ALLOW_THREADS; + } + + if (find != INVALID_HANDLE_VALUE) { + Py_BEGIN_ALLOW_THREADS; + FindVolumeClose(find); + Py_END_ALLOW_THREADS; + } + if (err && err != ERROR_NO_MORE_FILES) { + PyErr_SetFromWindowsErr(err); + Py_XDECREF(result); + result = NULL; + } + return result; +} + + +/*[clinic input] +os.listmounts + + volume: path_t + +Return a list containing mount points for a particular volume. + +'volume' should be a GUID path as returned from os.listvolumes. + +[clinic start generated code]*/ + +static PyObject * +os_listmounts_impl(PyObject *module, path_t *volume) +/*[clinic end generated code: output=06da49679de4512e input=a8a27178e3f67845]*/ +{ + wchar_t default_buffer[MAX_PATH + 1]; + DWORD buflen = Py_ARRAY_LENGTH(default_buffer); + LPWSTR buffer = default_buffer; + DWORD attributes; + PyObject *str = NULL; + PyObject *nullchar = NULL; + PyObject *result = NULL; + + /* Ensure we have a valid volume path before continuing */ + Py_BEGIN_ALLOW_THREADS + attributes = GetFileAttributesW(volume->wide); + Py_END_ALLOW_THREADS + if (attributes == INVALID_FILE_ATTRIBUTES && + GetLastError() == ERROR_UNRECOGNIZED_VOLUME) + { + return PyErr_SetFromWindowsErr(ERROR_UNRECOGNIZED_VOLUME); + } + + if (PySys_Audit("os.listmounts", "O", volume->object) < 0) { + return NULL; + } + + while (1) { + BOOL success; + Py_BEGIN_ALLOW_THREADS + success = GetVolumePathNamesForVolumeNameW(volume->wide, buffer, + buflen, &buflen); + Py_END_ALLOW_THREADS + if (success) { + break; + } + if (GetLastError() != ERROR_MORE_DATA) { + PyErr_SetFromWindowsErr(0); + goto exit; + } + if (buffer != default_buffer) { + PyMem_Free((void *)buffer); + } + buffer = (wchar_t*)PyMem_Malloc(sizeof(wchar_t) * buflen); + if (!buffer) { + PyErr_NoMemory(); + goto exit; + } + } + if (buflen < 2) { + result = PyList_New(0); + goto exit; + } + // buflen includes two null terminators, one for the last string + // and one for the array of strings. + str = PyUnicode_FromWideChar(buffer, buflen - 2); + nullchar = PyUnicode_FromStringAndSize("\0", 1); + if (str && nullchar) { + result = PyUnicode_Split(str, nullchar, -1); + } +exit: + if (buffer != default_buffer) { + PyMem_Free(buffer); + } + Py_XDECREF(nullchar); + Py_XDECREF(str); + return result; +} + + int _PyOS_getfullpathname(const wchar_t *path, wchar_t **abspath_p) { @@ -15252,6 +15443,9 @@ static PyMethodDef posix_methods[] = { OS_GETCWDB_METHODDEF OS_LINK_METHODDEF OS_LISTDIR_METHODDEF + OS_LISTDRIVES_METHODDEF + OS_LISTMOUNTS_METHODDEF + OS_LISTVOLUMES_METHODDEF OS_LSTAT_METHODDEF OS_MKDIR_METHODDEF OS_NICE_METHODDEF From 53dceb53ade15587b9cfd30c0a0942232517dee9 Mon Sep 17 00:00:00 2001 From: Owain Davies <116417456+OTheDev@users.noreply.github.com> Date: Fri, 10 Mar 2023 19:22:02 +0700 Subject: [PATCH 35/38] gh-86509: Add link to Lib/_threading_local.py in threading docs (#101824) --- Doc/library/threading.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index b352125551fa79..83ed48052704fb 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -272,7 +272,7 @@ The instance's values will be different for separate threads. A class that represents thread-local data. For more details and extensive examples, see the documentation string of the - :mod:`_threading_local` module. + :mod:`_threading_local` module: :source:`Lib/_threading_local.py`. .. _thread-objects: From 64bde502cf89963bc7382b03ea9e1c0967d22e35 Mon Sep 17 00:00:00 2001 From: Paul Ganssle <1377457+pganssle@users.noreply.github.com> Date: Fri, 10 Mar 2023 10:29:37 -0500 Subject: [PATCH 36/38] GH-102537: Handle check for PYTHONTZPATH failing in zoneinfo test (GH-102538) It is possible but unlikely for the `python_tzpath_context` function to fail between the start of the `try` block and the point where `os.environ.get` succeeds, in which case `old_env` will be undefined. In this case, we want to take no action. Practically speaking this will really only happen in an error condition anyway, so it doesn't really matter, but we should probably do it right anyway. --- Lib/test/test_zoneinfo/test_zoneinfo.py | 9 ++++++++- .../Tests/2023-03-08-13-54-20.gh-issue-102537.Vfplpb.rst | 2 ++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-03-08-13-54-20.gh-issue-102537.Vfplpb.rst diff --git a/Lib/test/test_zoneinfo/test_zoneinfo.py b/Lib/test/test_zoneinfo/test_zoneinfo.py index 82041a2b488334..ae921f7432c466 100644 --- a/Lib/test/test_zoneinfo/test_zoneinfo.py +++ b/Lib/test/test_zoneinfo/test_zoneinfo.py @@ -1543,13 +1543,20 @@ class TzPathTest(TzPathUserMixin, ZoneInfoTestBase): @contextlib.contextmanager def python_tzpath_context(value): path_var = "PYTHONTZPATH" + unset_env_sentinel = object() + old_env = unset_env_sentinel try: with OS_ENV_LOCK: old_env = os.environ.get(path_var, None) os.environ[path_var] = value yield finally: - if old_env is None: + if old_env is unset_env_sentinel: + # In this case, `old_env` was never retrieved from the + # environment for whatever reason, so there's no need to + # reset the environment TZPATH. + pass + elif old_env is None: del os.environ[path_var] else: os.environ[path_var] = old_env # pragma: nocover diff --git a/Misc/NEWS.d/next/Tests/2023-03-08-13-54-20.gh-issue-102537.Vfplpb.rst b/Misc/NEWS.d/next/Tests/2023-03-08-13-54-20.gh-issue-102537.Vfplpb.rst new file mode 100644 index 00000000000000..94d160dd4127a6 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-03-08-13-54-20.gh-issue-102537.Vfplpb.rst @@ -0,0 +1,2 @@ +Adjust the error handling strategy in +``test_zoneinfo.TzPathTest.python_tzpath_context``. Patch by Paul Ganssle. From 12226bec2588f925f4698e1130ce78e118343934 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 10 Mar 2023 15:41:32 +0000 Subject: [PATCH 37/38] gh-102519: Add doc updates for os.listdrives, listvolumes and listmounts (GH-102585) --- Doc/library/os.rst | 63 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 23ce98785bedfc..5b9f49be1fad55 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -2188,6 +2188,69 @@ features: Accepts a :term:`path-like object`. +.. function:: listdrives() + + Return a list containing the names of drives on a Windows system. + + A drive name typically looks like ``'C:\\'``. Not every drive name + will be associated with a volume, and some may be inaccessible for + a variety of reasons, including permissions, network connectivity + or missing media. This function does not test for access. + + May raise :exc:`OSError` if an error occurs collecting the drive + names. + + .. audit-event:: os.listdrives "" os.listdrives + + .. availability:: Windows + + .. versionadded:: 3.12 + + +.. function:: listmounts(volume) + + Return a list containing the mount points for a volume on a Windows + system. + + *volume* must be represented as a GUID path, like those returned by + :func:`os.listvolumes`. Volumes may be mounted in multiple locations + or not at all. In the latter case, the list will be empty. Mount + points that are not associated with a volume will not be returned by + this function. + + The mount points return by this function will be absolute paths, and + may be longer than the drive name. + + Raises :exc:`OSError` if the volume is not recognized or if an error + occurs collecting the paths. + + .. audit-event:: os.listmounts volume os.listmounts + + .. availability:: Windows + + .. versionadded:: 3.12 + + +.. function:: listvolumes() + + Return a list containing the volumes in the system. + + Volumes are typically represented as a GUID path that looks like + ``\\?\Volume{xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx}\``. Files can + usually be accessed through a GUID path, permissions allowing. + However, users are generally not familiar with them, and so the + recommended use of this function is to retrieve mount points + using :func:`os.listmounts`. + + May raise :exc:`OSError` if an error occurs collecting the volumes. + + .. audit-event:: os.listvolumes "" os.listvolumes + + .. availability:: Windows + + .. versionadded:: 3.12 + + .. function:: lstat(path, *, dir_fd=None) Perform the equivalent of an :c:func:`lstat` system call on the given path. From 90f1d777177e28b6c7b8d9ba751550e373d61b0a Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Fri, 10 Mar 2023 17:29:04 +0000 Subject: [PATCH 38/38] GH-80486: Fix handling of NTFS alternate data streams in pathlib (GH-102454) Co-authored-by: Maor Kleinberger --- Lib/pathlib.py | 8 ++++-- Lib/test/test_pathlib.py | 28 ++++++++++++++++++- .../2019-03-15-22-50-27.bpo-36305.Pbkv6u.rst | 2 ++ 3 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2019-03-15-22-50-27.bpo-36305.Pbkv6u.rst diff --git a/Lib/pathlib.py b/Lib/pathlib.py index d375529ff5f767..55c44f12e5a2fb 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -320,8 +320,9 @@ def _from_parsed_parts(cls, drv, root, parts): def _format_parsed_parts(cls, drv, root, parts): if drv or root: return drv + root + cls._flavour.sep.join(parts[1:]) - else: - return cls._flavour.sep.join(parts) + elif parts and cls._flavour.splitdrive(parts[0])[0]: + parts = ['.'] + parts + return cls._flavour.sep.join(parts) def __str__(self): """Return the string representation of the path, suitable for @@ -1188,7 +1189,8 @@ def expanduser(self): homedir = self._flavour.expanduser(self._parts[0]) if homedir[:1] == "~": raise RuntimeError("Could not determine home directory.") - return self._from_parts([homedir] + self._parts[1:]) + drv, root, parts = self._parse_parts((homedir,)) + return self._from_parsed_parts(drv, root, parts + self._parts[1:]) return self diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index f8e2f44d27fc1e..f05dead5886743 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -122,6 +122,13 @@ def test_parse_parts(self): # the second path is relative. check(['c:/a/b', 'c:x/y'], ('c:', '\\', ['c:\\', 'a', 'b', 'x', 'y'])) check(['c:/a/b', 'c:/x/y'], ('c:', '\\', ['c:\\', 'x', 'y'])) + # Paths to files with NTFS alternate data streams + check(['./c:s'], ('', '', ['c:s'])) + check(['cc:s'], ('', '', ['cc:s'])) + check(['C:c:s'], ('C:', '', ['C:', 'c:s'])) + check(['C:/c:s'], ('C:', '\\', ['C:\\', 'c:s'])) + check(['D:a', './c:b'], ('D:', '', ['D:', 'a', 'c:b'])) + check(['D:/a', './c:b'], ('D:', '\\', ['D:\\', 'a', 'c:b'])) # @@ -165,6 +172,7 @@ def test_constructor_common(self): self.assertEqual(P(P('a'), 'b'), P('a/b')) self.assertEqual(P(P('a'), P('b')), P('a/b')) self.assertEqual(P(P('a'), P('b'), P('c')), P(FakePath("a/b/c"))) + self.assertEqual(P(P('./a:b')), P('./a:b')) def test_bytes(self): P = self.cls @@ -814,7 +822,8 @@ class PureWindowsPathTest(_BasePurePathTest, unittest.TestCase): equivalences = _BasePurePathTest.equivalences.copy() equivalences.update({ - 'c:a': [ ('c:', 'a'), ('c:', 'a/'), ('/', 'c:', 'a') ], + './a:b': [ ('./a:b',) ], + 'c:a': [ ('c:', 'a'), ('c:', 'a/'), ('.', 'c:', 'a') ], 'c:/a': [ ('c:/', 'a'), ('c:', '/', 'a'), ('c:', '/a'), ('/z', 'c:/', 'a'), ('//x/y', 'c:/', 'a'), @@ -838,6 +847,7 @@ def test_str(self): self.assertEqual(str(p), '\\\\a\\b\\c\\d') def test_str_subclass(self): + self._check_str_subclass('.\\a:b') self._check_str_subclass('c:') self._check_str_subclass('c:a') self._check_str_subclass('c:a\\b.txt') @@ -1005,6 +1015,7 @@ def test_drive(self): self.assertEqual(P('//a/b').drive, '\\\\a\\b') self.assertEqual(P('//a/b/').drive, '\\\\a\\b') self.assertEqual(P('//a/b/c/d').drive, '\\\\a\\b') + self.assertEqual(P('./c:a').drive, '') def test_root(self): P = self.cls @@ -1341,6 +1352,14 @@ def test_join(self): self.assertEqual(pp, P('C:/a/b/x/y')) pp = p.joinpath('c:/x/y') self.assertEqual(pp, P('C:/x/y')) + # Joining with files with NTFS data streams => the filename should + # not be parsed as a drive letter + pp = p.joinpath(P('./d:s')) + self.assertEqual(pp, P('C:/a/b/d:s')) + pp = p.joinpath(P('./dd:s')) + self.assertEqual(pp, P('C:/a/b/dd:s')) + pp = p.joinpath(P('E:d:s')) + self.assertEqual(pp, P('E:d:s')) def test_div(self): # Basically the same as joinpath(). @@ -1361,6 +1380,11 @@ def test_div(self): # the second path is relative. self.assertEqual(p / 'c:x/y', P('C:/a/b/x/y')) self.assertEqual(p / 'c:/x/y', P('C:/x/y')) + # Joining with files with NTFS data streams => the filename should + # not be parsed as a drive letter + self.assertEqual(p / P('./d:s'), P('C:/a/b/d:s')) + self.assertEqual(p / P('./dd:s'), P('C:/a/b/dd:s')) + self.assertEqual(p / P('E:d:s'), P('E:d:s')) def test_is_reserved(self): P = self.cls @@ -1626,6 +1650,8 @@ def test_expanduser_common(self): self.assertEqual(p.expanduser(), p) p = P(P('').absolute().anchor) / '~' self.assertEqual(p.expanduser(), p) + p = P('~/a:b') + self.assertEqual(p.expanduser(), P(os.path.expanduser('~'), './a:b')) def test_exists(self): P = self.cls diff --git a/Misc/NEWS.d/next/Library/2019-03-15-22-50-27.bpo-36305.Pbkv6u.rst b/Misc/NEWS.d/next/Library/2019-03-15-22-50-27.bpo-36305.Pbkv6u.rst new file mode 100644 index 00000000000000..d9360496ac24cb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-03-15-22-50-27.bpo-36305.Pbkv6u.rst @@ -0,0 +1,2 @@ +Fix handling of Windows filenames that resemble drives, such as ``./a:b``, +in :mod:`pathlib`.