From 5d4d83130c1538586e559a64e3a2341794da92d9 Mon Sep 17 00:00:00 2001 From: Gary Donovan Date: Sat, 26 Nov 2022 05:03:20 +1100 Subject: [PATCH 01/35] Fix typo on inline comment for email.generator (GH-98210) Trivial change to comment - no issue or new entry necessary --- Lib/email/generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/email/generator.py b/Lib/email/generator.py index c9b121624e08d5..885e6ba98540a7 100644 --- a/Lib/email/generator.py +++ b/Lib/email/generator.py @@ -170,7 +170,7 @@ def _write(self, msg): # parameter. # # The way we do this, so as to make the _handle_*() methods simpler, - # is to cache any subpart writes into a buffer. The we write the + # is to cache any subpart writes into a buffer. Then we write the # headers and the buffer contents. That way, subpart handlers can # Do The Right Thing, and can still modify the Content-Type: header if # necessary. From 7d2dcc53d09fe903329926bf7bbfe460b1465dab Mon Sep 17 00:00:00 2001 From: Stanley <46876382+slateny@users.noreply.github.com> Date: Fri, 25 Nov 2022 11:10:22 -0800 Subject: [PATCH 02/35] gh-64019: Have attribute table in `inspect` docs link to module attributes instead of listing them (GH-98116) Co-authored-by: Michael Anckaert --- Doc/library/inspect.rst | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 44f1ae04c9e39e..9cb7a6f94e49cd 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -32,7 +32,7 @@ The :func:`getmembers` function retrieves the members of an object such as a class or module. The functions whose names begin with "is" are mainly provided as convenient choices for the second argument to :func:`getmembers`. They also help you determine when you can expect to find the following special -attributes: +attributes (see :ref:`import-mod-attrs` for module attributes): .. this function name is too big to fit in the ascii-art table below .. |coroutine-origin-link| replace:: :func:`sys.set_coroutine_origin_tracking_depth` @@ -40,11 +40,6 @@ attributes: +-----------+-------------------+---------------------------+ | Type | Attribute | Description | +===========+===================+===========================+ -| module | __doc__ | documentation string | -+-----------+-------------------+---------------------------+ -| | __file__ | filename (missing for | -| | | built-in modules) | -+-----------+-------------------+---------------------------+ | class | __doc__ | documentation string | +-----------+-------------------+---------------------------+ | | __name__ | name with which this | From ae234fbc5ce045066448f2f0cda2f1c3c7ddebea Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Fri, 25 Nov 2022 19:15:57 +0000 Subject: [PATCH 03/35] gh-99029: Fix handling of `PureWindowsPath('C:\').relative_to('C:')` (GH-99031) `relative_to()` now treats naked drive paths as relative. This brings its behaviour in line with other parts of pathlib, and with `ntpath.relpath()`, and so allows us to factor out the pathlib-specific implementation. --- Lib/pathlib.py | 56 +++++-------------- Lib/test/test_pathlib.py | 14 ++--- ...2-11-02-23-47-07.gh-issue-99029.7uCiIB.rst | 2 + 3 files changed, 20 insertions(+), 52 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-11-02-23-47-07.gh-issue-99029.7uCiIB.rst diff --git a/Lib/pathlib.py b/Lib/pathlib.py index bc57ae60e725b2..f31eb3010368d5 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -632,57 +632,27 @@ def relative_to(self, *other, walk_up=False): The *walk_up* parameter controls whether `..` may be used to resolve the path. """ - # For the purpose of this method, drive and root are considered - # separate parts, i.e.: - # Path('c:/').relative_to('c:') gives Path('/') - # Path('c:/').relative_to('/') raise ValueError if not other: raise TypeError("need at least one argument") - parts = self._parts - drv = self._drv - root = self._root - if root: - abs_parts = [drv, root] + parts[1:] - else: - abs_parts = parts - other_drv, other_root, other_parts = self._parse_args(other) - if other_root: - other_abs_parts = [other_drv, other_root] + other_parts[1:] - else: - other_abs_parts = other_parts - num_parts = len(other_abs_parts) - casefold = self._flavour.casefold_parts - num_common_parts = 0 - for part, other_part in zip(casefold(abs_parts), casefold(other_abs_parts)): - if part != other_part: + path_cls = type(self) + other = path_cls(*other) + for step, path in enumerate([other] + list(other.parents)): + if self.is_relative_to(path): break - num_common_parts += 1 - if walk_up: - failure = root != other_root - if drv or other_drv: - failure = casefold([drv]) != casefold([other_drv]) or (failure and num_parts > 1) - error_message = "{!r} is not on the same drive as {!r}" - up_parts = (num_parts-num_common_parts)*['..'] else: - failure = (root or drv) if num_parts == 0 else num_common_parts != num_parts - error_message = "{!r} is not in the subpath of {!r}" - up_parts = [] - error_message += " OR one path is relative and the other is absolute." - if failure: - formatted = self._format_parsed_parts(other_drv, other_root, other_parts) - raise ValueError(error_message.format(str(self), str(formatted))) - path_parts = up_parts + abs_parts[num_common_parts:] - new_root = root if num_common_parts == 1 else '' - return self._from_parsed_parts('', new_root, path_parts) + raise ValueError(f"{str(self)!r} and {str(other)!r} have different anchors") + if step and not walk_up: + raise ValueError(f"{str(self)!r} is not in the subpath of {str(other)!r}") + parts = ('..',) * step + self.parts[len(path.parts):] + return path_cls(*parts) def is_relative_to(self, *other): """Return True if the path is relative to another path or False. """ - try: - self.relative_to(*other) - return True - except ValueError: - return False + if not other: + raise TypeError("need at least one argument") + other = type(self)(*other) + return other == self or other in self.parents @property def parts(self): diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 94401e5429cdf2..1d01d3cbd91d14 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -1183,10 +1183,6 @@ def test_relative_to(self): self.assertRaises(ValueError, p.relative_to, P('/Foo'), walk_up=True) self.assertRaises(ValueError, p.relative_to, P('C:/Foo'), walk_up=True) p = P('C:/Foo/Bar') - self.assertEqual(p.relative_to(P('c:')), P('/Foo/Bar')) - self.assertEqual(p.relative_to('c:'), P('/Foo/Bar')) - self.assertEqual(str(p.relative_to(P('c:'))), '\\Foo\\Bar') - self.assertEqual(str(p.relative_to('c:')), '\\Foo\\Bar') self.assertEqual(p.relative_to(P('c:/')), P('Foo/Bar')) self.assertEqual(p.relative_to('c:/'), P('Foo/Bar')) self.assertEqual(p.relative_to(P('c:/foO')), P('Bar')) @@ -1194,10 +1190,6 @@ def test_relative_to(self): self.assertEqual(p.relative_to('c:/foO/'), P('Bar')) self.assertEqual(p.relative_to(P('c:/foO/baR')), P()) self.assertEqual(p.relative_to('c:/foO/baR'), P()) - self.assertEqual(p.relative_to(P('c:'), walk_up=True), P('/Foo/Bar')) - self.assertEqual(p.relative_to('c:', walk_up=True), P('/Foo/Bar')) - self.assertEqual(str(p.relative_to(P('c:'), walk_up=True)), '\\Foo\\Bar') - self.assertEqual(str(p.relative_to('c:', walk_up=True)), '\\Foo\\Bar') self.assertEqual(p.relative_to(P('c:/'), walk_up=True), P('Foo/Bar')) self.assertEqual(p.relative_to('c:/', walk_up=True), P('Foo/Bar')) self.assertEqual(p.relative_to(P('c:/foO'), walk_up=True), P('Bar')) @@ -1209,6 +1201,8 @@ def test_relative_to(self): self.assertEqual(p.relative_to('C:/Foo/Bar/Baz', walk_up=True), P('..')) self.assertEqual(p.relative_to('C:/Foo/Baz', walk_up=True), P('../Bar')) # Unrelated paths. + self.assertRaises(ValueError, p.relative_to, 'c:') + self.assertRaises(ValueError, p.relative_to, P('c:')) self.assertRaises(ValueError, p.relative_to, P('C:/Baz')) self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Bar/Baz')) self.assertRaises(ValueError, p.relative_to, P('C:/Foo/Baz')) @@ -1218,6 +1212,8 @@ def test_relative_to(self): self.assertRaises(ValueError, p.relative_to, P('/')) self.assertRaises(ValueError, p.relative_to, P('/Foo')) self.assertRaises(ValueError, p.relative_to, P('//C/Foo')) + self.assertRaises(ValueError, p.relative_to, 'c:', walk_up=True) + self.assertRaises(ValueError, p.relative_to, P('c:'), walk_up=True) self.assertRaises(ValueError, p.relative_to, P('C:Foo'), walk_up=True) self.assertRaises(ValueError, p.relative_to, P('d:'), walk_up=True) self.assertRaises(ValueError, p.relative_to, P('d:/'), walk_up=True) @@ -1275,13 +1271,13 @@ def test_is_relative_to(self): self.assertFalse(p.is_relative_to(P('C:Foo/Bar/Baz'))) self.assertFalse(p.is_relative_to(P('C:Foo/Baz'))) p = P('C:/Foo/Bar') - self.assertTrue(p.is_relative_to('c:')) self.assertTrue(p.is_relative_to(P('c:/'))) self.assertTrue(p.is_relative_to(P('c:/foO'))) self.assertTrue(p.is_relative_to('c:/foO/')) self.assertTrue(p.is_relative_to(P('c:/foO/baR'))) self.assertTrue(p.is_relative_to('c:/foO/baR')) # Unrelated paths. + self.assertFalse(p.is_relative_to('c:')) self.assertFalse(p.is_relative_to(P('C:/Baz'))) self.assertFalse(p.is_relative_to(P('C:/Foo/Bar/Baz'))) self.assertFalse(p.is_relative_to(P('C:/Foo/Baz'))) diff --git a/Misc/NEWS.d/next/Library/2022-11-02-23-47-07.gh-issue-99029.7uCiIB.rst b/Misc/NEWS.d/next/Library/2022-11-02-23-47-07.gh-issue-99029.7uCiIB.rst new file mode 100644 index 00000000000000..0bfba5e1e32662 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-02-23-47-07.gh-issue-99029.7uCiIB.rst @@ -0,0 +1,2 @@ +:meth:`pathlib.PurePath.relative_to()` now treats naked Windows drive paths +as relative. This brings its behaviour in line with other parts of pathlib. From 5556d3e02ca841b82b1eb42cc3974e0a3bbffaac Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 26 Nov 2022 00:30:37 +0100 Subject: [PATCH 04/35] gh-98724: Fix warnings on Py_SETREF() usage (#99781) Cast argument to the expected type. --- Modules/_curses_panel.c | 2 +- Objects/longobject.c | 2 +- Objects/typeobject.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/_curses_panel.c b/Modules/_curses_panel.c index cd408d6aa35994..2144345de01ba3 100644 --- a/Modules/_curses_panel.c +++ b/Modules/_curses_panel.c @@ -424,7 +424,7 @@ _curses_panel_panel_replace_impl(PyCursesPanelObject *self, PyErr_SetString(state->PyCursesError, "replace_panel() returned ERR"); return NULL; } - Py_SETREF(po->wo, Py_NewRef(win)); + Py_SETREF(po->wo, (PyCursesWindowObject*)Py_NewRef(win)); Py_RETURN_NONE; } diff --git a/Objects/longobject.c b/Objects/longobject.c index f4bd981e4b9870..c84b4d3f316d5d 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -4775,7 +4775,7 @@ long_pow(PyObject *v, PyObject *w, PyObject *x) * because we're primarily trying to cut overhead for small powers. */ assert(bi); /* else there is no significant bit */ - Py_SETREF(z, Py_NewRef(a)); + Py_SETREF(z, (PyLongObject*)Py_NewRef(a)); for (bit = 2; ; bit <<= 1) { if (bit > bi) { /* found the first bit */ assert((bi & bit) == 0); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index ad8a936fa7ce20..b993aa405f6b6a 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -9593,7 +9593,7 @@ super_init_impl(PyObject *self, PyTypeObject *type, PyObject *obj) { return -1; Py_INCREF(obj); } - Py_XSETREF(su->type, Py_NewRef(type)); + Py_XSETREF(su->type, (PyTypeObject*)Py_NewRef(type)); Py_XSETREF(su->obj, obj); Py_XSETREF(su->obj_type, obj_type); return 0; From a86d8545221b16e714ffe3bda5afafc1d4748d13 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Fri, 25 Nov 2022 19:03:16 -0500 Subject: [PATCH 05/35] Fix typo in `__match_args__` doc (#99785) A opy of #98549, whose author (@icecream17) uses a school computer that blocks the CLA site. I did not mention this in commit comment above so CLA bot does not pick up the name and request the CLA again. --- Doc/reference/datamodel.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 301f41f3952c96..fd682fcff02003 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -2823,7 +2823,7 @@ Customizing positional arguments in class pattern matching When using a class name in a pattern, positional arguments in the pattern are not allowed by default, i.e. ``case MyClass(x, y)`` is typically invalid without special -support in ``MyClass``. To be able to use that kind of patterns, the class needs to +support in ``MyClass``. To be able to use that kind of pattern, the class needs to define a *__match_args__* attribute. .. data:: object.__match_args__ From ec2b76aa8b7c6313293ff9c6814e8bc31e08fcaf Mon Sep 17 00:00:00 2001 From: TheShermanTanker <32636402+TheShermanTanker@users.noreply.github.com> Date: Sat, 26 Nov 2022 17:31:42 +0800 Subject: [PATCH 06/35] GH-95896: posixmodule.c: fix osdefs.h inclusion to not depend on compiler (#95897) Co-authored-by: Steve Dower --- Modules/posixmodule.c | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 8185517b06b5dd..95ecf1c7c4b28c 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -16,6 +16,9 @@ #ifdef MS_WINDOWS # include # include +# include // UNLEN +# include "osdefs.h" // SEP +# define HAVE_SYMLINK #endif #ifdef __VXWORKS__ @@ -426,18 +429,7 @@ extern char *ctermid_r(char *); # ifdef HAVE_PROCESS_H # include # endif -# ifndef IO_REPARSE_TAG_SYMLINK -# define IO_REPARSE_TAG_SYMLINK (0xA000000CL) -# endif -# ifndef IO_REPARSE_TAG_MOUNT_POINT -# define IO_REPARSE_TAG_MOUNT_POINT (0xA0000003L) -# endif -# include "osdefs.h" // SEP # include -# include -# include // ShellExecute() -# include // UNLEN -# define HAVE_SYMLINK #endif /* _MSC_VER */ #ifndef MAXPATHLEN From 47d673d81fc315069c14f9438ebe61fb70ef1ccc Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Sat, 26 Nov 2022 12:33:48 +0300 Subject: [PATCH 07/35] gh-99502: mention bytes-like objects as input in `secrets.compare_digest` (GH-99512) Now it is in sync with https://docs.python.org/3/library/hmac.html#hmac.compare_digest It is the same function, just re-exported. So, I guess they should mention the same input types. --- Doc/library/secrets.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Doc/library/secrets.rst b/Doc/library/secrets.rst index dc8e5f46fb581e..4405dfc0535973 100644 --- a/Doc/library/secrets.rst +++ b/Doc/library/secrets.rst @@ -128,7 +128,9 @@ Other functions .. function:: compare_digest(a, b) - Return ``True`` if strings *a* and *b* are equal, otherwise ``False``, + Return ``True`` if strings or + :term:`bytes-like objects ` + *a* and *b* are equal, otherwise ``False``, using a "constant-time compare" to reduce the risk of `timing attacks `_. See :func:`hmac.compare_digest` for additional details. From e35ca417fe81a64985c2b29e863ce418ae75b96e Mon Sep 17 00:00:00 2001 From: Sam James Date: Sat, 26 Nov 2022 13:08:49 +0000 Subject: [PATCH 08/35] gh-99086: Fix -Wstrict-prototypes, -Wimplicit-function-declaration warnings in configure.ac (#99406) Follow up to 12078e78f6e4a21f344e4eaff529e1ff3b97734f. --- ...2-11-24-02-58-10.gh-issue-99086.DV_4Br.rst | 1 + configure | 52 ++++++++++--------- configure.ac | 52 ++++++++++--------- 3 files changed, 57 insertions(+), 48 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst diff --git a/Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst b/Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst new file mode 100644 index 00000000000000..2dace165ca1ada --- /dev/null +++ b/Misc/NEWS.d/next/Build/2022-11-24-02-58-10.gh-issue-99086.DV_4Br.rst @@ -0,0 +1 @@ +Fix ``-Wimplicit-int``, ``-Wstrict-prototypes``, and ``-Wimplicit-function-declaration`` compiler warnings in :program:`configure` checks. diff --git a/configure b/configure index 047fd94219d5be..3f8daf9dad5fd8 100755 --- a/configure +++ b/configure @@ -6755,7 +6755,7 @@ if test "x$enable_profiling" = xyes; then CC="$CC -pg" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -int main() { return 0; } +int main(void) { return 0; } _ACEOF if ac_fn_c_try_link "$LINENO"; then : @@ -9220,7 +9220,7 @@ else void* routine(void* p){return NULL;} -int main(){ +int main(void){ pthread_t p; if(pthread_create(&p,NULL,routine,NULL)!=0) return 1; @@ -9275,7 +9275,7 @@ else void* routine(void* p){return NULL;} -int main(){ +int main(void){ pthread_t p; if(pthread_create(&p,NULL,routine,NULL)!=0) return 1; @@ -9324,7 +9324,7 @@ else void* routine(void* p){return NULL;} -int main(){ +int main(void){ pthread_t p; if(pthread_create(&p,NULL,routine,NULL)!=0) return 1; @@ -9373,7 +9373,7 @@ else void* routine(void* p){return NULL;} -int main(){ +int main(void){ pthread_t p; if(pthread_create(&p,NULL,routine,NULL)!=0) return 1; @@ -12233,7 +12233,7 @@ else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -int main() +int main(void) { char s[16]; int i, *p1, *p2; @@ -15042,6 +15042,7 @@ $as_echo_n "checking for pthread_create in -lpthread... " >&6; } /* end confdefs.h. */ #include +#include #include void * start_routine (void *arg) { exit (0); } @@ -15352,7 +15353,7 @@ else void *foo(void *parm) { return NULL; } - int main() { + int main(void) { pthread_attr_t attr; pthread_t id; if (pthread_attr_init(&attr)) return (-1); @@ -16954,7 +16955,7 @@ else #include #include -int main(int argc, char*argv[]) +int main(int argc, char *argv[]) { if(chflags(argv[0], 0) != 0) return 1; @@ -17003,7 +17004,7 @@ else #include #include -int main(int argc, char*argv[]) +int main(int argc, char *argv[]) { if(lchflags(argv[0], 0) != 0) return 1; @@ -19711,7 +19712,7 @@ else #include #include -int main() +int main(void) { int passive, gaierr, inet4 = 0, inet6 = 0; struct addrinfo hints, *ai, *aitop; @@ -20908,7 +20909,7 @@ else #include #include -int main() { +int main(void) { volatile double x, y, z; /* 1./(1-2**-53) -> 1+2**-52 (correct), 1.0 (double rounding) */ x = 0.99999999999999989; /* 1-2**-53 */ @@ -21687,7 +21688,7 @@ else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -int main() +int main(void) { return (((-1)>>3 == -1) ? 0 : 1); } @@ -22589,7 +22590,7 @@ else #include #include -int main() +int main(void) { int val1 = nice(1); if (val1 != -1 && val1 == nice(2)) @@ -22631,7 +22632,7 @@ else #include #include -int main() +int main(void) { struct pollfd poll_struct = { 42, POLLIN|POLLPRI|POLLOUT, 0 }; int poll_test; @@ -22688,7 +22689,7 @@ else extern char *tzname[]; #endif -int main() +int main(void) { /* Note that we need to ensure that not only does tzset(3) do 'something' with localtime, but it works as documented @@ -24350,9 +24351,10 @@ else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ +#include #include -#include -int main() { +#include +int main(void) { size_t len = -1; const char *str = "text"; len = mbstowcs(NULL, str, 0); @@ -24552,7 +24554,7 @@ else #include #include void foo(void *p, void *q) { memmove(p, q, 19); } -int main() { +int main(void) { char a[32] = "123456789000000000"; foo(&a[9], a); if (strcmp(a, "123456789123456789000000000") != 0) @@ -24607,7 +24609,7 @@ else ); return r; } - int main() { + int main(void) { int p = 8; if ((foo(&p) ? : p) != 6) return 1; @@ -24650,7 +24652,7 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext #include atomic_int int_var; atomic_uintptr_t uintptr_var; - int main() { + int main(void) { atomic_store_explicit(&int_var, 5, memory_order_relaxed); atomic_store_explicit(&uintptr_var, 0, memory_order_relaxed); int loaded_value = atomic_load_explicit(&int_var, memory_order_seq_cst); @@ -24691,7 +24693,7 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext int val; - int main() { + int main(void) { __atomic_store_n(&val, 1, __ATOMIC_SEQ_CST); (void)__atomic_load_n(&val, __ATOMIC_SEQ_CST); return 0; @@ -24767,7 +24769,7 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext #include - int main() { + int main(void) { struct dirent entry; return entry.d_type == DT_UNKNOWN; } @@ -24805,11 +24807,12 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ + #include #include #include #include - int main() { + int main(void) { char buffer[1]; const size_t buflen = sizeof(buffer); const int flags = GRND_NONBLOCK; @@ -24852,9 +24855,10 @@ cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ + #include #include - int main() { + int main(void) { char buffer[1]; const size_t buflen = sizeof(buffer); const int flags = 0; diff --git a/configure.ac b/configure.ac index 19f12e8d5402fb..734a4db8389915 100644 --- a/configure.ac +++ b/configure.ac @@ -1427,7 +1427,7 @@ AC_ARG_ENABLE(profiling, if test "x$enable_profiling" = xyes; then ac_save_cc="$CC" CC="$CC -pg" - AC_LINK_IFELSE([AC_LANG_SOURCE([[int main() { return 0; }]])], + AC_LINK_IFELSE([AC_LANG_SOURCE([[int main(void) { return 0; }]])], [], [enable_profiling=no]) CC="$ac_save_cc" @@ -2553,7 +2553,7 @@ AC_CACHE_CHECK([whether pthreads are available without options], void* routine(void* p){return NULL;} -int main(){ +int main(void){ pthread_t p; if(pthread_create(&p,NULL,routine,NULL)!=0) return 1; @@ -2586,7 +2586,7 @@ AC_RUN_IFELSE([AC_LANG_SOURCE([[ void* routine(void* p){return NULL;} -int main(){ +int main(void){ pthread_t p; if(pthread_create(&p,NULL,routine,NULL)!=0) return 1; @@ -2613,7 +2613,7 @@ AC_RUN_IFELSE([AC_LANG_SOURCE([[ void* routine(void* p){return NULL;} -int main(){ +int main(void){ pthread_t p; if(pthread_create(&p,NULL,routine,NULL)!=0) return 1; @@ -2640,7 +2640,7 @@ AC_RUN_IFELSE([AC_LANG_SOURCE([[ void* routine(void* p){return NULL;} -int main(){ +int main(void){ pthread_t p; if(pthread_create(&p,NULL,routine,NULL)!=0) return 1; @@ -3578,7 +3578,7 @@ esac # check for systems that require aligned memory access AC_CACHE_CHECK([aligned memory access is required], [ac_cv_aligned_required], [AC_RUN_IFELSE([AC_LANG_SOURCE([[ -int main() +int main(void) { char s[16]; int i, *p1, *p2; @@ -4292,6 +4292,7 @@ yes AC_MSG_CHECKING([for pthread_create in -lpthread]) AC_LINK_IFELSE([AC_LANG_PROGRAM([[ #include +#include #include void * start_routine (void *arg) { exit (0); }]], [[ @@ -4361,7 +4362,7 @@ if test "$posix_threads" = "yes"; then void *foo(void *parm) { return NULL; } - int main() { + int main(void) { pthread_attr_t attr; pthread_t id; if (pthread_attr_init(&attr)) return (-1); @@ -4898,7 +4899,7 @@ AC_CACHE_CHECK([for chflags], [ac_cv_have_chflags], [dnl AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include #include -int main(int argc, char*argv[]) +int main(int argc, char *argv[]) { if(chflags(argv[0], 0) != 0) return 1; @@ -4920,7 +4921,7 @@ AC_CACHE_CHECK([for lchflags], [ac_cv_have_lchflags], [dnl AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include #include -int main(int argc, char*argv[]) +int main(int argc, char *argv[]) { if(lchflags(argv[0], 0) != 0) return 1; @@ -5196,7 +5197,7 @@ AS_VAR_IF([ac_cv_func_getaddrinfo], [yes], [ #include #include -int main() +int main(void) { int passive, gaierr, inet4 = 0, inet6 = 0; struct addrinfo hints, *ai, *aitop; @@ -5612,7 +5613,7 @@ CC="$CC $BASECFLAGS" AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include #include -int main() { +int main(void) { volatile double x, y, z; /* 1./(1-2**-53) -> 1+2**-52 (correct), 1.0 (double rounding) */ x = 0.99999999999999989; /* 1-2**-53 */ @@ -5919,7 +5920,7 @@ fi], # or fills with zeros (like the Cray J90, according to Tim Peters). AC_CACHE_CHECK([whether right shift extends the sign bit], [ac_cv_rshift_extends_sign], [ AC_RUN_IFELSE([AC_LANG_SOURCE([[ -int main() +int main(void) { return (((-1)>>3 == -1) ? 0 : 1); } @@ -6118,7 +6119,7 @@ AC_CACHE_CHECK([for broken nice()], [ac_cv_broken_nice], [ AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include #include -int main() +int main(void) { int val1 = nice(1); if (val1 != -1 && val1 == nice(2)) @@ -6140,7 +6141,7 @@ AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include #include -int main() +int main(void) { struct pollfd poll_struct = { 42, POLLIN|POLLPRI|POLLOUT, 0 }; int poll_test; @@ -6176,7 +6177,7 @@ AC_RUN_IFELSE([AC_LANG_SOURCE([[ extern char *tzname[]; #endif -int main() +int main(void) { /* Note that we need to ensure that not only does tzset(3) do 'something' with localtime, but it works as documented @@ -6517,9 +6518,10 @@ AC_CHECK_TYPE(socklen_t,, AC_CACHE_CHECK([for broken mbstowcs], [ac_cv_broken_mbstowcs], AC_RUN_IFELSE([AC_LANG_SOURCE([[ +#include #include -#include -int main() { +#include +int main(void) { size_t len = -1; const char *str = "text"; len = mbstowcs(NULL, str, 0); @@ -6646,7 +6648,7 @@ AC_RUN_IFELSE([AC_LANG_SOURCE([[ #include #include void foo(void *p, void *q) { memmove(p, q, 19); } -int main() { +int main(void) { char a[32] = "123456789000000000"; foo(&a[9], a); if (strcmp(a, "123456789123456789000000000") != 0) @@ -6687,7 +6689,7 @@ if test "$ac_cv_gcc_asm_for_x87" = yes; then ); return r; } - int main() { + int main(void) { int p = 8; if ((foo(&p) ? : p) != 6) return 1; @@ -6715,7 +6717,7 @@ AC_LINK_IFELSE( #include atomic_int int_var; atomic_uintptr_t uintptr_var; - int main() { + int main(void) { atomic_store_explicit(&int_var, 5, memory_order_relaxed); atomic_store_explicit(&uintptr_var, 0, memory_order_relaxed); int loaded_value = atomic_load_explicit(&int_var, memory_order_seq_cst); @@ -6736,7 +6738,7 @@ AC_LINK_IFELSE( [ AC_LANG_SOURCE([[ int val; - int main() { + int main(void) { __atomic_store_n(&val, 1, __ATOMIC_SEQ_CST); (void)__atomic_load_n(&val, __ATOMIC_SEQ_CST); return 0; @@ -6777,7 +6779,7 @@ AC_LINK_IFELSE( AC_LANG_SOURCE([[ #include - int main() { + int main(void) { struct dirent entry; return entry.d_type == DT_UNKNOWN; } @@ -6795,11 +6797,12 @@ AC_CACHE_CHECK([for the Linux getrandom() syscall], [ac_cv_getrandom_syscall], [ AC_LINK_IFELSE( [ AC_LANG_SOURCE([[ + #include #include #include #include - int main() { + int main(void) { char buffer[1]; const size_t buflen = sizeof(buffer); const int flags = GRND_NONBLOCK; @@ -6822,9 +6825,10 @@ AC_CACHE_CHECK([for the getrandom() function], [ac_cv_func_getrandom], [ AC_LINK_IFELSE( [ AC_LANG_SOURCE([[ + #include #include - int main() { + int main(void) { char buffer[1]; const size_t buflen = sizeof(buffer); const int flags = 0; From dc063a25d29840d863b15c86fdab15b4a1894c73 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Nov 2022 08:28:49 -0500 Subject: [PATCH 09/35] gh-97966: Restore prior expectation that uname_result._fields and ._asdict would include the processor. (gh-98343) --- Lib/platform.py | 6 ++++-- Lib/test/test_platform.py | 8 ++++++++ .../Library/2022-10-16-18-52-00.gh-issue-97966.humlhz.rst | 2 ++ 3 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-10-16-18-52-00.gh-issue-97966.humlhz.rst diff --git a/Lib/platform.py b/Lib/platform.py index 9f5b317287530b..6745321e31c279 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -847,6 +847,8 @@ class uname_result( except when needed. """ + _fields = ('system', 'node', 'release', 'version', 'machine', 'processor') + @functools.cached_property def processor(self): return _unknown_as_blank(_Processor.get()) @@ -860,7 +862,7 @@ def __iter__(self): @classmethod def _make(cls, iterable): # override factory to affect length check - num_fields = len(cls._fields) + num_fields = len(cls._fields) - 1 result = cls.__new__(cls, *iterable) if len(result) != num_fields + 1: msg = f'Expected {num_fields} arguments, got {len(result)}' @@ -874,7 +876,7 @@ def __len__(self): return len(tuple(iter(self))) def __reduce__(self): - return uname_result, tuple(self)[:len(self._fields)] + return uname_result, tuple(self)[:len(self._fields) - 1] _uname_cache = None diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 9c03a89fd57d07..3992faf8e5cd5b 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -277,6 +277,14 @@ def test_uname_slices(self): self.assertEqual(res[:], expected) self.assertEqual(res[:5], expected[:5]) + def test_uname_fields(self): + self.assertIn('processor', platform.uname()._fields) + + def test_uname_asdict(self): + res = platform.uname()._asdict() + self.assertEqual(len(res), 6) + self.assertIn('processor', res) + @unittest.skipIf(sys.platform in ['win32', 'OpenVMS'], "uname -p not used") @support.requires_subprocess() def test_uname_processor(self): diff --git a/Misc/NEWS.d/next/Library/2022-10-16-18-52-00.gh-issue-97966.humlhz.rst b/Misc/NEWS.d/next/Library/2022-10-16-18-52-00.gh-issue-97966.humlhz.rst new file mode 100644 index 00000000000000..b725465ae4f0ef --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-16-18-52-00.gh-issue-97966.humlhz.rst @@ -0,0 +1,2 @@ +On ``uname_result``, restored expectation that ``_fields`` and ``_asdict`` +would include all six properties including ``processor``. From 7796d3179b71536dd1d2ca7fdbc1255bdb8cfb52 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Nov 2022 09:44:13 -0500 Subject: [PATCH 10/35] gh-98098: Create packages from zipfile and test_zipfile (gh-98103) * gh-98098: Move zipfile into a package. * Moved test_zipfile to a package * Extracted module for test_path. * Add blurb * Add jaraco as owner of zipfile.Path. * Synchronize with minor changes found at jaraco/zipp@d9e7f4352d. --- .github/CODEOWNERS | 3 + Lib/test/test_zipfile/__init__.py | 5 + .../test_core.py} | 420 +---------------- Lib/test/test_zipfile/test_path.py | 423 ++++++++++++++++++ Lib/{zipfile.py => zipfile/__init__.py} | 387 +--------------- Lib/zipfile/__main__.py | 77 ++++ Lib/zipfile/_path.py | 315 +++++++++++++ ...2-10-08-15-41-00.gh-issue-98098.DugpWi.rst | 2 + 8 files changed, 834 insertions(+), 798 deletions(-) create mode 100644 Lib/test/test_zipfile/__init__.py rename Lib/test/{test_zipfile.py => test_zipfile/test_core.py} (90%) create mode 100644 Lib/test/test_zipfile/test_path.py rename Lib/{zipfile.py => zipfile/__init__.py} (88%) create mode 100644 Lib/zipfile/__main__.py create mode 100644 Lib/zipfile/_path.py create mode 100644 Misc/NEWS.d/next/Library/2022-10-08-15-41-00.gh-issue-98098.DugpWi.rst diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 5f6d86209b842e..5d30c0928e5ab8 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -154,3 +154,6 @@ Lib/ast.py @isidentical # pathlib **/*pathlib* @brettcannon + +# zipfile.Path +**/*zipfile/*_path.py @jaraco diff --git a/Lib/test/test_zipfile/__init__.py b/Lib/test/test_zipfile/__init__.py new file mode 100644 index 00000000000000..4b16ecc31156a5 --- /dev/null +++ b/Lib/test/test_zipfile/__init__.py @@ -0,0 +1,5 @@ +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile/test_core.py similarity index 90% rename from Lib/test/test_zipfile.py rename to Lib/test/test_zipfile/test_core.py index 6f6f4bc26b0d40..bb0f1467735bcf 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile/test_core.py @@ -6,7 +6,6 @@ import os import pathlib import posixpath -import string import struct import subprocess import sys @@ -14,7 +13,6 @@ import unittest import unittest.mock as mock import zipfile -import functools from tempfile import TemporaryFile @@ -2715,13 +2713,13 @@ def tearDown(self): class ZipInfoTests(unittest.TestCase): def test_from_file(self): zi = zipfile.ZipInfo.from_file(__file__) - self.assertEqual(posixpath.basename(zi.filename), 'test_zipfile.py') + self.assertEqual(posixpath.basename(zi.filename), 'test_core.py') self.assertFalse(zi.is_dir()) self.assertEqual(zi.file_size, os.path.getsize(__file__)) def test_from_file_pathlike(self): zi = zipfile.ZipInfo.from_file(pathlib.Path(__file__)) - self.assertEqual(posixpath.basename(zi.filename), 'test_zipfile.py') + self.assertEqual(posixpath.basename(zi.filename), 'test_core.py') self.assertFalse(zi.is_dir()) self.assertEqual(zi.file_size, os.path.getsize(__file__)) @@ -2867,420 +2865,6 @@ def test_execute_zip64(self): self.assertIn(b'number in executable: 5', output) -# Poor man's technique to consume a (smallish) iterable. -consume = tuple - - -# from jaraco.itertools 5.0 -class jaraco: - class itertools: - class Counter: - def __init__(self, i): - self.count = 0 - self._orig_iter = iter(i) - - def __iter__(self): - return self - - def __next__(self): - result = next(self._orig_iter) - self.count += 1 - return result - - -def add_dirs(zf): - """ - Given a writable zip file zf, inject directory entries for - any directories implied by the presence of children. - """ - for name in zipfile.CompleteDirs._implied_dirs(zf.namelist()): - zf.writestr(name, b"") - return zf - - -def build_alpharep_fixture(): - """ - Create a zip file with this structure: - - . - ├── a.txt - ├── b - │ ├── c.txt - │ ├── d - │ │ └── e.txt - │ └── f.txt - └── g - └── h - └── i.txt - - This fixture has the following key characteristics: - - - a file at the root (a) - - a file two levels deep (b/d/e) - - multiple files in a directory (b/c, b/f) - - a directory containing only a directory (g/h) - - "alpha" because it uses alphabet - "rep" because it's a representative example - """ - data = io.BytesIO() - zf = zipfile.ZipFile(data, "w") - zf.writestr("a.txt", b"content of a") - zf.writestr("b/c.txt", b"content of c") - zf.writestr("b/d/e.txt", b"content of e") - zf.writestr("b/f.txt", b"content of f") - zf.writestr("g/h/i.txt", b"content of i") - zf.filename = "alpharep.zip" - return zf - - -def pass_alpharep(meth): - """ - Given a method, wrap it in a for loop that invokes method - with each subtest. - """ - - @functools.wraps(meth) - def wrapper(self): - for alpharep in self.zipfile_alpharep(): - meth(self, alpharep=alpharep) - - return wrapper - - -class TestPath(unittest.TestCase): - def setUp(self): - self.fixtures = contextlib.ExitStack() - self.addCleanup(self.fixtures.close) - - def zipfile_alpharep(self): - with self.subTest(): - yield build_alpharep_fixture() - with self.subTest(): - yield add_dirs(build_alpharep_fixture()) - - def zipfile_ondisk(self, alpharep): - tmpdir = pathlib.Path(self.fixtures.enter_context(temp_dir())) - buffer = alpharep.fp - alpharep.close() - path = tmpdir / alpharep.filename - with path.open("wb") as strm: - strm.write(buffer.getvalue()) - return path - - @pass_alpharep - def test_iterdir_and_types(self, alpharep): - root = zipfile.Path(alpharep) - assert root.is_dir() - a, b, g = root.iterdir() - assert a.is_file() - assert b.is_dir() - assert g.is_dir() - c, f, d = b.iterdir() - assert c.is_file() and f.is_file() - (e,) = d.iterdir() - assert e.is_file() - (h,) = g.iterdir() - (i,) = h.iterdir() - assert i.is_file() - - @pass_alpharep - def test_is_file_missing(self, alpharep): - root = zipfile.Path(alpharep) - assert not root.joinpath('missing.txt').is_file() - - @pass_alpharep - def test_iterdir_on_file(self, alpharep): - root = zipfile.Path(alpharep) - a, b, g = root.iterdir() - with self.assertRaises(ValueError): - a.iterdir() - - @pass_alpharep - def test_subdir_is_dir(self, alpharep): - root = zipfile.Path(alpharep) - assert (root / 'b').is_dir() - assert (root / 'b/').is_dir() - assert (root / 'g').is_dir() - assert (root / 'g/').is_dir() - - @pass_alpharep - def test_open(self, alpharep): - root = zipfile.Path(alpharep) - a, b, g = root.iterdir() - with a.open(encoding="utf-8") as strm: - data = strm.read() - assert data == "content of a" - - def test_open_write(self): - """ - If the zipfile is open for write, it should be possible to - write bytes or text to it. - """ - zf = zipfile.Path(zipfile.ZipFile(io.BytesIO(), mode='w')) - with zf.joinpath('file.bin').open('wb') as strm: - strm.write(b'binary contents') - with zf.joinpath('file.txt').open('w', encoding="utf-8") as strm: - strm.write('text file') - - def test_open_extant_directory(self): - """ - Attempting to open a directory raises IsADirectoryError. - """ - zf = zipfile.Path(add_dirs(build_alpharep_fixture())) - with self.assertRaises(IsADirectoryError): - zf.joinpath('b').open() - - @pass_alpharep - def test_open_binary_invalid_args(self, alpharep): - root = zipfile.Path(alpharep) - with self.assertRaises(ValueError): - root.joinpath('a.txt').open('rb', encoding='utf-8') - with self.assertRaises(ValueError): - root.joinpath('a.txt').open('rb', 'utf-8') - - def test_open_missing_directory(self): - """ - Attempting to open a missing directory raises FileNotFoundError. - """ - zf = zipfile.Path(add_dirs(build_alpharep_fixture())) - with self.assertRaises(FileNotFoundError): - zf.joinpath('z').open() - - @pass_alpharep - def test_read(self, alpharep): - root = zipfile.Path(alpharep) - a, b, g = root.iterdir() - assert a.read_text(encoding="utf-8") == "content of a" - assert a.read_bytes() == b"content of a" - - @pass_alpharep - def test_joinpath(self, alpharep): - root = zipfile.Path(alpharep) - a = root.joinpath("a.txt") - assert a.is_file() - e = root.joinpath("b").joinpath("d").joinpath("e.txt") - assert e.read_text(encoding="utf-8") == "content of e" - - @pass_alpharep - def test_joinpath_multiple(self, alpharep): - root = zipfile.Path(alpharep) - e = root.joinpath("b", "d", "e.txt") - assert e.read_text(encoding="utf-8") == "content of e" - - @pass_alpharep - def test_traverse_truediv(self, alpharep): - root = zipfile.Path(alpharep) - a = root / "a.txt" - assert a.is_file() - e = root / "b" / "d" / "e.txt" - assert e.read_text(encoding="utf-8") == "content of e" - - @pass_alpharep - def test_traverse_simplediv(self, alpharep): - """ - Disable the __future__.division when testing traversal. - """ - code = compile( - source="zipfile.Path(alpharep) / 'a'", - filename="(test)", - mode="eval", - dont_inherit=True, - ) - eval(code) - - @pass_alpharep - def test_pathlike_construction(self, alpharep): - """ - zipfile.Path should be constructable from a path-like object - """ - zipfile_ondisk = self.zipfile_ondisk(alpharep) - pathlike = pathlib.Path(str(zipfile_ondisk)) - zipfile.Path(pathlike) - - @pass_alpharep - def test_traverse_pathlike(self, alpharep): - root = zipfile.Path(alpharep) - root / pathlib.Path("a") - - @pass_alpharep - def test_parent(self, alpharep): - root = zipfile.Path(alpharep) - assert (root / 'a').parent.at == '' - assert (root / 'a' / 'b').parent.at == 'a/' - - @pass_alpharep - def test_dir_parent(self, alpharep): - root = zipfile.Path(alpharep) - assert (root / 'b').parent.at == '' - assert (root / 'b/').parent.at == '' - - @pass_alpharep - def test_missing_dir_parent(self, alpharep): - root = zipfile.Path(alpharep) - assert (root / 'missing dir/').parent.at == '' - - @pass_alpharep - def test_mutability(self, alpharep): - """ - If the underlying zipfile is changed, the Path object should - reflect that change. - """ - root = zipfile.Path(alpharep) - a, b, g = root.iterdir() - alpharep.writestr('foo.txt', 'foo') - alpharep.writestr('bar/baz.txt', 'baz') - assert any(child.name == 'foo.txt' for child in root.iterdir()) - assert (root / 'foo.txt').read_text(encoding="utf-8") == 'foo' - (baz,) = (root / 'bar').iterdir() - assert baz.read_text(encoding="utf-8") == 'baz' - - HUGE_ZIPFILE_NUM_ENTRIES = 2 ** 13 - - def huge_zipfile(self): - """Create a read-only zipfile with a huge number of entries entries.""" - strm = io.BytesIO() - zf = zipfile.ZipFile(strm, "w") - for entry in map(str, range(self.HUGE_ZIPFILE_NUM_ENTRIES)): - zf.writestr(entry, entry) - zf.mode = 'r' - return zf - - def test_joinpath_constant_time(self): - """ - Ensure joinpath on items in zipfile is linear time. - """ - root = zipfile.Path(self.huge_zipfile()) - entries = jaraco.itertools.Counter(root.iterdir()) - for entry in entries: - entry.joinpath('suffix') - # Check the file iterated all items - assert entries.count == self.HUGE_ZIPFILE_NUM_ENTRIES - - # @func_timeout.func_set_timeout(3) - def test_implied_dirs_performance(self): - data = ['/'.join(string.ascii_lowercase + str(n)) for n in range(10000)] - zipfile.CompleteDirs._implied_dirs(data) - - @pass_alpharep - def test_read_does_not_close(self, alpharep): - alpharep = self.zipfile_ondisk(alpharep) - with zipfile.ZipFile(alpharep) as file: - for rep in range(2): - zipfile.Path(file, 'a.txt').read_text(encoding="utf-8") - - @pass_alpharep - def test_subclass(self, alpharep): - class Subclass(zipfile.Path): - pass - - root = Subclass(alpharep) - assert isinstance(root / 'b', Subclass) - - @pass_alpharep - def test_filename(self, alpharep): - root = zipfile.Path(alpharep) - assert root.filename == pathlib.Path('alpharep.zip') - - @pass_alpharep - def test_root_name(self, alpharep): - """ - The name of the root should be the name of the zipfile - """ - root = zipfile.Path(alpharep) - assert root.name == 'alpharep.zip' == root.filename.name - - @pass_alpharep - def test_suffix(self, alpharep): - """ - The suffix of the root should be the suffix of the zipfile. - The suffix of each nested file is the final component's last suffix, if any. - Includes the leading period, just like pathlib.Path. - """ - root = zipfile.Path(alpharep) - assert root.suffix == '.zip' == root.filename.suffix - - b = root / "b.txt" - assert b.suffix == ".txt" - - c = root / "c" / "filename.tar.gz" - assert c.suffix == ".gz" - - d = root / "d" - assert d.suffix == "" - - @pass_alpharep - def test_suffixes(self, alpharep): - """ - The suffix of the root should be the suffix of the zipfile. - The suffix of each nested file is the final component's last suffix, if any. - Includes the leading period, just like pathlib.Path. - """ - root = zipfile.Path(alpharep) - assert root.suffixes == ['.zip'] == root.filename.suffixes - - b = root / 'b.txt' - assert b.suffixes == ['.txt'] - - c = root / 'c' / 'filename.tar.gz' - assert c.suffixes == ['.tar', '.gz'] - - d = root / 'd' - assert d.suffixes == [] - - e = root / '.hgrc' - assert e.suffixes == [] - - @pass_alpharep - def test_stem(self, alpharep): - """ - The final path component, without its suffix - """ - root = zipfile.Path(alpharep) - assert root.stem == 'alpharep' == root.filename.stem - - b = root / "b.txt" - assert b.stem == "b" - - c = root / "c" / "filename.tar.gz" - assert c.stem == "filename.tar" - - d = root / "d" - assert d.stem == "d" - - @pass_alpharep - def test_root_parent(self, alpharep): - root = zipfile.Path(alpharep) - assert root.parent == pathlib.Path('.') - root.root.filename = 'foo/bar.zip' - assert root.parent == pathlib.Path('foo') - - @pass_alpharep - def test_root_unnamed(self, alpharep): - """ - It is an error to attempt to get the name - or parent of an unnamed zipfile. - """ - alpharep.filename = None - root = zipfile.Path(alpharep) - with self.assertRaises(TypeError): - root.name - with self.assertRaises(TypeError): - root.parent - - # .name and .parent should still work on subs - sub = root / "b" - assert sub.name == "b" - assert sub.parent - - @pass_alpharep - def test_inheritance(self, alpharep): - cls = type('PathChild', (zipfile.Path,), {}) - for alpharep in self.zipfile_alpharep(): - file = cls(alpharep).joinpath('some dir').parent - assert isinstance(file, cls) - - class EncodedMetadataTests(unittest.TestCase): file_names = ['\u4e00', '\u4e8c', '\u4e09'] # Han 'one', 'two', 'three' file_content = [ diff --git a/Lib/test/test_zipfile/test_path.py b/Lib/test/test_zipfile/test_path.py new file mode 100644 index 00000000000000..3c62e9a0b0e65d --- /dev/null +++ b/Lib/test/test_zipfile/test_path.py @@ -0,0 +1,423 @@ +import io +import zipfile +import contextlib +import pathlib +import unittest +import string +import functools + +from test.support.os_helper import temp_dir + + +# Poor man's technique to consume a (smallish) iterable. +consume = tuple + + +# from jaraco.itertools 5.0 +class jaraco: + class itertools: + class Counter: + def __init__(self, i): + self.count = 0 + self._orig_iter = iter(i) + + def __iter__(self): + return self + + def __next__(self): + result = next(self._orig_iter) + self.count += 1 + return result + + +def add_dirs(zf): + """ + Given a writable zip file zf, inject directory entries for + any directories implied by the presence of children. + """ + for name in zipfile.CompleteDirs._implied_dirs(zf.namelist()): + zf.writestr(name, b"") + return zf + + +def build_alpharep_fixture(): + """ + Create a zip file with this structure: + + . + ├── a.txt + ├── b + │ ├── c.txt + │ ├── d + │ │ └── e.txt + │ └── f.txt + └── g + └── h + └── i.txt + + This fixture has the following key characteristics: + + - a file at the root (a) + - a file two levels deep (b/d/e) + - multiple files in a directory (b/c, b/f) + - a directory containing only a directory (g/h) + + "alpha" because it uses alphabet + "rep" because it's a representative example + """ + data = io.BytesIO() + zf = zipfile.ZipFile(data, "w") + zf.writestr("a.txt", b"content of a") + zf.writestr("b/c.txt", b"content of c") + zf.writestr("b/d/e.txt", b"content of e") + zf.writestr("b/f.txt", b"content of f") + zf.writestr("g/h/i.txt", b"content of i") + zf.filename = "alpharep.zip" + return zf + + +def pass_alpharep(meth): + """ + Given a method, wrap it in a for loop that invokes method + with each subtest. + """ + + @functools.wraps(meth) + def wrapper(self): + for alpharep in self.zipfile_alpharep(): + meth(self, alpharep=alpharep) + + return wrapper + + +class TestPath(unittest.TestCase): + def setUp(self): + self.fixtures = contextlib.ExitStack() + self.addCleanup(self.fixtures.close) + + def zipfile_alpharep(self): + with self.subTest(): + yield build_alpharep_fixture() + with self.subTest(): + yield add_dirs(build_alpharep_fixture()) + + def zipfile_ondisk(self, alpharep): + tmpdir = pathlib.Path(self.fixtures.enter_context(temp_dir())) + buffer = alpharep.fp + alpharep.close() + path = tmpdir / alpharep.filename + with path.open("wb") as strm: + strm.write(buffer.getvalue()) + return path + + @pass_alpharep + def test_iterdir_and_types(self, alpharep): + root = zipfile.Path(alpharep) + assert root.is_dir() + a, b, g = root.iterdir() + assert a.is_file() + assert b.is_dir() + assert g.is_dir() + c, f, d = b.iterdir() + assert c.is_file() and f.is_file() + (e,) = d.iterdir() + assert e.is_file() + (h,) = g.iterdir() + (i,) = h.iterdir() + assert i.is_file() + + @pass_alpharep + def test_is_file_missing(self, alpharep): + root = zipfile.Path(alpharep) + assert not root.joinpath('missing.txt').is_file() + + @pass_alpharep + def test_iterdir_on_file(self, alpharep): + root = zipfile.Path(alpharep) + a, b, g = root.iterdir() + with self.assertRaises(ValueError): + a.iterdir() + + @pass_alpharep + def test_subdir_is_dir(self, alpharep): + root = zipfile.Path(alpharep) + assert (root / 'b').is_dir() + assert (root / 'b/').is_dir() + assert (root / 'g').is_dir() + assert (root / 'g/').is_dir() + + @pass_alpharep + def test_open(self, alpharep): + root = zipfile.Path(alpharep) + a, b, g = root.iterdir() + with a.open(encoding="utf-8") as strm: + data = strm.read() + assert data == "content of a" + + def test_open_write(self): + """ + If the zipfile is open for write, it should be possible to + write bytes or text to it. + """ + zf = zipfile.Path(zipfile.ZipFile(io.BytesIO(), mode='w')) + with zf.joinpath('file.bin').open('wb') as strm: + strm.write(b'binary contents') + with zf.joinpath('file.txt').open('w', encoding="utf-8") as strm: + strm.write('text file') + + def test_open_extant_directory(self): + """ + Attempting to open a directory raises IsADirectoryError. + """ + zf = zipfile.Path(add_dirs(build_alpharep_fixture())) + with self.assertRaises(IsADirectoryError): + zf.joinpath('b').open() + + @pass_alpharep + def test_open_binary_invalid_args(self, alpharep): + root = zipfile.Path(alpharep) + with self.assertRaises(ValueError): + root.joinpath('a.txt').open('rb', encoding='utf-8') + with self.assertRaises(ValueError): + root.joinpath('a.txt').open('rb', 'utf-8') + + def test_open_missing_directory(self): + """ + Attempting to open a missing directory raises FileNotFoundError. + """ + zf = zipfile.Path(add_dirs(build_alpharep_fixture())) + with self.assertRaises(FileNotFoundError): + zf.joinpath('z').open() + + @pass_alpharep + def test_read(self, alpharep): + root = zipfile.Path(alpharep) + a, b, g = root.iterdir() + assert a.read_text(encoding="utf-8") == "content of a" + assert a.read_bytes() == b"content of a" + + @pass_alpharep + def test_joinpath(self, alpharep): + root = zipfile.Path(alpharep) + a = root.joinpath("a.txt") + assert a.is_file() + e = root.joinpath("b").joinpath("d").joinpath("e.txt") + assert e.read_text(encoding="utf-8") == "content of e" + + @pass_alpharep + def test_joinpath_multiple(self, alpharep): + root = zipfile.Path(alpharep) + e = root.joinpath("b", "d", "e.txt") + assert e.read_text(encoding="utf-8") == "content of e" + + @pass_alpharep + def test_traverse_truediv(self, alpharep): + root = zipfile.Path(alpharep) + a = root / "a.txt" + assert a.is_file() + e = root / "b" / "d" / "e.txt" + assert e.read_text(encoding="utf-8") == "content of e" + + @pass_alpharep + def test_traverse_simplediv(self, alpharep): + """ + Disable the __future__.division when testing traversal. + """ + code = compile( + source="zipfile.Path(alpharep) / 'a'", + filename="(test)", + mode="eval", + dont_inherit=True, + ) + eval(code) + + @pass_alpharep + def test_pathlike_construction(self, alpharep): + """ + zipfile.Path should be constructable from a path-like object + """ + zipfile_ondisk = self.zipfile_ondisk(alpharep) + pathlike = pathlib.Path(str(zipfile_ondisk)) + zipfile.Path(pathlike) + + @pass_alpharep + def test_traverse_pathlike(self, alpharep): + root = zipfile.Path(alpharep) + root / pathlib.Path("a") + + @pass_alpharep + def test_parent(self, alpharep): + root = zipfile.Path(alpharep) + assert (root / 'a').parent.at == '' + assert (root / 'a' / 'b').parent.at == 'a/' + + @pass_alpharep + def test_dir_parent(self, alpharep): + root = zipfile.Path(alpharep) + assert (root / 'b').parent.at == '' + assert (root / 'b/').parent.at == '' + + @pass_alpharep + def test_missing_dir_parent(self, alpharep): + root = zipfile.Path(alpharep) + assert (root / 'missing dir/').parent.at == '' + + @pass_alpharep + def test_mutability(self, alpharep): + """ + If the underlying zipfile is changed, the Path object should + reflect that change. + """ + root = zipfile.Path(alpharep) + a, b, g = root.iterdir() + alpharep.writestr('foo.txt', 'foo') + alpharep.writestr('bar/baz.txt', 'baz') + assert any(child.name == 'foo.txt' for child in root.iterdir()) + assert (root / 'foo.txt').read_text(encoding="utf-8") == 'foo' + (baz,) = (root / 'bar').iterdir() + assert baz.read_text(encoding="utf-8") == 'baz' + + HUGE_ZIPFILE_NUM_ENTRIES = 2**13 + + def huge_zipfile(self): + """Create a read-only zipfile with a huge number of entries entries.""" + strm = io.BytesIO() + zf = zipfile.ZipFile(strm, "w") + for entry in map(str, range(self.HUGE_ZIPFILE_NUM_ENTRIES)): + zf.writestr(entry, entry) + zf.mode = 'r' + return zf + + def test_joinpath_constant_time(self): + """ + Ensure joinpath on items in zipfile is linear time. + """ + root = zipfile.Path(self.huge_zipfile()) + entries = jaraco.itertools.Counter(root.iterdir()) + for entry in entries: + entry.joinpath('suffix') + # Check the file iterated all items + assert entries.count == self.HUGE_ZIPFILE_NUM_ENTRIES + + # @func_timeout.func_set_timeout(3) + def test_implied_dirs_performance(self): + data = ['/'.join(string.ascii_lowercase + str(n)) for n in range(10000)] + zipfile.CompleteDirs._implied_dirs(data) + + @pass_alpharep + def test_read_does_not_close(self, alpharep): + alpharep = self.zipfile_ondisk(alpharep) + with zipfile.ZipFile(alpharep) as file: + for rep in range(2): + zipfile.Path(file, 'a.txt').read_text(encoding="utf-8") + + @pass_alpharep + def test_subclass(self, alpharep): + class Subclass(zipfile.Path): + pass + + root = Subclass(alpharep) + assert isinstance(root / 'b', Subclass) + + @pass_alpharep + def test_filename(self, alpharep): + root = zipfile.Path(alpharep) + assert root.filename == pathlib.Path('alpharep.zip') + + @pass_alpharep + def test_root_name(self, alpharep): + """ + The name of the root should be the name of the zipfile + """ + root = zipfile.Path(alpharep) + assert root.name == 'alpharep.zip' == root.filename.name + + @pass_alpharep + def test_suffix(self, alpharep): + """ + The suffix of the root should be the suffix of the zipfile. + The suffix of each nested file is the final component's last suffix, if any. + Includes the leading period, just like pathlib.Path. + """ + root = zipfile.Path(alpharep) + assert root.suffix == '.zip' == root.filename.suffix + + b = root / "b.txt" + assert b.suffix == ".txt" + + c = root / "c" / "filename.tar.gz" + assert c.suffix == ".gz" + + d = root / "d" + assert d.suffix == "" + + @pass_alpharep + def test_suffixes(self, alpharep): + """ + The suffix of the root should be the suffix of the zipfile. + The suffix of each nested file is the final component's last suffix, if any. + Includes the leading period, just like pathlib.Path. + """ + root = zipfile.Path(alpharep) + assert root.suffixes == ['.zip'] == root.filename.suffixes + + b = root / 'b.txt' + assert b.suffixes == ['.txt'] + + c = root / 'c' / 'filename.tar.gz' + assert c.suffixes == ['.tar', '.gz'] + + d = root / 'd' + assert d.suffixes == [] + + e = root / '.hgrc' + assert e.suffixes == [] + + @pass_alpharep + def test_stem(self, alpharep): + """ + The final path component, without its suffix + """ + root = zipfile.Path(alpharep) + assert root.stem == 'alpharep' == root.filename.stem + + b = root / "b.txt" + assert b.stem == "b" + + c = root / "c" / "filename.tar.gz" + assert c.stem == "filename.tar" + + d = root / "d" + assert d.stem == "d" + + @pass_alpharep + def test_root_parent(self, alpharep): + root = zipfile.Path(alpharep) + assert root.parent == pathlib.Path('.') + root.root.filename = 'foo/bar.zip' + assert root.parent == pathlib.Path('foo') + + @pass_alpharep + def test_root_unnamed(self, alpharep): + """ + It is an error to attempt to get the name + or parent of an unnamed zipfile. + """ + alpharep.filename = None + root = zipfile.Path(alpharep) + with self.assertRaises(TypeError): + root.name + with self.assertRaises(TypeError): + root.parent + + # .name and .parent should still work on subs + sub = root / "b" + assert sub.name == "b" + assert sub.parent + + @pass_alpharep + def test_inheritance(self, alpharep): + cls = type('PathChild', (zipfile.Path,), {}) + for alpharep in self.zipfile_alpharep(): + file = cls(alpharep).joinpath('some dir').parent + assert isinstance(file, cls) diff --git a/Lib/zipfile.py b/Lib/zipfile/__init__.py similarity index 88% rename from Lib/zipfile.py rename to Lib/zipfile/__init__.py index 77b643caf9fc91..8f834267b28c2e 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile/__init__.py @@ -6,17 +6,13 @@ import binascii import importlib.util import io -import itertools import os -import posixpath import shutil import stat import struct import sys import threading import time -import contextlib -import pathlib try: import zlib # We may need its compression method @@ -2186,381 +2182,12 @@ def _compile(file, optimize=-1): return (fname, archivename) -def _parents(path): - """ - Given a path with elements separated by - posixpath.sep, generate all parents of that path. - - >>> list(_parents('b/d')) - ['b'] - >>> list(_parents('/b/d/')) - ['/b'] - >>> list(_parents('b/d/f/')) - ['b/d', 'b'] - >>> list(_parents('b')) - [] - >>> list(_parents('')) - [] - """ - return itertools.islice(_ancestry(path), 1, None) - - -def _ancestry(path): - """ - Given a path with elements separated by - posixpath.sep, generate all elements of that path - - >>> list(_ancestry('b/d')) - ['b/d', 'b'] - >>> list(_ancestry('/b/d/')) - ['/b/d', '/b'] - >>> list(_ancestry('b/d/f/')) - ['b/d/f', 'b/d', 'b'] - >>> list(_ancestry('b')) - ['b'] - >>> list(_ancestry('')) - [] - """ - path = path.rstrip(posixpath.sep) - while path and path != posixpath.sep: - yield path - path, tail = posixpath.split(path) - - -_dedupe = dict.fromkeys -"""Deduplicate an iterable in original order""" - - -def _difference(minuend, subtrahend): - """ - Return items in minuend not in subtrahend, retaining order - with O(1) lookup. - """ - return itertools.filterfalse(set(subtrahend).__contains__, minuend) - - -class CompleteDirs(ZipFile): - """ - A ZipFile subclass that ensures that implied directories - are always included in the namelist. - """ - - @staticmethod - def _implied_dirs(names): - parents = itertools.chain.from_iterable(map(_parents, names)) - as_dirs = (p + posixpath.sep for p in parents) - return _dedupe(_difference(as_dirs, names)) - - def namelist(self): - names = super(CompleteDirs, self).namelist() - return names + list(self._implied_dirs(names)) - - def _name_set(self): - return set(self.namelist()) - - def resolve_dir(self, name): - """ - If the name represents a directory, return that name - as a directory (with the trailing slash). - """ - names = self._name_set() - dirname = name + '/' - dir_match = name not in names and dirname in names - return dirname if dir_match else name - - @classmethod - def make(cls, source): - """ - Given a source (filename or zipfile), return an - appropriate CompleteDirs subclass. - """ - if isinstance(source, CompleteDirs): - return source - - if not isinstance(source, ZipFile): - return cls(source) - - # Only allow for FastLookup when supplied zipfile is read-only - if 'r' not in source.mode: - cls = CompleteDirs - - source.__class__ = cls - return source - - -class FastLookup(CompleteDirs): - """ - ZipFile subclass to ensure implicit - dirs exist and are resolved rapidly. - """ - - def namelist(self): - with contextlib.suppress(AttributeError): - return self.__names - self.__names = super(FastLookup, self).namelist() - return self.__names - - def _name_set(self): - with contextlib.suppress(AttributeError): - return self.__lookup - self.__lookup = super(FastLookup, self)._name_set() - return self.__lookup - - -class Path: - """ - A pathlib-compatible interface for zip files. - - Consider a zip file with this structure:: - - . - ├── a.txt - └── b - ├── c.txt - └── d - └── e.txt - - >>> data = io.BytesIO() - >>> zf = ZipFile(data, 'w') - >>> zf.writestr('a.txt', 'content of a') - >>> zf.writestr('b/c.txt', 'content of c') - >>> zf.writestr('b/d/e.txt', 'content of e') - >>> zf.filename = 'mem/abcde.zip' - - Path accepts the zipfile object itself or a filename - - >>> root = Path(zf) - - From there, several path operations are available. - - Directory iteration (including the zip file itself): +from ._path import ( # noqa: E402 + Path, - >>> a, b = root.iterdir() - >>> a - Path('mem/abcde.zip', 'a.txt') - >>> b - Path('mem/abcde.zip', 'b/') + # used privately for tests + CompleteDirs, # noqa: F401 +) - name property: - - >>> b.name - 'b' - - join with divide operator: - - >>> c = b / 'c.txt' - >>> c - Path('mem/abcde.zip', 'b/c.txt') - >>> c.name - 'c.txt' - - Read text: - - >>> c.read_text() - 'content of c' - - existence: - - >>> c.exists() - True - >>> (b / 'missing.txt').exists() - False - - Coercion to string: - - >>> import os - >>> str(c).replace(os.sep, posixpath.sep) - 'mem/abcde.zip/b/c.txt' - - At the root, ``name``, ``filename``, and ``parent`` - resolve to the zipfile. Note these attributes are not - valid and will raise a ``ValueError`` if the zipfile - has no filename. - - >>> root.name - 'abcde.zip' - >>> str(root.filename).replace(os.sep, posixpath.sep) - 'mem/abcde.zip' - >>> str(root.parent) - 'mem' - """ - - __repr = "{self.__class__.__name__}({self.root.filename!r}, {self.at!r})" - - def __init__(self, root, at=""): - """ - Construct a Path from a ZipFile or filename. - - Note: When the source is an existing ZipFile object, - its type (__class__) will be mutated to a - specialized type. If the caller wishes to retain the - original type, the caller should either create a - separate ZipFile object or pass a filename. - """ - self.root = FastLookup.make(root) - self.at = at - - def open(self, mode='r', *args, pwd=None, **kwargs): - """ - Open this entry as text or binary following the semantics - of ``pathlib.Path.open()`` by passing arguments through - to io.TextIOWrapper(). - """ - if self.is_dir(): - raise IsADirectoryError(self) - zip_mode = mode[0] - if not self.exists() and zip_mode == 'r': - raise FileNotFoundError(self) - stream = self.root.open(self.at, zip_mode, pwd=pwd) - if 'b' in mode: - if args or kwargs: - raise ValueError("encoding args invalid for binary operation") - return stream - else: - kwargs["encoding"] = io.text_encoding(kwargs.get("encoding")) - return io.TextIOWrapper(stream, *args, **kwargs) - - @property - def name(self): - return pathlib.Path(self.at).name or self.filename.name - - @property - def suffix(self): - return pathlib.Path(self.at).suffix or self.filename.suffix - - @property - def suffixes(self): - return pathlib.Path(self.at).suffixes or self.filename.suffixes - - @property - def stem(self): - return pathlib.Path(self.at).stem or self.filename.stem - - @property - def filename(self): - return pathlib.Path(self.root.filename).joinpath(self.at) - - def read_text(self, *args, **kwargs): - kwargs["encoding"] = io.text_encoding(kwargs.get("encoding")) - with self.open('r', *args, **kwargs) as strm: - return strm.read() - - def read_bytes(self): - with self.open('rb') as strm: - return strm.read() - - def _is_child(self, path): - return posixpath.dirname(path.at.rstrip("/")) == self.at.rstrip("/") - - def _next(self, at): - return self.__class__(self.root, at) - - def is_dir(self): - return not self.at or self.at.endswith("/") - - def is_file(self): - return self.exists() and not self.is_dir() - - def exists(self): - return self.at in self.root._name_set() - - def iterdir(self): - if not self.is_dir(): - raise ValueError("Can't listdir a file") - subs = map(self._next, self.root.namelist()) - return filter(self._is_child, subs) - - def __str__(self): - return posixpath.join(self.root.filename, self.at) - - def __repr__(self): - return self.__repr.format(self=self) - - def joinpath(self, *other): - next = posixpath.join(self.at, *other) - return self._next(self.root.resolve_dir(next)) - - __truediv__ = joinpath - - @property - def parent(self): - if not self.at: - return self.filename.parent - parent_at = posixpath.dirname(self.at.rstrip('/')) - if parent_at: - parent_at += '/' - return self._next(parent_at) - - -def main(args=None): - import argparse - - description = 'A simple command-line interface for zipfile module.' - parser = argparse.ArgumentParser(description=description) - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument('-l', '--list', metavar='', - help='Show listing of a zipfile') - group.add_argument('-e', '--extract', nargs=2, - metavar=('', ''), - help='Extract zipfile into target dir') - group.add_argument('-c', '--create', nargs='+', - metavar=('', ''), - help='Create zipfile from sources') - group.add_argument('-t', '--test', metavar='', - help='Test if a zipfile is valid') - parser.add_argument('--metadata-encoding', metavar='', - help='Specify encoding of member names for -l, -e and -t') - args = parser.parse_args(args) - - encoding = args.metadata_encoding - - if args.test is not None: - src = args.test - with ZipFile(src, 'r', metadata_encoding=encoding) as zf: - badfile = zf.testzip() - if badfile: - print("The following enclosed file is corrupted: {!r}".format(badfile)) - print("Done testing") - - elif args.list is not None: - src = args.list - with ZipFile(src, 'r', metadata_encoding=encoding) as zf: - zf.printdir() - - elif args.extract is not None: - src, curdir = args.extract - with ZipFile(src, 'r', metadata_encoding=encoding) as zf: - zf.extractall(curdir) - - elif args.create is not None: - if encoding: - print("Non-conforming encodings not supported with -c.", - file=sys.stderr) - sys.exit(1) - - zip_name = args.create.pop(0) - files = args.create - - def addToZip(zf, path, zippath): - if os.path.isfile(path): - zf.write(path, zippath, ZIP_DEFLATED) - elif os.path.isdir(path): - if zippath: - zf.write(path, zippath) - for nm in sorted(os.listdir(path)): - addToZip(zf, - os.path.join(path, nm), os.path.join(zippath, nm)) - # else: ignore - - with ZipFile(zip_name, 'w') as zf: - for path in files: - zippath = os.path.basename(path) - if not zippath: - zippath = os.path.basename(os.path.dirname(path)) - if zippath in ('', os.curdir, os.pardir): - zippath = '' - addToZip(zf, path, zippath) - - -if __name__ == "__main__": - main() +# used privately for tests +from .__main__ import main # noqa: F401, E402 diff --git a/Lib/zipfile/__main__.py b/Lib/zipfile/__main__.py new file mode 100644 index 00000000000000..a9e5fb1b8d72c4 --- /dev/null +++ b/Lib/zipfile/__main__.py @@ -0,0 +1,77 @@ +import sys +import os +from . import ZipFile, ZIP_DEFLATED + + +def main(args=None): + import argparse + + description = 'A simple command-line interface for zipfile module.' + parser = argparse.ArgumentParser(description=description) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('-l', '--list', metavar='', + help='Show listing of a zipfile') + group.add_argument('-e', '--extract', nargs=2, + metavar=('', ''), + help='Extract zipfile into target dir') + group.add_argument('-c', '--create', nargs='+', + metavar=('', ''), + help='Create zipfile from sources') + group.add_argument('-t', '--test', metavar='', + help='Test if a zipfile is valid') + parser.add_argument('--metadata-encoding', metavar='', + help='Specify encoding of member names for -l, -e and -t') + args = parser.parse_args(args) + + encoding = args.metadata_encoding + + if args.test is not None: + src = args.test + with ZipFile(src, 'r', metadata_encoding=encoding) as zf: + badfile = zf.testzip() + if badfile: + print("The following enclosed file is corrupted: {!r}".format(badfile)) + print("Done testing") + + elif args.list is not None: + src = args.list + with ZipFile(src, 'r', metadata_encoding=encoding) as zf: + zf.printdir() + + elif args.extract is not None: + src, curdir = args.extract + with ZipFile(src, 'r', metadata_encoding=encoding) as zf: + zf.extractall(curdir) + + elif args.create is not None: + if encoding: + print("Non-conforming encodings not supported with -c.", + file=sys.stderr) + sys.exit(1) + + zip_name = args.create.pop(0) + files = args.create + + def addToZip(zf, path, zippath): + if os.path.isfile(path): + zf.write(path, zippath, ZIP_DEFLATED) + elif os.path.isdir(path): + if zippath: + zf.write(path, zippath) + for nm in sorted(os.listdir(path)): + addToZip(zf, + os.path.join(path, nm), os.path.join(zippath, nm)) + # else: ignore + + with ZipFile(zip_name, 'w') as zf: + for path in files: + zippath = os.path.basename(path) + if not zippath: + zippath = os.path.basename(os.path.dirname(path)) + if zippath in ('', os.curdir, os.pardir): + zippath = '' + addToZip(zf, path, zippath) + + +if __name__ == "__main__": + main() diff --git a/Lib/zipfile/_path.py b/Lib/zipfile/_path.py new file mode 100644 index 00000000000000..67ef07a130d1ad --- /dev/null +++ b/Lib/zipfile/_path.py @@ -0,0 +1,315 @@ +import io +import posixpath +import zipfile +import itertools +import contextlib +import pathlib + + +__all__ = ['Path'] + + +def _parents(path): + """ + Given a path with elements separated by + posixpath.sep, generate all parents of that path. + + >>> list(_parents('b/d')) + ['b'] + >>> list(_parents('/b/d/')) + ['/b'] + >>> list(_parents('b/d/f/')) + ['b/d', 'b'] + >>> list(_parents('b')) + [] + >>> list(_parents('')) + [] + """ + return itertools.islice(_ancestry(path), 1, None) + + +def _ancestry(path): + """ + Given a path with elements separated by + posixpath.sep, generate all elements of that path + + >>> list(_ancestry('b/d')) + ['b/d', 'b'] + >>> list(_ancestry('/b/d/')) + ['/b/d', '/b'] + >>> list(_ancestry('b/d/f/')) + ['b/d/f', 'b/d', 'b'] + >>> list(_ancestry('b')) + ['b'] + >>> list(_ancestry('')) + [] + """ + path = path.rstrip(posixpath.sep) + while path and path != posixpath.sep: + yield path + path, tail = posixpath.split(path) + + +_dedupe = dict.fromkeys +"""Deduplicate an iterable in original order""" + + +def _difference(minuend, subtrahend): + """ + Return items in minuend not in subtrahend, retaining order + with O(1) lookup. + """ + return itertools.filterfalse(set(subtrahend).__contains__, minuend) + + +class CompleteDirs(zipfile.ZipFile): + """ + A ZipFile subclass that ensures that implied directories + are always included in the namelist. + """ + + @staticmethod + def _implied_dirs(names): + parents = itertools.chain.from_iterable(map(_parents, names)) + as_dirs = (p + posixpath.sep for p in parents) + return _dedupe(_difference(as_dirs, names)) + + def namelist(self): + names = super(CompleteDirs, self).namelist() + return names + list(self._implied_dirs(names)) + + def _name_set(self): + return set(self.namelist()) + + def resolve_dir(self, name): + """ + If the name represents a directory, return that name + as a directory (with the trailing slash). + """ + names = self._name_set() + dirname = name + '/' + dir_match = name not in names and dirname in names + return dirname if dir_match else name + + @classmethod + def make(cls, source): + """ + Given a source (filename or zipfile), return an + appropriate CompleteDirs subclass. + """ + if isinstance(source, CompleteDirs): + return source + + if not isinstance(source, zipfile.ZipFile): + return cls(source) + + # Only allow for FastLookup when supplied zipfile is read-only + if 'r' not in source.mode: + cls = CompleteDirs + + source.__class__ = cls + return source + + +class FastLookup(CompleteDirs): + """ + ZipFile subclass to ensure implicit + dirs exist and are resolved rapidly. + """ + + def namelist(self): + with contextlib.suppress(AttributeError): + return self.__names + self.__names = super(FastLookup, self).namelist() + return self.__names + + def _name_set(self): + with contextlib.suppress(AttributeError): + return self.__lookup + self.__lookup = super(FastLookup, self)._name_set() + return self.__lookup + + +class Path: + """ + A pathlib-compatible interface for zip files. + + Consider a zip file with this structure:: + + . + ├── a.txt + └── b + ├── c.txt + └── d + └── e.txt + + >>> data = io.BytesIO() + >>> zf = ZipFile(data, 'w') + >>> zf.writestr('a.txt', 'content of a') + >>> zf.writestr('b/c.txt', 'content of c') + >>> zf.writestr('b/d/e.txt', 'content of e') + >>> zf.filename = 'mem/abcde.zip' + + Path accepts the zipfile object itself or a filename + + >>> root = Path(zf) + + From there, several path operations are available. + + Directory iteration (including the zip file itself): + + >>> a, b = root.iterdir() + >>> a + Path('mem/abcde.zip', 'a.txt') + >>> b + Path('mem/abcde.zip', 'b/') + + name property: + + >>> b.name + 'b' + + join with divide operator: + + >>> c = b / 'c.txt' + >>> c + Path('mem/abcde.zip', 'b/c.txt') + >>> c.name + 'c.txt' + + Read text: + + >>> c.read_text() + 'content of c' + + existence: + + >>> c.exists() + True + >>> (b / 'missing.txt').exists() + False + + Coercion to string: + + >>> import os + >>> str(c).replace(os.sep, posixpath.sep) + 'mem/abcde.zip/b/c.txt' + + At the root, ``name``, ``filename``, and ``parent`` + resolve to the zipfile. Note these attributes are not + valid and will raise a ``ValueError`` if the zipfile + has no filename. + + >>> root.name + 'abcde.zip' + >>> str(root.filename).replace(os.sep, posixpath.sep) + 'mem/abcde.zip' + >>> str(root.parent) + 'mem' + """ + + __repr = "{self.__class__.__name__}({self.root.filename!r}, {self.at!r})" + + def __init__(self, root, at=""): + """ + Construct a Path from a ZipFile or filename. + + Note: When the source is an existing ZipFile object, + its type (__class__) will be mutated to a + specialized type. If the caller wishes to retain the + original type, the caller should either create a + separate ZipFile object or pass a filename. + """ + self.root = FastLookup.make(root) + self.at = at + + def open(self, mode='r', *args, pwd=None, **kwargs): + """ + Open this entry as text or binary following the semantics + of ``pathlib.Path.open()`` by passing arguments through + to io.TextIOWrapper(). + """ + if self.is_dir(): + raise IsADirectoryError(self) + zip_mode = mode[0] + if not self.exists() and zip_mode == 'r': + raise FileNotFoundError(self) + stream = self.root.open(self.at, zip_mode, pwd=pwd) + if 'b' in mode: + if args or kwargs: + raise ValueError("encoding args invalid for binary operation") + return stream + else: + kwargs["encoding"] = io.text_encoding(kwargs.get("encoding")) + return io.TextIOWrapper(stream, *args, **kwargs) + + @property + def name(self): + return pathlib.Path(self.at).name or self.filename.name + + @property + def suffix(self): + return pathlib.Path(self.at).suffix or self.filename.suffix + + @property + def suffixes(self): + return pathlib.Path(self.at).suffixes or self.filename.suffixes + + @property + def stem(self): + return pathlib.Path(self.at).stem or self.filename.stem + + @property + def filename(self): + return pathlib.Path(self.root.filename).joinpath(self.at) + + def read_text(self, *args, **kwargs): + kwargs["encoding"] = io.text_encoding(kwargs.get("encoding")) + with self.open('r', *args, **kwargs) as strm: + return strm.read() + + def read_bytes(self): + with self.open('rb') as strm: + return strm.read() + + def _is_child(self, path): + return posixpath.dirname(path.at.rstrip("/")) == self.at.rstrip("/") + + def _next(self, at): + return self.__class__(self.root, at) + + def is_dir(self): + return not self.at or self.at.endswith("/") + + def is_file(self): + return self.exists() and not self.is_dir() + + def exists(self): + return self.at in self.root._name_set() + + def iterdir(self): + if not self.is_dir(): + raise ValueError("Can't listdir a file") + subs = map(self._next, self.root.namelist()) + return filter(self._is_child, subs) + + def __str__(self): + return posixpath.join(self.root.filename, self.at) + + def __repr__(self): + return self.__repr.format(self=self) + + def joinpath(self, *other): + next = posixpath.join(self.at, *other) + return self._next(self.root.resolve_dir(next)) + + __truediv__ = joinpath + + @property + def parent(self): + if not self.at: + return self.filename.parent + parent_at = posixpath.dirname(self.at.rstrip('/')) + if parent_at: + parent_at += '/' + return self._next(parent_at) diff --git a/Misc/NEWS.d/next/Library/2022-10-08-15-41-00.gh-issue-98098.DugpWi.rst b/Misc/NEWS.d/next/Library/2022-10-08-15-41-00.gh-issue-98098.DugpWi.rst new file mode 100644 index 00000000000000..202275e16ea081 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-08-15-41-00.gh-issue-98098.DugpWi.rst @@ -0,0 +1,2 @@ +Created packages from zipfile and test_zipfile modules, separating +``zipfile.Path`` functionality. From 78365b8e283c78e23725748500f48dd2c2ca1161 Mon Sep 17 00:00:00 2001 From: Sam Ezeh Date: Sat, 26 Nov 2022 17:57:05 +0000 Subject: [PATCH 11/35] gh-91078: Return None from TarFile.next when the tarfile is empty (GH-91850) Co-authored-by: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> --- Lib/tarfile.py | 2 ++ Lib/test/test_tarfile.py | 12 ++++++++++++ .../2022-04-23-03-46-37.gh-issue-91078.87-hkp.rst | 1 + 3 files changed, 15 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2022-04-23-03-46-37.gh-issue-91078.87-hkp.rst diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 42100e9a39436e..b47015f5cb6be5 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -2339,6 +2339,8 @@ def next(self): # Advance the file pointer. if self.offset != self.fileobj.tell(): + if self.offset == 0: + return None self.fileobj.seek(self.offset - 1) if not self.fileobj.read(1): raise ReadError("unexpected end of data") diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 0868d5d6e90915..213932069201b9 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -734,6 +734,18 @@ def test_zlib_error_does_not_leak(self): with self.assertRaises(tarfile.ReadError): tarfile.open(self.tarname) + def test_next_on_empty_tarfile(self): + fd = io.BytesIO() + tf = tarfile.open(fileobj=fd, mode="w") + tf.close() + + fd.seek(0) + with tarfile.open(fileobj=fd, mode="r|") as tf: + self.assertEqual(tf.next(), None) + + fd.seek(0) + with tarfile.open(fileobj=fd, mode="r") as tf: + self.assertEqual(tf.next(), None) class MiscReadTest(MiscReadTestBase, unittest.TestCase): test_fail_comp = None diff --git a/Misc/NEWS.d/next/Library/2022-04-23-03-46-37.gh-issue-91078.87-hkp.rst b/Misc/NEWS.d/next/Library/2022-04-23-03-46-37.gh-issue-91078.87-hkp.rst new file mode 100644 index 00000000000000..e05d5e2a13146c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-04-23-03-46-37.gh-issue-91078.87-hkp.rst @@ -0,0 +1 @@ +:meth:`TarFile.next` now returns ``None`` when called on an empty tarfile. From 003f341e99234cf6088341e746ffef15e12ccda2 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Nov 2022 13:00:05 -0500 Subject: [PATCH 12/35] Fix zipfile packaging after GH-98103 (GH-99797) * Add zipfile and test_zipfile to list of packages. Fixes regression introduced in #98103. * Restore support for py -m test.test_zipfile --- Lib/test/test_zipfile/__main__.py | 7 +++++++ Makefile.pre.in | 2 ++ 2 files changed, 9 insertions(+) create mode 100644 Lib/test/test_zipfile/__main__.py diff --git a/Lib/test/test_zipfile/__main__.py b/Lib/test/test_zipfile/__main__.py new file mode 100644 index 00000000000000..e25ac946edffe4 --- /dev/null +++ b/Lib/test/test_zipfile/__main__.py @@ -0,0 +1,7 @@ +import unittest + +from . import load_tests # noqa: F401 + + +if __name__ == "__main__": + unittest.main() diff --git a/Makefile.pre.in b/Makefile.pre.in index 5c49af36d86736..f6df7a620deaed 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1978,6 +1978,7 @@ LIBSUBDIRS= asyncio \ wsgiref \ $(XMLLIBSUBDIRS) \ xmlrpc \ + zipfile \ zoneinfo \ __phello__ TESTSUBDIRS= idlelib/idle_test \ @@ -2051,6 +2052,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_tools \ test/test_ttk \ test/test_warnings test/test_warnings/data \ + test/test_zipfile \ test/test_zoneinfo test/test_zoneinfo/data \ test/test_unittest test/test_unittest/testmock \ test/tracedmodules \ From 5f8898216e7b67b7de6b0b1aad9277e88bcebfdb Mon Sep 17 00:00:00 2001 From: busywhitespace Date: Sat, 26 Nov 2022 19:01:08 +0100 Subject: [PATCH 13/35] gh-99795: Fix typo in importlib.resources.abc (GH-99796) Changing TraversableReader to TraversableResources at one place of the documentation. See #99795 for more details. --- Doc/library/importlib.resources.abc.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/importlib.resources.abc.rst b/Doc/library/importlib.resources.abc.rst index 57fffe0d905cbe..7747e89a833c02 100644 --- a/Doc/library/importlib.resources.abc.rst +++ b/Doc/library/importlib.resources.abc.rst @@ -145,7 +145,7 @@ :class:`importlib.resources.abc.ResourceReader` and provides concrete implementations of the :class:`importlib.resources.abc.ResourceReader`'s abstract methods. Therefore, any loader supplying - :class:`importlib.abc.TraversableReader` also supplies ResourceReader. + :class:`importlib.abc.TraversableResources` also supplies ResourceReader. Loaders that wish to support resource reading are expected to implement this interface. From 93f22d30eb7bf579d511b1866674bc1c2513dde9 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Nov 2022 13:05:41 -0500 Subject: [PATCH 14/35] gh-98108: Add limited pickleability to zipfile.Path (GH-98109) * gh-98098: Move zipfile into a package. * Moved test_zipfile to a package * Extracted module for test_path. * Add blurb * Add jaraco as owner of zipfile.Path. * Synchronize with minor changes found at jaraco/zipp@d9e7f4352d. * gh-98108: Sync with zipp 3.9.1 adding pickleability. --- Lib/test/test_zipfile/_functools.py | 9 ++++ Lib/test/test_zipfile/_itertools.py | 12 +++++ Lib/test/test_zipfile/_test_params.py | 39 +++++++++++++++ Lib/test/test_zipfile/test_path.py | 50 +++++++++++-------- Lib/zipfile/_path.py | 20 +++++++- ...2-10-08-19-20-33.gh-issue-98108.WUObqM.rst | 2 + 6 files changed, 110 insertions(+), 22 deletions(-) create mode 100644 Lib/test/test_zipfile/_functools.py create mode 100644 Lib/test/test_zipfile/_itertools.py create mode 100644 Lib/test/test_zipfile/_test_params.py create mode 100644 Misc/NEWS.d/next/Library/2022-10-08-19-20-33.gh-issue-98108.WUObqM.rst diff --git a/Lib/test/test_zipfile/_functools.py b/Lib/test/test_zipfile/_functools.py new file mode 100644 index 00000000000000..75f2b20e06d77f --- /dev/null +++ b/Lib/test/test_zipfile/_functools.py @@ -0,0 +1,9 @@ +import functools + + +# from jaraco.functools 3.5.2 +def compose(*funcs): + def compose_two(f1, f2): + return lambda *args, **kwargs: f1(f2(*args, **kwargs)) + + return functools.reduce(compose_two, funcs) diff --git a/Lib/test/test_zipfile/_itertools.py b/Lib/test/test_zipfile/_itertools.py new file mode 100644 index 00000000000000..559f3f111b88a3 --- /dev/null +++ b/Lib/test/test_zipfile/_itertools.py @@ -0,0 +1,12 @@ +# from more_itertools v8.13.0 +def always_iterable(obj, base_type=(str, bytes)): + if obj is None: + return iter(()) + + if (base_type is not None) and isinstance(obj, base_type): + return iter((obj,)) + + try: + return iter(obj) + except TypeError: + return iter((obj,)) diff --git a/Lib/test/test_zipfile/_test_params.py b/Lib/test/test_zipfile/_test_params.py new file mode 100644 index 00000000000000..bc95b4ebf4a168 --- /dev/null +++ b/Lib/test/test_zipfile/_test_params.py @@ -0,0 +1,39 @@ +import types +import functools + +from ._itertools import always_iterable + + +def parameterize(names, value_groups): + """ + Decorate a test method to run it as a set of subtests. + + Modeled after pytest.parametrize. + """ + + def decorator(func): + @functools.wraps(func) + def wrapped(self): + for values in value_groups: + resolved = map(Invoked.eval, always_iterable(values)) + params = dict(zip(always_iterable(names), resolved)) + with self.subTest(**params): + func(self, **params) + + return wrapped + + return decorator + + +class Invoked(types.SimpleNamespace): + """ + Wrap a function to be invoked for each usage. + """ + + @classmethod + def wrap(cls, func): + return cls(func=func) + + @classmethod + def eval(cls, cand): + return cand.func() if isinstance(cand, cls) else cand diff --git a/Lib/test/test_zipfile/test_path.py b/Lib/test/test_zipfile/test_path.py index 3c62e9a0b0e65d..02253c59e959fb 100644 --- a/Lib/test/test_zipfile/test_path.py +++ b/Lib/test/test_zipfile/test_path.py @@ -4,7 +4,12 @@ import pathlib import unittest import string -import functools +import pickle +import itertools + +from ._test_params import parameterize, Invoked +from ._functools import compose + from test.support.os_helper import temp_dir @@ -76,18 +81,12 @@ def build_alpharep_fixture(): return zf -def pass_alpharep(meth): - """ - Given a method, wrap it in a for loop that invokes method - with each subtest. - """ - - @functools.wraps(meth) - def wrapper(self): - for alpharep in self.zipfile_alpharep(): - meth(self, alpharep=alpharep) +alpharep_generators = [ + Invoked.wrap(build_alpharep_fixture), + Invoked.wrap(compose(add_dirs, build_alpharep_fixture)), +] - return wrapper +pass_alpharep = parameterize(['alpharep'], alpharep_generators) class TestPath(unittest.TestCase): @@ -95,12 +94,6 @@ def setUp(self): self.fixtures = contextlib.ExitStack() self.addCleanup(self.fixtures.close) - def zipfile_alpharep(self): - with self.subTest(): - yield build_alpharep_fixture() - with self.subTest(): - yield add_dirs(build_alpharep_fixture()) - def zipfile_ondisk(self, alpharep): tmpdir = pathlib.Path(self.fixtures.enter_context(temp_dir())) buffer = alpharep.fp @@ -418,6 +411,21 @@ def test_root_unnamed(self, alpharep): @pass_alpharep def test_inheritance(self, alpharep): cls = type('PathChild', (zipfile.Path,), {}) - for alpharep in self.zipfile_alpharep(): - file = cls(alpharep).joinpath('some dir').parent - assert isinstance(file, cls) + file = cls(alpharep).joinpath('some dir').parent + assert isinstance(file, cls) + + @parameterize( + ['alpharep', 'path_type', 'subpath'], + itertools.product( + alpharep_generators, + [str, pathlib.Path], + ['', 'b/'], + ), + ) + def test_pickle(self, alpharep, path_type, subpath): + zipfile_ondisk = path_type(self.zipfile_ondisk(alpharep)) + + saved_1 = pickle.dumps(zipfile.Path(zipfile_ondisk, at=subpath)) + restored_1 = pickle.loads(saved_1) + first, *rest = restored_1.iterdir() + assert first.read_text().startswith('content of ') diff --git a/Lib/zipfile/_path.py b/Lib/zipfile/_path.py index 67ef07a130d1ad..aea17b65b6aa2d 100644 --- a/Lib/zipfile/_path.py +++ b/Lib/zipfile/_path.py @@ -62,7 +62,25 @@ def _difference(minuend, subtrahend): return itertools.filterfalse(set(subtrahend).__contains__, minuend) -class CompleteDirs(zipfile.ZipFile): +class InitializedState: + """ + Mix-in to save the initialization state for pickling. + """ + + def __init__(self, *args, **kwargs): + self.__args = args + self.__kwargs = kwargs + super().__init__(*args, **kwargs) + + def __getstate__(self): + return self.__args, self.__kwargs + + def __setstate__(self, state): + args, kwargs = state + super().__init__(*args, **kwargs) + + +class CompleteDirs(InitializedState, zipfile.ZipFile): """ A ZipFile subclass that ensures that implied directories are always included in the namelist. diff --git a/Misc/NEWS.d/next/Library/2022-10-08-19-20-33.gh-issue-98108.WUObqM.rst b/Misc/NEWS.d/next/Library/2022-10-08-19-20-33.gh-issue-98108.WUObqM.rst new file mode 100644 index 00000000000000..7e962580dda228 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-08-19-20-33.gh-issue-98108.WUObqM.rst @@ -0,0 +1,2 @@ +``zipfile.Path`` is now pickleable if its initialization parameters were +pickleable (e.g. for file system paths). From 7f005749b27c7b9108ea24e5c0ff25068910b75c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Nov 2022 16:57:20 -0500 Subject: [PATCH 15/35] gh-88330: Add more detail about what is a resource. (#99801) --- Doc/library/importlib.resources.rst | 14 +++++++++++--- .../2022-11-26-15-51-23.gh-issue-88330.B_wFq8.rst | 1 + 2 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Documentation/2022-11-26-15-51-23.gh-issue-88330.B_wFq8.rst diff --git a/Doc/library/importlib.resources.rst b/Doc/library/importlib.resources.rst index 827e7d8d5aced4..399191301a3614 100644 --- a/Doc/library/importlib.resources.rst +++ b/Doc/library/importlib.resources.rst @@ -11,9 +11,17 @@ .. versionadded:: 3.7 This module leverages Python's import system to provide access to *resources* -within *packages*. If you can import a package, you can access resources -within that package. Resources can be opened or read, in either binary or -text mode. +within *packages*. + +"Resources" are file-like resources associated with a module or package in +Python. The resources may be contained directly in a package or within a +subdirectory contained in that package. Resources may be text or binary. As a +result, Python module sources (.py) of a package and compilation artifacts +(pycache) are technically de-facto resources of that package. In practice, +however, resources are primarily those non-Python artifacts exposed +specifically by the package author. + +Resources can be opened or read in either binary or text mode. Resources are roughly akin to files inside directories, though it's important to keep in mind that this is just a metaphor. Resources and packages **do diff --git a/Misc/NEWS.d/next/Documentation/2022-11-26-15-51-23.gh-issue-88330.B_wFq8.rst b/Misc/NEWS.d/next/Documentation/2022-11-26-15-51-23.gh-issue-88330.B_wFq8.rst new file mode 100644 index 00000000000000..0f242eecc31258 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2022-11-26-15-51-23.gh-issue-88330.B_wFq8.rst @@ -0,0 +1 @@ +Improved the description of what a resource is in importlib.resources docs. From 25bc115df9d0e82309852609a83b5ab7f804cdc1 Mon Sep 17 00:00:00 2001 From: Ivan Savov Date: Sat, 26 Nov 2022 17:24:04 -0500 Subject: [PATCH 16/35] gh-89682: [doc] reword docstring of __contains__ to clarify that it returns a bool (GH-29043) --- .../Documentation/2022-11-26-21-43-05.gh-issue-89682.DhKoTM.rst | 1 + Objects/typeobject.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Documentation/2022-11-26-21-43-05.gh-issue-89682.DhKoTM.rst diff --git a/Misc/NEWS.d/next/Documentation/2022-11-26-21-43-05.gh-issue-89682.DhKoTM.rst b/Misc/NEWS.d/next/Documentation/2022-11-26-21-43-05.gh-issue-89682.DhKoTM.rst new file mode 100644 index 00000000000000..46be065b653952 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2022-11-26-21-43-05.gh-issue-89682.DhKoTM.rst @@ -0,0 +1 @@ +Reworded docstring of the default ``__contains__`` to clarify that it returns a :class:`bool`. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index b993aa405f6b6a..a4974a1b4f7113 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8741,7 +8741,7 @@ static pytype_slotdef slotdefs[] = { SQSLOT(__delitem__, sq_ass_item, slot_sq_ass_item, wrap_sq_delitem, "__delitem__($self, key, /)\n--\n\nDelete self[key]."), SQSLOT(__contains__, sq_contains, slot_sq_contains, wrap_objobjproc, - "__contains__($self, key, /)\n--\n\nReturn key in self."), + "__contains__($self, key, /)\n--\n\nReturn bool(key in self)."), SQSLOT(__iadd__, sq_inplace_concat, NULL, wrap_binaryfunc, "__iadd__($self, value, /)\n--\n\nImplement self+=value."), From 024ac542d738f56b36bdeb3517a10e93da5acab9 Mon Sep 17 00:00:00 2001 From: Nick Drozd Date: Sat, 26 Nov 2022 16:33:25 -0600 Subject: [PATCH 17/35] bpo-45975: Simplify some while-loops with walrus operator (GH-29347) --- Lib/_pyio.py | 5 +---- Lib/base64.py | 15 +++------------ Lib/ctypes/_aix.py | 8 ++------ Lib/email/parser.py | 5 +---- Lib/ftplib.py | 10 ++-------- Lib/http/client.py | 16 +++------------- Lib/http/cookiejar.py | 9 ++------- Lib/mailbox.py | 5 +---- Lib/mailcap.py | 4 +--- Lib/mimetypes.py | 5 +---- Lib/pstats.py | 2 -- Lib/pydoc.py | 4 +--- Lib/quopri.py | 9 ++------- Lib/shlex.py | 5 +---- Lib/shutil.py | 5 +---- Lib/smtplib.py | 5 +---- Lib/socketserver.py | 3 +-- Lib/tarfile.py | 12 +++--------- Lib/test/test_lzma.py | 20 ++++---------------- Lib/urllib/request.py | 10 ++-------- Lib/wsgiref/handlers.py | 5 +---- Lib/wsgiref/validate.py | 5 +---- Lib/xdrlib.py | 4 +--- Lib/xml/dom/expatbuilder.py | 5 +---- Lib/xml/etree/ElementTree.py | 5 +---- Lib/xml/sax/xmlreader.py | 4 +--- Lib/xmlrpc/client.py | 5 +---- Lib/xmlrpc/server.py | 4 +--- 28 files changed, 41 insertions(+), 153 deletions(-) diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 163cf9de279ff0..7f247ff47c9e61 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -638,10 +638,7 @@ def read(self, size=-1): def readall(self): """Read until EOF, using multiple read() call.""" res = bytearray() - while True: - data = self.read(DEFAULT_BUFFER_SIZE) - if not data: - break + while data := self.read(DEFAULT_BUFFER_SIZE): res += data if res: return bytes(res) diff --git a/Lib/base64.py b/Lib/base64.py index 30796a6fd6d05a..95dc7b0086051b 100755 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -508,14 +508,8 @@ def b85decode(b): def encode(input, output): """Encode a file; input and output are binary files.""" - while True: - s = input.read(MAXBINSIZE) - if not s: - break - while len(s) < MAXBINSIZE: - ns = input.read(MAXBINSIZE-len(s)) - if not ns: - break + while s := input.read(MAXBINSIZE): + while len(s) < MAXBINSIZE and (ns := input.read(MAXBINSIZE-len(s))): s += ns line = binascii.b2a_base64(s) output.write(line) @@ -523,10 +517,7 @@ def encode(input, output): def decode(input, output): """Decode a file; input and output are binary files.""" - while True: - line = input.readline() - if not line: - break + while line := input.readline(): s = binascii.a2b_base64(line) output.write(s) diff --git a/Lib/ctypes/_aix.py b/Lib/ctypes/_aix.py index fc3e95cbcc88a5..ee790f713a9ee3 100644 --- a/Lib/ctypes/_aix.py +++ b/Lib/ctypes/_aix.py @@ -108,12 +108,8 @@ def get_ld_headers(file): p = Popen(["/usr/bin/dump", f"-X{AIX_ABI}", "-H", file], universal_newlines=True, stdout=PIPE, stderr=DEVNULL) # be sure to read to the end-of-file - getting all entries - while True: - ld_header = get_ld_header(p) - if ld_header: - ldr_headers.append((ld_header, get_ld_header_info(p))) - else: - break + while ld_header := get_ld_header(p): + ldr_headers.append((ld_header, get_ld_header_info(p))) p.stdout.close() p.wait() return ldr_headers diff --git a/Lib/email/parser.py b/Lib/email/parser.py index 7db4da1ff081c1..e94d455baa5262 100644 --- a/Lib/email/parser.py +++ b/Lib/email/parser.py @@ -49,10 +49,7 @@ def parse(self, fp, headersonly=False): feedparser = FeedParser(self._class, policy=self.policy) if headersonly: feedparser._set_headersonly() - while True: - data = fp.read(8192) - if not data: - break + while data := fp.read(8192): feedparser.feed(data) return feedparser.close() diff --git a/Lib/ftplib.py b/Lib/ftplib.py index c7ca8f632e1bd4..a56e0c3085701b 100644 --- a/Lib/ftplib.py +++ b/Lib/ftplib.py @@ -434,10 +434,7 @@ def retrbinary(self, cmd, callback, blocksize=8192, rest=None): """ self.voidcmd('TYPE I') with self.transfercmd(cmd, rest) as conn: - while 1: - data = conn.recv(blocksize) - if not data: - break + while data := conn.recv(blocksize): callback(data) # shutdown ssl layer if _SSLSocket is not None and isinstance(conn, _SSLSocket): @@ -496,10 +493,7 @@ def storbinary(self, cmd, fp, blocksize=8192, callback=None, rest=None): """ self.voidcmd('TYPE I') with self.transfercmd(cmd, rest) as conn: - while 1: - buf = fp.read(blocksize) - if not buf: - break + while buf := fp.read(blocksize): conn.sendall(buf) if callback: callback(buf) diff --git a/Lib/http/client.py b/Lib/http/client.py index 0a3e950c669622..15c5cf634cf508 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -578,11 +578,7 @@ def _read_chunked(self, amt=None): assert self.chunked != _UNKNOWN value = [] try: - while True: - chunk_left = self._get_chunk_left() - if chunk_left is None: - break - + while (chunk_left := self._get_chunk_left()) is not None: if amt is not None and amt <= chunk_left: value.append(self._safe_read(amt)) self.chunk_left = chunk_left - amt @@ -998,10 +994,7 @@ def send(self, data): encode = self._is_textIO(data) if encode and self.debuglevel > 0: print("encoding file using iso-8859-1") - while 1: - datablock = data.read(self.blocksize) - if not datablock: - break + while datablock := data.read(self.blocksize): if encode: datablock = datablock.encode("iso-8859-1") sys.audit("http.client.send", self, datablock) @@ -1031,10 +1024,7 @@ def _read_readable(self, readable): encode = self._is_textIO(readable) if encode and self.debuglevel > 0: print("encoding file using iso-8859-1") - while True: - datablock = readable.read(self.blocksize) - if not datablock: - break + while datablock := readable.read(self.blocksize): if encode: datablock = datablock.encode("iso-8859-1") yield datablock diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py index 65c45e2b17dfc0..b0161a86fdbb51 100644 --- a/Lib/http/cookiejar.py +++ b/Lib/http/cookiejar.py @@ -1915,9 +1915,7 @@ def _really_load(self, f, filename, ignore_discard, ignore_expires): "comment", "commenturl") try: - while 1: - line = f.readline() - if line == "": break + while (line := f.readline()) != "": if not line.startswith(header): continue line = line[len(header):].strip() @@ -2017,12 +2015,9 @@ def _really_load(self, f, filename, ignore_discard, ignore_expires): filename) try: - while 1: - line = f.readline() + while (line := f.readline()) != "": rest = {} - if line == "": break - # httponly is a cookie flag as defined in rfc6265 # when encoded in a netscape cookie file, # the line is prepended with "#HttpOnly_" diff --git a/Lib/mailbox.py b/Lib/mailbox.py index 70da07ed2e9e8b..59834a2b3b5243 100644 --- a/Lib/mailbox.py +++ b/Lib/mailbox.py @@ -1956,10 +1956,7 @@ def readlines(self, sizehint=None): def __iter__(self): """Iterate over lines.""" - while True: - line = self.readline() - if not line: - return + while line := self.readline(): yield line def tell(self): diff --git a/Lib/mailcap.py b/Lib/mailcap.py index 7278ea7051fccf..2f4656e854b3bb 100644 --- a/Lib/mailcap.py +++ b/Lib/mailcap.py @@ -90,9 +90,7 @@ def _readmailcapfile(fp, lineno): the viewing command is stored with the key "view". """ caps = {} - while 1: - line = fp.readline() - if not line: break + while line := fp.readline(): # Ignore comments and blank lines if line[0] == '#' or line.strip() == '': continue diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index 3224363a3f2bfb..37228de4828de5 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -217,10 +217,7 @@ def readfp(self, fp, strict=True): list of standard types, else to the list of non-standard types. """ - while 1: - line = fp.readline() - if not line: - break + while line := fp.readline(): words = line.split() for i in range(len(words)): if words[i][0] == '#': diff --git a/Lib/pstats.py b/Lib/pstats.py index 80408313e8b27f..51bcca84188740 100644 --- a/Lib/pstats.py +++ b/Lib/pstats.py @@ -223,8 +223,6 @@ def get_sort_arg_defs(self): for word, tup in self.sort_arg_dict_default.items(): fragment = word while fragment: - if not fragment: - break if fragment in dict: bad_list[fragment] = 0 break diff --git a/Lib/pydoc.py b/Lib/pydoc.py index c79ec77a8c09e4..0a693f45230c93 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -686,9 +686,7 @@ def markup(self, text, escape=None, funcs={}, classes={}, methods={}): r'RFC[- ]?(\d+)|' r'PEP[- ]?(\d+)|' r'(self\.)?(\w+))') - while True: - match = pattern.search(text, here) - if not match: break + while match := pattern.search(text, here): start, end = match.span() results.append(escape(text[here:start])) diff --git a/Lib/quopri.py b/Lib/quopri.py index 08899c5cb73a30..f36cf7b3951cda 100755 --- a/Lib/quopri.py +++ b/Lib/quopri.py @@ -67,10 +67,7 @@ def write(s, output=output, lineEnd=b'\n'): output.write(s + lineEnd) prevline = None - while 1: - line = input.readline() - if not line: - break + while line := input.readline(): outline = [] # Strip off any readline induced trailing newline stripped = b'' @@ -126,9 +123,7 @@ def decode(input, output, header=False): return new = b'' - while 1: - line = input.readline() - if not line: break + while line := input.readline(): i, n = 0, len(line) if n > 0 and line[n-1:n] == b'\n': partial = 0; n = n-1 diff --git a/Lib/shlex.py b/Lib/shlex.py index a91c9b022627b1..f4821616b62a0f 100644 --- a/Lib/shlex.py +++ b/Lib/shlex.py @@ -333,10 +333,7 @@ def quote(s): def _print_tokens(lexer): - while 1: - tt = lexer.get_token() - if not tt: - break + while tt := lexer.get_token(): print("Token: " + repr(tt)) if __name__ == '__main__': diff --git a/Lib/shutil.py b/Lib/shutil.py index f372406a6c51a8..867925aa10cc04 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -194,10 +194,7 @@ def copyfileobj(fsrc, fdst, length=0): # Localize variable access to minimize overhead. fsrc_read = fsrc.read fdst_write = fdst.write - while True: - buf = fsrc_read(length) - if not buf: - break + while buf := fsrc_read(length): fdst_write(buf) def _samefile(src, dst): diff --git a/Lib/smtplib.py b/Lib/smtplib.py index 05d2f8ccd73c98..18c91746fd7bf2 100755 --- a/Lib/smtplib.py +++ b/Lib/smtplib.py @@ -1099,10 +1099,7 @@ def prompt(prompt): toaddrs = prompt("To").split(',') print("Enter message, end with ^D:") msg = '' - while 1: - line = sys.stdin.readline() - if not line: - break + while line := sys.stdin.readline(): msg = msg + line print("Message length is %d" % len(msg)) diff --git a/Lib/socketserver.py b/Lib/socketserver.py index 30a5cfa59fe05b..842d526b011911 100644 --- a/Lib/socketserver.py +++ b/Lib/socketserver.py @@ -292,8 +292,7 @@ def handle_request(self): selector.register(self, selectors.EVENT_READ) while True: - ready = selector.select(timeout) - if ready: + if selector.select(timeout): return self._handle_request_noblock() else: if timeout is not None: diff --git a/Lib/tarfile.py b/Lib/tarfile.py index b47015f5cb6be5..d686435d90ad1b 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -1262,11 +1262,7 @@ def _proc_pax(self, tarfile): # the newline. keyword and value are both UTF-8 encoded strings. regex = re.compile(br"(\d+) ([^=]+)=") pos = 0 - while True: - match = regex.match(buf, pos) - if not match: - break - + while match := regex.match(buf, pos): length, keyword = match.groups() length = int(length) if length == 0: @@ -2418,10 +2414,8 @@ def _load(self): """Read through the entire archive file and look for readable members. """ - while True: - tarinfo = self.next() - if tarinfo is None: - break + while self.next() is not None: + pass self._loaded = True def _check(self, mode=None): diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py index 145c8cfced4080..18f474ba2a8bdc 100644 --- a/Lib/test/test_lzma.py +++ b/Lib/test/test_lzma.py @@ -825,10 +825,7 @@ def test_read_0(self): def test_read_10(self): with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: chunks = [] - while True: - result = f.read(10) - if not result: - break + while result := f.read(10): self.assertLessEqual(len(result), 10) chunks.append(result) self.assertEqual(b"".join(chunks), INPUT) @@ -911,10 +908,7 @@ def test_read_bad_data(self): def test_read1(self): with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: blocks = [] - while True: - result = f.read1() - if not result: - break + while result := f.read1(): blocks.append(result) self.assertEqual(b"".join(blocks), INPUT) self.assertEqual(f.read1(), b"") @@ -926,10 +920,7 @@ def test_read1_0(self): def test_read1_10(self): with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: blocks = [] - while True: - result = f.read1(10) - if not result: - break + while result := f.read1(10): blocks.append(result) self.assertEqual(b"".join(blocks), INPUT) self.assertEqual(f.read1(), b"") @@ -937,10 +928,7 @@ def test_read1_10(self): def test_read1_multistream(self): with LZMAFile(BytesIO(COMPRESSED_XZ * 5)) as f: blocks = [] - while True: - result = f.read1() - if not result: - break + while result := f.read1(): blocks.append(result) self.assertEqual(b"".join(blocks), INPUT * 5) self.assertEqual(f.read1(), b"") diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 278aa3a14bfeea..151034e6a81bf9 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -265,10 +265,7 @@ def urlretrieve(url, filename=None, reporthook=None, data=None): if reporthook: reporthook(blocknum, bs, size) - while True: - block = fp.read(bs) - if not block: - break + while block := fp.read(bs): read += len(block) tfp.write(block) blocknum += 1 @@ -1847,10 +1844,7 @@ def retrieve(self, url, filename=None, reporthook=None, data=None): size = int(headers["Content-Length"]) if reporthook: reporthook(blocknum, bs, size) - while 1: - block = fp.read(bs) - if not block: - break + while block := fp.read(bs): read += len(block) tfp.write(block) blocknum += 1 diff --git a/Lib/wsgiref/handlers.py b/Lib/wsgiref/handlers.py index cd0916dc5553fb..cafe872c7aae9b 100644 --- a/Lib/wsgiref/handlers.py +++ b/Lib/wsgiref/handlers.py @@ -475,10 +475,7 @@ def _write(self,data): from warnings import warn warn("SimpleHandler.stdout.write() should not do partial writes", DeprecationWarning) - while True: - data = data[result:] - if not data: - break + while data := data[result:]: result = self.stdout.write(data) def _flush(self): diff --git a/Lib/wsgiref/validate.py b/Lib/wsgiref/validate.py index 6044e320a474c6..1a1853cd63a0d2 100644 --- a/Lib/wsgiref/validate.py +++ b/Lib/wsgiref/validate.py @@ -214,10 +214,7 @@ def readlines(self, *args): return lines def __iter__(self): - while 1: - line = self.readline() - if not line: - return + while line := self.readline(): yield line def close(self): diff --git a/Lib/xdrlib.py b/Lib/xdrlib.py index b56ffa59b73dcb..f8c2c18228da4d 100644 --- a/Lib/xdrlib.py +++ b/Lib/xdrlib.py @@ -224,9 +224,7 @@ def unpack_string(self): def unpack_list(self, unpack_item): list = [] - while 1: - x = self.unpack_uint() - if x == 0: break + while (x := self.unpack_uint()) != 0: if x != 1: raise ConversionError('0 or 1 expected, got %r' % (x,)) item = unpack_item() diff --git a/Lib/xml/dom/expatbuilder.py b/Lib/xml/dom/expatbuilder.py index 199c22d0af347e..7dd667bf3fbe04 100644 --- a/Lib/xml/dom/expatbuilder.py +++ b/Lib/xml/dom/expatbuilder.py @@ -200,10 +200,7 @@ def parseFile(self, file): parser = self.getParser() first_buffer = True try: - while 1: - buffer = file.read(16*1024) - if not buffer: - break + while buffer := file.read(16*1024): parser.Parse(buffer, False) if first_buffer and self.document.documentElement: self._setup_subset(buffer) diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index ebbe2b703bfd8f..df5d5191126ae1 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -566,10 +566,7 @@ def parse(self, source, parser=None): # it with chunks. self._root = parser._parse_whole(source) return self._root - while True: - data = source.read(65536) - if not data: - break + while data := source.read(65536): parser.feed(data) self._root = parser.close() return self._root diff --git a/Lib/xml/sax/xmlreader.py b/Lib/xml/sax/xmlreader.py index 716f22840414e6..e906121d23b9ef 100644 --- a/Lib/xml/sax/xmlreader.py +++ b/Lib/xml/sax/xmlreader.py @@ -120,10 +120,8 @@ def parse(self, source): file = source.getCharacterStream() if file is None: file = source.getByteStream() - buffer = file.read(self._bufsize) - while buffer: + while buffer := file.read(self._bufsize): self.feed(buffer) - buffer = file.read(self._bufsize) self.close() def feed(self, data): diff --git a/Lib/xmlrpc/client.py b/Lib/xmlrpc/client.py index bef23f4505e03c..ea8da766cb5a7e 100644 --- a/Lib/xmlrpc/client.py +++ b/Lib/xmlrpc/client.py @@ -1339,10 +1339,7 @@ def parse_response(self, response): p, u = self.getparser() - while 1: - data = stream.read(1024) - if not data: - break + while data := stream.read(1024): if self.verbose: print("body:", repr(data)) p.feed(data) diff --git a/Lib/xmlrpc/server.py b/Lib/xmlrpc/server.py index 0c4b558045a9f4..4dddb1d10e08bd 100644 --- a/Lib/xmlrpc/server.py +++ b/Lib/xmlrpc/server.py @@ -720,9 +720,7 @@ def markup(self, text, escape=None, funcs={}, classes={}, methods={}): r'RFC[- ]?(\d+)|' r'PEP[- ]?(\d+)|' r'(self\.)?((?:\w|\.)+))\b') - while 1: - match = pattern.search(text, here) - if not match: break + while match := pattern.search(text, here): start, end = match.span() results.append(escape(text[here:start])) From 191708c56cf45e0e1c98a0e7292ffa67b7e3b09c Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Sun, 27 Nov 2022 11:24:48 +0530 Subject: [PATCH 18/35] GH-66285: fix forking in asyncio (#99769) Closes #66285 --- Lib/asyncio/events.py | 11 +++ Lib/test/test_asyncio/test_unix_events.py | 98 +++++++++++++++++++ ...2-11-17-10-56-47.gh-issue-66285.KvjlaB.rst | 1 + 3 files changed, 110 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2022-11-17-10-56-47.gh-issue-66285.KvjlaB.rst diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index a327ba54a323a8..2836bbcc463fe5 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -13,6 +13,7 @@ import contextvars import os +import signal import socket import subprocess import sys @@ -842,3 +843,13 @@ def set_child_watcher(watcher): _c_get_running_loop = get_running_loop _c_get_event_loop = get_event_loop _c__get_event_loop = _get_event_loop + + +if hasattr(os, 'fork'): + def on_fork(): + # Reset the loop and wakeupfd in the forked child process. + if _event_loop_policy is not None: + _event_loop_policy._local = BaseDefaultEventLoopPolicy._Local() + signal.set_wakeup_fd(-1) + + os.register_at_fork(after_in_child=on_fork) diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index 93e8611f184d25..092edb215854b7 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -3,6 +3,7 @@ import contextlib import errno import io +import multiprocessing import os import pathlib import signal @@ -15,6 +16,8 @@ import warnings from test.support import os_helper from test.support import socket_helper +from test.support import wait_process +from test.support import hashlib_helper if sys.platform == 'win32': raise unittest.SkipTest('UNIX only') @@ -1867,5 +1870,100 @@ async def runner(): wsock.close() +@unittest.skipUnless(hasattr(os, 'fork'), 'requires os.fork()') +class TestFork(unittest.IsolatedAsyncioTestCase): + + async def test_fork_not_share_event_loop(self): + # The forked process should not share the event loop with the parent + loop = asyncio.get_running_loop() + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + pid = os.fork() + if pid == 0: + # child + try: + loop = asyncio.get_event_loop_policy().get_event_loop() + os.write(w, str(id(loop)).encode()) + finally: + os._exit(0) + else: + # parent + child_loop = int(os.read(r, 100).decode()) + self.assertNotEqual(child_loop, id(loop)) + wait_process(pid, exitcode=0) + + @hashlib_helper.requires_hashdigest('md5') + def test_fork_signal_handling(self): + # Sending signal to the forked process should not affect the parent + # process + ctx = multiprocessing.get_context('fork') + manager = ctx.Manager() + self.addCleanup(manager.shutdown) + child_started = manager.Event() + child_handled = manager.Event() + parent_handled = manager.Event() + + def child_main(): + signal.signal(signal.SIGTERM, lambda *args: child_handled.set()) + child_started.set() + time.sleep(1) + + async def main(): + loop = asyncio.get_running_loop() + loop.add_signal_handler(signal.SIGTERM, lambda *args: parent_handled.set()) + + process = ctx.Process(target=child_main) + process.start() + child_started.wait() + os.kill(process.pid, signal.SIGTERM) + process.join() + + async def func(): + await asyncio.sleep(0.1) + return 42 + + # Test parent's loop is still functional + self.assertEqual(await asyncio.create_task(func()), 42) + + asyncio.run(main()) + + self.assertFalse(parent_handled.is_set()) + self.assertTrue(child_handled.is_set()) + + @hashlib_helper.requires_hashdigest('md5') + def test_fork_asyncio_run(self): + ctx = multiprocessing.get_context('fork') + manager = ctx.Manager() + self.addCleanup(manager.shutdown) + result = manager.Value('i', 0) + + async def child_main(): + await asyncio.sleep(0.1) + result.value = 42 + + process = ctx.Process(target=lambda: asyncio.run(child_main())) + process.start() + process.join() + + self.assertEqual(result.value, 42) + + @hashlib_helper.requires_hashdigest('md5') + def test_fork_asyncio_subprocess(self): + ctx = multiprocessing.get_context('fork') + manager = ctx.Manager() + self.addCleanup(manager.shutdown) + result = manager.Value('i', 1) + + async def child_main(): + proc = await asyncio.create_subprocess_exec(sys.executable, '-c', 'pass') + result.value = await proc.wait() + + process = ctx.Process(target=lambda: asyncio.run(child_main())) + process.start() + process.join() + + self.assertEqual(result.value, 0) + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2022-11-17-10-56-47.gh-issue-66285.KvjlaB.rst b/Misc/NEWS.d/next/Library/2022-11-17-10-56-47.gh-issue-66285.KvjlaB.rst new file mode 100644 index 00000000000000..ebd82173882726 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-17-10-56-47.gh-issue-66285.KvjlaB.rst @@ -0,0 +1 @@ +Fix :mod:`asyncio` to not share event loop and signal wakeupfd in forked processes. Patch by Kumar Aditya. From 62a5dc13e941d01beb215db4218a10977914ab55 Mon Sep 17 00:00:00 2001 From: Zackery Spytz Date: Sat, 26 Nov 2022 22:27:41 -0800 Subject: [PATCH 19/35] bpo-43327: Fix the docs for PyImport_ImportFrozenModuleObject() (#24659) The docs stated that PyImport_ImportFrozenModuleObject() returns a new reference, but it actually returns an int. Co-authored-by: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> --- Doc/data/refcounts.dat | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 51ccacf13f9e3b..349c4dd5be3d81 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -1018,10 +1018,10 @@ PyImport_Import:PyObject*::+1: PyImport_Import:PyObject*:name:0: PyImport_ImportFrozenModule:int::: -PyImport_ImportFrozenModule:const char*::: +PyImport_ImportFrozenModule:const char*:name:: PyImport_ImportFrozenModuleObject:int::: -PyImport_ImportFrozenModuleObject:PyObject*::+1: +PyImport_ImportFrozenModuleObject:PyObject*:name:+1: PyImport_ImportModule:PyObject*::+1: PyImport_ImportModule:const char*:name:: From 22860dbbc8b53954055847d2bb036af68b4ea409 Mon Sep 17 00:00:00 2001 From: George Zhang Date: Sun, 27 Nov 2022 01:38:39 -0500 Subject: [PATCH 20/35] doc: Remove backslashes in doctest grammar docs (#29346) --- Doc/library/doctest.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index 75c6ee289a91e9..c106d5a3383a5e 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -696,10 +696,10 @@ special Python comments following an example's source code: .. productionlist:: doctest directive: "#" "doctest:" `directive_options` - directive_options: `directive_option` ("," `directive_option`)\* + directive_options: `directive_option` ("," `directive_option`)* directive_option: `on_or_off` `directive_option_name` - on_or_off: "+" \| "-" - directive_option_name: "DONT_ACCEPT_BLANKLINE" \| "NORMALIZE_WHITESPACE" \| ... + on_or_off: "+" | "-" + directive_option_name: "DONT_ACCEPT_BLANKLINE" | "NORMALIZE_WHITESPACE" | ... Whitespace is not allowed between the ``+`` or ``-`` and the directive option name. The directive option name can be any of the option flag names explained From 65629399bcfe2a6606b8201d190877f7f54e6be5 Mon Sep 17 00:00:00 2001 From: Brad Wolfe Date: Sun, 27 Nov 2022 11:25:12 +0100 Subject: [PATCH 21/35] gh-85988: Change documentation for sys.float_info.rounds (GH-99675) * Change documentation for sys.float_info.rounds Change the documentation for sys.float_info.rounds to remove references to C99 section 5.2.4.2.2 and instead place the available values inline. * Correction to previous documentation change Newlines were not preserved in generated HTML on previous commit. I have changes the list to a comma-separated list of values and their meanings. * Clarify source for value of FLT_ROUNDS Clarify the source of the FLT_ROUNDS value and change 'floating-point addition' to 'floating-point arithmetic' to indicate that the rounding mode applies to all arithmetic operations. --- Doc/library/sys.rst | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index d54ecd75a2628f..428ce51165c9b5 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -604,12 +604,18 @@ always available. +---------------------+----------------+--------------------------------------------------+ | :const:`radix` | FLT_RADIX | radix of exponent representation | +---------------------+----------------+--------------------------------------------------+ - | :const:`rounds` | FLT_ROUNDS | integer constant representing the rounding mode | - | | | used for arithmetic operations. This reflects | - | | | the value of the system FLT_ROUNDS macro at | - | | | interpreter startup time. See section 5.2.4.2.2 | - | | | of the C99 standard for an explanation of the | - | | | possible values and their meanings. | + | :const:`rounds` | FLT_ROUNDS | integer representing the rounding mode for | + | | | floating-point arithmetic. This reflects the | + | | | value of the system FLT_ROUNDS macro at | + | | | interpreter startup time: | + | | | ``-1`` indeterminable, | + | | | ``0`` toward zero, | + | | | ``1`` to nearest, | + | | | ``2`` toward positive infinity, | + | | | ``3`` toward negative infinity | + | | | | + | | | All other values for FLT_ROUNDS characterize | + | | | implementation-defined rounding behavior. | +---------------------+----------------+--------------------------------------------------+ The attribute :attr:`sys.float_info.dig` needs further explanation. If From 9c9f085e9a1d1464376ea421e5c96472ca11c3b4 Mon Sep 17 00:00:00 2001 From: Yonatan Goldschmidt Date: Sun, 27 Nov 2022 12:39:23 +0200 Subject: [PATCH 22/35] Remove unused local variables in inspect.py (#24218) --- Lib/inspect.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/inspect.py b/Lib/inspect.py index d0015aa202044e..311a3f7e04b6a3 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -656,7 +656,7 @@ def classify_class_attrs(cls): if name == '__dict__': raise Exception("__dict__ is special, don't want the proxy") get_obj = getattr(cls, name) - except Exception as exc: + except Exception: pass else: homecls = getattr(get_obj, "__objclass__", homecls) @@ -1310,7 +1310,6 @@ def getargs(co): nkwargs = co.co_kwonlyargcount args = list(names[:nargs]) kwonlyargs = list(names[nargs:nargs+nkwargs]) - step = 0 nargs += nkwargs varargs = None From d08fb257698e3475d6f69bb808211d39e344e5b2 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Sun, 27 Nov 2022 11:56:14 +0100 Subject: [PATCH 23/35] GH-87235: Make sure "python /dev/fd/9 9 header_offset: - raise ZipImportError(f'bad local header offset: {archive!r}', path=archive) - file_offset += arc_offset - - try: - name = fp.read(name_size) + fp.seek(-END_CENTRAL_DIR_SIZE, 2) + header_position = fp.tell() + buffer = fp.read(END_CENTRAL_DIR_SIZE) except OSError: raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) - if len(name) != name_size: + if len(buffer) != END_CENTRAL_DIR_SIZE: raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) - # On Windows, calling fseek to skip over the fields we don't use is - # slower than reading the data because fseek flushes stdio's - # internal buffers. See issue #8745. + if buffer[:4] != STRING_END_ARCHIVE: + # Bad: End of Central Dir signature + # Check if there's a comment. + try: + fp.seek(0, 2) + file_size = fp.tell() + except OSError: + raise ZipImportError(f"can't read Zip file: {archive!r}", + path=archive) + max_comment_start = max(file_size - MAX_COMMENT_LEN - + END_CENTRAL_DIR_SIZE, 0) + try: + fp.seek(max_comment_start) + data = fp.read() + except OSError: + raise ZipImportError(f"can't read Zip file: {archive!r}", + path=archive) + pos = data.rfind(STRING_END_ARCHIVE) + if pos < 0: + raise ZipImportError(f'not a Zip file: {archive!r}', + path=archive) + buffer = data[pos:pos+END_CENTRAL_DIR_SIZE] + if len(buffer) != END_CENTRAL_DIR_SIZE: + raise ZipImportError(f"corrupt Zip file: {archive!r}", + path=archive) + header_position = file_size - len(data) + pos + + header_size = _unpack_uint32(buffer[12:16]) + header_offset = _unpack_uint32(buffer[16:20]) + if header_position < header_size: + raise ZipImportError(f'bad central directory size: {archive!r}', path=archive) + if header_position < header_offset: + raise ZipImportError(f'bad central directory offset: {archive!r}', path=archive) + header_position -= header_size + arc_offset = header_position - header_offset + if arc_offset < 0: + raise ZipImportError(f'bad central directory size or offset: {archive!r}', path=archive) + + files = {} + # Start of Central Directory + count = 0 try: - if len(fp.read(header_size - name_size)) != header_size - name_size: - raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) + fp.seek(header_position) except OSError: raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) + while True: + buffer = fp.read(46) + if len(buffer) < 4: + raise EOFError('EOF read where not expected') + # Start of file header + if buffer[:4] != b'PK\x01\x02': + break # Bad: Central Dir File Header + if len(buffer) != 46: + raise EOFError('EOF read where not expected') + flags = _unpack_uint16(buffer[8:10]) + compress = _unpack_uint16(buffer[10:12]) + time = _unpack_uint16(buffer[12:14]) + date = _unpack_uint16(buffer[14:16]) + crc = _unpack_uint32(buffer[16:20]) + data_size = _unpack_uint32(buffer[20:24]) + file_size = _unpack_uint32(buffer[24:28]) + name_size = _unpack_uint16(buffer[28:30]) + extra_size = _unpack_uint16(buffer[30:32]) + comment_size = _unpack_uint16(buffer[32:34]) + file_offset = _unpack_uint32(buffer[42:46]) + header_size = name_size + extra_size + comment_size + if file_offset > header_offset: + raise ZipImportError(f'bad local header offset: {archive!r}', path=archive) + file_offset += arc_offset - if flags & 0x800: - # UTF-8 file names extension - name = name.decode() - else: - # Historical ZIP filename encoding try: - name = name.decode('ascii') - except UnicodeDecodeError: - name = name.decode('latin1').translate(cp437_table) - - name = name.replace('/', path_sep) - path = _bootstrap_external._path_join(archive, name) - t = (path, compress, data_size, file_size, file_offset, time, date, crc) - files[name] = t - count += 1 + name = fp.read(name_size) + except OSError: + raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) + if len(name) != name_size: + raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) + # On Windows, calling fseek to skip over the fields we don't use is + # slower than reading the data because fseek flushes stdio's + # internal buffers. See issue #8745. + try: + if len(fp.read(header_size - name_size)) != header_size - name_size: + raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) + except OSError: + raise ZipImportError(f"can't read Zip file: {archive!r}", path=archive) + + if flags & 0x800: + # UTF-8 file names extension + name = name.decode() + else: + # Historical ZIP filename encoding + try: + name = name.decode('ascii') + except UnicodeDecodeError: + name = name.decode('latin1').translate(cp437_table) + + name = name.replace('/', path_sep) + path = _bootstrap_external._path_join(archive, name) + t = (path, compress, data_size, file_size, file_offset, time, date, crc) + files[name] = t + count += 1 + finally: + fp.seek(start_offset) _bootstrap._verbose_message('zipimport: found {} names in {!r}', count, archive) return files diff --git a/Misc/NEWS.d/next/macOS/2022-11-25-09-23-20.gh-issue-87235.SifjCD.rst b/Misc/NEWS.d/next/macOS/2022-11-25-09-23-20.gh-issue-87235.SifjCD.rst new file mode 100644 index 00000000000000..3111e4975e87b3 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2022-11-25-09-23-20.gh-issue-87235.SifjCD.rst @@ -0,0 +1 @@ +On macOS ``python3 /dev/fd/9 9 Date: Sun, 27 Nov 2022 06:01:02 -0500 Subject: [PATCH 24/35] gh-99815: remove unused 'invalid' sentinel value and code that checks for it in inspect.signature parsing (GH-21104) --- Lib/inspect.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/Lib/inspect.py b/Lib/inspect.py index 311a3f7e04b6a3..a896fcda31d1dd 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2184,7 +2184,6 @@ def _signature_fromstr(cls, obj, s, skip_bound_arg=True): parameters = [] empty = Parameter.empty - invalid = object() module = None module_dict = {} @@ -2234,17 +2233,12 @@ def visit_Name(self, node): def p(name_node, default_node, default=empty): name = parse_name(name_node) - if name is invalid: - return None if default_node and default_node is not _empty: try: default_node = RewriteSymbolics().visit(default_node) - o = ast.literal_eval(default_node) + default = ast.literal_eval(default_node) except ValueError: - o = invalid - if o is invalid: return None - default = o if o is not invalid else default parameters.append(Parameter(name, kind, default=default, annotation=empty)) # non-keyword-only parameters From 2653b82c1a44371ad0da6b5a1101abbda4acd2d3 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Sun, 27 Nov 2022 14:15:26 +0300 Subject: [PATCH 25/35] gh-99677: Deduplicate self-type in `mro` in `inspect._getmembers` (#99678) Closes #99677 --- Lib/inspect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/inspect.py b/Lib/inspect.py index a896fcda31d1dd..31ac888126b57c 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -537,7 +537,7 @@ def _getmembers(object, predicate, getter): processed = set() names = dir(object) if isclass(object): - mro = (object,) + getmro(object) + mro = getmro(object) # add any DynamicClassAttributes to the list of names if object is a class; # this may result in duplicate entries if, for example, a virtual # attribute with the same name as a DynamicClassAttribute exists From 969620d59ab12fc55d0e757a6fbee6aff29830ea Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 27 Nov 2022 19:48:29 +0200 Subject: [PATCH 26/35] Docs: Move .PHONY to each section to avoid copy/paste omissions (#99396) --- Doc/Makefile | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/Doc/Makefile b/Doc/Makefile index b09a9d754fb5aa..3d484ac3ae7937 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -21,10 +21,7 @@ PAPEROPT_letter = -D latex_elements.papersize=letterpaper ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees $(PAPEROPT_$(PAPER)) -j auto \ $(SPHINXOPTS) $(SPHINXERRORHANDLING) . build/$(BUILDER) $(SOURCES) -.PHONY: help build html htmlhelp latex text texinfo epub changes linkcheck \ - coverage doctest pydoc-topics htmlview clean clean-venv venv dist check serve \ - autobuild-dev autobuild-dev-html autobuild-stable autobuild-stable-html - +.PHONY: help help: @echo "Please use \`make ' where is one of" @echo " clean to remove build files" @@ -44,6 +41,7 @@ help: @echo " dist to create a \"dist\" directory with archived docs for download" @echo " check to run a check for frequent markup errors" +.PHONY: build build: -mkdir -p build # Look first for a Misc/NEWS file (building from a source release tarball @@ -70,38 +68,46 @@ build: $(SPHINXBUILD) $(ALLSPHINXOPTS) @echo +.PHONY: html html: BUILDER = html html: build @echo "Build finished. The HTML pages are in build/html." +.PHONY: htmlhelp htmlhelp: BUILDER = htmlhelp htmlhelp: build @echo "Build finished; now you can run HTML Help Workshop with the" \ "build/htmlhelp/pydoc.hhp project file." +.PHONY: latex latex: BUILDER = latex latex: build @echo "Build finished; the LaTeX files are in build/latex." @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ "run these through (pdf)latex." +.PHONY: text text: BUILDER = text text: build @echo "Build finished; the text files are in build/text." +.PHONY: texinfo texinfo: BUILDER = texinfo texinfo: build @echo "Build finished; the python.texi file is in build/texinfo." @echo "Run \`make info' in that directory to run it through makeinfo." +.PHONY: epub epub: BUILDER = epub epub: build @echo "Build finished; the epub files are in build/epub." +.PHONY: changes changes: BUILDER = changes changes: build @echo "The overview file is in build/changes." +.PHONY: linkcheck linkcheck: BUILDER = linkcheck linkcheck: @$(MAKE) build BUILDER=$(BUILDER) || { \ @@ -109,10 +115,12 @@ linkcheck: "or in build/$(BUILDER)/output.txt"; \ false; } +.PHONY: coverage coverage: BUILDER = coverage coverage: build @echo "Coverage finished; see c.txt and python.txt in build/coverage" +.PHONY: doctest doctest: BUILDER = doctest doctest: @$(MAKE) build BUILDER=$(BUILDER) || { \ @@ -120,20 +128,25 @@ doctest: "results in build/doctest/output.txt"; \ false; } +.PHONY: pydoc-topics pydoc-topics: BUILDER = pydoc-topics pydoc-topics: build @echo "Building finished; now run this:" \ "cp build/pydoc-topics/topics.py ../Lib/pydoc_data/topics.py" +.PHONY: htmlview htmlview: html $(PYTHON) -c "import os, webbrowser; webbrowser.open('file://' + os.path.realpath('build/html/index.html'))" +.PHONY: clean clean: clean-venv -rm -rf build/* +.PHONY: clean-venv clean-venv: rm -rf $(VENVDIR) +.PHONY: venv venv: @if [ -d $(VENVDIR) ] ; then \ echo "venv already exists."; \ @@ -145,6 +158,7 @@ venv: echo "The venv has been created in the $(VENVDIR) directory"; \ fi +.PHONY: dist dist: rm -rf dist mkdir -p dist @@ -199,12 +213,14 @@ dist: rm -r dist/python-$(DISTVERSION)-docs-texinfo rm dist/python-$(DISTVERSION)-docs-texinfo.tar +.PHONY: check check: # Check the docs and NEWS files with sphinx-lint. # Ignore the tools and venv dirs and check that the default role is not used. $(SPHINXLINT) -i tools -i $(VENVDIR) --enable default-role $(SPHINXLINT) --enable default-role ../Misc/NEWS.d/next/ +.PHONY: serve serve: @echo "The serve target was removed, use htmlview instead (see bpo-36329)" @@ -216,15 +232,18 @@ serve: # output files) # for development releases: always build +.PHONY: autobuild-dev autobuild-dev: make dist SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' # for quick rebuilds (HTML only) +.PHONY: autobuild-dev-html autobuild-dev-html: make html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' # for stable releases: only build if not in pre-release stage (alpha, beta) # release candidate downloads are okay, since the stable tree can be in that stage +.PHONY: autobuild-stable autobuild-stable: @case $(DISTVERSION) in *[ab]*) \ echo "Not building; $(DISTVERSION) is not a release version."; \ @@ -232,6 +251,7 @@ autobuild-stable: esac @make autobuild-dev +.PHONY: autobuild-stable-html autobuild-stable-html: @case $(DISTVERSION) in *[ab]*) \ echo "Not building; $(DISTVERSION) is not a release version."; \ From dfc2732a57e3ea6603d62f769d4f9c80be726fa4 Mon Sep 17 00:00:00 2001 From: Sam Ezeh Date: Sun, 27 Nov 2022 17:58:39 +0000 Subject: [PATCH 27/35] gh-91340: Document multiprocessing.set_start_method force parameter (GH-32339) #91340 https://bugs.python.org/issue47184 Automerge-Triggered-By: GH:kumaraditya303 --- Doc/library/multiprocessing.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 5516084780673f..b5ceeb796f8f2f 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -1089,10 +1089,14 @@ Miscellaneous .. versionchanged:: 3.11 Accepts a :term:`path-like object`. -.. function:: set_start_method(method) +.. function:: set_start_method(method, force=False) Set the method which should be used to start child processes. - *method* can be ``'fork'``, ``'spawn'`` or ``'forkserver'``. + The *method* argument can be ``'fork'``, ``'spawn'`` or ``'forkserver'``. + Raises :exc:`RuntimeError` if the start method has already been set and *force* + is not ``True``. If *method* is ``None`` and *force* is ``True`` then the start + method is set to ``None``. If *method* is ``None`` and *force* is ``False`` + then the context is set to the default context. Note that this should be called at most once, and it should be protected inside the ``if __name__ == '__main__'`` clause of the From 276643e207d44c53b87a8108d5b00982defcce1e Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 27 Nov 2022 22:08:30 +0100 Subject: [PATCH 28/35] Docs: both sqlite3 "point examples" now adapt to str (#99823) --- Doc/library/sqlite3.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 0dac2312b2feb1..7e2235b285b814 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -2105,7 +2105,7 @@ The following example illustrates the implicit and explicit approaches: return f"Point({self.x}, {self.y})" def adapt_point(point): - return f"{point.x};{point.y}".encode("utf-8") + return f"{point.x};{point.y}" def convert_point(s): x, y = list(map(float, s.split(b";"))) From 594de165bf2f21d6b28eb17003ea78fc20c0ffed Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Mon, 28 Nov 2022 09:49:10 +0300 Subject: [PATCH 29/35] gh-51524: Fix bug when calling trace.CoverageResults with valid infile (#99629) Co-authored-by: Terry Jan Reedy --- Lib/test/test_trace.py | 10 ++++++++++ Lib/trace.py | 2 +- .../2022-11-21-17-56-18.gh-issue-51524.nTykx8.rst | 1 + 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2022-11-21-17-56-18.gh-issue-51524.nTykx8.rst diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py index 5f712111ca14e0..fad2b3b8379ffc 100644 --- a/Lib/test/test_trace.py +++ b/Lib/test/test_trace.py @@ -1,4 +1,5 @@ import os +from pickle import dump import sys from test.support import captured_stdout from test.support.os_helper import (TESTFN, rmtree, unlink) @@ -412,6 +413,15 @@ def test_issue9936(self): self.assertIn(modname, coverage) self.assertEqual(coverage[modname], (5, 100)) + def test_coverageresults_update(self): + # Update empty CoverageResults with a non-empty infile. + infile = TESTFN + '-infile' + with open(infile, 'wb') as f: + dump(({}, {}, {'caller': 1}), f, protocol=1) + self.addCleanup(unlink, infile) + results = trace.CoverageResults({}, {}, infile, {}) + self.assertEqual(results.callers, {'caller': 1}) + ### Tests that don't mess with sys.settrace and can be traced ### themselves TODO: Skip tests that do mess with sys.settrace when ### regrtest is invoked with -T option. diff --git a/Lib/trace.py b/Lib/trace.py index 2cf3643878d4b8..213e46517d683d 100755 --- a/Lib/trace.py +++ b/Lib/trace.py @@ -172,7 +172,7 @@ def __init__(self, counts=None, calledfuncs=None, infile=None, try: with open(self.infile, 'rb') as f: counts, calledfuncs, callers = pickle.load(f) - self.update(self.__class__(counts, calledfuncs, callers)) + self.update(self.__class__(counts, calledfuncs, callers=callers)) except (OSError, EOFError, ValueError) as err: print(("Skipping counts file %r: %s" % (self.infile, err)), file=sys.stderr) diff --git a/Misc/NEWS.d/next/Library/2022-11-21-17-56-18.gh-issue-51524.nTykx8.rst b/Misc/NEWS.d/next/Library/2022-11-21-17-56-18.gh-issue-51524.nTykx8.rst new file mode 100644 index 00000000000000..63fe7b8a3a3254 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-21-17-56-18.gh-issue-51524.nTykx8.rst @@ -0,0 +1 @@ +Fix bug when calling trace.CoverageResults with valid infile. From 219696abb240607d3f807853c4c180825e60716e Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 28 Nov 2022 09:22:08 +0100 Subject: [PATCH 30/35] gh-99249: Clarify "read-only" slots tp_bases & tp_mro (GH-99342) These slots are marked "should be treated as read-only" in the table at the start of the document. That doesn't say anything about setting them in the static struct. `tp_bases` docs did say that it should be ``NULL`` (TIL!). If you ignore that, seemingly nothing bad happens. However, some slots may not be inherited, depending on which sub-slot structs are present. (FWIW, NumPy sets tp_bases and is affected by the quirk -- though to be fair, its DUAL_INHERIT code probably predates tp_bases docs, and also the result happens to be benign.) This patch makes things explicit. It also makes the summary table legend easier to scan. Co-authored-by: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> --- Doc/c-api/typeobj.rst | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 4c462f46056739..8f8869ec668a8d 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -149,10 +149,16 @@ Quick Reference +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ .. [#slots] - A slot name in parentheses indicates it is (effectively) deprecated. - Names in angle brackets should be treated as read-only. - Names in square brackets are for internal use only. - "" (as a prefix) means the field is required (must be non-``NULL``). + + **()**: A slot name in parentheses indicates it is (effectively) deprecated. + + **<>**: Names in angle brackets should be initially set to ``NULL`` and + treated as read-only. + + **[]**: Names in square brackets are for internal use only. + + **** (as a prefix) means the field is required (must be non-``NULL``). + .. [#cols] Columns: **"O"**: set on :c:type:`PyBaseObject_Type` @@ -1923,8 +1929,19 @@ and :c:type:`PyType_Type` effectively act as defaults.) Tuple of base types. - This is set for types created by a class statement. It should be ``NULL`` for - statically defined types. + This field should be set to ``NULL`` and treated as read-only. + Python will fill it in when the type is :c:func:`initialized `. + + For dynamically created classes, the ``Py_tp_bases`` + :c:type:`slot ` can be used instead of the *bases* argument + of :c:func:`PyType_FromSpecWithBases`. + The argument form is preferred. + + .. warning:: + + Multiple inheritance does not work well for statically defined types. + If you set ``tp_bases`` to a tuple, Python will not raise an error, + but some slots will only be inherited from the first base. **Inheritance:** @@ -1936,6 +1953,8 @@ and :c:type:`PyType_Type` effectively act as defaults.) Tuple containing the expanded set of base types, starting with the type itself and ending with :class:`object`, in Method Resolution Order. + This field should be set to ``NULL`` and treated as read-only. + Python will fill it in when the type is :c:func:`initialized `. **Inheritance:** From 492dc02b01828f346dd62412fefc654e781de923 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Mon, 28 Nov 2022 10:41:24 +0100 Subject: [PATCH 31/35] bpo-41825: restructure docs for the os.wait*() family (GH-22356) --- Doc/library/os.rst | 235 +++++++++++------- .../2020-09-22-12-32-16.bpo-41825.npcaCb.rst | 3 + 2 files changed, 145 insertions(+), 93 deletions(-) create mode 100644 Misc/NEWS.d/next/Documentation/2020-09-22-12-32-16.bpo-41825.npcaCb.rst diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 775aa32df99a46..b06f9bbcd831c2 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -4491,6 +4491,9 @@ written in Python, such as a mail server's external command delivery program. number is zero); the high bit of the low byte is set if a core file was produced. + If there are no children that could be waited for, :exc:`ChildProcessError` + is raised. + :func:`waitstatus_to_exitcode` can be used to convert the exit status into an exit code. @@ -4498,76 +4501,40 @@ written in Python, such as a mail server's external command delivery program. .. seealso:: - :func:`waitpid` can be used to wait for the completion of a specific - child process and has more options. + The other :func:`!wait*` functions documented below can be used to wait for the + completion of a specific child process and have more options. + :func:`waitpid` is the only one also available on Windows. -.. function:: waitid(idtype, id, options, /) - Wait for the completion of one or more child processes. - *idtype* can be :data:`P_PID`, :data:`P_PGID`, :data:`P_ALL`, or - :data:`P_PIDFD` on Linux. - *id* specifies the pid to wait on. - *options* is constructed from the ORing of one or more of :data:`WEXITED`, - :data:`WSTOPPED` or :data:`WCONTINUED` and additionally may be ORed with - :data:`WNOHANG` or :data:`WNOWAIT`. The return value is an object - representing the data contained in the :c:type:`siginfo_t` structure, namely: - :attr:`si_pid`, :attr:`si_uid`, :attr:`si_signo`, :attr:`si_status`, - :attr:`si_code` or ``None`` if :data:`WNOHANG` is specified and there are no - children in a waitable state. - - .. availability:: Unix, not Emscripten, not WASI. - - .. versionadded:: 3.3 - -.. data:: P_PID - P_PGID - P_ALL - - These are the possible values for *idtype* in :func:`waitid`. They affect - how *id* is interpreted. - - .. availability:: Unix, not Emscripten, not WASI. - - .. versionadded:: 3.3 - -.. data:: P_PIDFD - - This is a Linux-specific *idtype* that indicates that *id* is a file - descriptor that refers to a process. - - .. availability:: Linux >= 5.4 - - .. versionadded:: 3.9 - -.. data:: WEXITED - WSTOPPED - WNOWAIT +.. function:: waitid(idtype, id, options, /) - Flags that can be used in *options* in :func:`waitid` that specify what - child signal to wait for. + Wait for the completion of a child process. - .. availability:: Unix, not Emscripten, not WASI. + *idtype* can be :data:`P_PID`, :data:`P_PGID`, :data:`P_ALL`, or (on Linux) :data:`P_PIDFD`. + The interpretation of *id* depends on it; see their individual descriptions. - .. versionadded:: 3.3 + *options* is an OR combination of flags. At least one of :data:`WEXITED`, + :data:`WSTOPPED` or :data:`WCONTINUED` is required; + :data:`WNOHANG` and :data:`WNOWAIT` are additional optional flags. + The return value is an object representing the data contained in the + :c:type:`!siginfo_t` structure with the following attributes: -.. data:: CLD_EXITED - CLD_KILLED - CLD_DUMPED - CLD_TRAPPED - CLD_STOPPED - CLD_CONTINUED + * :attr:`!si_pid` (process ID) + * :attr:`!si_uid` (real user ID of the child) + * :attr:`!si_signo` (always :data:`~signal.SIGCHLD`) + * :attr:`!si_status` (the exit status or signal number, depending on :attr:`!si_code`) + * :attr:`!si_code` (see :data:`CLD_EXITED` for possible values) - These are the possible values for :attr:`si_code` in the result returned by - :func:`waitid`. + If :data:`WNOHANG` is specified and there are no matching children in the + requested state, ``None`` is returned. + Otherwise, if there are no matching children + that could be waited for, :exc:`ChildProcessError` is raised. .. availability:: Unix, not Emscripten, not WASI. .. versionadded:: 3.3 - .. versionchanged:: 3.9 - Added :data:`CLD_KILLED` and :data:`CLD_STOPPED` values. - .. function:: waitpid(pid, options, /) @@ -4585,8 +4552,11 @@ written in Python, such as a mail server's external command delivery program. ``-1``, status is requested for any process in the process group ``-pid`` (the absolute value of *pid*). - An :exc:`OSError` is raised with the value of errno when the syscall - returns -1. + *options* is an OR combination of flags. If it contains :data:`WNOHANG` and + there are no matching children in the requested state, ``(0, 0)`` is + returned. Otherwise, if there are no matching children that could be waited + for, :exc:`ChildProcessError` is raised. Other options that can be used are + :data:`WUNTRACED` and :data:`WCONTINUED`. On Windows: Wait for completion of a process given by process handle *pid*, and return a tuple containing *pid*, and its exit status shifted left by 8 bits @@ -4599,7 +4569,7 @@ written in Python, such as a mail server's external command delivery program. :func:`waitstatus_to_exitcode` can be used to convert the exit status into an exit code. - .. availability:: Unix, not Emscripten, not WASI. + .. availability:: Unix, Windows, not Emscripten, not WASI. .. versionchanged:: 3.5 If the system call is interrupted and the signal handler does not raise an @@ -4612,9 +4582,9 @@ written in Python, such as a mail server's external command delivery program. Similar to :func:`waitpid`, except no process id argument is given and a 3-element tuple containing the child's process id, exit status indication, and resource usage information is returned. Refer to - :mod:`resource`.\ :func:`~resource.getrusage` for details on resource usage - information. The option argument is the same as that provided to - :func:`waitpid` and :func:`wait4`. + :func:`resource.getrusage` for details on resource usage information. The + *options* argument is the same as that provided to :func:`waitpid` and + :func:`wait4`. :func:`waitstatus_to_exitcode` can be used to convert the exit status into an exitcode. @@ -4625,10 +4595,10 @@ written in Python, such as a mail server's external command delivery program. .. function:: wait4(pid, options) Similar to :func:`waitpid`, except a 3-element tuple, containing the child's - process id, exit status indication, and resource usage information is returned. - Refer to :mod:`resource`.\ :func:`~resource.getrusage` for details on - resource usage information. The arguments to :func:`wait4` are the same - as those provided to :func:`waitpid`. + process id, exit status indication, and resource usage information is + returned. Refer to :func:`resource.getrusage` for details on resource usage + information. The arguments to :func:`wait4` are the same as those provided + to :func:`waitpid`. :func:`waitstatus_to_exitcode` can be used to convert the exit status into an exitcode. @@ -4636,6 +4606,111 @@ written in Python, such as a mail server's external command delivery program. .. availability:: Unix, not Emscripten, not WASI. +.. data:: P_PID + P_PGID + P_ALL + P_PIDFD + + These are the possible values for *idtype* in :func:`waitid`. They affect + how *id* is interpreted: + + * :data:`!P_PID` - wait for the child whose PID is *id*. + * :data:`!P_PGID` - wait for any child whose progress group ID is *id*. + * :data:`!P_ALL` - wait for any child; *id* is ignored. + * :data:`!P_PIDFD` - wait for the child identified by the file descriptor + *id* (a process file descriptor created with :func:`pidfd_open`). + + .. availability:: Unix, not Emscripten, not WASI. + + .. note:: :data:`!P_PIDFD` is only available on Linux >= 5.4. + + .. versionadded:: 3.3 + .. versionadded:: 3.9 + The :data:`!P_PIDFD` constant. + + +.. data:: WCONTINUED + + This *options* flag for :func:`waitpid`, :func:`wait3`, :func:`wait4`, and + :func:`waitid` causes child processes to be reported if they have been + continued from a job control stop since they were last reported. + + .. availability:: Unix, not Emscripten, not WASI. + + +.. data:: WEXITED + + This *options* flag for :func:`waitid` causes child processes that have terminated to + be reported. + + The other ``wait*`` functions always report children that have terminated, + so this option is not available for them. + + .. availability:: Unix, not Emscripten, not WASI. + + .. versionadded:: 3.3 + + +.. data:: WSTOPPED + + This *options* flag for :func:`waitid` causes child processes that have been stopped + by the delivery of a signal to be reported. + + This option is not available for the other ``wait*`` functions. + + .. availability:: Unix, not Emscripten, not WASI. + + .. versionadded:: 3.3 + + +.. data:: WUNTRACED + + This *options* flag for :func:`waitpid`, :func:`wait3`, and :func:`wait4` causes + child processes to also be reported if they have been stopped but their + current state has not been reported since they were stopped. + + This option is not available for :func:`waitid`. + + .. availability:: Unix, not Emscripten, not WASI. + + +.. data:: WNOHANG + + This *options* flag causes :func:`waitpid`, :func:`wait3`, :func:`wait4`, and + :func:`waitid` to return right away if no child process status is available + immediately. + + .. availability:: Unix, not Emscripten, not WASI. + + +.. data:: WNOWAIT + + This *options* flag causes :func:`waitid` to leave the child in a waitable state, so that + a later :func:`!wait*` call can be used to retrieve the child status information again. + + This option is not available for the other ``wait*`` functions. + + .. availability:: Unix, not Emscripten, not WASI. + + +.. data:: CLD_EXITED + CLD_KILLED + CLD_DUMPED + CLD_TRAPPED + CLD_STOPPED + CLD_CONTINUED + + These are the possible values for :attr:`!si_code` in the result returned by + :func:`waitid`. + + .. availability:: Unix, not Emscripten, not WASI. + + .. versionadded:: 3.3 + + .. versionchanged:: 3.9 + Added :data:`CLD_KILLED` and :data:`CLD_STOPPED` values. + + .. function:: waitstatus_to_exitcode(status) Convert a wait status to an exit code. @@ -4668,32 +4743,6 @@ written in Python, such as a mail server's external command delivery program. .. versionadded:: 3.9 -.. data:: WNOHANG - - The option for :func:`waitpid` to return immediately if no child process status - is available immediately. The function returns ``(0, 0)`` in this case. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. data:: WCONTINUED - - This option causes child processes to be reported if they have been continued - from a job control stop since their status was last reported. - - .. availability:: Unix, not Emscripten, not WASI. - - Some Unix systems. - - -.. data:: WUNTRACED - - This option causes child processes to be reported if they have been stopped but - their current state has not been reported since they were stopped. - - .. availability:: Unix, not Emscripten, not WASI. - - The following functions take a process status code as returned by :func:`system`, :func:`wait`, or :func:`waitpid` as a parameter. They may be used to determine the disposition of a process. diff --git a/Misc/NEWS.d/next/Documentation/2020-09-22-12-32-16.bpo-41825.npcaCb.rst b/Misc/NEWS.d/next/Documentation/2020-09-22-12-32-16.bpo-41825.npcaCb.rst new file mode 100644 index 00000000000000..390b4a9824c793 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2020-09-22-12-32-16.bpo-41825.npcaCb.rst @@ -0,0 +1,3 @@ +Restructured the documentation for the :func:`os.wait* ` family of functions, +and improved the docs for :func:`os.waitid` with more explanation of the +possible argument constants. From 53eef27133c1da395b3b4d7ce0ab1d5b743ffb41 Mon Sep 17 00:00:00 2001 From: Zackery Spytz Date: Mon, 28 Nov 2022 02:46:40 -0800 Subject: [PATCH 32/35] bpo-31718: Fix io.IncrementalNewlineDecoder SystemErrors and segfaults (#18640) Co-authored-by: Oren Milman Co-authored-by: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> --- Lib/test/test_io.py | 10 ++++++- .../2020-02-23-23-48-15.bpo-31718.sXko5e.rst | 3 ++ Modules/_io/textio.c | 28 +++++++++++++------ 3 files changed, 32 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-02-23-23-48-15.bpo-31718.sXko5e.rst diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index c927f15aafef72..c5f2e5060a546d 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -3945,7 +3945,15 @@ def test_translate(self): self.assertEqual(decoder.decode(b"\r\r\n"), "\r\r\n") class CIncrementalNewlineDecoderTest(IncrementalNewlineDecoderTest): - pass + @support.cpython_only + def test_uninitialized(self): + uninitialized = self.IncrementalNewlineDecoder.__new__( + self.IncrementalNewlineDecoder) + self.assertRaises(ValueError, uninitialized.decode, b'bar') + self.assertRaises(ValueError, uninitialized.getstate) + self.assertRaises(ValueError, uninitialized.setstate, (b'foo', 0)) + self.assertRaises(ValueError, uninitialized.reset) + class PyIncrementalNewlineDecoderTest(IncrementalNewlineDecoderTest): pass diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-02-23-23-48-15.bpo-31718.sXko5e.rst b/Misc/NEWS.d/next/Core and Builtins/2020-02-23-23-48-15.bpo-31718.sXko5e.rst new file mode 100644 index 00000000000000..dd96c9e20d8759 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-02-23-23-48-15.bpo-31718.sXko5e.rst @@ -0,0 +1,3 @@ +Raise :exc:`ValueError` instead of :exc:`SystemError` when methods of +uninitialized :class:`io.IncrementalNewlineDecoder` objects are called. +Patch by Oren Milman. diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index ff903e9341de27..3091f6efafccd4 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -231,15 +231,16 @@ _io_IncrementalNewlineDecoder___init___impl(nldecoder_object *self, PyObject *errors) /*[clinic end generated code: output=fbd04d443e764ec2 input=89db6b19c6b126bf]*/ { - self->decoder = Py_NewRef(decoder); if (errors == NULL) { - self->errors = Py_NewRef(&_Py_ID(strict)); + errors = Py_NewRef(&_Py_ID(strict)); } else { - self->errors = Py_NewRef(errors); + errors = Py_NewRef(errors); } + Py_XSETREF(self->errors, errors); + Py_XSETREF(self->decoder, Py_NewRef(decoder)); self->translate = translate ? 1 : 0; self->seennl = 0; self->pendingcr = 0; @@ -274,6 +275,13 @@ check_decoded(PyObject *decoded) return 0; } +#define CHECK_INITIALIZED_DECODER(self) \ + if (self->errors == NULL) { \ + PyErr_SetString(PyExc_ValueError, \ + "IncrementalNewlineDecoder.__init__() not called"); \ + return NULL; \ + } + #define SEEN_CR 1 #define SEEN_LF 2 #define SEEN_CRLF 4 @@ -287,11 +295,7 @@ _PyIncrementalNewlineDecoder_decode(PyObject *myself, Py_ssize_t output_len; nldecoder_object *self = (nldecoder_object *) myself; - if (self->decoder == NULL) { - PyErr_SetString(PyExc_ValueError, - "IncrementalNewlineDecoder.__init__ not called"); - return NULL; - } + CHECK_INITIALIZED_DECODER(self); /* decode input (with the eventual \r from a previous pass) */ if (self->decoder != Py_None) { @@ -502,6 +506,8 @@ _io_IncrementalNewlineDecoder_getstate_impl(nldecoder_object *self) PyObject *buffer; unsigned long long flag; + CHECK_INITIALIZED_DECODER(self); + if (self->decoder != Py_None) { PyObject *state = PyObject_CallMethodNoArgs(self->decoder, &_Py_ID(getstate)); @@ -546,6 +552,8 @@ _io_IncrementalNewlineDecoder_setstate(nldecoder_object *self, PyObject *buffer; unsigned long long flag; + CHECK_INITIALIZED_DECODER(self); + if (!PyTuple_Check(state)) { PyErr_SetString(PyExc_TypeError, "state argument must be a tuple"); return NULL; @@ -576,6 +584,8 @@ static PyObject * _io_IncrementalNewlineDecoder_reset_impl(nldecoder_object *self) /*[clinic end generated code: output=32fa40c7462aa8ff input=728678ddaea776df]*/ { + CHECK_INITIALIZED_DECODER(self); + self->seennl = 0; self->pendingcr = 0; if (self->decoder != Py_None) @@ -587,6 +597,8 @@ _io_IncrementalNewlineDecoder_reset_impl(nldecoder_object *self) static PyObject * incrementalnewlinedecoder_newlines_get(nldecoder_object *self, void *context) { + CHECK_INITIALIZED_DECODER(self); + switch (self->seennl) { case SEEN_CR: return PyUnicode_FromString("\r"); From 02f72b8b938e301bbaaf0142547014e074bd564c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 28 Nov 2022 16:40:08 +0100 Subject: [PATCH 33/35] gh-89653: PEP 670: Convert macros to functions (#99843) Convert macros to static inline functions to avoid macro pitfalls, like duplication of side effects: * DK_ENTRIES() * DK_UNICODE_ENTRIES() * PyCode_GetNumFree() * PyFloat_AS_DOUBLE() * PyInstanceMethod_GET_FUNCTION() * PyMemoryView_GET_BASE() * PyMemoryView_GET_BUFFER() * PyMethod_GET_FUNCTION() * PyMethod_GET_SELF() * PySet_GET_SIZE() * _PyHeapType_GET_MEMBERS() Changes: * PyCode_GetNumFree() casts PyCode_GetNumFree.co_nfreevars from int to Py_ssize_t to be future proof, and because Py_ssize_t is commonly used in the C API. * PyCode_GetNumFree() doesn't cast its argument: the replaced macro already required the exact type PyCodeObject*. * Add assertions in some functions using "CAST" macros to check the arguments type when Python is built with assertions (debug build). * Remove an outdated comment in unicodeobject.h. --- Include/cpython/classobject.h | 34 ++++++++++++++++++++++---------- Include/cpython/code.h | 7 ++++++- Include/cpython/floatobject.h | 10 ++++++++-- Include/cpython/memoryobject.h | 13 ++++++++++-- Include/cpython/setobject.h | 9 +++++++-- Include/cpython/unicodeobject.h | 2 -- Include/internal/pycore_dict.h | 21 ++++++++++++++------ Include/internal/pycore_object.h | 5 +++-- 8 files changed, 74 insertions(+), 27 deletions(-) diff --git a/Include/cpython/classobject.h b/Include/cpython/classobject.h index 051041965002a3..d7c9ddd1336c46 100644 --- a/Include/cpython/classobject.h +++ b/Include/cpython/classobject.h @@ -26,12 +26,20 @@ PyAPI_FUNC(PyObject *) PyMethod_New(PyObject *, PyObject *); PyAPI_FUNC(PyObject *) PyMethod_Function(PyObject *); PyAPI_FUNC(PyObject *) PyMethod_Self(PyObject *); -/* Macros for direct access to these values. Type checks are *not* - done, so use with care. */ -#define PyMethod_GET_FUNCTION(meth) \ - (((PyMethodObject *)(meth)) -> im_func) -#define PyMethod_GET_SELF(meth) \ - (((PyMethodObject *)(meth)) -> im_self) +#define _PyMethod_CAST(meth) \ + (assert(PyMethod_Check(meth)), _Py_CAST(PyMethodObject*, meth)) + +/* Static inline functions for direct access to these values. + Type checks are *not* done, so use with care. */ +static inline PyObject* PyMethod_GET_FUNCTION(PyObject *meth) { + return _PyMethod_CAST(meth)->im_func; +} +#define PyMethod_GET_FUNCTION(meth) PyMethod_GET_FUNCTION(_PyObject_CAST(meth)) + +static inline PyObject* PyMethod_GET_SELF(PyObject *meth) { + return _PyMethod_CAST(meth)->im_self; +} +#define PyMethod_GET_SELF(meth) PyMethod_GET_SELF(_PyObject_CAST(meth)) typedef struct { PyObject_HEAD @@ -45,10 +53,16 @@ PyAPI_DATA(PyTypeObject) PyInstanceMethod_Type; PyAPI_FUNC(PyObject *) PyInstanceMethod_New(PyObject *); PyAPI_FUNC(PyObject *) PyInstanceMethod_Function(PyObject *); -/* Macros for direct access to these values. Type checks are *not* - done, so use with care. */ -#define PyInstanceMethod_GET_FUNCTION(meth) \ - (((PyInstanceMethodObject *)(meth)) -> func) +#define _PyInstanceMethod_CAST(meth) \ + (assert(PyInstanceMethod_Check(meth)), \ + _Py_CAST(PyInstanceMethodObject*, meth)) + +/* Static inline function for direct access to these values. + Type checks are *not* done, so use with care. */ +static inline PyObject* PyInstanceMethod_GET_FUNCTION(PyObject *meth) { + return _PyInstanceMethod_CAST(meth)->func; +} +#define PyInstanceMethod_GET_FUNCTION(meth) PyInstanceMethod_GET_FUNCTION(_PyObject_CAST(meth)) #ifdef __cplusplus } diff --git a/Include/cpython/code.h b/Include/cpython/code.h index ebac0b12a461bc..15b464fe2ee95c 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -147,7 +147,12 @@ struct PyCodeObject _PyCode_DEF(1); PyAPI_DATA(PyTypeObject) PyCode_Type; #define PyCode_Check(op) Py_IS_TYPE((op), &PyCode_Type) -#define PyCode_GetNumFree(op) ((op)->co_nfreevars) + +static inline Py_ssize_t PyCode_GetNumFree(PyCodeObject *op) { + assert(PyCode_Check(op)); + return op->co_nfreevars; +} + #define _PyCode_CODE(CO) ((_Py_CODEUNIT *)(CO)->co_code_adaptive) #define _PyCode_NBYTES(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT)) diff --git a/Include/cpython/floatobject.h b/Include/cpython/floatobject.h index 7795d9f83f05cb..127093098bfe64 100644 --- a/Include/cpython/floatobject.h +++ b/Include/cpython/floatobject.h @@ -7,9 +7,15 @@ typedef struct { double ob_fval; } PyFloatObject; -// Macro version of PyFloat_AsDouble() trading safety for speed. +#define _PyFloat_CAST(op) \ + (assert(PyFloat_Check(op)), _Py_CAST(PyFloatObject*, op)) + +// Static inline version of PyFloat_AsDouble() trading safety for speed. // It doesn't check if op is a double object. -#define PyFloat_AS_DOUBLE(op) (((PyFloatObject *)(op))->ob_fval) +static inline double PyFloat_AS_DOUBLE(PyObject *op) { + return _PyFloat_CAST(op)->ob_fval; +} +#define PyFloat_AS_DOUBLE(op) PyFloat_AS_DOUBLE(_PyObject_CAST(op)) PyAPI_FUNC(int) PyFloat_Pack2(double x, char *p, int le); diff --git a/Include/cpython/memoryobject.h b/Include/cpython/memoryobject.h index e2a1e168e463b8..deab3cc89f726e 100644 --- a/Include/cpython/memoryobject.h +++ b/Include/cpython/memoryobject.h @@ -36,7 +36,16 @@ typedef struct { Py_ssize_t ob_array[1]; /* shape, strides, suboffsets */ } PyMemoryViewObject; +#define _PyMemoryView_CAST(op) _Py_CAST(PyMemoryViewObject*, op) + /* Get a pointer to the memoryview's private copy of the exporter's buffer. */ -#define PyMemoryView_GET_BUFFER(op) (&((PyMemoryViewObject *)(op))->view) +static inline Py_buffer* PyMemoryView_GET_BUFFER(PyObject *op) { + return (&_PyMemoryView_CAST(op)->view); +} +#define PyMemoryView_GET_BUFFER(op) PyMemoryView_GET_BUFFER(_PyObject_CAST(op)) + /* Get a pointer to the exporting object (this may be NULL!). */ -#define PyMemoryView_GET_BASE(op) (((PyMemoryViewObject *)(op))->view.obj) +static inline PyObject* PyMemoryView_GET_BASE(PyObject *op) { + return _PyMemoryView_CAST(op)->view.obj; +} +#define PyMemoryView_GET_BASE(op) PyMemoryView_GET_BASE(_PyObject_CAST(op)) diff --git a/Include/cpython/setobject.h b/Include/cpython/setobject.h index b4443a678b7e77..20fd63eaae56e2 100644 --- a/Include/cpython/setobject.h +++ b/Include/cpython/setobject.h @@ -58,8 +58,13 @@ typedef struct { PyObject *weakreflist; /* List of weak references */ } PySetObject; -#define PySet_GET_SIZE(so) \ - (assert(PyAnySet_Check(so)), (((PySetObject *)(so))->used)) +#define _PySet_CAST(so) \ + (assert(PyAnySet_Check(so)), _Py_CAST(PySetObject*, so)) + +static inline Py_ssize_t PySet_GET_SIZE(PyObject *so) { + return _PySet_CAST(so)->used; +} +#define PySet_GET_SIZE(so) PySet_GET_SIZE(_PyObject_CAST(so)) PyAPI_DATA(PyObject *) _PySet_Dummy; diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index 86eeab67275ec8..a75336f590e81b 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -231,8 +231,6 @@ enum PyUnicode_Kind { // new compiler warnings on "kind < PyUnicode_KIND(str)" (compare signed and // unsigned numbers) where kind type is an int or on // "unsigned int kind = PyUnicode_KIND(str)" (cast signed to unsigned). -// Only declare the function as static inline function in the limited C API -// version 3.12 which is stricter. #define PyUnicode_KIND(op) (_PyASCIIObject_CAST(op)->state.kind) /* Return a void pointer to the raw unicode buffer. */ diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 25bd3bffb2e7aa..04b7084901303d 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -128,12 +128,21 @@ struct _dictvalues { #else #define DK_SIZE(dk) (1<dk_kind == DICT_KEYS_GENERAL), \ - (PyDictKeyEntry*)(&((int8_t*)((dk)->dk_indices))[(size_t)1 << (dk)->dk_log2_index_bytes])) -#define DK_UNICODE_ENTRIES(dk) \ - (assert((dk)->dk_kind != DICT_KEYS_GENERAL), \ - (PyDictUnicodeEntry*)(&((int8_t*)((dk)->dk_indices))[(size_t)1 << (dk)->dk_log2_index_bytes])) + +static inline void* _DK_ENTRIES(PyDictKeysObject *dk) { + int8_t *indices = (int8_t*)(dk->dk_indices); + size_t index = (size_t)1 << dk->dk_log2_index_bytes; + return (&indices[index]); +} +static inline PyDictKeyEntry* DK_ENTRIES(PyDictKeysObject *dk) { + assert(dk->dk_kind == DICT_KEYS_GENERAL); + return (PyDictKeyEntry*)_DK_ENTRIES(dk); +} +static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) { + assert(dk->dk_kind != DICT_KEYS_GENERAL); + return (PyDictUnicodeEntry*)_DK_ENTRIES(dk); +} + #define DK_IS_UNICODE(dk) ((dk)->dk_kind != DICT_KEYS_GENERAL) #define DICT_VERSION_INCREMENT (1 << DICT_MAX_WATCHERS) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 8b78f79e950e92..33c8c0b75ea742 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -355,8 +355,9 @@ extern int _PyType_HasSubclasses(PyTypeObject *); extern PyObject* _PyType_GetSubclasses(PyTypeObject *); // Access macro to the members which are floating "behind" the object -#define _PyHeapType_GET_MEMBERS(etype) \ - ((PyMemberDef *)(((char *)(etype)) + Py_TYPE(etype)->tp_basicsize)) +static inline PyMemberDef* _PyHeapType_GET_MEMBERS(PyHeapTypeObject *etype) { + return (PyMemberDef*)((char*)etype + Py_TYPE(etype)->tp_basicsize); +} PyAPI_FUNC(PyObject *) _PyObject_LookupSpecial(PyObject *, PyObject *); From 65417988a589e6edfeada83227a8b0884a64af4f Mon Sep 17 00:00:00 2001 From: David Miguel Susano Pinto Date: Mon, 28 Nov 2022 16:05:21 +0000 Subject: [PATCH 34/35] Grammatical improvements for ctypes 'winmode' documentation (GH-19167) --- Doc/library/ctypes.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 71e5545ffe47c6..fd5df875ed74d5 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -1440,8 +1440,8 @@ copy of the windows error code. The *winmode* parameter is used on Windows to specify how the library is loaded (since *mode* is ignored). It takes any value that is valid for the Win32 API -``LoadLibraryEx`` flags parameter. When omitted, the default is to use the flags -that result in the most secure DLL load to avoiding issues such as DLL +``LoadLibraryEx`` flags parameter. When omitted, the default is to use the +flags that result in the most secure DLL load, which avoids issues such as DLL hijacking. Passing the full path to the DLL is the safest way to ensure the correct library and dependencies are loaded. From 7bae15cf37239d4d345e09cc318bd82d03ec30cd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 28 Nov 2022 17:42:22 +0100 Subject: [PATCH 35/35] Use _Py_RVALUE() in macros (#99844) The following macros are modified to use _Py_RVALUE(), so they can no longer be used as l-value: * DK_LOG_SIZE() * _PyCode_CODE() * _PyList_ITEMS() * _PyTuple_ITEMS() * _Py_SLIST_HEAD() * _Py_SLIST_ITEM_NEXT() _PyCode_CODE() is private and other macros are part of the internal C API. --- Include/cpython/code.h | 2 +- Include/internal/pycore_dict.h | 2 +- Include/internal/pycore_hashtable.h | 4 ++-- Include/internal/pycore_list.h | 2 +- Include/internal/pycore_tuple.h | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 15b464fe2ee95c..fd57e0035bc09a 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -153,7 +153,7 @@ static inline Py_ssize_t PyCode_GetNumFree(PyCodeObject *op) { return op->co_nfreevars; } -#define _PyCode_CODE(CO) ((_Py_CODEUNIT *)(CO)->co_code_adaptive) +#define _PyCode_CODE(CO) _Py_RVALUE((_Py_CODEUNIT *)(CO)->co_code_adaptive) #define _PyCode_NBYTES(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT)) /* Public interface */ diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 04b7084901303d..2b3b56b343ad99 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -122,7 +122,7 @@ struct _dictvalues { PyObject *values[1]; }; -#define DK_LOG_SIZE(dk) ((dk)->dk_log2_size) +#define DK_LOG_SIZE(dk) _Py_RVALUE((dk)->dk_log2_size) #if SIZEOF_VOID_P > 4 #define DK_SIZE(dk) (((int64_t)1)<next) +#define _Py_SLIST_ITEM_NEXT(ITEM) _Py_RVALUE(((_Py_slist_item_t *)(ITEM))->next) -#define _Py_SLIST_HEAD(SLIST) (((_Py_slist_t *)(SLIST))->head) +#define _Py_SLIST_HEAD(SLIST) _Py_RVALUE(((_Py_slist_t *)(SLIST))->head) /* _Py_hashtable: table entry */ diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h index 691d13bc8d9ffa..628267cc8a9618 100644 --- a/Include/internal/pycore_list.h +++ b/Include/internal/pycore_list.h @@ -35,7 +35,7 @@ struct _Py_list_state { #endif }; -#define _PyList_ITEMS(op) (_PyList_CAST(op)->ob_item) +#define _PyList_ITEMS(op) _Py_RVALUE(_PyList_CAST(op)->ob_item) extern int _PyList_AppendTakeRefListResize(PyListObject *self, PyObject *newitem); diff --git a/Include/internal/pycore_tuple.h b/Include/internal/pycore_tuple.h index 1efe4fa2bdef94..504c36338d9e96 100644 --- a/Include/internal/pycore_tuple.h +++ b/Include/internal/pycore_tuple.h @@ -62,7 +62,7 @@ struct _Py_tuple_state { #endif }; -#define _PyTuple_ITEMS(op) (_PyTuple_CAST(op)->ob_item) +#define _PyTuple_ITEMS(op) _Py_RVALUE(_PyTuple_CAST(op)->ob_item) extern PyObject *_PyTuple_FromArray(PyObject *const *, Py_ssize_t); extern PyObject *_PyTuple_FromArraySteal(PyObject *const *, Py_ssize_t);