From bee5aefdd77573d962db0d7a16f3bc0404406f76 Mon Sep 17 00:00:00 2001 From: Thomas Wouters Date: Wed, 17 Apr 2019 11:08:12 +0200 Subject: [PATCH 001/527] Initial stab at using libgc: switch to using GC_malloc, disable pymalloc by default, skip calling GC_free (to test libgc actually deleting things on its own). --- Include/Python.h | 7 +++++++ Lib/test/test_capi.py | 3 +++ Makefile.pre.in | 13 +++++++------ Modules/_tracemalloc.c | 4 ++-- Modules/posixmodule.c | 8 ++++++++ Objects/obmalloc.c | 9 +++++---- Python/pylifecycle.c | 6 ++++++ aclocal.m4 | 4 ++-- configure | 44 ++++++++++++++++++++++++++++++++++++++++-- configure.ac | 19 +++++++++++++++++- 10 files changed, 100 insertions(+), 17 deletions(-) diff --git a/Include/Python.h b/Include/Python.h index 55b06aeea9846f..493962ae046152 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -60,6 +60,13 @@ */ #include +/* libgc doesn't appear to have a config.h for these defines, so assume + * GC_THREADS support is enabled. + */ +#define GC_THREADS 1 +/* Include gc.h for its intercepts of thread functions. */ +#include + #include "pyport.h" #include "pymacro.h" diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 31dab6a423e25a..2ba8f5f0d2a87e 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -540,12 +540,15 @@ def check_pyobject_is_freed(self, func): code = code.format(func=func) assert_python_ok('-c', code, PYTHONMALLOC=self.PYTHONMALLOC) + @unittest.skipUnless(support.with_pymalloc(), 'need pymalloc') def test_pyobject_is_freed_uninitialized(self): self.check_pyobject_is_freed('pyobject_uninitialized') + @unittest.skipUnless(support.with_pymalloc(), 'need pymalloc') def test_pyobject_is_freed_forbidden_bytes(self): self.check_pyobject_is_freed('pyobject_forbidden_bytes') + @unittest.skipUnless(support.with_pymalloc(), 'need pymalloc') def test_pyobject_is_freed_free(self): self.check_pyobject_is_freed('pyobject_freed') diff --git a/Makefile.pre.in b/Makefile.pre.in index 75eb66be3c0135..6f62abe1c5cfbf 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -231,7 +231,8 @@ INSTSONAME= @INSTSONAME@ LIBS= @LIBS@ LIBM= @LIBM@ LIBC= @LIBC@ -SYSLIBS= $(LIBM) $(LIBC) +LIBGC= @LIBGC@ +SYSLIBS= $(LIBM) $(LIBC) $(LIBGC) SHLIBS= @SHLIBS@ DLINCLDIR= @DLINCLDIR@ @@ -560,7 +561,7 @@ clinic: check-clean-src $(srcdir)/Modules/_blake2/blake2s_impl.c # Build the interpreter $(BUILDPYTHON): Programs/python.o $(LIBRARY) $(LDLIBRARY) $(PY3LIBRARY) - $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) + $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LIBGC) platform: $(BUILDPYTHON) pybuilddir.txt $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform @@ -615,21 +616,21 @@ $(LIBRARY): $(LIBRARY_OBJS) libpython$(LDVERSION).so: $(LIBRARY_OBJS) $(DTRACE_OBJS) if test $(INSTSONAME) != $(LDLIBRARY); then \ - $(BLDSHARED) -Wl,-h$(INSTSONAME) -o $(INSTSONAME) $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM); \ + $(BLDSHARED) -Wl,-h$(INSTSONAME) -o $(INSTSONAME) $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBGC) $(LIBC) $(LIBM); \ $(LN) -f $(INSTSONAME) $@; \ else \ - $(BLDSHARED) -o $@ $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM); \ + $(BLDSHARED) -o $@ $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBGC) $(LIBC) $(LIBM); \ fi libpython3.so: libpython$(LDVERSION).so $(BLDSHARED) $(NO_AS_NEEDED) -o $@ -Wl,-h$@ $^ libpython$(LDVERSION).dylib: $(LIBRARY_OBJS) - $(CC) -dynamiclib -Wl,-single_module $(PY_CORE_LDFLAGS) -undefined dynamic_lookup -Wl,-install_name,$(prefix)/lib/libpython$(LDVERSION).dylib -Wl,-compatibility_version,$(VERSION) -Wl,-current_version,$(VERSION) -o $@ $(LIBRARY_OBJS) $(DTRACE_OBJS) $(SHLIBS) $(LIBC) $(LIBM); \ + $(CC) -dynamiclib -Wl,-single_module $(PY_CORE_LDFLAGS) -undefined dynamic_lookup -Wl,-install_name,$(prefix)/lib/libpython$(LDVERSION).dylib -Wl,-compatibility_version,$(VERSION) -Wl,-current_version,$(VERSION) -o $@ $(LIBRARY_OBJS) $(DTRACE_OBJS) $(SHLIBS) $(LIBGC) $(LIBC) $(LIBM); \ libpython$(VERSION).sl: $(LIBRARY_OBJS) - $(LDSHARED) -o $@ $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM) + $(LDSHARED) -o $@ $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBGC) $(LIBC) $(LIBM) # Copy up the gdb python hooks into a position where they can be automatically # loaded by gdb during Lib/test/test_gdb.py diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index c5d5671032ebf1..e93bd446d811e6 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -273,13 +273,13 @@ hashtable_new(size_t key_size, size_t data_size, static void* raw_malloc(size_t size) { - return allocators.raw.malloc(allocators.raw.ctx, size); + return malloc(size); } static void raw_free(void *ptr) { - allocators.raw.free(allocators.raw.ctx, ptr); + free(ptr); } diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 221f7101b21356..e3904aa578b943 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -407,11 +407,16 @@ PyOS_BeforeFork(void) run_at_forkers(_PyInterpreterState_Get()->before_forkers, 1); _PyImport_AcquireLock(); + /* GC atfork_prepare needs to be called last, as it disables GC. */ + GC_atfork_prepare(); } void PyOS_AfterFork_Parent(void) { + /* GC atfork functions need to be called first, before allocations can + * happen. */ + GC_atfork_parent(); if (_PyImport_ReleaseLock() <= 0) Py_FatalError("failed releasing import lock after fork"); @@ -422,6 +427,9 @@ void PyOS_AfterFork_Child(void) { _PyRuntimeState *runtime = &_PyRuntime; + /* GC atfork functions need to be called first, before allocations can + * happen. */ + GC_atfork_child(); _PyGILState_Reinit(runtime); _PyInterpreterState_DeleteExceptMain(runtime); PyEval_ReInitThreads(); diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 7cfd28965967f9..f2e4d7dc8c41dd 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -1,5 +1,6 @@ #include "Python.h" #include "pycore_pymem.h" +#include "gc/gc.h" #include @@ -96,7 +97,7 @@ _PyMem_RawMalloc(void *ctx, size_t size) To solve these problems, allocate an extra byte. */ if (size == 0) size = 1; - return malloc(size); + return GC_MALLOC(size); } static void * @@ -110,7 +111,7 @@ _PyMem_RawCalloc(void *ctx, size_t nelem, size_t elsize) nelem = 1; elsize = 1; } - return calloc(nelem, elsize); + return GC_MALLOC(nelem * elsize); } static void * @@ -118,13 +119,13 @@ _PyMem_RawRealloc(void *ctx, void *ptr, size_t size) { if (size == 0) size = 1; - return realloc(ptr, size); + return GC_REALLOC(ptr, size); } static void _PyMem_RawFree(void *ctx, void *ptr) { - free(ptr); + /* GC_FREE(ptr); */ } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index bd4d1d92662a69..61764ae32d5b59 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2,6 +2,8 @@ #include "Python.h" +#include + #include "Python-ast.h" #undef Yield /* undefine macro conflicting with */ #include "pycore_coreconfig.h" @@ -84,6 +86,8 @@ _PyRuntime_Initialize(void) } runtime_initialized = 1; + GC_INIT(); + return _PyRuntimeState_Init(&_PyRuntime); } @@ -535,6 +539,8 @@ pycore_create_interpreter(_PyRuntimeState *runtime, /* Create the GIL */ PyEval_InitThreads(); + GC_allow_register_threads(); + return _Py_INIT_OK(); } diff --git a/aclocal.m4 b/aclocal.m4 index 85f00dd5fac7f2..030e6877de9f7c 100644 --- a/aclocal.m4 +++ b/aclocal.m4 @@ -1,6 +1,6 @@ -# generated automatically by aclocal 1.16.1 -*- Autoconf -*- +# generated automatically by aclocal 1.15.1 -*- Autoconf -*- -# Copyright (C) 1996-2018 Free Software Foundation, Inc. +# Copyright (C) 1996-2017 Free Software Foundation, Inc. # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, diff --git a/configure b/configure index e6e4007351591a..d636b71b5ac9cb 100755 --- a/configure +++ b/configure @@ -635,6 +635,7 @@ LIBPYTHON EXT_SUFFIX ALT_SOABI SOABI +LIBGC LIBC LIBM HAVE_GETHOSTBYNAME @@ -785,6 +786,7 @@ infodir docdir oldincludedir includedir +runstatedir localstatedir sharedstatedir sysconfdir @@ -840,6 +842,7 @@ with_valgrind with_dtrace with_libm with_libc +with_libgc enable_big_digits with_computed_gotos with_ensurepip @@ -897,6 +900,7 @@ datadir='${datarootdir}' sysconfdir='${prefix}/etc' sharedstatedir='${prefix}/com' localstatedir='${prefix}/var' +runstatedir='${localstatedir}/run' includedir='${prefix}/include' oldincludedir='/usr/include' docdir='${datarootdir}/doc/${PACKAGE_TARNAME}' @@ -1149,6 +1153,15 @@ do | -silent | --silent | --silen | --sile | --sil) silent=yes ;; + -runstatedir | --runstatedir | --runstatedi | --runstated \ + | --runstate | --runstat | --runsta | --runst | --runs \ + | --run | --ru | --r) + ac_prev=runstatedir ;; + -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \ + | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \ + | --run=* | --ru=* | --r=*) + runstatedir=$ac_optarg ;; + -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) ac_prev=sbindir ;; -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ @@ -1286,7 +1299,7 @@ fi for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \ datadir sysconfdir sharedstatedir localstatedir includedir \ oldincludedir docdir infodir htmldir dvidir pdfdir psdir \ - libdir localedir mandir + libdir localedir mandir runstatedir do eval ac_val=\$$ac_var # Remove trailing slashes. @@ -1439,6 +1452,7 @@ Fine tuning of the installation directories: --sysconfdir=DIR read-only single-machine data [PREFIX/etc] --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] --localstatedir=DIR modifiable single-machine data [PREFIX/var] + --runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run] --libdir=DIR object code libraries [EPREFIX/lib] --includedir=DIR C header files [PREFIX/include] --oldincludedir=DIR C header files for non-gcc [/usr/include] @@ -1537,6 +1551,7 @@ Optional Packages: --with(out)-dtrace disable/enable DTrace support --with-libm=STRING math library --with-libc=STRING C library + --with-libgc=STRING BDW GC library --with(out)-computed-gotos Use computed gotos in evaluation loop (enabled by default on supported compilers) @@ -11251,7 +11266,7 @@ fi if test -z "$with_pymalloc" then - with_pymalloc="yes" + with_pymalloc="no" fi if test "$with_pymalloc" != "no" then @@ -14190,6 +14205,31 @@ $as_echo "default LIBC=\"$LIBC\"" >&6; } fi +# check for --with-libgc=... + +LIBGC="-lgc" +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-libgc=STRING" >&5 +$as_echo_n "checking for --with-libgc=STRING... " >&6; } + +# Check whether --with-libgc was given. +if test "${with_libgc+set}" = set; then : + withval=$with_libgc; +if test "$withval" = no +then LIBGC= + { $as_echo "$as_me:${as_lineno-$LINENO}: result: force LIBGC empty" >&5 +$as_echo "force LIBGC empty" >&6; } +elif test "$withval" != yes +then LIBGC=$withval + { $as_echo "$as_me:${as_lineno-$LINENO}: result: set LIBGC=\"$withval\"" >&5 +$as_echo "set LIBGC=\"$withval\"" >&6; } +else as_fn_error $? "proper usage is --with-libgc=STRING" "$LINENO" 5 +fi +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: default LIBGC=\"$LIBGC\"" >&5 +$as_echo "default LIBGC=\"$LIBGC\"" >&6; } +fi + + # ************************************** # * Check for gcc x64 inline assembler * # ************************************** diff --git a/configure.ac b/configure.ac index a02597da2db5ea..af09695af8bf64 100644 --- a/configure.ac +++ b/configure.ac @@ -3401,7 +3401,7 @@ AC_ARG_WITH(pymalloc, if test -z "$with_pymalloc" then - with_pymalloc="yes" + with_pymalloc="no" fi if test "$with_pymalloc" != "no" then @@ -4313,6 +4313,23 @@ else AC_MSG_ERROR([proper usage is --with-libc=STRING]) fi], [AC_MSG_RESULT(default LIBC="$LIBC")]) +# check for --with-libgc=... +AC_SUBST(LIBGC) +LIBGC="-lgc" +AC_MSG_CHECKING(for --with-libgc=STRING) +AC_ARG_WITH(libgc, + AS_HELP_STRING([--with-libgc=STRING], [BDW GC library]), +[ +if test "$withval" = no +then LIBGC= + AC_MSG_RESULT(force LIBGC empty) +elif test "$withval" != yes +then LIBGC=$withval + AC_MSG_RESULT(set LIBGC="$withval") +else AC_MSG_ERROR([proper usage is --with-libgc=STRING]) +fi], +[AC_MSG_RESULT(default LIBGC="$LIBGC")]) + # ************************************** # * Check for gcc x64 inline assembler * # ************************************** From 85c8ff3a4ff7e79c2f747ce1384622816c778ffe Mon Sep 17 00:00:00 2001 From: Thomas Wouters Date: Thu, 25 Apr 2019 12:11:58 +0200 Subject: [PATCH 002/527] Plug Python object finalization into libgc finalizers. This doesn't get rid of refcounting, just makes sure libgc sees enough references to live objects (by aborting if it tries to finalize an object with a non-zero refcount) and also act as a refleak-detector. --- Include/Python.h | 1 + Include/cpython/objimpl.h | 1 + Include/object.h | 2 + Include/objimpl.h | 123 ++++++++++++++++++++------------------ Objects/object.c | 31 +++++++--- Objects/obmalloc.c | 23 ++++++- Python/pylifecycle.c | 1 + 7 files changed, 115 insertions(+), 67 deletions(-) diff --git a/Include/Python.h b/Include/Python.h index 493962ae046152..32bde235117dd9 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -64,6 +64,7 @@ * GC_THREADS support is enabled. */ #define GC_THREADS 1 +/* #define GC_DEBUG 1 */ /* Include gc.h for its intercepts of thread functions. */ #include diff --git a/Include/cpython/objimpl.h b/Include/cpython/objimpl.h index f121922bc42ced..87d8ff493523e1 100644 --- a/Include/cpython/objimpl.h +++ b/Include/cpython/objimpl.h @@ -54,6 +54,7 @@ typedef struct { } PyGC_Head; #define _Py_AS_GC(o) ((PyGC_Head *)(o)-1) +#define _Py_FROM_GC(o) ((PyObject *)((PyGC_Head *)(o)+1)) /* True if the object is currently tracked by the GC. */ #define _PyObject_GC_IS_TRACKED(o) (_Py_AS_GC(o)->_gc_next != 0) diff --git a/Include/object.h b/Include/object.h index 13e88a6dc6f02a..2db179a18fc732 100644 --- a/Include/object.h +++ b/Include/object.h @@ -442,6 +442,8 @@ static inline void _Py_ForgetReference(PyObject *op) PyAPI_FUNC(void) _Py_Dealloc(PyObject *); +PyAPI_FUNC(void) _Py_Dealloc_finalizer(void *, void *); +PyAPI_FUNC(void) _Py_Dealloc_GC_finalizer(void *, void *); static inline void _Py_INCREF(PyObject *op) { diff --git a/Include/objimpl.h b/Include/objimpl.h index 2337d8a56c774e..63056ad5a9502a 100644 --- a/Include/objimpl.h +++ b/Include/objimpl.h @@ -110,6 +110,58 @@ PyAPI_FUNC(void) PyObject_Free(void *ptr); #define PyObject_DEL PyObject_Free +/* + * Garbage Collection Support + * ========================== + */ + +/* C equivalent of gc.collect() which ignores the state of gc.enabled. */ +PyAPI_FUNC(Py_ssize_t) PyGC_Collect(void); + +/* Test if a type has a GC head */ +#define PyType_IS_GC(t) PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC) + +PyAPI_FUNC(PyVarObject *) _PyObject_GC_Resize(PyVarObject *, Py_ssize_t); +#define PyObject_GC_Resize(type, op, n) \ + ( (type *) _PyObject_GC_Resize(_PyVarObject_CAST(op), (n)) ) + + + +PyAPI_FUNC(PyObject *) _PyObject_GC_New(PyTypeObject *); +PyAPI_FUNC(PyVarObject *) _PyObject_GC_NewVar(PyTypeObject *, Py_ssize_t); + +/* Tell the GC to track this object. + * + * See also private _PyObject_GC_TRACK() macro. */ +PyAPI_FUNC(void) PyObject_GC_Track(void *); + +/* Tell the GC to stop tracking this object. + * + * See also private _PyObject_GC_UNTRACK() macro. */ +PyAPI_FUNC(void) PyObject_GC_UnTrack(void *); + +PyAPI_FUNC(void) PyObject_GC_Del(void *); + +#define PyObject_GC_New(type, typeobj) \ + ( (type *) _PyObject_GC_New(typeobj) ) +#define PyObject_GC_NewVar(type, typeobj, n) \ + ( (type *) _PyObject_GC_NewVar((typeobj), (n)) ) + + +/* Utility macro to help write tp_traverse functions. + * To use this macro, the tp_traverse function must name its arguments + * "visit" and "arg". This is intended to keep tp_traverse functions + * looking as much alike as possible. + */ +#define Py_VISIT(op) \ + do { \ + if (op) { \ + int vret = visit(_PyObject_CAST(op), arg); \ + if (vret) \ + return vret; \ + } \ + } while (0) + /* * Generic object allocator interface * ================================== @@ -127,6 +179,12 @@ PyAPI_FUNC(PyVarObject *) _PyObject_NewVar(PyTypeObject *, Py_ssize_t); #define PyObject_NewVar(type, typeobj, n) \ ( (type *) _PyObject_NewVar((typeobj), (n)) ) +#ifndef Py_LIMITED_API +# define Py_CPYTHON_OBJIMPL_H +# include "cpython/objimpl.h" +# undef Py_CPYTHON_OBJIMPL_H +#endif + /* Inline functions trading binary compatibility for speed: PyObject_INIT() is the fast version of PyObject_Init(), and PyObject_INIT_VAR() is the fast version of PyObject_InitVar. @@ -142,6 +200,13 @@ _PyObject_INIT(PyObject *op, PyTypeObject *typeobj) Py_INCREF(typeobj); } _Py_NewReference(op); + if (GC_is_heap_ptr(op)) { + if (PyType_IS_GC(typeobj)) { + GC_REGISTER_FINALIZER(_Py_AS_GC(op), _Py_Dealloc_GC_finalizer, NULL, NULL, NULL); + } else { + GC_REGISTER_FINALIZER(op, _Py_Dealloc_finalizer, NULL, NULL, NULL); + } + } return op; } @@ -220,64 +285,6 @@ _PyObject_INIT_VAR(PyVarObject *op, PyTypeObject *typeobj, Py_ssize_t size) -/* - * Garbage Collection Support - * ========================== - */ - -/* C equivalent of gc.collect() which ignores the state of gc.enabled. */ -PyAPI_FUNC(Py_ssize_t) PyGC_Collect(void); - -/* Test if a type has a GC head */ -#define PyType_IS_GC(t) PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC) - -PyAPI_FUNC(PyVarObject *) _PyObject_GC_Resize(PyVarObject *, Py_ssize_t); -#define PyObject_GC_Resize(type, op, n) \ - ( (type *) _PyObject_GC_Resize(_PyVarObject_CAST(op), (n)) ) - - - -PyAPI_FUNC(PyObject *) _PyObject_GC_New(PyTypeObject *); -PyAPI_FUNC(PyVarObject *) _PyObject_GC_NewVar(PyTypeObject *, Py_ssize_t); - -/* Tell the GC to track this object. - * - * See also private _PyObject_GC_TRACK() macro. */ -PyAPI_FUNC(void) PyObject_GC_Track(void *); - -/* Tell the GC to stop tracking this object. - * - * See also private _PyObject_GC_UNTRACK() macro. */ -PyAPI_FUNC(void) PyObject_GC_UnTrack(void *); - -PyAPI_FUNC(void) PyObject_GC_Del(void *); - -#define PyObject_GC_New(type, typeobj) \ - ( (type *) _PyObject_GC_New(typeobj) ) -#define PyObject_GC_NewVar(type, typeobj, n) \ - ( (type *) _PyObject_GC_NewVar((typeobj), (n)) ) - - -/* Utility macro to help write tp_traverse functions. - * To use this macro, the tp_traverse function must name its arguments - * "visit" and "arg". This is intended to keep tp_traverse functions - * looking as much alike as possible. - */ -#define Py_VISIT(op) \ - do { \ - if (op) { \ - int vret = visit(_PyObject_CAST(op), arg); \ - if (vret) \ - return vret; \ - } \ - } while (0) - -#ifndef Py_LIMITED_API -# define Py_CPYTHON_OBJIMPL_H -# include "cpython/objimpl.h" -# undef Py_CPYTHON_OBJIMPL_H -#endif - #ifdef __cplusplus } #endif diff --git a/Objects/object.c b/Objects/object.c index 732f9ccefa8774..83da0086330b35 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -251,13 +251,7 @@ PyObject_Init(PyObject *op, PyTypeObject *tp) { if (op == NULL) return PyErr_NoMemory(); - /* Any changes should be reflected in PyObject_INIT (objimpl.h) */ - Py_TYPE(op) = tp; - if (PyType_GetFlags(tp) & Py_TPFLAGS_HEAPTYPE) { - Py_INCREF(tp); - } - _Py_NewReference(op); - return op; + return PyObject_INIT(op, tp); } PyVarObject * @@ -2207,6 +2201,29 @@ _Py_Dealloc(PyObject *op) (*dealloc)(op); } +void +_Py_Dealloc_GC_finalizer(void *_op, void *unused) +{ + PyObject *op = _Py_FROM_GC(_op); + assert(PyObject_IS_GC(op)); + _Py_Dealloc_finalizer((void *)op, (void *)_op); +} + +void +_Py_Dealloc_finalizer(void *_op, void *is_gc) +{ + PyObject *op = (PyObject *)_op; + assert(is_gc == NULL ? !_Py_IS_GC(op) || _Py_IS_GC(op)); + if (op->ob_refcnt != 0) { + fprintf(stderr, "object %p refcount leak (%ld)\n", op, op->ob_refcnt); + abort(); + } else { + fprintf(stderr, "object %p not freed (refcount 0)\n", op); + abort(); + } + _Py_Dealloc(op); +} + #ifdef __cplusplus } #endif diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index f2e4d7dc8c41dd..297917c6c1bd70 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -117,15 +117,34 @@ _PyMem_RawCalloc(void *ctx, size_t nelem, size_t elsize) static void * _PyMem_RawRealloc(void *ctx, void *ptr, size_t size) { + GC_finalization_proc fn; + + if (ptr == NULL) + return _PyMem_RawMalloc(ctx, size); + if (size == 0) size = 1; - return GC_REALLOC(ptr, size); + if (!GC_is_heap_ptr(ptr)) { + abort(); + } + GC_REGISTER_FINALIZER(ptr, NULL, NULL, &fn, NULL); + ptr = GC_REALLOC(ptr, size); + if (fn != NULL) { + GC_REGISTER_FINALIZER(ptr, fn, NULL, NULL, NULL); + } + return ptr; } static void _PyMem_RawFree(void *ctx, void *ptr) { - /* GC_FREE(ptr); */ + if (ptr == NULL) + return; + if (!GC_is_heap_ptr(ptr)) { + abort(); + } + GC_REGISTER_FINALIZER(ptr, NULL, NULL, NULL, NULL); +// GC_FREE(ptr); } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 61764ae32d5b59..a0edc8780db140 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -86,6 +86,7 @@ _PyRuntime_Initialize(void) } runtime_initialized = 1; + GC_set_all_interior_pointers(1); GC_INIT(); return _PyRuntimeState_Init(&_PyRuntime); From 0725e21d45f48c7950392e8e8fff8c41c09a3dbf Mon Sep 17 00:00:00 2001 From: Thomas Wouters Date: Tue, 30 Apr 2019 17:51:00 -0700 Subject: [PATCH 003/527] Fix assert. --- Objects/object.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/object.c b/Objects/object.c index 83da0086330b35..8e8b1542d17b58 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2213,7 +2213,7 @@ void _Py_Dealloc_finalizer(void *_op, void *is_gc) { PyObject *op = (PyObject *)_op; - assert(is_gc == NULL ? !_Py_IS_GC(op) || _Py_IS_GC(op)); + assert(is_gc == NULL ? !PyObject_IS_GC(op) : PyObject_IS_GC(op)); if (op->ob_refcnt != 0) { fprintf(stderr, "object %p refcount leak (%ld)\n", op, op->ob_refcnt); abort(); From 0302e8f1696144132c83286182b19742b33c7a2b Mon Sep 17 00:00:00 2001 From: Thomas Wouters Date: Tue, 30 Apr 2019 17:52:46 -0700 Subject: [PATCH 004/527] Fix -x dev by disable memory poisoning in the debug allocator, since we read the memory after the call to DebugFree (in the finalizer). --- Objects/obmalloc.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 297917c6c1bd70..0f3525f26d165f 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -2123,7 +2123,9 @@ _PyMem_DebugRawFree(void *ctx, void *p) _PyMem_DebugCheckAddress(api->api_id, p); nbytes = read_size_t(q); nbytes += PYMEM_DEBUG_EXTRA_BYTES; - memset(q, DEADBYTE, nbytes); + // TODO(thomas@python.org): figure out alternative that works with + // _Py_Dealloc_finalize, probably by writing magic values to refcnt. + // memset(q, DEADBYTE, nbytes); api->alloc.free(api->alloc.ctx, q); } From abebc0e11117d6930f3d9567abb7dbc1face2597 Mon Sep 17 00:00:00 2001 From: Thomas Wouters Date: Fri, 3 May 2019 11:27:56 -0700 Subject: [PATCH 005/527] Stop calling _Py_Dealloc from Py_DECREF (and get rid of it entirely). This means all finalization is done by libgc. It currently segfaults in cyclic-gc. --- Doc/c-api/refcounting.rst | 2 +- Include/cpython/object.h | 12 ------------ Include/object.h | 4 ---- Objects/object.c | 10 ++-------- PC/python3.def | 1 - 5 files changed, 3 insertions(+), 26 deletions(-) diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst index 225a1feb250684..d5ea1be806b0ba 100644 --- a/Doc/c-api/refcounting.rst +++ b/Doc/c-api/refcounting.rst @@ -68,6 +68,6 @@ simply exported function versions of :c:func:`Py_XINCREF` and :c:func:`Py_XDECREF`, respectively. The following functions or macros are only for use within the interpreter core: -:c:func:`_Py_Dealloc`, :c:func:`_Py_ForgetReference`, :c:func:`_Py_NewReference`, +:c:func:`_Py_ForgetReference`, :c:func:`_Py_NewReference`, as well as the global variable :c:data:`_Py_RefTotal`. diff --git a/Include/cpython/object.h b/Include/cpython/object.h index ba52a4835823b6..3ade5d0b7c0bdc 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -332,18 +332,6 @@ _PyObject_GenericSetAttrWithDict(PyObject *, PyObject *, #define PyType_HasFeature(t,f) (((t)->tp_flags & (f)) != 0) -static inline void _Py_Dealloc_inline(PyObject *op) -{ - destructor dealloc = Py_TYPE(op)->tp_dealloc; -#ifdef Py_TRACE_REFS - _Py_ForgetReference(op); -#else - _Py_INC_TPFREES(op); -#endif - (*dealloc)(op); -} -#define _Py_Dealloc(op) _Py_Dealloc_inline(op) - /* Safely decref `op` and set `op` to `op2`. * diff --git a/Include/object.h b/Include/object.h index 2db179a18fc732..ea6dc12ff5eb34 100644 --- a/Include/object.h +++ b/Include/object.h @@ -441,7 +441,6 @@ static inline void _Py_ForgetReference(PyObject *op) #endif /* !Py_TRACE_REFS */ -PyAPI_FUNC(void) _Py_Dealloc(PyObject *); PyAPI_FUNC(void) _Py_Dealloc_finalizer(void *, void *); PyAPI_FUNC(void) _Py_Dealloc_GC_finalizer(void *, void *); @@ -466,9 +465,6 @@ static inline void _Py_DECREF(const char *filename, int lineno, } #endif } - else { - _Py_Dealloc(op); - } } #define Py_DECREF(op) _Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op)) diff --git a/Objects/object.c b/Objects/object.c index 8e8b1542d17b58..7717ba25b2595c 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2187,10 +2187,8 @@ _PyObject_AssertFailed(PyObject *obj, const char *expr, const char *msg, } -#undef _Py_Dealloc - void -_Py_Dealloc(PyObject *op) +_GC_Py_Dealloc(PyObject *op) { destructor dealloc = Py_TYPE(op)->tp_dealloc; #ifdef Py_TRACE_REFS @@ -2216,12 +2214,8 @@ _Py_Dealloc_finalizer(void *_op, void *is_gc) assert(is_gc == NULL ? !PyObject_IS_GC(op) : PyObject_IS_GC(op)); if (op->ob_refcnt != 0) { fprintf(stderr, "object %p refcount leak (%ld)\n", op, op->ob_refcnt); - abort(); - } else { - fprintf(stderr, "object %p not freed (refcount 0)\n", op); - abort(); } - _Py_Dealloc(op); + _GC_Py_Dealloc(op); } #ifdef __cplusplus diff --git a/PC/python3.def b/PC/python3.def index e317864d0cd802..5cb683839f9705 100644 --- a/PC/python3.def +++ b/PC/python3.def @@ -793,7 +793,6 @@ EXPORTS _Py_BuildValue_SizeT=python38._Py_BuildValue_SizeT _Py_CheckRecursionLimit=python38._Py_CheckRecursionLimit DATA _Py_CheckRecursiveCall=python38._Py_CheckRecursiveCall - _Py_Dealloc=python38._Py_Dealloc _Py_EllipsisObject=python38._Py_EllipsisObject DATA _Py_FalseStruct=python38._Py_FalseStruct DATA _Py_NoneStruct=python38._Py_NoneStruct DATA From ad2d559d19d164e36e2ab0cfaa8e08a54f7e2d04 Mon Sep 17 00:00:00 2001 From: Thomas Wouters Date: Sat, 4 May 2019 06:20:04 -0700 Subject: [PATCH 006/527] Hackishly turn off cylic-GC. --- Include/cpython/objimpl.h | 6 +- Include/internal/pycore_object.h | 28 +----- Include/objimpl.h | 8 +- Modules/gcmodule.c | 147 +++---------------------------- Objects/object.c | 6 +- Objects/obmalloc.c | 6 +- 6 files changed, 23 insertions(+), 178 deletions(-) diff --git a/Include/cpython/objimpl.h b/Include/cpython/objimpl.h index 87d8ff493523e1..ed3a2a524dceaf 100644 --- a/Include/cpython/objimpl.h +++ b/Include/cpython/objimpl.h @@ -38,9 +38,7 @@ PyAPI_FUNC(Py_ssize_t) _PyGC_CollectIfEnabled(void); /* Test if an object has a GC head */ -#define PyObject_IS_GC(o) \ - (PyType_IS_GC(Py_TYPE(o)) \ - && (Py_TYPE(o)->tp_is_gc == NULL || Py_TYPE(o)->tp_is_gc(o))) +#define PyObject_IS_GC(o) (0) /* GC information is stored BEFORE the object structure. */ typedef struct { @@ -57,7 +55,7 @@ typedef struct { #define _Py_FROM_GC(o) ((PyObject *)((PyGC_Head *)(o)+1)) /* True if the object is currently tracked by the GC. */ -#define _PyObject_GC_IS_TRACKED(o) (_Py_AS_GC(o)->_gc_next != 0) +#define _PyObject_GC_IS_TRACKED(o) (0) /* True if the object may be tracked by the GC in the future, or already is. This can be useful to implement some optimizations. */ diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 81548f819198e3..4c3f0d8c136bb3 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -28,21 +28,7 @@ PyAPI_FUNC(int) _PyDict_CheckConsistency(PyObject *mp, int check_content); static inline void _PyObject_GC_TRACK_impl(const char *filename, int lineno, PyObject *op) { - _PyObject_ASSERT_FROM(op, !_PyObject_GC_IS_TRACKED(op), - "object already tracked by the garbage collector", - filename, lineno, "_PyObject_GC_TRACK"); - - PyGC_Head *gc = _Py_AS_GC(op); - _PyObject_ASSERT_FROM(op, - (gc->_gc_prev & _PyGC_PREV_MASK_COLLECTING) == 0, - "object is in generation which is garbage collected", - filename, lineno, "_PyObject_GC_TRACK"); - - PyGC_Head *last = (PyGC_Head*)(_PyRuntime.gc.generation0->_gc_prev); - _PyGCHead_SET_NEXT(last, gc); - _PyGCHead_SET_PREV(gc, last); - _PyGCHead_SET_NEXT(gc, _PyRuntime.gc.generation0); - _PyRuntime.gc.generation0->_gc_prev = (uintptr_t)gc; + return; } #define _PyObject_GC_TRACK(op) \ @@ -60,17 +46,7 @@ static inline void _PyObject_GC_TRACK_impl(const char *filename, int lineno, static inline void _PyObject_GC_UNTRACK_impl(const char *filename, int lineno, PyObject *op) { - _PyObject_ASSERT_FROM(op, _PyObject_GC_IS_TRACKED(op), - "object not tracked by the garbage collector", - filename, lineno, "_PyObject_GC_UNTRACK"); - - PyGC_Head *gc = _Py_AS_GC(op); - PyGC_Head *prev = _PyGCHead_PREV(gc); - PyGC_Head *next = _PyGCHead_NEXT(gc); - _PyGCHead_SET_NEXT(prev, next); - _PyGCHead_SET_PREV(next, prev); - gc->_gc_next = 0; - gc->_gc_prev &= _PyGC_PREV_MASK_FINALIZED; + return; } #define _PyObject_GC_UNTRACK(op) \ diff --git a/Include/objimpl.h b/Include/objimpl.h index 63056ad5a9502a..3fc804626840f6 100644 --- a/Include/objimpl.h +++ b/Include/objimpl.h @@ -119,7 +119,7 @@ PyAPI_FUNC(void) PyObject_Free(void *ptr); PyAPI_FUNC(Py_ssize_t) PyGC_Collect(void); /* Test if a type has a GC head */ -#define PyType_IS_GC(t) PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC) +#define PyType_IS_GC(t) (0) PyAPI_FUNC(PyVarObject *) _PyObject_GC_Resize(PyVarObject *, Py_ssize_t); #define PyObject_GC_Resize(type, op, n) \ @@ -201,11 +201,7 @@ _PyObject_INIT(PyObject *op, PyTypeObject *typeobj) } _Py_NewReference(op); if (GC_is_heap_ptr(op)) { - if (PyType_IS_GC(typeobj)) { - GC_REGISTER_FINALIZER(_Py_AS_GC(op), _Py_Dealloc_GC_finalizer, NULL, NULL, NULL); - } else { - GC_REGISTER_FINALIZER(op, _Py_Dealloc_finalizer, NULL, NULL, NULL); - } + GC_REGISTER_FINALIZER_IGNORE_SELF(op, _Py_Dealloc_finalizer, NULL, NULL, NULL); } return op; } diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index be9b73a8446073..0ff066dd881ac4 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -1795,26 +1795,8 @@ PyInit_gc(void) Py_ssize_t PyGC_Collect(void) { - struct _gc_runtime_state *state = &_PyRuntime.gc; - if (!state->enabled) { - return 0; - } - - Py_ssize_t n; - if (state->collecting) { - /* already collecting, don't do anything */ - n = 0; - } - else { - PyObject *exc, *value, *tb; - state->collecting = 1; - PyErr_Fetch(&exc, &value, &tb); - n = collect_with_callback(state, NUM_GENERATIONS - 1); - PyErr_Restore(exc, value, tb); - state->collecting = 0; - } - - return n; + GC_gcollect(); + return 0; } Py_ssize_t @@ -1826,78 +1808,19 @@ _PyGC_CollectIfEnabled(void) Py_ssize_t _PyGC_CollectNoFail(void) { - assert(!PyErr_Occurred()); - - struct _gc_runtime_state *state = &_PyRuntime.gc; - Py_ssize_t n; - - /* Ideally, this function is only called on interpreter shutdown, - and therefore not recursively. Unfortunately, when there are daemon - threads, a daemon thread can start a cyclic garbage collection - during interpreter shutdown (and then never finish it). - See http://bugs.python.org/issue8713#msg195178 for an example. - */ - if (state->collecting) { - n = 0; - } - else { - state->collecting = 1; - n = collect(state, NUM_GENERATIONS - 1, NULL, NULL, 1); - state->collecting = 0; - } - return n; + GC_gcollect(); + return 0; } void _PyGC_DumpShutdownStats(_PyRuntimeState *runtime) { - struct _gc_runtime_state *state = &runtime->gc; - if (!(state->debug & DEBUG_SAVEALL) - && state->garbage != NULL && PyList_GET_SIZE(state->garbage) > 0) { - const char *message; - if (state->debug & DEBUG_UNCOLLECTABLE) - message = "gc: %zd uncollectable objects at " \ - "shutdown"; - else - message = "gc: %zd uncollectable objects at " \ - "shutdown; use gc.set_debug(gc.DEBUG_UNCOLLECTABLE) to list them"; - /* PyErr_WarnFormat does too many things and we are at shutdown, - the warnings module's dependencies (e.g. linecache) may be gone - already. */ - if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0, - "gc", NULL, message, - PyList_GET_SIZE(state->garbage))) - PyErr_WriteUnraisable(NULL); - if (state->debug & DEBUG_UNCOLLECTABLE) { - PyObject *repr = NULL, *bytes = NULL; - repr = PyObject_Repr(state->garbage); - if (!repr || !(bytes = PyUnicode_EncodeFSDefault(repr))) - PyErr_WriteUnraisable(state->garbage); - else { - PySys_WriteStderr( - " %s\n", - PyBytes_AS_STRING(bytes) - ); - } - Py_XDECREF(repr); - Py_XDECREF(bytes); - } - } + // GC_dump(); } void _PyGC_Fini(_PyRuntimeState *runtime) { - struct _gc_runtime_state *state = &runtime->gc; - Py_CLEAR(state->garbage); - Py_CLEAR(state->callbacks); -} - -/* for debugging */ -void -_PyGC_Dump(PyGC_Head *g) -{ - _PyObject_Dump(FROM_GC(g)); } /* extension modules might be compiled with GC support so these @@ -1906,57 +1829,24 @@ _PyGC_Dump(PyGC_Head *g) void PyObject_GC_Track(void *op_raw) { - PyObject *op = _PyObject_CAST(op_raw); - if (_PyObject_GC_IS_TRACKED(op)) { - _PyObject_ASSERT_FAILED_MSG(op, - "object already tracked " - "by the garbage collector"); - } - _PyObject_GC_TRACK(op); } void PyObject_GC_UnTrack(void *op_raw) { - PyObject *op = _PyObject_CAST(op_raw); - /* Obscure: the Py_TRASHCAN mechanism requires that we be able to - * call PyObject_GC_UnTrack twice on an object. - */ - if (_PyObject_GC_IS_TRACKED(op)) { - _PyObject_GC_UNTRACK(op); - } } static PyObject * -_PyObject_GC_Alloc(int use_calloc, size_t basicsize) +_PyObject_GC_Alloc(int use_calloc, size_t size) { struct _gc_runtime_state *state = &_PyRuntime.gc; PyObject *op; - PyGC_Head *g; - size_t size; - if (basicsize > PY_SSIZE_T_MAX - sizeof(PyGC_Head)) - return PyErr_NoMemory(); - size = sizeof(PyGC_Head) + basicsize; if (use_calloc) - g = (PyGC_Head *)PyObject_Calloc(1, size); + op = PyObject_Calloc(1, size); else - g = (PyGC_Head *)PyObject_Malloc(size); - if (g == NULL) + op = PyObject_Malloc(size); + if (op == NULL) return PyErr_NoMemory(); - assert(((uintptr_t)g & 3) == 0); // g must be aligned 4bytes boundary - g->_gc_next = 0; - g->_gc_prev = 0; - state->generations[0].count++; /* number of allocated GC objects */ - if (state->generations[0].count > state->generations[0].threshold && - state->enabled && - state->generations[0].threshold && - !state->collecting && - !PyErr_Occurred()) { - state->collecting = 1; - collect_generations(state); - state->collecting = 0; - } - op = FROM_GC(g); return op; } @@ -2002,16 +1892,10 @@ PyVarObject * _PyObject_GC_Resize(PyVarObject *op, Py_ssize_t nitems) { const size_t basicsize = _PyObject_VAR_SIZE(Py_TYPE(op), nitems); - _PyObject_ASSERT((PyObject *)op, !_PyObject_GC_IS_TRACKED(op)); - if (basicsize > PY_SSIZE_T_MAX - sizeof(PyGC_Head)) { - return (PyVarObject *)PyErr_NoMemory(); - } - PyGC_Head *g = AS_GC(op); - g = (PyGC_Head *)PyObject_REALLOC(g, sizeof(PyGC_Head) + basicsize); - if (g == NULL) + op = PyObject_REALLOC(op, basicsize); + if (op == NULL) return (PyVarObject *)PyErr_NoMemory(); - op = (PyVarObject *) FROM_GC(g); Py_SIZE(op) = nitems; return op; } @@ -2019,13 +1903,4 @@ _PyObject_GC_Resize(PyVarObject *op, Py_ssize_t nitems) void PyObject_GC_Del(void *op) { - PyGC_Head *g = AS_GC(op); - if (_PyObject_GC_IS_TRACKED(op)) { - gc_list_remove(g); - } - struct _gc_runtime_state *state = &_PyRuntime.gc; - if (state->generations[0].count > 0) { - state->generations[0].count--; - } - PyObject_FREE(g); } diff --git a/Objects/object.c b/Objects/object.c index 7717ba25b2595c..60debb2c3dea5c 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -312,10 +312,10 @@ PyObject_CallFinalizerFromDealloc(PyObject *self) Py_ssize_t refcnt; /* Temporarily resurrect the object. */ - if (self->ob_refcnt != 0) { +/* if (self->ob_refcnt != 0) { Py_FatalError("PyObject_CallFinalizerFromDealloc called on " "object with a non-zero refcount"); - } + } */ self->ob_refcnt = 1; PyObject_CallFinalizer(self); @@ -2213,7 +2213,7 @@ _Py_Dealloc_finalizer(void *_op, void *is_gc) PyObject *op = (PyObject *)_op; assert(is_gc == NULL ? !PyObject_IS_GC(op) : PyObject_IS_GC(op)); if (op->ob_refcnt != 0) { - fprintf(stderr, "object %p refcount leak (%ld)\n", op, op->ob_refcnt); + // fprintf(stderr, "object %p refcount leak (%ld)\n", op, op->ob_refcnt); } _GC_Py_Dealloc(op); } diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 0f3525f26d165f..e3363003095fe1 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -127,10 +127,10 @@ _PyMem_RawRealloc(void *ctx, void *ptr, size_t size) if (!GC_is_heap_ptr(ptr)) { abort(); } - GC_REGISTER_FINALIZER(ptr, NULL, NULL, &fn, NULL); + GC_REGISTER_FINALIZER_IGNORE_SELF(ptr, NULL, NULL, &fn, NULL); ptr = GC_REALLOC(ptr, size); if (fn != NULL) { - GC_REGISTER_FINALIZER(ptr, fn, NULL, NULL, NULL); + GC_REGISTER_FINALIZER_IGNORE_SELF(ptr, fn, NULL, NULL, NULL); } return ptr; } @@ -143,7 +143,7 @@ _PyMem_RawFree(void *ctx, void *ptr) if (!GC_is_heap_ptr(ptr)) { abort(); } - GC_REGISTER_FINALIZER(ptr, NULL, NULL, NULL, NULL); + GC_REGISTER_FINALIZER_IGNORE_SELF(ptr, NULL, NULL, NULL, NULL); // GC_FREE(ptr); } From 10ce09060a258e089f5125ea7b51f559ad45a8dd Mon Sep 17 00:00:00 2001 From: Thomas Wouters Date: Wed, 17 Apr 2019 11:08:12 +0200 Subject: [PATCH 007/527] Initial stab at using libgc: switch to using GC_malloc, disable pymalloc by default, skip calling GC_free (to test libgc actually deleting things on its own). --- Include/Python.h | 7 +++++++ Lib/test/test_capi.py | 3 +++ Makefile.pre.in | 13 +++++++------ Modules/_tracemalloc.c | 4 ++-- Modules/posixmodule.c | 8 ++++++++ Objects/obmalloc.c | 9 +++++---- Python/pylifecycle.c | 6 ++++++ aclocal.m4 | 4 ++-- configure | 44 ++++++++++++++++++++++++++++++++++++++++-- configure.ac | 19 +++++++++++++++++- 10 files changed, 100 insertions(+), 17 deletions(-) diff --git a/Include/Python.h b/Include/Python.h index 55b06aeea9846f..493962ae046152 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -60,6 +60,13 @@ */ #include +/* libgc doesn't appear to have a config.h for these defines, so assume + * GC_THREADS support is enabled. + */ +#define GC_THREADS 1 +/* Include gc.h for its intercepts of thread functions. */ +#include + #include "pyport.h" #include "pymacro.h" diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 31dab6a423e25a..2ba8f5f0d2a87e 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -540,12 +540,15 @@ def check_pyobject_is_freed(self, func): code = code.format(func=func) assert_python_ok('-c', code, PYTHONMALLOC=self.PYTHONMALLOC) + @unittest.skipUnless(support.with_pymalloc(), 'need pymalloc') def test_pyobject_is_freed_uninitialized(self): self.check_pyobject_is_freed('pyobject_uninitialized') + @unittest.skipUnless(support.with_pymalloc(), 'need pymalloc') def test_pyobject_is_freed_forbidden_bytes(self): self.check_pyobject_is_freed('pyobject_forbidden_bytes') + @unittest.skipUnless(support.with_pymalloc(), 'need pymalloc') def test_pyobject_is_freed_free(self): self.check_pyobject_is_freed('pyobject_freed') diff --git a/Makefile.pre.in b/Makefile.pre.in index 75eb66be3c0135..6f62abe1c5cfbf 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -231,7 +231,8 @@ INSTSONAME= @INSTSONAME@ LIBS= @LIBS@ LIBM= @LIBM@ LIBC= @LIBC@ -SYSLIBS= $(LIBM) $(LIBC) +LIBGC= @LIBGC@ +SYSLIBS= $(LIBM) $(LIBC) $(LIBGC) SHLIBS= @SHLIBS@ DLINCLDIR= @DLINCLDIR@ @@ -560,7 +561,7 @@ clinic: check-clean-src $(srcdir)/Modules/_blake2/blake2s_impl.c # Build the interpreter $(BUILDPYTHON): Programs/python.o $(LIBRARY) $(LDLIBRARY) $(PY3LIBRARY) - $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) + $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LIBGC) platform: $(BUILDPYTHON) pybuilddir.txt $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform @@ -615,21 +616,21 @@ $(LIBRARY): $(LIBRARY_OBJS) libpython$(LDVERSION).so: $(LIBRARY_OBJS) $(DTRACE_OBJS) if test $(INSTSONAME) != $(LDLIBRARY); then \ - $(BLDSHARED) -Wl,-h$(INSTSONAME) -o $(INSTSONAME) $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM); \ + $(BLDSHARED) -Wl,-h$(INSTSONAME) -o $(INSTSONAME) $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBGC) $(LIBC) $(LIBM); \ $(LN) -f $(INSTSONAME) $@; \ else \ - $(BLDSHARED) -o $@ $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM); \ + $(BLDSHARED) -o $@ $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBGC) $(LIBC) $(LIBM); \ fi libpython3.so: libpython$(LDVERSION).so $(BLDSHARED) $(NO_AS_NEEDED) -o $@ -Wl,-h$@ $^ libpython$(LDVERSION).dylib: $(LIBRARY_OBJS) - $(CC) -dynamiclib -Wl,-single_module $(PY_CORE_LDFLAGS) -undefined dynamic_lookup -Wl,-install_name,$(prefix)/lib/libpython$(LDVERSION).dylib -Wl,-compatibility_version,$(VERSION) -Wl,-current_version,$(VERSION) -o $@ $(LIBRARY_OBJS) $(DTRACE_OBJS) $(SHLIBS) $(LIBC) $(LIBM); \ + $(CC) -dynamiclib -Wl,-single_module $(PY_CORE_LDFLAGS) -undefined dynamic_lookup -Wl,-install_name,$(prefix)/lib/libpython$(LDVERSION).dylib -Wl,-compatibility_version,$(VERSION) -Wl,-current_version,$(VERSION) -o $@ $(LIBRARY_OBJS) $(DTRACE_OBJS) $(SHLIBS) $(LIBGC) $(LIBC) $(LIBM); \ libpython$(VERSION).sl: $(LIBRARY_OBJS) - $(LDSHARED) -o $@ $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM) + $(LDSHARED) -o $@ $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBGC) $(LIBC) $(LIBM) # Copy up the gdb python hooks into a position where they can be automatically # loaded by gdb during Lib/test/test_gdb.py diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index c5d5671032ebf1..e93bd446d811e6 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -273,13 +273,13 @@ hashtable_new(size_t key_size, size_t data_size, static void* raw_malloc(size_t size) { - return allocators.raw.malloc(allocators.raw.ctx, size); + return malloc(size); } static void raw_free(void *ptr) { - allocators.raw.free(allocators.raw.ctx, ptr); + free(ptr); } diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 221f7101b21356..e3904aa578b943 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -407,11 +407,16 @@ PyOS_BeforeFork(void) run_at_forkers(_PyInterpreterState_Get()->before_forkers, 1); _PyImport_AcquireLock(); + /* GC atfork_prepare needs to be called last, as it disables GC. */ + GC_atfork_prepare(); } void PyOS_AfterFork_Parent(void) { + /* GC atfork functions need to be called first, before allocations can + * happen. */ + GC_atfork_parent(); if (_PyImport_ReleaseLock() <= 0) Py_FatalError("failed releasing import lock after fork"); @@ -422,6 +427,9 @@ void PyOS_AfterFork_Child(void) { _PyRuntimeState *runtime = &_PyRuntime; + /* GC atfork functions need to be called first, before allocations can + * happen. */ + GC_atfork_child(); _PyGILState_Reinit(runtime); _PyInterpreterState_DeleteExceptMain(runtime); PyEval_ReInitThreads(); diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 7cfd28965967f9..f2e4d7dc8c41dd 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -1,5 +1,6 @@ #include "Python.h" #include "pycore_pymem.h" +#include "gc/gc.h" #include @@ -96,7 +97,7 @@ _PyMem_RawMalloc(void *ctx, size_t size) To solve these problems, allocate an extra byte. */ if (size == 0) size = 1; - return malloc(size); + return GC_MALLOC(size); } static void * @@ -110,7 +111,7 @@ _PyMem_RawCalloc(void *ctx, size_t nelem, size_t elsize) nelem = 1; elsize = 1; } - return calloc(nelem, elsize); + return GC_MALLOC(nelem * elsize); } static void * @@ -118,13 +119,13 @@ _PyMem_RawRealloc(void *ctx, void *ptr, size_t size) { if (size == 0) size = 1; - return realloc(ptr, size); + return GC_REALLOC(ptr, size); } static void _PyMem_RawFree(void *ctx, void *ptr) { - free(ptr); + /* GC_FREE(ptr); */ } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index bd4d1d92662a69..61764ae32d5b59 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2,6 +2,8 @@ #include "Python.h" +#include + #include "Python-ast.h" #undef Yield /* undefine macro conflicting with */ #include "pycore_coreconfig.h" @@ -84,6 +86,8 @@ _PyRuntime_Initialize(void) } runtime_initialized = 1; + GC_INIT(); + return _PyRuntimeState_Init(&_PyRuntime); } @@ -535,6 +539,8 @@ pycore_create_interpreter(_PyRuntimeState *runtime, /* Create the GIL */ PyEval_InitThreads(); + GC_allow_register_threads(); + return _Py_INIT_OK(); } diff --git a/aclocal.m4 b/aclocal.m4 index 85f00dd5fac7f2..030e6877de9f7c 100644 --- a/aclocal.m4 +++ b/aclocal.m4 @@ -1,6 +1,6 @@ -# generated automatically by aclocal 1.16.1 -*- Autoconf -*- +# generated automatically by aclocal 1.15.1 -*- Autoconf -*- -# Copyright (C) 1996-2018 Free Software Foundation, Inc. +# Copyright (C) 1996-2017 Free Software Foundation, Inc. # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, diff --git a/configure b/configure index e6e4007351591a..d636b71b5ac9cb 100755 --- a/configure +++ b/configure @@ -635,6 +635,7 @@ LIBPYTHON EXT_SUFFIX ALT_SOABI SOABI +LIBGC LIBC LIBM HAVE_GETHOSTBYNAME @@ -785,6 +786,7 @@ infodir docdir oldincludedir includedir +runstatedir localstatedir sharedstatedir sysconfdir @@ -840,6 +842,7 @@ with_valgrind with_dtrace with_libm with_libc +with_libgc enable_big_digits with_computed_gotos with_ensurepip @@ -897,6 +900,7 @@ datadir='${datarootdir}' sysconfdir='${prefix}/etc' sharedstatedir='${prefix}/com' localstatedir='${prefix}/var' +runstatedir='${localstatedir}/run' includedir='${prefix}/include' oldincludedir='/usr/include' docdir='${datarootdir}/doc/${PACKAGE_TARNAME}' @@ -1149,6 +1153,15 @@ do | -silent | --silent | --silen | --sile | --sil) silent=yes ;; + -runstatedir | --runstatedir | --runstatedi | --runstated \ + | --runstate | --runstat | --runsta | --runst | --runs \ + | --run | --ru | --r) + ac_prev=runstatedir ;; + -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \ + | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \ + | --run=* | --ru=* | --r=*) + runstatedir=$ac_optarg ;; + -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) ac_prev=sbindir ;; -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ @@ -1286,7 +1299,7 @@ fi for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \ datadir sysconfdir sharedstatedir localstatedir includedir \ oldincludedir docdir infodir htmldir dvidir pdfdir psdir \ - libdir localedir mandir + libdir localedir mandir runstatedir do eval ac_val=\$$ac_var # Remove trailing slashes. @@ -1439,6 +1452,7 @@ Fine tuning of the installation directories: --sysconfdir=DIR read-only single-machine data [PREFIX/etc] --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] --localstatedir=DIR modifiable single-machine data [PREFIX/var] + --runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run] --libdir=DIR object code libraries [EPREFIX/lib] --includedir=DIR C header files [PREFIX/include] --oldincludedir=DIR C header files for non-gcc [/usr/include] @@ -1537,6 +1551,7 @@ Optional Packages: --with(out)-dtrace disable/enable DTrace support --with-libm=STRING math library --with-libc=STRING C library + --with-libgc=STRING BDW GC library --with(out)-computed-gotos Use computed gotos in evaluation loop (enabled by default on supported compilers) @@ -11251,7 +11266,7 @@ fi if test -z "$with_pymalloc" then - with_pymalloc="yes" + with_pymalloc="no" fi if test "$with_pymalloc" != "no" then @@ -14190,6 +14205,31 @@ $as_echo "default LIBC=\"$LIBC\"" >&6; } fi +# check for --with-libgc=... + +LIBGC="-lgc" +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-libgc=STRING" >&5 +$as_echo_n "checking for --with-libgc=STRING... " >&6; } + +# Check whether --with-libgc was given. +if test "${with_libgc+set}" = set; then : + withval=$with_libgc; +if test "$withval" = no +then LIBGC= + { $as_echo "$as_me:${as_lineno-$LINENO}: result: force LIBGC empty" >&5 +$as_echo "force LIBGC empty" >&6; } +elif test "$withval" != yes +then LIBGC=$withval + { $as_echo "$as_me:${as_lineno-$LINENO}: result: set LIBGC=\"$withval\"" >&5 +$as_echo "set LIBGC=\"$withval\"" >&6; } +else as_fn_error $? "proper usage is --with-libgc=STRING" "$LINENO" 5 +fi +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: default LIBGC=\"$LIBGC\"" >&5 +$as_echo "default LIBGC=\"$LIBGC\"" >&6; } +fi + + # ************************************** # * Check for gcc x64 inline assembler * # ************************************** diff --git a/configure.ac b/configure.ac index a02597da2db5ea..af09695af8bf64 100644 --- a/configure.ac +++ b/configure.ac @@ -3401,7 +3401,7 @@ AC_ARG_WITH(pymalloc, if test -z "$with_pymalloc" then - with_pymalloc="yes" + with_pymalloc="no" fi if test "$with_pymalloc" != "no" then @@ -4313,6 +4313,23 @@ else AC_MSG_ERROR([proper usage is --with-libc=STRING]) fi], [AC_MSG_RESULT(default LIBC="$LIBC")]) +# check for --with-libgc=... +AC_SUBST(LIBGC) +LIBGC="-lgc" +AC_MSG_CHECKING(for --with-libgc=STRING) +AC_ARG_WITH(libgc, + AS_HELP_STRING([--with-libgc=STRING], [BDW GC library]), +[ +if test "$withval" = no +then LIBGC= + AC_MSG_RESULT(force LIBGC empty) +elif test "$withval" != yes +then LIBGC=$withval + AC_MSG_RESULT(set LIBGC="$withval") +else AC_MSG_ERROR([proper usage is --with-libgc=STRING]) +fi], +[AC_MSG_RESULT(default LIBGC="$LIBGC")]) + # ************************************** # * Check for gcc x64 inline assembler * # ************************************** From 031557c1d366ec6c41aee279983de6327fc82a81 Mon Sep 17 00:00:00 2001 From: Thomas Wouters Date: Thu, 25 Apr 2019 12:11:58 +0200 Subject: [PATCH 008/527] Plug Python object finalization into libgc finalizers. This doesn't get rid of refcounting, just makes sure libgc sees enough references to live objects (by aborting if it tries to finalize an object with a non-zero refcount) and also act as a refleak-detector. --- Include/Python.h | 1 + Include/cpython/objimpl.h | 1 + Include/object.h | 2 + Include/objimpl.h | 123 ++++++++++++++++++++------------------ Objects/object.c | 31 +++++++--- Objects/obmalloc.c | 23 ++++++- Python/pylifecycle.c | 1 + 7 files changed, 115 insertions(+), 67 deletions(-) diff --git a/Include/Python.h b/Include/Python.h index 493962ae046152..32bde235117dd9 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -64,6 +64,7 @@ * GC_THREADS support is enabled. */ #define GC_THREADS 1 +/* #define GC_DEBUG 1 */ /* Include gc.h for its intercepts of thread functions. */ #include diff --git a/Include/cpython/objimpl.h b/Include/cpython/objimpl.h index f121922bc42ced..87d8ff493523e1 100644 --- a/Include/cpython/objimpl.h +++ b/Include/cpython/objimpl.h @@ -54,6 +54,7 @@ typedef struct { } PyGC_Head; #define _Py_AS_GC(o) ((PyGC_Head *)(o)-1) +#define _Py_FROM_GC(o) ((PyObject *)((PyGC_Head *)(o)+1)) /* True if the object is currently tracked by the GC. */ #define _PyObject_GC_IS_TRACKED(o) (_Py_AS_GC(o)->_gc_next != 0) diff --git a/Include/object.h b/Include/object.h index 13e88a6dc6f02a..2db179a18fc732 100644 --- a/Include/object.h +++ b/Include/object.h @@ -442,6 +442,8 @@ static inline void _Py_ForgetReference(PyObject *op) PyAPI_FUNC(void) _Py_Dealloc(PyObject *); +PyAPI_FUNC(void) _Py_Dealloc_finalizer(void *, void *); +PyAPI_FUNC(void) _Py_Dealloc_GC_finalizer(void *, void *); static inline void _Py_INCREF(PyObject *op) { diff --git a/Include/objimpl.h b/Include/objimpl.h index 2337d8a56c774e..63056ad5a9502a 100644 --- a/Include/objimpl.h +++ b/Include/objimpl.h @@ -110,6 +110,58 @@ PyAPI_FUNC(void) PyObject_Free(void *ptr); #define PyObject_DEL PyObject_Free +/* + * Garbage Collection Support + * ========================== + */ + +/* C equivalent of gc.collect() which ignores the state of gc.enabled. */ +PyAPI_FUNC(Py_ssize_t) PyGC_Collect(void); + +/* Test if a type has a GC head */ +#define PyType_IS_GC(t) PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC) + +PyAPI_FUNC(PyVarObject *) _PyObject_GC_Resize(PyVarObject *, Py_ssize_t); +#define PyObject_GC_Resize(type, op, n) \ + ( (type *) _PyObject_GC_Resize(_PyVarObject_CAST(op), (n)) ) + + + +PyAPI_FUNC(PyObject *) _PyObject_GC_New(PyTypeObject *); +PyAPI_FUNC(PyVarObject *) _PyObject_GC_NewVar(PyTypeObject *, Py_ssize_t); + +/* Tell the GC to track this object. + * + * See also private _PyObject_GC_TRACK() macro. */ +PyAPI_FUNC(void) PyObject_GC_Track(void *); + +/* Tell the GC to stop tracking this object. + * + * See also private _PyObject_GC_UNTRACK() macro. */ +PyAPI_FUNC(void) PyObject_GC_UnTrack(void *); + +PyAPI_FUNC(void) PyObject_GC_Del(void *); + +#define PyObject_GC_New(type, typeobj) \ + ( (type *) _PyObject_GC_New(typeobj) ) +#define PyObject_GC_NewVar(type, typeobj, n) \ + ( (type *) _PyObject_GC_NewVar((typeobj), (n)) ) + + +/* Utility macro to help write tp_traverse functions. + * To use this macro, the tp_traverse function must name its arguments + * "visit" and "arg". This is intended to keep tp_traverse functions + * looking as much alike as possible. + */ +#define Py_VISIT(op) \ + do { \ + if (op) { \ + int vret = visit(_PyObject_CAST(op), arg); \ + if (vret) \ + return vret; \ + } \ + } while (0) + /* * Generic object allocator interface * ================================== @@ -127,6 +179,12 @@ PyAPI_FUNC(PyVarObject *) _PyObject_NewVar(PyTypeObject *, Py_ssize_t); #define PyObject_NewVar(type, typeobj, n) \ ( (type *) _PyObject_NewVar((typeobj), (n)) ) +#ifndef Py_LIMITED_API +# define Py_CPYTHON_OBJIMPL_H +# include "cpython/objimpl.h" +# undef Py_CPYTHON_OBJIMPL_H +#endif + /* Inline functions trading binary compatibility for speed: PyObject_INIT() is the fast version of PyObject_Init(), and PyObject_INIT_VAR() is the fast version of PyObject_InitVar. @@ -142,6 +200,13 @@ _PyObject_INIT(PyObject *op, PyTypeObject *typeobj) Py_INCREF(typeobj); } _Py_NewReference(op); + if (GC_is_heap_ptr(op)) { + if (PyType_IS_GC(typeobj)) { + GC_REGISTER_FINALIZER(_Py_AS_GC(op), _Py_Dealloc_GC_finalizer, NULL, NULL, NULL); + } else { + GC_REGISTER_FINALIZER(op, _Py_Dealloc_finalizer, NULL, NULL, NULL); + } + } return op; } @@ -220,64 +285,6 @@ _PyObject_INIT_VAR(PyVarObject *op, PyTypeObject *typeobj, Py_ssize_t size) -/* - * Garbage Collection Support - * ========================== - */ - -/* C equivalent of gc.collect() which ignores the state of gc.enabled. */ -PyAPI_FUNC(Py_ssize_t) PyGC_Collect(void); - -/* Test if a type has a GC head */ -#define PyType_IS_GC(t) PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC) - -PyAPI_FUNC(PyVarObject *) _PyObject_GC_Resize(PyVarObject *, Py_ssize_t); -#define PyObject_GC_Resize(type, op, n) \ - ( (type *) _PyObject_GC_Resize(_PyVarObject_CAST(op), (n)) ) - - - -PyAPI_FUNC(PyObject *) _PyObject_GC_New(PyTypeObject *); -PyAPI_FUNC(PyVarObject *) _PyObject_GC_NewVar(PyTypeObject *, Py_ssize_t); - -/* Tell the GC to track this object. - * - * See also private _PyObject_GC_TRACK() macro. */ -PyAPI_FUNC(void) PyObject_GC_Track(void *); - -/* Tell the GC to stop tracking this object. - * - * See also private _PyObject_GC_UNTRACK() macro. */ -PyAPI_FUNC(void) PyObject_GC_UnTrack(void *); - -PyAPI_FUNC(void) PyObject_GC_Del(void *); - -#define PyObject_GC_New(type, typeobj) \ - ( (type *) _PyObject_GC_New(typeobj) ) -#define PyObject_GC_NewVar(type, typeobj, n) \ - ( (type *) _PyObject_GC_NewVar((typeobj), (n)) ) - - -/* Utility macro to help write tp_traverse functions. - * To use this macro, the tp_traverse function must name its arguments - * "visit" and "arg". This is intended to keep tp_traverse functions - * looking as much alike as possible. - */ -#define Py_VISIT(op) \ - do { \ - if (op) { \ - int vret = visit(_PyObject_CAST(op), arg); \ - if (vret) \ - return vret; \ - } \ - } while (0) - -#ifndef Py_LIMITED_API -# define Py_CPYTHON_OBJIMPL_H -# include "cpython/objimpl.h" -# undef Py_CPYTHON_OBJIMPL_H -#endif - #ifdef __cplusplus } #endif diff --git a/Objects/object.c b/Objects/object.c index cb727943cb342e..d342575be4e3b0 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -251,13 +251,7 @@ PyObject_Init(PyObject *op, PyTypeObject *tp) { if (op == NULL) return PyErr_NoMemory(); - /* Any changes should be reflected in PyObject_INIT (objimpl.h) */ - Py_TYPE(op) = tp; - if (PyType_GetFlags(tp) & Py_TPFLAGS_HEAPTYPE) { - Py_INCREF(tp); - } - _Py_NewReference(op); - return op; + return PyObject_INIT(op, tp); } PyVarObject * @@ -2207,6 +2201,29 @@ _Py_Dealloc(PyObject *op) (*dealloc)(op); } +void +_Py_Dealloc_GC_finalizer(void *_op, void *unused) +{ + PyObject *op = _Py_FROM_GC(_op); + assert(PyObject_IS_GC(op)); + _Py_Dealloc_finalizer((void *)op, (void *)_op); +} + +void +_Py_Dealloc_finalizer(void *_op, void *is_gc) +{ + PyObject *op = (PyObject *)_op; + assert(is_gc == NULL ? !_Py_IS_GC(op) || _Py_IS_GC(op)); + if (op->ob_refcnt != 0) { + fprintf(stderr, "object %p refcount leak (%ld)\n", op, op->ob_refcnt); + abort(); + } else { + fprintf(stderr, "object %p not freed (refcount 0)\n", op); + abort(); + } + _Py_Dealloc(op); +} + #ifdef __cplusplus } #endif diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index f2e4d7dc8c41dd..297917c6c1bd70 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -117,15 +117,34 @@ _PyMem_RawCalloc(void *ctx, size_t nelem, size_t elsize) static void * _PyMem_RawRealloc(void *ctx, void *ptr, size_t size) { + GC_finalization_proc fn; + + if (ptr == NULL) + return _PyMem_RawMalloc(ctx, size); + if (size == 0) size = 1; - return GC_REALLOC(ptr, size); + if (!GC_is_heap_ptr(ptr)) { + abort(); + } + GC_REGISTER_FINALIZER(ptr, NULL, NULL, &fn, NULL); + ptr = GC_REALLOC(ptr, size); + if (fn != NULL) { + GC_REGISTER_FINALIZER(ptr, fn, NULL, NULL, NULL); + } + return ptr; } static void _PyMem_RawFree(void *ctx, void *ptr) { - /* GC_FREE(ptr); */ + if (ptr == NULL) + return; + if (!GC_is_heap_ptr(ptr)) { + abort(); + } + GC_REGISTER_FINALIZER(ptr, NULL, NULL, NULL, NULL); +// GC_FREE(ptr); } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 61764ae32d5b59..a0edc8780db140 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -86,6 +86,7 @@ _PyRuntime_Initialize(void) } runtime_initialized = 1; + GC_set_all_interior_pointers(1); GC_INIT(); return _PyRuntimeState_Init(&_PyRuntime); From d36bb296b2aea6572679b35a209a166a949bf293 Mon Sep 17 00:00:00 2001 From: Thomas Wouters Date: Tue, 30 Apr 2019 17:51:00 -0700 Subject: [PATCH 009/527] Fix assert. --- Objects/object.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/object.c b/Objects/object.c index d342575be4e3b0..5a74ab3b06a604 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2213,7 +2213,7 @@ void _Py_Dealloc_finalizer(void *_op, void *is_gc) { PyObject *op = (PyObject *)_op; - assert(is_gc == NULL ? !_Py_IS_GC(op) || _Py_IS_GC(op)); + assert(is_gc == NULL ? !PyObject_IS_GC(op) : PyObject_IS_GC(op)); if (op->ob_refcnt != 0) { fprintf(stderr, "object %p refcount leak (%ld)\n", op, op->ob_refcnt); abort(); From 54bca9834ce3dc5b23a17c7c6b7431021c48cd8c Mon Sep 17 00:00:00 2001 From: Thomas Wouters Date: Tue, 30 Apr 2019 17:52:46 -0700 Subject: [PATCH 010/527] Fix -x dev by disable memory poisoning in the debug allocator, since we read the memory after the call to DebugFree (in the finalizer). --- Objects/obmalloc.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 297917c6c1bd70..0f3525f26d165f 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -2123,7 +2123,9 @@ _PyMem_DebugRawFree(void *ctx, void *p) _PyMem_DebugCheckAddress(api->api_id, p); nbytes = read_size_t(q); nbytes += PYMEM_DEBUG_EXTRA_BYTES; - memset(q, DEADBYTE, nbytes); + // TODO(thomas@python.org): figure out alternative that works with + // _Py_Dealloc_finalize, probably by writing magic values to refcnt. + // memset(q, DEADBYTE, nbytes); api->alloc.free(api->alloc.ctx, q); } From 2beb723edb57923cedf737e06b9a231494064191 Mon Sep 17 00:00:00 2001 From: Thomas Wouters Date: Fri, 3 May 2019 11:27:56 -0700 Subject: [PATCH 011/527] Stop calling _Py_Dealloc from Py_DECREF (and get rid of it entirely). This means all finalization is done by libgc. It currently segfaults in cyclic-gc. --- Doc/c-api/refcounting.rst | 2 +- Include/cpython/object.h | 12 ------------ Include/object.h | 4 ---- Objects/object.c | 10 ++-------- PC/python3.def | 1 - 5 files changed, 3 insertions(+), 26 deletions(-) diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst index 225a1feb250684..d5ea1be806b0ba 100644 --- a/Doc/c-api/refcounting.rst +++ b/Doc/c-api/refcounting.rst @@ -68,6 +68,6 @@ simply exported function versions of :c:func:`Py_XINCREF` and :c:func:`Py_XDECREF`, respectively. The following functions or macros are only for use within the interpreter core: -:c:func:`_Py_Dealloc`, :c:func:`_Py_ForgetReference`, :c:func:`_Py_NewReference`, +:c:func:`_Py_ForgetReference`, :c:func:`_Py_NewReference`, as well as the global variable :c:data:`_Py_RefTotal`. diff --git a/Include/cpython/object.h b/Include/cpython/object.h index ba52a4835823b6..3ade5d0b7c0bdc 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -332,18 +332,6 @@ _PyObject_GenericSetAttrWithDict(PyObject *, PyObject *, #define PyType_HasFeature(t,f) (((t)->tp_flags & (f)) != 0) -static inline void _Py_Dealloc_inline(PyObject *op) -{ - destructor dealloc = Py_TYPE(op)->tp_dealloc; -#ifdef Py_TRACE_REFS - _Py_ForgetReference(op); -#else - _Py_INC_TPFREES(op); -#endif - (*dealloc)(op); -} -#define _Py_Dealloc(op) _Py_Dealloc_inline(op) - /* Safely decref `op` and set `op` to `op2`. * diff --git a/Include/object.h b/Include/object.h index 2db179a18fc732..ea6dc12ff5eb34 100644 --- a/Include/object.h +++ b/Include/object.h @@ -441,7 +441,6 @@ static inline void _Py_ForgetReference(PyObject *op) #endif /* !Py_TRACE_REFS */ -PyAPI_FUNC(void) _Py_Dealloc(PyObject *); PyAPI_FUNC(void) _Py_Dealloc_finalizer(void *, void *); PyAPI_FUNC(void) _Py_Dealloc_GC_finalizer(void *, void *); @@ -466,9 +465,6 @@ static inline void _Py_DECREF(const char *filename, int lineno, } #endif } - else { - _Py_Dealloc(op); - } } #define Py_DECREF(op) _Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op)) diff --git a/Objects/object.c b/Objects/object.c index 5a74ab3b06a604..c13e6147773169 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2187,10 +2187,8 @@ _PyObject_AssertFailed(PyObject *obj, const char *expr, const char *msg, } -#undef _Py_Dealloc - void -_Py_Dealloc(PyObject *op) +_GC_Py_Dealloc(PyObject *op) { destructor dealloc = Py_TYPE(op)->tp_dealloc; #ifdef Py_TRACE_REFS @@ -2216,12 +2214,8 @@ _Py_Dealloc_finalizer(void *_op, void *is_gc) assert(is_gc == NULL ? !PyObject_IS_GC(op) : PyObject_IS_GC(op)); if (op->ob_refcnt != 0) { fprintf(stderr, "object %p refcount leak (%ld)\n", op, op->ob_refcnt); - abort(); - } else { - fprintf(stderr, "object %p not freed (refcount 0)\n", op); - abort(); } - _Py_Dealloc(op); + _GC_Py_Dealloc(op); } #ifdef __cplusplus diff --git a/PC/python3.def b/PC/python3.def index e317864d0cd802..5cb683839f9705 100644 --- a/PC/python3.def +++ b/PC/python3.def @@ -793,7 +793,6 @@ EXPORTS _Py_BuildValue_SizeT=python38._Py_BuildValue_SizeT _Py_CheckRecursionLimit=python38._Py_CheckRecursionLimit DATA _Py_CheckRecursiveCall=python38._Py_CheckRecursiveCall - _Py_Dealloc=python38._Py_Dealloc _Py_EllipsisObject=python38._Py_EllipsisObject DATA _Py_FalseStruct=python38._Py_FalseStruct DATA _Py_NoneStruct=python38._Py_NoneStruct DATA From ba862da94478f5cac92a94e32d2cd09184f6852f Mon Sep 17 00:00:00 2001 From: Thomas Wouters Date: Sat, 4 May 2019 06:20:04 -0700 Subject: [PATCH 012/527] Hackishly turn off cylic-GC. --- Include/cpython/objimpl.h | 6 +- Include/internal/pycore_object.h | 28 +----- Include/objimpl.h | 8 +- Modules/gcmodule.c | 147 +++---------------------------- Objects/object.c | 6 +- Objects/obmalloc.c | 6 +- 6 files changed, 23 insertions(+), 178 deletions(-) diff --git a/Include/cpython/objimpl.h b/Include/cpython/objimpl.h index 87d8ff493523e1..ed3a2a524dceaf 100644 --- a/Include/cpython/objimpl.h +++ b/Include/cpython/objimpl.h @@ -38,9 +38,7 @@ PyAPI_FUNC(Py_ssize_t) _PyGC_CollectIfEnabled(void); /* Test if an object has a GC head */ -#define PyObject_IS_GC(o) \ - (PyType_IS_GC(Py_TYPE(o)) \ - && (Py_TYPE(o)->tp_is_gc == NULL || Py_TYPE(o)->tp_is_gc(o))) +#define PyObject_IS_GC(o) (0) /* GC information is stored BEFORE the object structure. */ typedef struct { @@ -57,7 +55,7 @@ typedef struct { #define _Py_FROM_GC(o) ((PyObject *)((PyGC_Head *)(o)+1)) /* True if the object is currently tracked by the GC. */ -#define _PyObject_GC_IS_TRACKED(o) (_Py_AS_GC(o)->_gc_next != 0) +#define _PyObject_GC_IS_TRACKED(o) (0) /* True if the object may be tracked by the GC in the future, or already is. This can be useful to implement some optimizations. */ diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 81548f819198e3..4c3f0d8c136bb3 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -28,21 +28,7 @@ PyAPI_FUNC(int) _PyDict_CheckConsistency(PyObject *mp, int check_content); static inline void _PyObject_GC_TRACK_impl(const char *filename, int lineno, PyObject *op) { - _PyObject_ASSERT_FROM(op, !_PyObject_GC_IS_TRACKED(op), - "object already tracked by the garbage collector", - filename, lineno, "_PyObject_GC_TRACK"); - - PyGC_Head *gc = _Py_AS_GC(op); - _PyObject_ASSERT_FROM(op, - (gc->_gc_prev & _PyGC_PREV_MASK_COLLECTING) == 0, - "object is in generation which is garbage collected", - filename, lineno, "_PyObject_GC_TRACK"); - - PyGC_Head *last = (PyGC_Head*)(_PyRuntime.gc.generation0->_gc_prev); - _PyGCHead_SET_NEXT(last, gc); - _PyGCHead_SET_PREV(gc, last); - _PyGCHead_SET_NEXT(gc, _PyRuntime.gc.generation0); - _PyRuntime.gc.generation0->_gc_prev = (uintptr_t)gc; + return; } #define _PyObject_GC_TRACK(op) \ @@ -60,17 +46,7 @@ static inline void _PyObject_GC_TRACK_impl(const char *filename, int lineno, static inline void _PyObject_GC_UNTRACK_impl(const char *filename, int lineno, PyObject *op) { - _PyObject_ASSERT_FROM(op, _PyObject_GC_IS_TRACKED(op), - "object not tracked by the garbage collector", - filename, lineno, "_PyObject_GC_UNTRACK"); - - PyGC_Head *gc = _Py_AS_GC(op); - PyGC_Head *prev = _PyGCHead_PREV(gc); - PyGC_Head *next = _PyGCHead_NEXT(gc); - _PyGCHead_SET_NEXT(prev, next); - _PyGCHead_SET_PREV(next, prev); - gc->_gc_next = 0; - gc->_gc_prev &= _PyGC_PREV_MASK_FINALIZED; + return; } #define _PyObject_GC_UNTRACK(op) \ diff --git a/Include/objimpl.h b/Include/objimpl.h index 63056ad5a9502a..3fc804626840f6 100644 --- a/Include/objimpl.h +++ b/Include/objimpl.h @@ -119,7 +119,7 @@ PyAPI_FUNC(void) PyObject_Free(void *ptr); PyAPI_FUNC(Py_ssize_t) PyGC_Collect(void); /* Test if a type has a GC head */ -#define PyType_IS_GC(t) PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC) +#define PyType_IS_GC(t) (0) PyAPI_FUNC(PyVarObject *) _PyObject_GC_Resize(PyVarObject *, Py_ssize_t); #define PyObject_GC_Resize(type, op, n) \ @@ -201,11 +201,7 @@ _PyObject_INIT(PyObject *op, PyTypeObject *typeobj) } _Py_NewReference(op); if (GC_is_heap_ptr(op)) { - if (PyType_IS_GC(typeobj)) { - GC_REGISTER_FINALIZER(_Py_AS_GC(op), _Py_Dealloc_GC_finalizer, NULL, NULL, NULL); - } else { - GC_REGISTER_FINALIZER(op, _Py_Dealloc_finalizer, NULL, NULL, NULL); - } + GC_REGISTER_FINALIZER_IGNORE_SELF(op, _Py_Dealloc_finalizer, NULL, NULL, NULL); } return op; } diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index be9b73a8446073..0ff066dd881ac4 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -1795,26 +1795,8 @@ PyInit_gc(void) Py_ssize_t PyGC_Collect(void) { - struct _gc_runtime_state *state = &_PyRuntime.gc; - if (!state->enabled) { - return 0; - } - - Py_ssize_t n; - if (state->collecting) { - /* already collecting, don't do anything */ - n = 0; - } - else { - PyObject *exc, *value, *tb; - state->collecting = 1; - PyErr_Fetch(&exc, &value, &tb); - n = collect_with_callback(state, NUM_GENERATIONS - 1); - PyErr_Restore(exc, value, tb); - state->collecting = 0; - } - - return n; + GC_gcollect(); + return 0; } Py_ssize_t @@ -1826,78 +1808,19 @@ _PyGC_CollectIfEnabled(void) Py_ssize_t _PyGC_CollectNoFail(void) { - assert(!PyErr_Occurred()); - - struct _gc_runtime_state *state = &_PyRuntime.gc; - Py_ssize_t n; - - /* Ideally, this function is only called on interpreter shutdown, - and therefore not recursively. Unfortunately, when there are daemon - threads, a daemon thread can start a cyclic garbage collection - during interpreter shutdown (and then never finish it). - See http://bugs.python.org/issue8713#msg195178 for an example. - */ - if (state->collecting) { - n = 0; - } - else { - state->collecting = 1; - n = collect(state, NUM_GENERATIONS - 1, NULL, NULL, 1); - state->collecting = 0; - } - return n; + GC_gcollect(); + return 0; } void _PyGC_DumpShutdownStats(_PyRuntimeState *runtime) { - struct _gc_runtime_state *state = &runtime->gc; - if (!(state->debug & DEBUG_SAVEALL) - && state->garbage != NULL && PyList_GET_SIZE(state->garbage) > 0) { - const char *message; - if (state->debug & DEBUG_UNCOLLECTABLE) - message = "gc: %zd uncollectable objects at " \ - "shutdown"; - else - message = "gc: %zd uncollectable objects at " \ - "shutdown; use gc.set_debug(gc.DEBUG_UNCOLLECTABLE) to list them"; - /* PyErr_WarnFormat does too many things and we are at shutdown, - the warnings module's dependencies (e.g. linecache) may be gone - already. */ - if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0, - "gc", NULL, message, - PyList_GET_SIZE(state->garbage))) - PyErr_WriteUnraisable(NULL); - if (state->debug & DEBUG_UNCOLLECTABLE) { - PyObject *repr = NULL, *bytes = NULL; - repr = PyObject_Repr(state->garbage); - if (!repr || !(bytes = PyUnicode_EncodeFSDefault(repr))) - PyErr_WriteUnraisable(state->garbage); - else { - PySys_WriteStderr( - " %s\n", - PyBytes_AS_STRING(bytes) - ); - } - Py_XDECREF(repr); - Py_XDECREF(bytes); - } - } + // GC_dump(); } void _PyGC_Fini(_PyRuntimeState *runtime) { - struct _gc_runtime_state *state = &runtime->gc; - Py_CLEAR(state->garbage); - Py_CLEAR(state->callbacks); -} - -/* for debugging */ -void -_PyGC_Dump(PyGC_Head *g) -{ - _PyObject_Dump(FROM_GC(g)); } /* extension modules might be compiled with GC support so these @@ -1906,57 +1829,24 @@ _PyGC_Dump(PyGC_Head *g) void PyObject_GC_Track(void *op_raw) { - PyObject *op = _PyObject_CAST(op_raw); - if (_PyObject_GC_IS_TRACKED(op)) { - _PyObject_ASSERT_FAILED_MSG(op, - "object already tracked " - "by the garbage collector"); - } - _PyObject_GC_TRACK(op); } void PyObject_GC_UnTrack(void *op_raw) { - PyObject *op = _PyObject_CAST(op_raw); - /* Obscure: the Py_TRASHCAN mechanism requires that we be able to - * call PyObject_GC_UnTrack twice on an object. - */ - if (_PyObject_GC_IS_TRACKED(op)) { - _PyObject_GC_UNTRACK(op); - } } static PyObject * -_PyObject_GC_Alloc(int use_calloc, size_t basicsize) +_PyObject_GC_Alloc(int use_calloc, size_t size) { struct _gc_runtime_state *state = &_PyRuntime.gc; PyObject *op; - PyGC_Head *g; - size_t size; - if (basicsize > PY_SSIZE_T_MAX - sizeof(PyGC_Head)) - return PyErr_NoMemory(); - size = sizeof(PyGC_Head) + basicsize; if (use_calloc) - g = (PyGC_Head *)PyObject_Calloc(1, size); + op = PyObject_Calloc(1, size); else - g = (PyGC_Head *)PyObject_Malloc(size); - if (g == NULL) + op = PyObject_Malloc(size); + if (op == NULL) return PyErr_NoMemory(); - assert(((uintptr_t)g & 3) == 0); // g must be aligned 4bytes boundary - g->_gc_next = 0; - g->_gc_prev = 0; - state->generations[0].count++; /* number of allocated GC objects */ - if (state->generations[0].count > state->generations[0].threshold && - state->enabled && - state->generations[0].threshold && - !state->collecting && - !PyErr_Occurred()) { - state->collecting = 1; - collect_generations(state); - state->collecting = 0; - } - op = FROM_GC(g); return op; } @@ -2002,16 +1892,10 @@ PyVarObject * _PyObject_GC_Resize(PyVarObject *op, Py_ssize_t nitems) { const size_t basicsize = _PyObject_VAR_SIZE(Py_TYPE(op), nitems); - _PyObject_ASSERT((PyObject *)op, !_PyObject_GC_IS_TRACKED(op)); - if (basicsize > PY_SSIZE_T_MAX - sizeof(PyGC_Head)) { - return (PyVarObject *)PyErr_NoMemory(); - } - PyGC_Head *g = AS_GC(op); - g = (PyGC_Head *)PyObject_REALLOC(g, sizeof(PyGC_Head) + basicsize); - if (g == NULL) + op = PyObject_REALLOC(op, basicsize); + if (op == NULL) return (PyVarObject *)PyErr_NoMemory(); - op = (PyVarObject *) FROM_GC(g); Py_SIZE(op) = nitems; return op; } @@ -2019,13 +1903,4 @@ _PyObject_GC_Resize(PyVarObject *op, Py_ssize_t nitems) void PyObject_GC_Del(void *op) { - PyGC_Head *g = AS_GC(op); - if (_PyObject_GC_IS_TRACKED(op)) { - gc_list_remove(g); - } - struct _gc_runtime_state *state = &_PyRuntime.gc; - if (state->generations[0].count > 0) { - state->generations[0].count--; - } - PyObject_FREE(g); } diff --git a/Objects/object.c b/Objects/object.c index c13e6147773169..54a827430dfc7c 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -312,10 +312,10 @@ PyObject_CallFinalizerFromDealloc(PyObject *self) Py_ssize_t refcnt; /* Temporarily resurrect the object. */ - if (self->ob_refcnt != 0) { +/* if (self->ob_refcnt != 0) { Py_FatalError("PyObject_CallFinalizerFromDealloc called on " "object with a non-zero refcount"); - } + } */ self->ob_refcnt = 1; PyObject_CallFinalizer(self); @@ -2213,7 +2213,7 @@ _Py_Dealloc_finalizer(void *_op, void *is_gc) PyObject *op = (PyObject *)_op; assert(is_gc == NULL ? !PyObject_IS_GC(op) : PyObject_IS_GC(op)); if (op->ob_refcnt != 0) { - fprintf(stderr, "object %p refcount leak (%ld)\n", op, op->ob_refcnt); + // fprintf(stderr, "object %p refcount leak (%ld)\n", op, op->ob_refcnt); } _GC_Py_Dealloc(op); } diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 0f3525f26d165f..e3363003095fe1 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -127,10 +127,10 @@ _PyMem_RawRealloc(void *ctx, void *ptr, size_t size) if (!GC_is_heap_ptr(ptr)) { abort(); } - GC_REGISTER_FINALIZER(ptr, NULL, NULL, &fn, NULL); + GC_REGISTER_FINALIZER_IGNORE_SELF(ptr, NULL, NULL, &fn, NULL); ptr = GC_REALLOC(ptr, size); if (fn != NULL) { - GC_REGISTER_FINALIZER(ptr, fn, NULL, NULL, NULL); + GC_REGISTER_FINALIZER_IGNORE_SELF(ptr, fn, NULL, NULL, NULL); } return ptr; } @@ -143,7 +143,7 @@ _PyMem_RawFree(void *ctx, void *ptr) if (!GC_is_heap_ptr(ptr)) { abort(); } - GC_REGISTER_FINALIZER(ptr, NULL, NULL, NULL, NULL); + GC_REGISTER_FINALIZER_IGNORE_SELF(ptr, NULL, NULL, NULL, NULL); // GC_FREE(ptr); } From 2609aa9c7ba580df74823dec3165e4d1e954d3a3 Mon Sep 17 00:00:00 2001 From: Thomas Wouters Date: Wed, 8 May 2019 11:05:04 -0700 Subject: [PATCH 013/527] Fix weakrefs (mostly) by using hidden pointers for the pointer from the weakref to the object, from the weakref to the next/prev weakref in the linked list of weakrefs, and from the object to the first weakref in the linked list. We don't need to use disappearing links or long links (from libgc) because we already take care of clearing these links when any of these objects go away (in tp_dealloc). --- Include/Python.h | 2 +- Include/cpython/objimpl.h | 8 ++- Include/weakrefobject.h | 22 ++----- Modules/_weakref.c | 21 ++++--- Modules/gcmodule.c | 9 +-- Objects/object.c | 2 + Objects/typeobject.c | 17 +++-- Objects/weakrefobject.c | 127 +++++++++++++++++++++++--------------- 8 files changed, 116 insertions(+), 92 deletions(-) diff --git a/Include/Python.h b/Include/Python.h index 32bde235117dd9..9fdb3852bbeeac 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -64,7 +64,7 @@ * GC_THREADS support is enabled. */ #define GC_THREADS 1 -/* #define GC_DEBUG 1 */ +#define GC_DEBUG 1 /* Include gc.h for its intercepts of thread functions. */ #include diff --git a/Include/cpython/objimpl.h b/Include/cpython/objimpl.h index ed3a2a524dceaf..abb150b885ab18 100644 --- a/Include/cpython/objimpl.h +++ b/Include/cpython/objimpl.h @@ -105,7 +105,13 @@ PyAPI_FUNC(PyObject *) _PyObject_GC_Calloc(size_t size); #define PyType_SUPPORTS_WEAKREFS(t) ((t)->tp_weaklistoffset > 0) #define PyObject_GET_WEAKREFS_LISTPTR(o) \ - ((PyObject **) (((char *) (o)) + Py_TYPE(o)->tp_weaklistoffset)) + ((GC_hidden_pointer *) (((char *) (o)) + Py_TYPE(o)->tp_weaklistoffset)) + +#define _Py_HIDE_POINTER(p) \ + ((GC_hidden_pointer)((p) == NULL ? 0 : GC_HIDE_POINTER(p))) + +#define _Py_REVEAL_POINTER(p) \ + ((PyObject *)((p) == 0 ? NULL : GC_REVEAL_POINTER(p))) #ifdef __cplusplus } diff --git a/Include/weakrefobject.h b/Include/weakrefobject.h index 17051568f3a6e9..ceca9551811e7e 100644 --- a/Include/weakrefobject.h +++ b/Include/weakrefobject.h @@ -17,10 +17,9 @@ struct _PyWeakReference { PyObject_HEAD /* The object to which this is a weak reference, or Py_None if none. - * Note that this is a stealth reference: wr_object's refcount is - * not incremented to reflect this pointer. + * Note that this is a hidden pointer. */ - PyObject *wr_object; + GC_hidden_pointer wr_object; /* A callable to invoke when wr_object dies, or NULL if none. */ PyObject *wr_callback; @@ -35,8 +34,8 @@ struct _PyWeakReference { * If wr_object goes away, wr_object is set to Py_None, and these pointers * have no meaning then. */ - PyWeakReference *wr_prev; - PyWeakReference *wr_next; + GC_hidden_pointer wr_prev; + GC_hidden_pointer wr_next; }; #endif @@ -67,18 +66,9 @@ PyAPI_FUNC(Py_ssize_t) _PyWeakref_GetWeakrefCount(PyWeakReference *head); PyAPI_FUNC(void) _PyWeakref_ClearRef(PyWeakReference *self); #endif -/* Explanation for the Py_REFCNT() check: when a weakref's target is part - of a long chain of deallocations which triggers the trashcan mechanism, - clearing the weakrefs can be delayed long after the target's refcount - has dropped to zero. In the meantime, code accessing the weakref will - be able to "see" the target object even though it is supposed to be - unreachable. See issue #16602. */ - -#define PyWeakref_GET_OBJECT(ref) \ - (Py_REFCNT(((PyWeakReference *)(ref))->wr_object) > 0 \ - ? ((PyWeakReference *)(ref))->wr_object \ - : Py_None) +PyAPI_FUNC(PyObject *) _PyWeakref_GET_OBJECT(PyWeakReference *self); +#define PyWeakref_GET_OBJECT(ref) _PyWeakref_GET_OBJECT((PyWeakReference *)ref) #ifdef __cplusplus } diff --git a/Modules/_weakref.c b/Modules/_weakref.c index c1238e00d35f4a..8b12545e37aca6 100644 --- a/Modules/_weakref.c +++ b/Modules/_weakref.c @@ -1,9 +1,6 @@ #include "Python.h" -#define GET_WEAKREFS_LISTPTR(o) \ - ((PyWeakReference **) PyObject_GET_WEAKREFS_LISTPTR(o)) - /*[clinic input] module _weakref [clinic start generated code]*/ @@ -25,13 +22,14 @@ static Py_ssize_t _weakref_getweakrefcount_impl(PyObject *module, PyObject *object) /*[clinic end generated code: output=301806d59558ff3e input=cedb69711b6a2507]*/ { - PyWeakReference **list; + GC_hidden_pointer *list; if (!PyType_SUPPORTS_WEAKREFS(Py_TYPE(object))) return 0; - list = GET_WEAKREFS_LISTPTR(object); - return _PyWeakref_GetWeakrefCount(*list); + list = PyObject_GET_WEAKREFS_LISTPTR(object); + return _PyWeakref_GetWeakrefCount( + (PyWeakReference *)_Py_REVEAL_POINTER(*list)); } @@ -85,17 +83,20 @@ weakref_getweakrefs(PyObject *self, PyObject *object) PyObject *result = NULL; if (PyType_SUPPORTS_WEAKREFS(Py_TYPE(object))) { - PyWeakReference **list = GET_WEAKREFS_LISTPTR(object); - Py_ssize_t count = _PyWeakref_GetWeakrefCount(*list); + GC_hidden_pointer *list = PyObject_GET_WEAKREFS_LISTPTR(object); + Py_ssize_t count = _PyWeakref_GetWeakrefCount( + (PyWeakReference *)_Py_REVEAL_POINTER(*list)); result = PyList_New(count); if (result != NULL) { - PyWeakReference *current = *list; + PyWeakReference *current = + (PyWeakReference *)_Py_REVEAL_POINTER(*list); Py_ssize_t i; for (i = 0; i < count; ++i) { PyList_SET_ITEM(result, i, (PyObject *) current); Py_INCREF(current); - current = current->wr_next; + current = + (PyWeakReference *)_Py_REVEAL_POINTER(current->wr_next); } } } diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 0ff066dd881ac4..167e5c47304736 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -671,7 +671,7 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) * pass completes. */ for (gc = GC_NEXT(unreachable); gc != unreachable; gc = next) { - PyWeakReference **wrlist; + GC_hidden_pointer *wrlist; op = FROM_GC(gc); next = GC_NEXT(gc); @@ -680,14 +680,15 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) continue; /* It supports weakrefs. Does it have any? */ - wrlist = (PyWeakReference **) - PyObject_GET_WEAKREFS_LISTPTR(op); + wrlist = PyObject_GET_WEAKREFS_LISTPTR(op); /* `op` may have some weakrefs. March over the list, clear * all the weakrefs, and move the weakrefs with callbacks * that must be called into wrcb_to_call. */ - for (wr = *wrlist; wr != NULL; wr = *wrlist) { + for (wr = (PyWeakReference *)_Py_REVEAL_POINTER(*wrlist); + wr != NULL; + wr = (PyWeakReference *)_Py_REVEAL_POINTER(*wrlist)) { PyGC_Head *wrasgc; /* AS_GC(wr) */ /* _PyWeakref_ClearRef clears the weakref but leaves diff --git a/Objects/object.c b/Objects/object.c index 54a827430dfc7c..f9216b63329cd5 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2190,6 +2190,7 @@ _PyObject_AssertFailed(PyObject *obj, const char *expr, const char *msg, void _GC_Py_Dealloc(PyObject *op) { + PyGILState_STATE state = PyGILState_Ensure(); destructor dealloc = Py_TYPE(op)->tp_dealloc; #ifdef Py_TRACE_REFS _Py_ForgetReference(op); @@ -2197,6 +2198,7 @@ _GC_Py_Dealloc(PyObject *op) _Py_INC_TPFREES(op); #endif (*dealloc)(op); + PyGILState_Release(state); } void diff --git a/Objects/typeobject.c b/Objects/typeobject.c index eeaae1f9f78947..cd8735d0848834 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1225,10 +1225,10 @@ subtype_dealloc(PyObject *self) being finalized that has already been destroyed. */ if (type->tp_weaklistoffset && !base->tp_weaklistoffset) { /* Modeled after GET_WEAKREFS_LISTPTR() */ - PyWeakReference **list = (PyWeakReference **) \ - PyObject_GET_WEAKREFS_LISTPTR(self); - while (*list) - _PyWeakref_ClearRef(*list); + GC_hidden_pointer *list = PyObject_GET_WEAKREFS_LISTPTR(self); + PyWeakReference *wr; + while ((wr = GC_REVEAL_POINTER(*list)) != NULL) + _PyWeakref_ClearRef(wr); } } @@ -2215,7 +2215,7 @@ subtype_setdict(PyObject *obj, PyObject *value, void *context) static PyObject * subtype_getweakref(PyObject *obj, void *context) { - PyObject **weaklistptr; + GC_hidden_pointer *weaklistptr; PyObject *result; PyTypeObject *type = Py_TYPE(obj); @@ -2229,11 +2229,10 @@ subtype_getweakref(PyObject *obj, void *context) _PyObject_ASSERT((PyObject *)type, ((type->tp_weaklistoffset + sizeof(PyObject *)) <= (size_t)(type->tp_basicsize))); - weaklistptr = (PyObject **)((char *)obj + type->tp_weaklistoffset); - if (*weaklistptr == NULL) + weaklistptr = PyObject_GET_WEAKREFS_LISTPTR(obj); + result = GC_REVEAL_POINTER(*weaklistptr); + if (result == NULL) result = Py_None; - else - result = *weaklistptr; Py_INCREF(result); return result; } diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index ff6d92254f7fe2..f9864c32d37e89 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -1,10 +1,27 @@ #include "Python.h" #include "structmember.h" +#define GET_NEXT(o) ((PyWeakReference *)_Py_REVEAL_POINTER((o)->wr_next)) +#define GET_PREV(o) ((PyWeakReference *)_Py_REVEAL_POINTER((o)->wr_prev)) +#define SET_NEXT(o, v) (((PyWeakReference *)(o))->wr_next = _Py_HIDE_POINTER(v)) +#define SET_PREV(o, v) (((PyWeakReference *)(o))->wr_prev = _Py_HIDE_POINTER(v)) -#define GET_WEAKREFS_LISTPTR(o) \ - ((PyWeakReference **) PyObject_GET_WEAKREFS_LISTPTR(o)) +#define REVEAL(p) ((PyWeakReference *)_Py_REVEAL_POINTER(p)) +static void * +get_object_callback(void *_self) +{ + PyWeakReference *self = (PyWeakReference *)_self; + if (self->wr_object == (GC_hidden_pointer)NULL) + return (void *)Py_None; + return GC_REVEAL_POINTER(self->wr_object); +} + +PyObject * +_PyWeakref_GET_OBJECT(PyWeakReference *self) +{ + return (PyObject *)GC_call_with_alloc_lock(get_object_callback, self); +} Py_ssize_t _PyWeakref_GetWeakrefCount(PyWeakReference *head) @@ -13,7 +30,7 @@ _PyWeakref_GetWeakrefCount(PyWeakReference *head) while (head != NULL) { ++count; - head = head->wr_next; + head = GET_NEXT(head); } return count; } @@ -23,9 +40,9 @@ static void init_weakref(PyWeakReference *self, PyObject *ob, PyObject *callback) { self->hash = -1; - self->wr_object = ob; - self->wr_prev = NULL; - self->wr_next = NULL; + self->wr_object = GC_HIDE_POINTER(ob); + self->wr_prev = 0; + self->wr_next = 0; Py_XINCREF(callback); self->wr_callback = callback; } @@ -53,22 +70,26 @@ static void clear_weakref(PyWeakReference *self) { PyObject *callback = self->wr_callback; + PyObject *ob = PyWeakref_GET_OBJECT(self); - if (self->wr_object != Py_None) { - PyWeakReference **list = GET_WEAKREFS_LISTPTR(self->wr_object); + if (ob != Py_None) { + GC_hidden_pointer *list = PyObject_GET_WEAKREFS_LISTPTR(ob); + PyWeakReference *ref; - if (*list == self) + if (REVEAL(*list) == self) /* If 'self' is the end of the list (and thus self->wr_next == NULL) then the weakref list itself (and thus the value of *list) will end up being set to NULL. */ *list = self->wr_next; - self->wr_object = Py_None; - if (self->wr_prev != NULL) - self->wr_prev->wr_next = self->wr_next; - if (self->wr_next != NULL) - self->wr_next->wr_prev = self->wr_prev; - self->wr_prev = NULL; - self->wr_next = NULL; + self->wr_object = _Py_HIDE_POINTER(Py_None); + ref = GET_PREV(self); + if (ref != NULL) + ref->wr_next = self->wr_next; + ref = GET_NEXT(self); + if (ref != NULL) + ref->wr_prev = self->wr_prev; + self->wr_prev = 0; + self->wr_next = 0; } if (callback != NULL) { Py_DECREF(callback); @@ -105,6 +126,9 @@ static void weakref_dealloc(PyObject *self) { PyObject_GC_UnTrack(self); + // Hack around the fact that other parts of weakrefs check refcnt to + // check for objects that are in the process of deallocating... + self->ob_refcnt = 0; clear_weakref((PyWeakReference *) self); Py_TYPE(self)->tp_free(self); } @@ -229,7 +253,7 @@ get_basic_refs(PyWeakReference *head, little. */ if (PyWeakref_CheckRefExact(head)) { *refp = head; - head = head->wr_next; + head = GET_NEXT(head); } if (head != NULL && head->wr_callback == NULL @@ -244,26 +268,26 @@ get_basic_refs(PyWeakReference *head, static void insert_after(PyWeakReference *newref, PyWeakReference *prev) { - newref->wr_prev = prev; - newref->wr_next = prev->wr_next; - if (prev->wr_next != NULL) - prev->wr_next->wr_prev = newref; - prev->wr_next = newref; + SET_PREV(newref, prev); + SET_NEXT(newref, GET_NEXT(prev)); + if (GET_NEXT(prev) != NULL) + SET_PREV(GET_NEXT(prev), newref); + SET_NEXT(prev, newref); } /* Insert 'newref' at the head of the list; 'list' points to the variable * that stores the head. */ static void -insert_head(PyWeakReference *newref, PyWeakReference **list) +insert_head(PyWeakReference *newref, GC_hidden_pointer *list) { - PyWeakReference *next = *list; + PyWeakReference *next = REVEAL(*list); - newref->wr_prev = NULL; - newref->wr_next = next; + newref->wr_prev = 0; + SET_NEXT(newref, next); if (next != NULL) - next->wr_prev = newref; - *list = newref; + SET_PREV(next, newref); + *list = _Py_HIDE_POINTER(newref); } static int @@ -281,7 +305,7 @@ weakref___new__(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (parse_weakref_init_args("__new__", args, kwargs, &ob, &callback)) { PyWeakReference *ref, *proxy; - PyWeakReference **list; + GC_hidden_pointer *list; if (!PyType_SUPPORTS_WEAKREFS(Py_TYPE(ob))) { PyErr_Format(PyExc_TypeError, @@ -291,8 +315,8 @@ weakref___new__(PyTypeObject *type, PyObject *args, PyObject *kwargs) } if (callback == Py_None) callback = NULL; - list = GET_WEAKREFS_LISTPTR(ob); - get_basic_refs(*list, &ref, &proxy); + list = PyObject_GET_WEAKREFS_LISTPTR(ob); + get_basic_refs(REVEAL(*list), &ref, &proxy); if (callback == NULL && type == &_PyWeakref_RefType) { if (ref != NULL) { /* We can re-use an existing reference. */ @@ -315,7 +339,7 @@ weakref___new__(PyTypeObject *type, PyObject *args, PyObject *kwargs) else { PyWeakReference *prev; - get_basic_refs(*list, &ref, &proxy); + get_basic_refs(REVEAL(*list), &ref, &proxy); prev = (proxy == NULL) ? ref : proxy; if (prev == NULL) insert_head(self, list); @@ -738,7 +762,7 @@ PyObject * PyWeakref_NewRef(PyObject *ob, PyObject *callback) { PyWeakReference *result = NULL; - PyWeakReference **list; + GC_hidden_pointer *list; PyWeakReference *ref, *proxy; if (!PyType_SUPPORTS_WEAKREFS(Py_TYPE(ob))) { @@ -747,8 +771,8 @@ PyWeakref_NewRef(PyObject *ob, PyObject *callback) Py_TYPE(ob)->tp_name); return NULL; } - list = GET_WEAKREFS_LISTPTR(ob); - get_basic_refs(*list, &ref, &proxy); + list = PyObject_GET_WEAKREFS_LISTPTR(ob); + get_basic_refs(REVEAL(*list), &ref, &proxy); if (callback == Py_None) callback = NULL; if (callback == NULL) @@ -764,7 +788,7 @@ PyWeakref_NewRef(PyObject *ob, PyObject *callback) them. */ result = new_weakref(ob, callback); if (result != NULL) { - get_basic_refs(*list, &ref, &proxy); + get_basic_refs(REVEAL(*list), &ref, &proxy); if (callback == NULL) { if (ref == NULL) insert_head(result, list); @@ -797,7 +821,7 @@ PyObject * PyWeakref_NewProxy(PyObject *ob, PyObject *callback) { PyWeakReference *result = NULL; - PyWeakReference **list; + GC_hidden_pointer *list; PyWeakReference *ref, *proxy; if (!PyType_SUPPORTS_WEAKREFS(Py_TYPE(ob))) { @@ -806,8 +830,8 @@ PyWeakref_NewProxy(PyObject *ob, PyObject *callback) Py_TYPE(ob)->tp_name); return NULL; } - list = GET_WEAKREFS_LISTPTR(ob); - get_basic_refs(*list, &ref, &proxy); + list = PyObject_GET_WEAKREFS_LISTPTR(ob); + get_basic_refs(REVEAL(*list), &ref, &proxy); if (callback == Py_None) callback = NULL; if (callback == NULL) @@ -829,7 +853,7 @@ PyWeakref_NewProxy(PyObject *ob, PyObject *callback) Py_TYPE(result) = &_PyWeakref_CallableProxyType; else Py_TYPE(result) = &_PyWeakref_ProxyType; - get_basic_refs(*list, &ref, &proxy); + get_basic_refs(REVEAL(*list), &ref, &proxy); if (callback == NULL) { if (proxy != NULL) { /* Someone else added a proxy without a callback @@ -891,23 +915,24 @@ handle_callback(PyWeakReference *ref, PyObject *callback) void PyObject_ClearWeakRefs(PyObject *object) { - PyWeakReference **list; + GC_hidden_pointer *list; if (object == NULL - || !PyType_SUPPORTS_WEAKREFS(Py_TYPE(object)) - || object->ob_refcnt != 0) { + || !PyType_SUPPORTS_WEAKREFS(Py_TYPE(object))) { PyErr_BadInternalCall(); return; } - list = GET_WEAKREFS_LISTPTR(object); + list = PyObject_GET_WEAKREFS_LISTPTR(object); + PyWeakReference *current = REVEAL(*list); /* Remove the callback-less basic and proxy references */ - if (*list != NULL && (*list)->wr_callback == NULL) { - clear_weakref(*list); - if (*list != NULL && (*list)->wr_callback == NULL) - clear_weakref(*list); + if (current != NULL && current->wr_callback == NULL) { + clear_weakref(current); + current = REVEAL(*list); + if (current != NULL && current->wr_callback == NULL) + clear_weakref(current); } - if (*list != NULL) { - PyWeakReference *current = *list; + current = REVEAL(*list); + if (current != NULL) { Py_ssize_t count = _PyWeakref_GetWeakrefCount(current); PyObject *err_type, *err_value, *err_tb; @@ -934,7 +959,7 @@ PyObject_ClearWeakRefs(PyObject *object) } for (i = 0; i < count; ++i) { - PyWeakReference *next = current->wr_next; + PyWeakReference *next = GET_NEXT(current); if (((PyObject *)current)->ob_refcnt > 0) { From 539547623700351afea7ea0eaf26a11447ee82a7 Mon Sep 17 00:00:00 2001 From: Thomas Wouters Date: Wed, 8 May 2019 14:11:37 -0700 Subject: [PATCH 014/527] Fix merge glitch. --- Objects/object.c | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/Objects/object.c b/Objects/object.c index 82e9a3bdde2b28..f9216b63329cd5 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2220,25 +2220,6 @@ _Py_Dealloc_finalizer(void *_op, void *is_gc) _GC_Py_Dealloc(op); } -void -_Py_Dealloc_GC_finalizer(void *_op, void *unused) -{ - PyObject *op = _Py_FROM_GC(_op); - assert(PyObject_IS_GC(op)); - _Py_Dealloc_finalizer((void *)op, (void *)_op); -} - -void -_Py_Dealloc_finalizer(void *_op, void *is_gc) -{ - PyObject *op = (PyObject *)_op; - assert(is_gc == NULL ? !PyObject_IS_GC(op) : PyObject_IS_GC(op)); - if (op->ob_refcnt != 0) { - // fprintf(stderr, "object %p refcount leak (%ld)\n", op, op->ob_refcnt); - } - _GC_Py_Dealloc(op); -} - #ifdef __cplusplus } #endif From 267a3897b91bd70657d82165746a7c0e6adb3723 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Tue, 4 Jun 2019 21:55:17 +0200 Subject: [PATCH 015/527] Post v3.8.0b1 --- Include/patchlevel.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 881558bb44f612..4a52f833a52550 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -23,7 +23,7 @@ #define PY_RELEASE_SERIAL 1 /* Version as a string */ -#define PY_VERSION "3.8.0b1" +#define PY_VERSION "3.8.0b1+" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. From 99a5178cd143cc880f081dc5f53903fefa378681 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 4 Jun 2019 16:21:19 -0700 Subject: [PATCH 016/527] Doc: Python 3.9 in sidebar and version switcher. (GH-13824) (cherry picked from commit 59e7bbcaa4d0d556591f774c5ea4869c41fa95b0) Co-authored-by: Julien Palard --- Doc/tools/static/switchers.js | 3 ++- Doc/tools/templates/indexsidebar.html | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Doc/tools/static/switchers.js b/Doc/tools/static/switchers.js index 346b31494e60f9..fa298a76b0fe10 100644 --- a/Doc/tools/static/switchers.js +++ b/Doc/tools/static/switchers.js @@ -10,7 +10,8 @@ '(?:release/\\d.\\d[\\x\\d\\.]*)']; var all_versions = { - '3.8': 'dev (3.8)', + '3.9': 'dev (3.9)', + '3.8': 'pre (3.8)', '3.7': '3.7', '3.6': '3.6', '3.5': '3.5', diff --git a/Doc/tools/templates/indexsidebar.html b/Doc/tools/templates/indexsidebar.html index 3666af92f0da47..4fd7423430ca81 100644 --- a/Doc/tools/templates/indexsidebar.html +++ b/Doc/tools/templates/indexsidebar.html @@ -2,7 +2,8 @@

{% trans %}Download{% endtrans %}

{% trans %}Download these documents{% endtrans %}

{% trans %}Docs by version{% endtrans %}

diff --git a/Lib/idlelib/help.py b/Lib/idlelib/help.py index e19d3615016f2f..ba2c6ec554812b 100644 --- a/Lib/idlelib/help.py +++ b/Lib/idlelib/help.py @@ -62,6 +62,7 @@ def __init__(self, text): self.simplelist = False # simple list (no double spacing) self.toc = [] # pair headers with text indexes for toc self.header = '' # text within header tags for toc + self.prevtag = None # info about previous tag (was opener, tag) def indent(self, amt=1): self.level += amt @@ -78,8 +79,11 @@ def handle_starttag(self, tag, attrs): self.show = True # start of main content elif tag == 'div' and class_ == 'sphinxsidebar': self.show = False # end of main content - elif tag == 'p' and class_ != 'first': - s = '\n\n' + elif tag == 'p' and self.prevtag and not self.prevtag[0]: + # begin a new block for

tags after a closed tag + # avoid extra lines, e.g. after

 tags
+            lastline = self.text.get('end-1c linestart', 'end-1c')
+            s = '\n\n' if lastline and not lastline.isspace() else '\n'
         elif tag == 'span' and class_ == 'pre':
             self.chartags = 'pre'
         elif tag == 'span' and class_ == 'versionmodified':
@@ -120,6 +124,7 @@ def handle_starttag(self, tag, attrs):
             self.tags = tag
         if self.show:
             self.text.insert('end', s, (self.tags, self.chartags))
+        self.prevtag = (True, tag)
 
     def handle_endtag(self, tag):
         "Handle endtags in help.html."
@@ -139,6 +144,7 @@ def handle_endtag(self, tag):
             self.tags = ''
         elif tag in ['ul', 'dd', 'ol']:
             self.indent(amt=-1)
+        self.prevtag = (False, tag)
 
     def handle_data(self, data):
         "Handle date segments in help.html."

From 4dd1c9d9c2bca4744c70c9556b7051f4465ede3e Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
 <31488909+miss-islington@users.noreply.github.com>
Date: Tue, 3 Sep 2019 20:03:37 -0700
Subject: [PATCH 501/527] closes bpo-37966: Fully implement the UAX GH-15
 quick-check algorithm. (GH-15558)

The purpose of the `unicodedata.is_normalized` function is to answer
the question `str == unicodedata.normalized(form, str)` more
efficiently than writing just that, by using the "quick check"
optimization described in the Unicode standard in UAX GH-15.

However, it turns out the code doesn't implement the full algorithm
from the standard, and as a result we often miss the optimization and
end up having to compute the whole normalized string after all.

Implement the standard's algorithm.  This greatly speeds up
`unicodedata.is_normalized` in many cases where our partial variant
of quick-check had been returning MAYBE and the standard algorithm
returns NO.

At a quick test on my desktop, the existing code takes about 4.4 ms/MB
(so 4.4 ns per byte) when the partial quick-check returns MAYBE and it
has to do the slow normalize-and-compare:

  $ build.base/python -m timeit -s 'import unicodedata; s = "\uf900"*500000' \
      -- 'unicodedata.is_normalized("NFD", s)'
  50 loops, best of 5: 4.39 msec per loop

With this patch, it gets the answer instantly (58 ns) on the same 1 MB
string:

  $ build.dev/python -m timeit -s 'import unicodedata; s = "\uf900"*500000' \
      -- 'unicodedata.is_normalized("NFD", s)'
  5000000 loops, best of 5: 58.2 nsec per loop

This restores a small optimization that the original version of this
code had for the `unicodedata.normalize` use case.

With this, that case is actually faster than in master!

$ build.base/python -m timeit -s 'import unicodedata; s = "\u0338"*500000' \
    -- 'unicodedata.normalize("NFD", s)'
500 loops, best of 5: 561 usec per loop

$ build.dev/python -m timeit -s 'import unicodedata; s = "\u0338"*500000' \
    -- 'unicodedata.normalize("NFD", s)'
500 loops, best of 5: 512 usec per loop
(cherry picked from commit 2f09413947d1ce0043de62ed2346f9a2b4e5880b)

Co-authored-by: Greg Price 
---
 Doc/whatsnew/3.8.rst                          |  5 +-
 Lib/test/test_unicodedata.py                  |  2 +
 .../2019-08-27-21-21-36.bpo-37966.5OBLez.rst  |  3 +
 Modules/unicodedata.c                         | 75 +++++++++++++------
 4 files changed, 59 insertions(+), 26 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Core and Builtins/2019-08-27-21-21-36.bpo-37966.5OBLez.rst

diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst
index bcdb60d86d8553..4a1362d943c809 100644
--- a/Doc/whatsnew/3.8.rst
+++ b/Doc/whatsnew/3.8.rst
@@ -1090,8 +1090,9 @@ unicodedata
   `_ release.
 
 * New function :func:`~unicodedata.is_normalized` can be used to verify a string
-  is in a specific normal form. (Contributed by Max Belanger and David Euresti in
-  :issue:`32285`).
+  is in a specific normal form, often much faster than by actually normalizing
+  the string.  (Contributed by Max Belanger, David Euresti, and Greg Price in
+  :issue:`32285` and :issue:`37966`).
 
 
 unittest
diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py
index a52b6de547fbc9..07d717688b0c59 100644
--- a/Lib/test/test_unicodedata.py
+++ b/Lib/test/test_unicodedata.py
@@ -220,6 +220,8 @@ def test_issue29456(self):
         self.assertEqual(self.db.normalize('NFC', u11a7_str_a), u11a7_str_b)
         self.assertEqual(self.db.normalize('NFC', u11c3_str_a), u11c3_str_b)
 
+    # For tests of unicodedata.is_normalized / self.db.is_normalized ,
+    # see test_normalization.py .
 
     def test_east_asian_width(self):
         eaw = self.db.east_asian_width
diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-08-27-21-21-36.bpo-37966.5OBLez.rst b/Misc/NEWS.d/next/Core and Builtins/2019-08-27-21-21-36.bpo-37966.5OBLez.rst
new file mode 100644
index 00000000000000..6b9d69c5b3a9a4
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-08-27-21-21-36.bpo-37966.5OBLez.rst	
@@ -0,0 +1,3 @@
+The implementation of :func:`~unicodedata.is_normalized` has been greatly
+sped up on strings that aren't normalized, by implementing the full
+normalization-quick-check algorithm from the Unicode standard.
diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c
index ae0d4e46f9a409..5e8ba602d66848 100644
--- a/Modules/unicodedata.c
+++ b/Modules/unicodedata.c
@@ -19,6 +19,8 @@
 #include "ucnhash.h"
 #include "structmember.h"
 
+#include 
+
 _Py_IDENTIFIER(NFC);
 _Py_IDENTIFIER(NFD);
 _Py_IDENTIFIER(NFKC);
@@ -775,25 +777,40 @@ nfc_nfkc(PyObject *self, PyObject *input, int k)
     return result;
 }
 
-typedef enum {YES, NO, MAYBE} NormalMode;
-
-/* Return YES if the input is certainly normalized, NO or MAYBE if it might not be. */
-static NormalMode
-is_normalized(PyObject *self, PyObject *input, int nfc, int k)
+// This needs to match the logic in makeunicodedata.py
+// which constructs the quickcheck data.
+typedef enum {YES = 0, MAYBE = 1, NO = 2} QuickcheckResult;
+
+/* Run the Unicode normalization "quickcheck" algorithm.
+ *
+ * Return YES or NO if quickcheck determines the input is certainly
+ * normalized or certainly not, and MAYBE if quickcheck is unable to
+ * tell.
+ *
+ * If `yes_only` is true, then return MAYBE as soon as we determine
+ * the answer is not YES.
+ *
+ * For background and details on the algorithm, see UAX #15:
+ *   https://www.unicode.org/reports/tr15/#Detecting_Normalization_Forms
+ */
+static QuickcheckResult
+is_normalized_quickcheck(PyObject *self, PyObject *input,
+                         int nfc, int k, bool yes_only)
 {
-    Py_ssize_t i, len;
-    int kind;
-    void *data;
-    unsigned char prev_combining = 0, quickcheck_mask;
-
     /* An older version of the database is requested, quickchecks must be
        disabled. */
     if (self && UCD_Check(self))
         return NO;
 
-    /* The two quickcheck bits at this shift mean 0=Yes, 1=Maybe, 2=No,
-       as described in http://unicode.org/reports/tr15/#Annex8. */
-    quickcheck_mask = 3 << ((nfc ? 4 : 0) + (k ? 2 : 0));
+    Py_ssize_t i, len;
+    int kind;
+    void *data;
+    unsigned char prev_combining = 0;
+
+    /* The two quickcheck bits at this shift have type QuickcheckResult. */
+    int quickcheck_shift = (nfc ? 4 : 0) + (k ? 2 : 0);
+
+    QuickcheckResult result = YES; /* certainly normalized, unless we find something */
 
     i = 0;
     kind = PyUnicode_KIND(input);
@@ -802,16 +819,26 @@ is_normalized(PyObject *self, PyObject *input, int nfc, int k)
     while (i < len) {
         Py_UCS4 ch = PyUnicode_READ(kind, data, i++);
         const _PyUnicode_DatabaseRecord *record = _getrecord_ex(ch);
-        unsigned char combining = record->combining;
-        unsigned char quickcheck = record->normalization_quick_check;
 
-        if (quickcheck & quickcheck_mask)
-            return MAYBE; /* this string might need normalization */
+        unsigned char combining = record->combining;
         if (combining && prev_combining > combining)
             return NO; /* non-canonical sort order, not normalized */
         prev_combining = combining;
+
+        unsigned char quickcheck_whole = record->normalization_quick_check;
+        if (yes_only) {
+            if (quickcheck_whole & (3 << quickcheck_shift))
+                return MAYBE;
+        } else {
+            switch ((quickcheck_whole >> quickcheck_shift) & 3) {
+            case NO:
+              return NO;
+            case MAYBE:
+              result = MAYBE; /* this string might need normalization */
+            }
+        }
     }
-    return YES; /* certainly normalized */
+    return result;
 }
 
 /*[clinic input]
@@ -844,7 +871,7 @@ unicodedata_UCD_is_normalized_impl(PyObject *self, PyObject *form,
     PyObject *result;
     int nfc = 0;
     int k = 0;
-    NormalMode m;
+    QuickcheckResult m;
 
     PyObject *cmp;
     int match = 0;
@@ -867,7 +894,7 @@ unicodedata_UCD_is_normalized_impl(PyObject *self, PyObject *form,
         return NULL;
     }
 
-    m = is_normalized(self, input, nfc, k);
+    m = is_normalized_quickcheck(self, input, nfc, k, false);
 
     if (m == MAYBE) {
         cmp = (nfc ? nfc_nfkc : nfd_nfkd)(self, input, k);
@@ -913,28 +940,28 @@ unicodedata_UCD_normalize_impl(PyObject *self, PyObject *form,
     }
 
     if (_PyUnicode_EqualToASCIIId(form, &PyId_NFC)) {
-        if (is_normalized(self, input, 1, 0) == YES) {
+        if (is_normalized_quickcheck(self, input, 1, 0, true) == YES) {
             Py_INCREF(input);
             return input;
         }
         return nfc_nfkc(self, input, 0);
     }
     if (_PyUnicode_EqualToASCIIId(form, &PyId_NFKC)) {
-        if (is_normalized(self, input, 1, 1) == YES) {
+        if (is_normalized_quickcheck(self, input, 1, 1, true) == YES) {
             Py_INCREF(input);
             return input;
         }
         return nfc_nfkc(self, input, 1);
     }
     if (_PyUnicode_EqualToASCIIId(form, &PyId_NFD)) {
-        if (is_normalized(self, input, 0, 0) == YES) {
+        if (is_normalized_quickcheck(self, input, 0, 0, true) == YES) {
             Py_INCREF(input);
             return input;
         }
         return nfd_nfkd(self, input, 0);
     }
     if (_PyUnicode_EqualToASCIIId(form, &PyId_NFKD)) {
-        if (is_normalized(self, input, 0, 1) == YES) {
+        if (is_normalized_quickcheck(self, input, 0, 1, true) == YES) {
             Py_INCREF(input);
             return input;
         }

From 5e194f57c08898cb8da457c6584402a2be2c828d Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
 <31488909+miss-islington@users.noreply.github.com>
Date: Tue, 3 Sep 2019 23:10:45 -0700
Subject: [PATCH 502/527] Fix grammar in asyncio-dev.rst (GH-15672)

Automerge-Triggered-By: @ned-deily
(cherry picked from commit 675d17cec49ed7f7eb01d1bc3ae6999c728e070d)

Co-authored-by: Roger Iyengar 
---
 Doc/library/asyncio-dev.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst
index b7288036192979..101e7817a95e98 100644
--- a/Doc/library/asyncio-dev.rst
+++ b/Doc/library/asyncio-dev.rst
@@ -119,7 +119,7 @@ all concurrent asyncio Tasks and IO operations would be delayed
 by 1 second.
 
 An executor can be used to run a task in a different thread or even in
-a different process to avoid blocking block the OS thread with the
+a different process to avoid blocking the OS thread with the
 event loop.  See the :meth:`loop.run_in_executor` method for more
 details.
 

From cad7abf8abe657b696b9c8deb4b727e0cefaf36d Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
 <31488909+miss-islington@users.noreply.github.com>
Date: Wed, 4 Sep 2019 15:18:05 -0700
Subject: [PATCH 503/527] bpo-38030: Fix os.stat failures on block devices on
 Windows (GH-15681)

(cherry picked from commit 772ec0fad57412daa53d16d7019b6b2fe6e94942)

Co-authored-by: Steve Dower 
---
 Lib/test/test_os.py                           |  8 ++++++++
 .../2019-09-04-14-01-08.bpo-38030._USdtk.rst  |  1 +
 Modules/posixmodule.c                         | 19 +++++++++++++------
 3 files changed, 22 insertions(+), 6 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Windows/2019-09-04-14-01-08.bpo-38030._USdtk.rst

diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py
index 440cd6c1cf73c3..8ff0296fad0bb4 100644
--- a/Lib/test/test_os.py
+++ b/Lib/test/test_os.py
@@ -591,6 +591,14 @@ def test_access_denied(self):
         result = os.stat(fname)
         self.assertNotEqual(result.st_size, 0)
 
+    @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests")
+    def test_stat_block_device(self):
+        # bpo-38030: os.stat fails for block devices
+        # Test a filename like "//./C:"
+        fname = "//./" + os.path.splitdrive(os.getcwd())[0]
+        result = os.stat(fname)
+        self.assertEqual(result.st_mode, stat.S_IFBLK)
+
 
 class UtimeTests(unittest.TestCase):
     def setUp(self):
diff --git a/Misc/NEWS.d/next/Windows/2019-09-04-14-01-08.bpo-38030._USdtk.rst b/Misc/NEWS.d/next/Windows/2019-09-04-14-01-08.bpo-38030._USdtk.rst
new file mode 100644
index 00000000000000..f1be8a1e1c86f0
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-09-04-14-01-08.bpo-38030._USdtk.rst
@@ -0,0 +1 @@
+Fixes :func:`os.stat` failing for block devices on Windows
diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
index b9e8c0d94c20e1..771c5616152cfd 100644
--- a/Modules/posixmodule.c
+++ b/Modules/posixmodule.c
@@ -1793,13 +1793,13 @@ win32_xstat_impl(const wchar_t *path, struct _Py_stat_struct *result,
             case ERROR_INVALID_PARAMETER:
             case ERROR_INVALID_FUNCTION:
             case ERROR_NOT_SUPPORTED:
-                retval = -1;
+                /* Volumes and physical disks are block devices, e.g.
+                   \\.\C: and \\.\PhysicalDrive0. */
+                memset(result, 0, sizeof(*result));
+                result->st_mode = 0x6000; /* S_IFBLK */
                 goto cleanup;
             }
-            /* Volumes and physical disks are block devices, e.g.
-               \\.\C: and \\.\PhysicalDrive0. */
-            memset(result, 0, sizeof(*result));
-            result->st_mode = 0x6000; /* S_IFBLK */
+            retval = -1;
             goto cleanup;
         }
     }
@@ -1826,7 +1826,14 @@ win32_xstat_impl(const wchar_t *path, struct _Py_stat_struct *result,
 
 cleanup:
     if (hFile != INVALID_HANDLE_VALUE) {
-        CloseHandle(hFile);
+        /* Preserve last error if we are failing */
+        error = retval ? GetLastError() : 0;
+        if (!CloseHandle(hFile)) {
+            retval = -1;
+        } else if (retval) {
+            /* Restore last error */
+            SetLastError(error);
+        }
     }
 
     return retval;

From 29825a33926db63bb73601ba9e76ecd3c6cfa0f6 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
 <31488909+miss-islington@users.noreply.github.com>
Date: Wed, 4 Sep 2019 17:39:34 -0700
Subject: [PATCH 504/527] Fix idlelib.help comments (GH-15669)

(cherry picked from commit 6cd9666ce93658ae91f07b396aa6932b362a61d3)

Co-authored-by: Terry Jan Reedy 
---
 Lib/idlelib/help.py | 49 +++++++++++++++++++++++----------------------
 1 file changed, 25 insertions(+), 24 deletions(-)

diff --git a/Lib/idlelib/help.py b/Lib/idlelib/help.py
index ba2c6ec554812b..9f63ea0d3990e6 100644
--- a/Lib/idlelib/help.py
+++ b/Lib/idlelib/help.py
@@ -50,21 +50,22 @@ class HelpParser(HTMLParser):
     """
     def __init__(self, text):
         HTMLParser.__init__(self, convert_charrefs=True)
-        self.text = text         # text widget we're rendering into
-        self.tags = ''           # current block level text tags to apply
-        self.chartags = ''       # current character level text tags
-        self.show = False        # used so we exclude page navigation
-        self.hdrlink = False     # used so we don't show header links
-        self.level = 0           # indentation level
-        self.pre = False         # displaying preformatted text
-        self.hprefix = ''        # prefix such as '25.5' to strip from headings
-        self.nested_dl = False   # if we're in a nested 
- self.simplelist = False # simple list (no double spacing) - self.toc = [] # pair headers with text indexes for toc - self.header = '' # text within header tags for toc - self.prevtag = None # info about previous tag (was opener, tag) + self.text = text # Text widget we're rendering into. + self.tags = '' # Current block level text tags to apply. + self.chartags = '' # Current character level text tags. + self.show = False # Exclude html page navigation. + self.hdrlink = False # Exclude html header links. + self.level = 0 # Track indentation level. + self.pre = False # Displaying preformatted text? + self.hprefix = '' # Heading prefix (like '25.5'?) to remove. + self.nested_dl = False # In a nested
? + self.simplelist = False # In a simple list (no double spacing)? + self.toc = [] # Pair headers with text indexes for toc. + self.header = '' # Text within header tags for toc. + self.prevtag = None # Previous tag info (opener?, tag). def indent(self, amt=1): + "Change indent (+1, 0, -1) and tags." self.level += amt self.tags = '' if self.level == 0 else 'l'+str(self.level) @@ -76,12 +77,12 @@ def handle_starttag(self, tag, attrs): class_ = v s = '' if tag == 'div' and class_ == 'section': - self.show = True # start of main content + self.show = True # Start main content. elif tag == 'div' and class_ == 'sphinxsidebar': - self.show = False # end of main content + self.show = False # End main content. elif tag == 'p' and self.prevtag and not self.prevtag[0]: - # begin a new block for

tags after a closed tag - # avoid extra lines, e.g. after

 tags
+            # Begin a new block for 

tags after a closed tag. + # Avoid extra lines, e.g. after

 tags.
             lastline = self.text.get('end-1c linestart', 'end-1c')
             s = '\n\n' if lastline and not lastline.isspace() else '\n'
         elif tag == 'span' and class_ == 'pre':
@@ -103,7 +104,7 @@ def handle_starttag(self, tag, attrs):
         elif tag == 'li':
             s = '\n* ' if self.simplelist else '\n\n* '
         elif tag == 'dt':
-            s = '\n\n' if not self.nested_dl else '\n'  # avoid extra line
+            s = '\n\n' if not self.nested_dl else '\n'  # Avoid extra line.
             self.nested_dl = False
         elif tag == 'dd':
             self.indent()
@@ -129,12 +130,13 @@ def handle_starttag(self, tag, attrs):
     def handle_endtag(self, tag):
         "Handle endtags in help.html."
         if tag in ['h1', 'h2', 'h3']:
-            self.indent(0)  # clear tag, reset indent
+            assert self.level == 0
             if self.show:
                 indent = ('        ' if tag == 'h3' else
                           '    ' if tag == 'h2' else
                           '')
                 self.toc.append((indent+self.header, self.text.index('insert')))
+            self.tags = ''
         elif tag in ['span', 'em']:
             self.chartags = ''
         elif tag == 'a':
@@ -143,7 +145,7 @@ def handle_endtag(self, tag):
             self.pre = False
             self.tags = ''
         elif tag in ['ul', 'dd', 'ol']:
-            self.indent(amt=-1)
+            self.indent(-1)
         self.prevtag = (False, tag)
 
     def handle_data(self, data):
@@ -169,7 +171,7 @@ def __init__(self, parent, filename):
         "Configure tags and feed file to parser."
         uwide = idleConf.GetOption('main', 'EditorWindow', 'width', type='int')
         uhigh = idleConf.GetOption('main', 'EditorWindow', 'height', type='int')
-        uhigh = 3 * uhigh // 4  # lines average 4/3 of editor line height
+        uhigh = 3 * uhigh // 4  # Lines average 4/3 of editor line height.
         Text.__init__(self, parent, wrap='word', highlightthickness=0,
                       padx=5, borderwidth=0, width=uwide, height=uhigh)
 
@@ -209,7 +211,6 @@ class HelpFrame(Frame):
     "Display html text, scrollbar, and toc."
     def __init__(self, parent, filename):
         Frame.__init__(self, parent)
-        # keep references to widgets for test access.
         self.text = text = HelpText(self, filename)
         self['background'] = text['background']
         self.toc = toc = self.toc_menu(text)
@@ -217,7 +218,7 @@ def __init__(self, parent, filename):
         text['yscrollcommand'] = scroll.set
 
         self.rowconfigure(0, weight=1)
-        self.columnconfigure(1, weight=1)  # text
+        self.columnconfigure(1, weight=1)  # Only expand the text widget.
         toc.grid(row=0, column=0, sticky='nw')
         text.grid(row=0, column=1, sticky='nsew')
         scroll.grid(row=0, column=2, sticky='ns')
@@ -279,7 +280,7 @@ def show_idlehelp(parent):
     "Create HelpWindow; called from Idle Help event handler."
     filename = join(abspath(dirname(__file__)), 'help.html')
     if not isfile(filename):
-        # try copy_strip, present message
+        # Try copy_strip, present message.
         return
     HelpWindow(parent, filename, 'IDLE Help (%s)' % python_version())
 

From 6d7a786d2e4b48a6b50614e042ace9ff996f0238 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
 <31488909+miss-islington@users.noreply.github.com>
Date: Wed, 4 Sep 2019 17:54:59 -0700
Subject: [PATCH 505/527] bpo-22347: Update mimetypes.guess_type to allow
 proper parsing of URLs (GH-15522)

https://bugs.python.org/issue22347
(cherry picked from commit 87bd2071c756188b6cd577889fb1682831142ceb)

Co-authored-by: Dong-hee Na 
---
 Lib/mimetypes.py                                          | 3 ++-
 Lib/test/test_mimetypes.py                                | 8 ++++++++
 Lib/test/test_urllib2.py                                  | 2 +-
 .../next/Library/2019-08-27-01-03-26.bpo-22347._TRpYr.rst | 2 ++
 4 files changed, 13 insertions(+), 2 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Library/2019-08-27-01-03-26.bpo-22347._TRpYr.rst

diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py
index 01a16fdf9aa1b3..f38005c9d29598 100644
--- a/Lib/mimetypes.py
+++ b/Lib/mimetypes.py
@@ -114,7 +114,8 @@ def guess_type(self, url, strict=True):
         but non-standard types.
         """
         url = os.fspath(url)
-        scheme, url = urllib.parse._splittype(url)
+        p = urllib.parse.urlparse(url)
+        scheme, url = p.scheme, p.path
         if scheme == 'data':
             # syntax of data URLs:
             # dataurl   := "data:" [ mediatype ] [ ";base64" ] "," data
diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py
index bfd5eeedaa77b4..7761c3fe867a7e 100644
--- a/Lib/test/test_mimetypes.py
+++ b/Lib/test/test_mimetypes.py
@@ -51,6 +51,14 @@ def test_non_standard_types(self):
         eq(self.db.guess_type('foo.xul', strict=False), ('text/xul', None))
         eq(self.db.guess_extension('image/jpg', strict=False), '.jpg')
 
+    def test_url(self):
+        result = self.db.guess_type('http://host.html')
+        msg = 'URL only has a host name, not a file'
+        self.assertSequenceEqual(result, (None, None), msg)
+        result = self.db.guess_type('http://example.com/host.html')
+        msg = 'Should be text/html'
+        self.assertSequenceEqual(result, ('text/html', None), msg)
+
     def test_guess_all_types(self):
         eq = self.assertEqual
         unless = self.assertTrue
diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py
index c6d275e16b097f..810ee5e6ea28bf 100644
--- a/Lib/test/test_urllib2.py
+++ b/Lib/test/test_urllib2.py
@@ -744,7 +744,7 @@ def connect_ftp(self, user, passwd, host, port, dirs,
              ["foo", "bar"], "", None),
             ("ftp://localhost/baz.gif;type=a",
              "localhost", ftplib.FTP_PORT, "", "", "A",
-             [], "baz.gif", None),  # XXX really this should guess image/gif
+             [], "baz.gif", "image/gif"),
             ]:
             req = Request(url)
             req.timeout = None
diff --git a/Misc/NEWS.d/next/Library/2019-08-27-01-03-26.bpo-22347._TRpYr.rst b/Misc/NEWS.d/next/Library/2019-08-27-01-03-26.bpo-22347._TRpYr.rst
new file mode 100644
index 00000000000000..1a3c19938217c4
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-08-27-01-03-26.bpo-22347._TRpYr.rst
@@ -0,0 +1,2 @@
+Update mimetypes.guess_type to allow proper parsing of URLs with only a host name.
+Patch by Dong-hee Na.

From 6ad0a2c45f78020f7994e47620c1cf7b225f8197 Mon Sep 17 00:00:00 2001
From: Abhilash Raj 
Date: Wed, 4 Sep 2019 18:20:40 -0700
Subject: [PATCH 506/527] [3.8] bpo-37764: Fix infinite loop when parsing
 unstructured email headers. (GH-15239) (GH-15686)

Fixes a case in which email._header_value_parser.get_unstructured hangs the system for some invalid headers. This covers the cases in which the header contains either:
- a case without trailing whitespace
- an invalid encoded word

https://bugs.python.org/issue37764

This fix should also be backported to 3.7 and 3.8

https://bugs.python.org/issue37764
(cherry picked from commit c5b242f87f31286ad38991bc3868cf4cfbf2b681)

Co-authored-by: Ashwin Ramaswami 
---
 Lib/email/_header_value_parser.py             | 19 ++++++++++++++---
 .../test_email/test__header_value_parser.py   | 16 ++++++++++++++
 Lib/test/test_email/test_email.py             | 21 +++++++++++++++++++
 Misc/ACKS                                     |  1 +
 .../2019-08-27-01-13-05.bpo-37764.qv67PQ.rst  |  1 +
 5 files changed, 55 insertions(+), 3 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Security/2019-08-27-01-13-05.bpo-37764.qv67PQ.rst

diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py
index b5003943ab0d97..16c19907d68d59 100644
--- a/Lib/email/_header_value_parser.py
+++ b/Lib/email/_header_value_parser.py
@@ -935,6 +935,10 @@ def __str__(self):
         return ''
 
 
+class _InvalidEwError(errors.HeaderParseError):
+    """Invalid encoded word found while parsing headers."""
+
+
 # XXX these need to become classes and used as instances so
 # that a program can't change them in a parse tree and screw
 # up other parse trees.  Maybe should have  tests for that, too.
@@ -1039,7 +1043,10 @@ def get_encoded_word(value):
         raise errors.HeaderParseError(
             "expected encoded word but found {}".format(value))
     remstr = ''.join(remainder)
-    if len(remstr) > 1 and remstr[0] in hexdigits and remstr[1] in hexdigits:
+    if (len(remstr) > 1 and
+        remstr[0] in hexdigits and
+        remstr[1] in hexdigits and
+        tok.count('?') < 2):
         # The ? after the CTE was followed by an encoded word escape (=XX).
         rest, *remainder = remstr.split('?=', 1)
         tok = tok + '?=' + rest
@@ -1051,7 +1058,7 @@ def get_encoded_word(value):
     try:
         text, charset, lang, defects = _ew.decode('=?' + tok + '?=')
     except ValueError:
-        raise errors.HeaderParseError(
+        raise _InvalidEwError(
             "encoded word format invalid: '{}'".format(ew.cte))
     ew.charset = charset
     ew.lang = lang
@@ -1101,9 +1108,12 @@ def get_unstructured(value):
             token, value = get_fws(value)
             unstructured.append(token)
             continue
+        valid_ew = True
         if value.startswith('=?'):
             try:
                 token, value = get_encoded_word(value)
+            except _InvalidEwError:
+                valid_ew = False
             except errors.HeaderParseError:
                 # XXX: Need to figure out how to register defects when
                 # appropriate here.
@@ -1125,7 +1135,10 @@ def get_unstructured(value):
         # Split in the middle of an atom if there is a rfc2047 encoded word
         # which does not have WSP on both sides. The defect will be registered
         # the next time through the loop.
-        if rfc2047_matcher.search(tok):
+        # This needs to only be performed when the encoded word is valid;
+        # otherwise, performing it on an invalid encoded word can cause
+        # the parser to go in an infinite loop.
+        if valid_ew and rfc2047_matcher.search(tok):
             tok, *remainder = value.partition('=?')
         vtext = ValueTerminal(tok, 'vtext')
         _validate_xtext(vtext)
diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py
index bad4333dbc4351..dd33b065c804bc 100644
--- a/Lib/test/test_email/test__header_value_parser.py
+++ b/Lib/test/test_email/test__header_value_parser.py
@@ -383,6 +383,22 @@ def test_get_unstructured_ew_without_trailing_whitespace(self):
             [errors.InvalidHeaderDefect],
             '')
 
+    def test_get_unstructured_without_trailing_whitespace_hang_case(self):
+        self._test_get_x(self._get_unst,
+            '=?utf-8?q?somevalue?=aa',
+            'somevalueaa',
+            'somevalueaa',
+            [errors.InvalidHeaderDefect],
+            '')
+
+    def test_get_unstructured_invalid_ew(self):
+        self._test_get_x(self._get_unst,
+            '=?utf-8?q?=somevalue?=',
+            '=?utf-8?q?=somevalue?=',
+            '=?utf-8?q?=somevalue?=',
+            [],
+            '')
+
     # get_qp_ctext
 
     def test_get_qp_ctext_only(self):
diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py
index aa775881c5521a..5414cf070cc12f 100644
--- a/Lib/test/test_email/test_email.py
+++ b/Lib/test/test_email/test_email.py
@@ -5381,6 +5381,27 @@ def test_rfc2231_unencoded_then_encoded_segments(self):
         eq(language, 'en-us')
         eq(s, 'My Document For You')
 
+    def test_should_not_hang_on_invalid_ew_messages(self):
+        messages = ["""From: user@host.com
+To: user@host.com
+Bad-Header:
+ =?us-ascii?Q?LCSwrV11+IB0rSbSker+M9vWR7wEDSuGqmHD89Gt=ea0nJFSaiz4vX3XMJPT4vrE?=
+ =?us-ascii?Q?xGUZeOnp0o22pLBB7CYLH74Js=wOlK6Tfru2U47qR?=
+ =?us-ascii?Q?72OfyEY2p2=2FrA9xNFyvH+fBTCmazxwzF8nGkK6D?=
+
+Hello!
+""", """From: ����� �������� 
+To: "xxx" 
+Subject:   ��� ���������� ����� ����� � ��������� �� ����
+MIME-Version: 1.0
+Content-Type: text/plain; charset="windows-1251";
+Content-Transfer-Encoding: 8bit
+
+�� ����� � ���� ������ ��� ��������
+"""]
+        for m in messages:
+            with self.subTest(m=m):
+                msg = email.message_from_string(m)
 
 
 # Tests to ensure that signed parts of an email are completely preserved, as
diff --git a/Misc/ACKS b/Misc/ACKS
index 24e327a5f86e99..def874b0071e6f 100644
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -1330,6 +1330,7 @@ Burton Radons
 Abhilash Raj
 Shorya Raj
 Dhushyanth Ramasamy
+Ashwin Ramaswami
 Jeff Ramnani
 Bayard Randel
 Varpu Rantala
diff --git a/Misc/NEWS.d/next/Security/2019-08-27-01-13-05.bpo-37764.qv67PQ.rst b/Misc/NEWS.d/next/Security/2019-08-27-01-13-05.bpo-37764.qv67PQ.rst
new file mode 100644
index 00000000000000..27fa8e192f0c07
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2019-08-27-01-13-05.bpo-37764.qv67PQ.rst
@@ -0,0 +1 @@
+Fixes email._header_value_parser.get_unstructured going into an infinite loop for a specific case in which the email header does not have trailing whitespace, and the case in which it contains an invalid encoded word. Patch by Ashwin Ramaswami.
\ No newline at end of file

From 9c2654d1aa85968fede1b888fba86aebc06c5be6 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
 <31488909+miss-islington@users.noreply.github.com>
Date: Wed, 4 Sep 2019 18:53:47 -0700
Subject: [PATCH 507/527] bpo-37902: IDLE: Add scrolling for IDLE browsers.
 (GH-15368)

Modify the wheel event handler so it can also be used for module, path, and stack browsers.
Patch by George Zhang.
(cherry picked from commit 2cd902585815582eb059e3b40e014ebe4e7fdee7)

Co-authored-by: GeeTransit 
---
 Lib/idlelib/NEWS.txt                          |  3 ++
 Lib/idlelib/editor.py                         | 25 +++------------
 Lib/idlelib/idle_test/test_multicall.py       |  8 +++++
 Lib/idlelib/idle_test/test_tree.py            | 29 ++++++++++++++++-
 Lib/idlelib/tree.py                           | 31 +++++++++++++++++++
 Misc/ACKS                                     |  1 +
 .../2019-08-21-16-02-49.bpo-37902._R_adE.rst  |  2 ++
 7 files changed, 78 insertions(+), 21 deletions(-)
 create mode 100644 Misc/NEWS.d/next/IDLE/2019-08-21-16-02-49.bpo-37902._R_adE.rst

diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt
index 47c2291d237727..c9e846a6fba667 100644
--- a/Lib/idlelib/NEWS.txt
+++ b/Lib/idlelib/NEWS.txt
@@ -3,6 +3,9 @@ Released on 2019-10-20?
 ======================================
 
 
+bpo-37092: Add mousewheel scrolling for IDLE module, path, and stack
+browsers.  Patch by George Zhang.
+
 bpo-35771: To avoid occasional spurious test_idle failures on slower
 machines, increase the ``hover_delay`` in test_tooltip.
 
diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py
index 793ed3afaed0ff..5cbf704ab27b6a 100644
--- a/Lib/idlelib/editor.py
+++ b/Lib/idlelib/editor.py
@@ -26,6 +26,7 @@
 from idlelib import query
 from idlelib import replace
 from idlelib import search
+from idlelib.tree import wheel_event
 from idlelib import window
 
 # The default tab setting for a Text widget, in average-width characters.
@@ -151,9 +152,10 @@ def __init__(self, flist=None, filename=None, key=None, root=None):
         else:
             # Elsewhere, use right-click for popup menus.
             text.bind("<3>",self.right_menu_event)
-        text.bind('', self.mousescroll)
-        text.bind('', self.mousescroll)
-        text.bind('', self.mousescroll)
+
+        text.bind('', wheel_event)
+        text.bind('', wheel_event)
+        text.bind('', wheel_event)
         text.bind('', self.handle_winconfig)
         text.bind("<>", self.cut)
         text.bind("<>", self.copy)
@@ -502,23 +504,6 @@ def handle_yview(self, event, *args):
         self.text.yview(event, *args)
         return 'break'
 
-    def mousescroll(self, event):
-        """Handle scrollwheel event.
-
-        For wheel up, event.delta = 120*n on Windows, -1*n on darwin,
-        where n can be > 1 if one scrolls fast.  Flicking the wheel
-        generates up to maybe 20 events with n up to 10 or more 1.
-        Macs use wheel down (delta = 1*n) to scroll up, so positive
-        delta means to scroll up on both systems.
-
-        X-11 sends Control-Button-4 event instead.
-        """
-        up = {EventType.MouseWheel: event.delta > 0,
-              EventType.Button: event.num == 4}
-        lines = -5 if up[event.type] else 5
-        self.text.yview_scroll(lines, 'units')
-        return 'break'
-
     rmenu = None
 
     def right_menu_event(self, event):
diff --git a/Lib/idlelib/idle_test/test_multicall.py b/Lib/idlelib/idle_test/test_multicall.py
index 68156a743d7b9b..ba582bb3ca51b4 100644
--- a/Lib/idlelib/idle_test/test_multicall.py
+++ b/Lib/idlelib/idle_test/test_multicall.py
@@ -35,6 +35,14 @@ def test_init(self):
         mctext = self.mc(self.root)
         self.assertIsInstance(mctext._MultiCall__binders, list)
 
+    def test_yview(self):
+        # Added for tree.wheel_event
+        # (it depends on yview to not be overriden)
+        mc = self.mc
+        self.assertIs(mc.yview, Text.yview)
+        mctext = self.mc(self.root)
+        self.assertIs(mctext.yview.__func__, Text.yview)
+
 
 if __name__ == '__main__':
     unittest.main(verbosity=2)
diff --git a/Lib/idlelib/idle_test/test_tree.py b/Lib/idlelib/idle_test/test_tree.py
index 9be9abee361f08..b3e4c10cf9e38e 100644
--- a/Lib/idlelib/idle_test/test_tree.py
+++ b/Lib/idlelib/idle_test/test_tree.py
@@ -4,7 +4,7 @@
 import unittest
 from test.support import requires
 requires('gui')
-from tkinter import Tk
+from tkinter import Tk, EventType, SCROLL
 
 
 class TreeTest(unittest.TestCase):
@@ -29,5 +29,32 @@ def test_init(self):
         node.expand()
 
 
+class TestScrollEvent(unittest.TestCase):
+
+    def test_wheel_event(self):
+        # Fake widget class containing `yview` only.
+        class _Widget:
+            def __init__(widget, *expected):
+                widget.expected = expected
+            def yview(widget, *args):
+                self.assertTupleEqual(widget.expected, args)
+        # Fake event class
+        class _Event:
+            pass
+        #        (type, delta, num, amount)
+        tests = ((EventType.MouseWheel, 120, -1, -5),
+                 (EventType.MouseWheel, -120, -1, 5),
+                 (EventType.ButtonPress, -1, 4, -5),
+                 (EventType.ButtonPress, -1, 5, 5))
+
+        event = _Event()
+        for ty, delta, num, amount in tests:
+            event.type = ty
+            event.delta = delta
+            event.num = num
+            res = tree.wheel_event(event, _Widget(SCROLL, amount, "units"))
+            self.assertEqual(res, "break")
+
+
 if __name__ == '__main__':
     unittest.main(verbosity=2)
diff --git a/Lib/idlelib/tree.py b/Lib/idlelib/tree.py
index 21426cbb33e0da..6229be4e5a8ad5 100644
--- a/Lib/idlelib/tree.py
+++ b/Lib/idlelib/tree.py
@@ -56,6 +56,30 @@ def listicons(icondir=ICONDIR):
             column = 0
     root.images = images
 
+def wheel_event(event, widget=None):
+    """Handle scrollwheel event.
+
+    For wheel up, event.delta = 120*n on Windows, -1*n on darwin,
+    where n can be > 1 if one scrolls fast.  Flicking the wheel
+    generates up to maybe 20 events with n up to 10 or more 1.
+    Macs use wheel down (delta = 1*n) to scroll up, so positive
+    delta means to scroll up on both systems.
+
+    X-11 sends Control-Button-4,5 events instead.
+
+    The widget parameter is needed so browser label bindings can pass
+    the underlying canvas.
+
+    This function depends on widget.yview to not be overridden by
+    a subclass.
+    """
+    up = {EventType.MouseWheel: event.delta > 0,
+          EventType.ButtonPress: event.num == 4}
+    lines = -5 if up[event.type] else 5
+    widget = event.widget if widget is None else widget
+    widget.yview(SCROLL, lines, 'units')
+    return 'break'
+
 
 class TreeNode:
 
@@ -260,6 +284,9 @@ def drawtext(self):
                                        anchor="nw", window=self.label)
         self.label.bind("<1>", self.select_or_edit)
         self.label.bind("", self.flip)
+        self.label.bind("", lambda e: wheel_event(e, self.canvas))
+        self.label.bind("", lambda e: wheel_event(e, self.canvas))
+        self.label.bind("", lambda e: wheel_event(e, self.canvas))
         self.text_id = id
 
     def select_or_edit(self, event=None):
@@ -410,6 +437,7 @@ def GetSubList(self):
 # A canvas widget with scroll bars and some useful bindings
 
 class ScrolledCanvas:
+
     def __init__(self, master, **opts):
         if 'yscrollincrement' not in opts:
             opts['yscrollincrement'] = 17
@@ -431,6 +459,9 @@ def __init__(self, master, **opts):
         self.canvas.bind("", self.page_down)
         self.canvas.bind("", self.unit_up)
         self.canvas.bind("", self.unit_down)
+        self.canvas.bind("", wheel_event)
+        self.canvas.bind("", wheel_event)
+        self.canvas.bind("", wheel_event)
         #if isinstance(master, Toplevel) or isinstance(master, Tk):
         self.canvas.bind("", self.zoom_height)
         self.canvas.focus_set()
diff --git a/Misc/ACKS b/Misc/ACKS
index def874b0071e6f..290f8ea33269b3 100644
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -1863,6 +1863,7 @@ Nickolai Zeldovich
 Yuxiao Zeng
 Uwe Zessin
 Cheng Zhang
+George Zhang
 Kai Zhu
 Tarek Ziadé
 Jelle Zijlstra
diff --git a/Misc/NEWS.d/next/IDLE/2019-08-21-16-02-49.bpo-37902._R_adE.rst b/Misc/NEWS.d/next/IDLE/2019-08-21-16-02-49.bpo-37902._R_adE.rst
new file mode 100644
index 00000000000000..24b4142484695c
--- /dev/null
+++ b/Misc/NEWS.d/next/IDLE/2019-08-21-16-02-49.bpo-37902._R_adE.rst
@@ -0,0 +1,2 @@
+Add mousewheel scrolling for IDLE module, path, and stack browsers.
+Patch by George Zhang.

From bdcbb83c6640c2b30d0e1a2a497e9ba34cc53b26 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
 <31488909+miss-islington@users.noreply.github.com>
Date: Wed, 4 Sep 2019 21:25:59 -0700
Subject: [PATCH 508/527] bpo-38026: fix inspect.getattr_static (GH-15676)

It should avoid dynamic lookup including `isinstance`.

This is a regression caused by GH-5351.
(cherry picked from commit 8f9cc8771ffb8d0e21be287eaed42ae06087acca)

Co-authored-by: Inada Naoki 
---
 Lib/inspect.py                                                | 4 ++--
 .../next/Library/2019-09-04-20-34-14.bpo-38026.0LLRX-.rst     | 2 ++
 2 files changed, 4 insertions(+), 2 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Library/2019-09-04-20-34-14.bpo-38026.0LLRX-.rst

diff --git a/Lib/inspect.py b/Lib/inspect.py
index 99a580bd2f2350..a616f2d49b7d96 100644
--- a/Lib/inspect.py
+++ b/Lib/inspect.py
@@ -1558,7 +1558,7 @@ def _shadowed_dict(klass):
         except KeyError:
             pass
         else:
-            if not (isinstance(class_dict, types.GetSetDescriptorType) and
+            if not (type(class_dict) is types.GetSetDescriptorType and
                     class_dict.__name__ == "__dict__" and
                     class_dict.__objclass__ is entry):
                 return class_dict
@@ -1580,7 +1580,7 @@ def getattr_static(obj, attr, default=_sentinel):
         klass = type(obj)
         dict_attr = _shadowed_dict(klass)
         if (dict_attr is _sentinel or
-            isinstance(dict_attr, types.MemberDescriptorType)):
+            type(dict_attr) is types.MemberDescriptorType):
             instance_result = _check_instance(obj, attr)
     else:
         klass = obj
diff --git a/Misc/NEWS.d/next/Library/2019-09-04-20-34-14.bpo-38026.0LLRX-.rst b/Misc/NEWS.d/next/Library/2019-09-04-20-34-14.bpo-38026.0LLRX-.rst
new file mode 100644
index 00000000000000..166cd867514cae
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-09-04-20-34-14.bpo-38026.0LLRX-.rst
@@ -0,0 +1,2 @@
+Fixed :func:`inspect.getattr_static` used ``isinstance`` while it should
+avoid dynamic lookup.

From dafbe3265657dc5a5c46e762080023f0aa25ec58 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
 <31488909+miss-islington@users.noreply.github.com>
Date: Thu, 5 Sep 2019 00:42:22 -0700
Subject: [PATCH 509/527] bpo-36324:  Apply review comments from Allen Downey
 (GH-15693) (GH-15694)

(cherry picked from commit e4810b2a6c1d0db1a27ad046831b8fa3b57967a4)

Co-authored-by: Raymond Hettinger 
---
 Doc/library/statistics.rst | 129 +++++++++++++++++++------------------
 Lib/statistics.py          |  38 +++++------
 Misc/ACKS                  |   1 +
 3 files changed, 83 insertions(+), 85 deletions(-)

diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst
index cbe2b8223faf9b..9f9fc86b3a1350 100644
--- a/Doc/library/statistics.rst
+++ b/Doc/library/statistics.rst
@@ -26,10 +26,10 @@ numeric (:class:`Real`-valued) data.
    Unless explicitly noted otherwise, these functions support :class:`int`,
    :class:`float`, :class:`decimal.Decimal` and :class:`fractions.Fraction`.
    Behaviour with other types (whether in the numeric tower or not) is
-   currently unsupported.  Mixed types are also undefined and
-   implementation-dependent.  If your input data consists of mixed types,
-   you may be able to use :func:`map` to ensure a consistent result, e.g.
-   ``map(float, input_data)``.
+   currently unsupported.  Collections with a mix of types are also undefined
+   and implementation-dependent.  If your input data consists of mixed types,
+   you may be able to use :func:`map` to ensure a consistent result, for
+   example: ``map(float, input_data)``.
 
 Averages and measures of central location
 -----------------------------------------
@@ -102,11 +102,9 @@ However, for reading convenience, most of the examples show sorted sequences.
    .. note::
 
       The mean is strongly affected by outliers and is not a robust estimator
-      for central location: the mean is not necessarily a typical example of the
-      data points.  For more robust, although less efficient, measures of
-      central location, see :func:`median` and :func:`mode`.  (In this case,
-      "efficient" refers to statistical efficiency rather than computational
-      efficiency.)
+      for central location: the mean is not necessarily a typical example of
+      the data points.  For more robust measures of central location, see
+      :func:`median` and :func:`mode`.
 
       The sample mean gives an unbiased estimate of the true population mean,
       which means that, taken on average over all the possible samples,
@@ -120,9 +118,8 @@ However, for reading convenience, most of the examples show sorted sequences.
    Convert *data* to floats and compute the arithmetic mean.
 
    This runs faster than the :func:`mean` function and it always returns a
-   :class:`float`.  The result is highly accurate but not as perfect as
-   :func:`mean`.  If the input dataset is empty, raises a
-   :exc:`StatisticsError`.
+   :class:`float`.  The *data* may be a sequence or iterator.  If the input
+   dataset is empty, raises a :exc:`StatisticsError`.
 
    .. doctest::
 
@@ -136,15 +133,20 @@ However, for reading convenience, most of the examples show sorted sequences.
 
    Convert *data* to floats and compute the geometric mean.
 
+   The geometric mean indicates the central tendency or typical value of the
+   *data* using the product of the values (as opposed to the arithmetic mean
+   which uses their sum).
+
    Raises a :exc:`StatisticsError` if the input dataset is empty,
    if it contains a zero, or if it contains a negative value.
+   The *data* may be a sequence or iterator.
 
    No special efforts are made to achieve exact results.
    (However, this may change in the future.)
 
    .. doctest::
 
-      >>> round(geometric_mean([54, 24, 36]), 9)
+      >>> round(geometric_mean([54, 24, 36]), 1)
       36.0
 
    .. versionadded:: 3.8
@@ -174,7 +176,7 @@ However, for reading convenience, most of the examples show sorted sequences.
       3.6
 
    Using the arithmetic mean would give an average of about 5.167, which
-   is too high.
+   is well over the aggregate P/E ratio.
 
    :exc:`StatisticsError` is raised if *data* is empty, or any element
    is less than zero.
@@ -312,10 +314,10 @@ However, for reading convenience, most of the examples show sorted sequences.
    The mode (when it exists) is the most typical value and serves as a
    measure of central location.
 
-   If there are multiple modes, returns the first one encountered in the *data*.
-   If the smallest or largest of multiple modes is desired instead, use
-   ``min(multimode(data))`` or ``max(multimode(data))``.  If the input *data* is
-   empty, :exc:`StatisticsError` is raised.
+   If there are multiple modes with the same frequency, returns the first one
+   encountered in the *data*.  If the smallest or largest of those is
+   desired instead, use ``min(multimode(data))`` or ``max(multimode(data))``.
+   If the input *data* is empty, :exc:`StatisticsError` is raised.
 
    ``mode`` assumes discrete data, and returns a single value. This is the
    standard treatment of the mode as commonly taught in schools:
@@ -325,8 +327,8 @@ However, for reading convenience, most of the examples show sorted sequences.
       >>> mode([1, 1, 2, 3, 3, 3, 3, 4])
       3
 
-   The mode is unique in that it is the only statistic which also applies
-   to nominal (non-numeric) data:
+   The mode is unique in that it is the only statistic in this package that
+   also applies to nominal (non-numeric) data:
 
    .. doctest::
 
@@ -368,15 +370,16 @@ However, for reading convenience, most of the examples show sorted sequences.
 
 .. function:: pvariance(data, mu=None)
 
-   Return the population variance of *data*, a non-empty iterable of real-valued
-   numbers.  Variance, or second moment about the mean, is a measure of the
-   variability (spread or dispersion) of data.  A large variance indicates that
-   the data is spread out; a small variance indicates it is clustered closely
-   around the mean.
+   Return the population variance of *data*, a non-empty sequence or iterator
+   of real-valued numbers.  Variance, or second moment about the mean, is a
+   measure of the variability (spread or dispersion) of data.  A large
+   variance indicates that the data is spread out; a small variance indicates
+   it is clustered closely around the mean.
 
-   If the optional second argument *mu* is given, it should be the mean of
-   *data*.  If it is missing or ``None`` (the default), the mean is
-   automatically calculated.
+   If the optional second argument *mu* is given, it is typically the mean of
+   the *data*.  It can also be used to compute the second moment around a
+   point that is not the mean.  If it is missing or ``None`` (the default),
+   the arithmetic mean is automatically calculated.
 
    Use this function to calculate the variance from the entire population.  To
    estimate the variance from a sample, the :func:`variance` function is usually
@@ -401,10 +404,6 @@ However, for reading convenience, most of the examples show sorted sequences.
       >>> pvariance(data, mu)
       1.25
 
-   This function does not attempt to verify that you have passed the actual mean
-   as *mu*.  Using arbitrary values for *mu* may lead to invalid or impossible
-   results.
-
    Decimals and Fractions are supported:
 
    .. doctest::
@@ -423,11 +422,11 @@ However, for reading convenience, most of the examples show sorted sequences.
       σ².  When called on a sample instead, this is the biased sample variance
       s², also known as variance with N degrees of freedom.
 
-      If you somehow know the true population mean μ, you may use this function
-      to calculate the variance of a sample, giving the known population mean as
-      the second argument.  Provided the data points are representative
-      (e.g. independent and identically distributed), the result will be an
-      unbiased estimate of the population variance.
+      If you somehow know the true population mean μ, you may use this
+      function to calculate the variance of a sample, giving the known
+      population mean as the second argument.  Provided the data points are a
+      random sample of the population, the result will be an unbiased estimate
+      of the population variance.
 
 
 .. function:: stdev(data, xbar=None)
@@ -502,19 +501,19 @@ However, for reading convenience, most of the examples show sorted sequences.
       :func:`pvariance` function as the *mu* parameter to get the variance of a
       sample.
 
-.. function:: quantiles(dist, *, n=4, method='exclusive')
+.. function:: quantiles(data, *, n=4, method='exclusive')
 
-   Divide *dist* into *n* continuous intervals with equal probability.
+   Divide *data* into *n* continuous intervals with equal probability.
    Returns a list of ``n - 1`` cut points separating the intervals.
 
    Set *n* to 4 for quartiles (the default).  Set *n* to 10 for deciles.  Set
    *n* to 100 for percentiles which gives the 99 cuts points that separate
-   *dist* in to 100 equal sized groups.  Raises :exc:`StatisticsError` if *n*
+   *data* in to 100 equal sized groups.  Raises :exc:`StatisticsError` if *n*
    is not least 1.
 
-   The *dist* can be any iterable containing sample data or it can be an
+   The *data* can be any iterable containing sample data or it can be an
    instance of a class that defines an :meth:`~inv_cdf` method.  For meaningful
-   results, the number of data points in *dist* should be larger than *n*.
+   results, the number of data points in *data* should be larger than *n*.
    Raises :exc:`StatisticsError` if there are not at least two data points.
 
    For sample data, the cut points are linearly interpolated from the
@@ -523,7 +522,7 @@ However, for reading convenience, most of the examples show sorted sequences.
    cut-point will evaluate to ``104``.
 
    The *method* for computing quantiles can be varied depending on
-   whether the data in *dist* includes or excludes the lowest and
+   whether the data in *data* includes or excludes the lowest and
    highest possible values from the population.
 
    The default *method* is "exclusive" and is used for data sampled from
@@ -535,14 +534,14 @@ However, for reading convenience, most of the examples show sorted sequences.
 
    Setting the *method* to "inclusive" is used for describing population
    data or for samples that are known to include the most extreme values
-   from the population.  The minimum value in *dist* is treated as the 0th
+   from the population.  The minimum value in *data* is treated as the 0th
    percentile and the maximum value is treated as the 100th percentile.
    The portion of the population falling below the *i-th* of *m* sorted
    data points is computed as ``(i - 1) / (m - 1)``.  Given 11 sample
    values, the method sorts them and assigns the following percentiles:
    0%, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%.
 
-   If *dist* is an instance of a class that defines an
+   If *data* is an instance of a class that defines an
    :meth:`~inv_cdf` method, setting *method* has no effect.
 
    .. doctest::
@@ -580,7 +579,7 @@ A single exception is defined:
 :class:`NormalDist` is a tool for creating and manipulating normal
 distributions of a `random variable
 `_.  It is a
-composite class that treats the mean and standard deviation of data
+class that treats the mean and standard deviation of data
 measurements as a single entity.
 
 Normal distributions arise from the `Central Limit Theorem
@@ -616,13 +615,14 @@ of applications in statistics.
 
     .. classmethod:: NormalDist.from_samples(data)
 
-       Makes a normal distribution instance computed from sample data.  The
-       *data* can be any :term:`iterable` and should consist of values that
-       can be converted to type :class:`float`.
+       Makes a normal distribution instance with *mu* and *sigma* parameters
+       estimated from the *data* using :func:`fmean` and :func:`stdev`.
 
-       If *data* does not contain at least two elements, raises
-       :exc:`StatisticsError` because it takes at least one point to estimate
-       a central value and at least two points to estimate dispersion.
+       The *data* can be any :term:`iterable` and should consist of values
+       that can be converted to type :class:`float`.  If *data* does not
+       contain at least two elements, raises :exc:`StatisticsError` because it
+       takes at least one point to estimate a central value and at least two
+       points to estimate dispersion.
 
     .. method:: NormalDist.samples(n, *, seed=None)
 
@@ -636,10 +636,10 @@ of applications in statistics.
     .. method:: NormalDist.pdf(x)
 
        Using a `probability density function (pdf)
-       `_,
-       compute the relative likelihood that a random variable *X* will be near
-       the given value *x*.  Mathematically, it is the ratio ``P(x <= X <
-       x+dx) / dx``.
+       `_, compute
+       the relative likelihood that a random variable *X* will be near the
+       given value *x*.  Mathematically, it is the limit of the ratio ``P(x <=
+       X < x+dx) / dx`` as *dx* approaches zero.
 
        The relative likelihood is computed as the probability of a sample
        occurring in a narrow range divided by the width of the range (hence
@@ -667,8 +667,10 @@ of applications in statistics.
 
     .. method:: NormalDist.overlap(other)
 
-       Returns a value between 0.0 and 1.0 giving the overlapping area for
-       the two probability density functions.
+       Measures the agreement between two normal probability distributions.
+       Returns a value between 0.0 and 1.0 giving `the overlapping area for
+       the two probability density functions
+       `_.
 
     Instances of :class:`NormalDist` support addition, subtraction,
     multiplication and division by a constant.  These operations
@@ -740,12 +742,11 @@ Carlo simulation `_:
     ...     return (3*x + 7*x*y - 5*y) / (11 * z)
     ...
     >>> n = 100_000
-    >>> seed = 86753099035768
-    >>> X = NormalDist(10, 2.5).samples(n, seed=seed)
-    >>> Y = NormalDist(15, 1.75).samples(n, seed=seed)
-    >>> Z = NormalDist(50, 1.25).samples(n, seed=seed)
-    >>> NormalDist.from_samples(map(model, X, Y, Z))     # doctest: +SKIP
-    NormalDist(mu=1.8661894803304777, sigma=0.65238717376862)
+    >>> X = NormalDist(10, 2.5).samples(n, seed=3652260728)
+    >>> Y = NormalDist(15, 1.75).samples(n, seed=4582495471)
+    >>> Z = NormalDist(50, 1.25).samples(n, seed=6582483453)
+    >>> quantiles(map(model, X, Y, Z))       # doctest: +SKIP
+    [1.4591308524824727, 1.8035946855390597, 2.175091447274739]
 
 Normal distributions commonly arise in machine learning problems.
 
diff --git a/Lib/statistics.py b/Lib/statistics.py
index c7d6568145e0fa..4b172662770fb0 100644
--- a/Lib/statistics.py
+++ b/Lib/statistics.py
@@ -322,7 +322,6 @@ def fmean(data):
     """Convert data to floats and compute the arithmetic mean.
 
     This runs faster than the mean() function and it always returns a float.
-    The result is highly accurate but not as perfect as mean().
     If the input dataset is empty, it raises a StatisticsError.
 
     >>> fmean([3.5, 4.0, 5.25])
@@ -538,15 +537,16 @@ def mode(data):
     ``mode`` assumes discrete data, and returns a single value. This is the
     standard treatment of the mode as commonly taught in schools:
 
-    >>> mode([1, 1, 2, 3, 3, 3, 3, 4])
-    3
+        >>> mode([1, 1, 2, 3, 3, 3, 3, 4])
+        3
 
     This also works with nominal (non-numeric) data:
 
-    >>> mode(["red", "blue", "blue", "red", "green", "red", "red"])
-    'red'
+        >>> mode(["red", "blue", "blue", "red", "green", "red", "red"])
+        'red'
 
-    If there are multiple modes, return the first one encountered.
+    If there are multiple modes with same frequency, return the first one
+    encountered:
 
         >>> mode(['red', 'red', 'green', 'blue', 'blue'])
         'red'
@@ -615,28 +615,28 @@ def multimode(data):
 # position is that fewer options make for easier choices and that
 # external packages can be used for anything more advanced.
 
-def quantiles(dist, /, *, n=4, method='exclusive'):
-    """Divide *dist* into *n* continuous intervals with equal probability.
+def quantiles(data, /, *, n=4, method='exclusive'):
+    """Divide *data* into *n* continuous intervals with equal probability.
 
     Returns a list of (n - 1) cut points separating the intervals.
 
     Set *n* to 4 for quartiles (the default).  Set *n* to 10 for deciles.
     Set *n* to 100 for percentiles which gives the 99 cuts points that
-    separate *dist* in to 100 equal sized groups.
+    separate *data* in to 100 equal sized groups.
 
-    The *dist* can be any iterable containing sample data or it can be
+    The *data* can be any iterable containing sample data or it can be
     an instance of a class that defines an inv_cdf() method.  For sample
     data, the cut points are linearly interpolated between data points.
 
-    If *method* is set to *inclusive*, *dist* is treated as population
+    If *method* is set to *inclusive*, *data* is treated as population
     data.  The minimum value is treated as the 0th percentile and the
     maximum value is treated as the 100th percentile.
     """
     if n < 1:
         raise StatisticsError('n must be at least 1')
-    if hasattr(dist, 'inv_cdf'):
-        return [dist.inv_cdf(i / n) for i in range(1, n)]
-    data = sorted(dist)
+    if hasattr(data, 'inv_cdf'):
+        return [data.inv_cdf(i / n) for i in range(1, n)]
+    data = sorted(data)
     ld = len(data)
     if ld < 2:
         raise StatisticsError('must have at least two data points')
@@ -745,7 +745,7 @@ def variance(data, xbar=None):
 def pvariance(data, mu=None):
     """Return the population variance of ``data``.
 
-    data should be an iterable of Real-valued numbers, with at least one
+    data should be a sequence or iterator of Real-valued numbers, with at least one
     value. The optional argument mu, if given, should be the mean of
     the data. If it is missing or None, the mean is automatically calculated.
 
@@ -766,10 +766,6 @@ def pvariance(data, mu=None):
     >>> pvariance(data, mu)
     1.25
 
-    This function does not check that ``mu`` is actually the mean of ``data``.
-    Giving arbitrary values for ``mu`` may lead to invalid or impossible
-    results.
-
     Decimals and Fractions are supported:
 
     >>> from decimal import Decimal as D
@@ -913,8 +909,8 @@ def __init__(self, mu=0.0, sigma=1.0):
         "NormalDist where mu is the mean and sigma is the standard deviation."
         if sigma < 0.0:
             raise StatisticsError('sigma must be non-negative')
-        self._mu = mu
-        self._sigma = sigma
+        self._mu = float(mu)
+        self._sigma = float(sigma)
 
     @classmethod
     def from_samples(cls, data):
diff --git a/Misc/ACKS b/Misc/ACKS
index 290f8ea33269b3..55aab6b6b2b324 100644
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -414,6 +414,7 @@ Dima Dorfman
 Yves Dorfsman
 Michael Dorman
 Steve Dower
+Allen Downey
 Cesar Douady
 Dean Draayer
 Fred L. Drake, Jr.

From f5649bfe7622447b302ef55e4db3a96b5840f8e8 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
 <31488909+miss-islington@users.noreply.github.com>
Date: Thu, 5 Sep 2019 01:10:40 -0700
Subject: [PATCH 510/527] bpo-36324:  Apply review comment from Jake Vanderplas
 (GH-15695) (GH-15696)

(cherry picked from commit 9b51570ffd0494c07dafe10c7d2afe865754694c)

Co-authored-by: Raymond Hettinger 
---
 Doc/library/statistics.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst
index 9f9fc86b3a1350..2ddd393a459368 100644
--- a/Doc/library/statistics.rst
+++ b/Doc/library/statistics.rst
@@ -727,9 +727,9 @@ Find the `quartiles `_ and `deciles
 
 .. doctest::
 
-    >>> [round(sat.inv_cdf(p)) for p in (0.25, 0.50, 0.75)]
+    >>> list(map(round, quantiles(sat)))
     [928, 1060, 1192]
-    >>> [round(sat.inv_cdf(p / 10)) for p in range(1, 10)]
+    >>> list(map(round, quantiles(sat, n=10)))
     [810, 896, 958, 1011, 1060, 1109, 1162, 1224, 1310]
 
 To estimate the distribution for a model than isn't easy to solve

From 7eaeddad75cc736d327b8ece9380ef6ee23a0d9a Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
 <31488909+miss-islington@users.noreply.github.com>
Date: Thu, 5 Sep 2019 04:17:41 -0700
Subject: [PATCH 511/527] Correct minor gramatical mistake in sys.settrace doc
 (GH-15637)

(cherry picked from commit 3038e87ba848023470f571242a8bb5a206c24430)

Co-authored-by: Andre Delfino 
---
 Doc/library/sys.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst
index 09a987ca32ace9..be1af371d428a5 100644
--- a/Doc/library/sys.rst
+++ b/Doc/library/sys.rst
@@ -1276,7 +1276,8 @@ always available.
 
    The trace function is invoked (with *event* set to ``'call'``) whenever a new
    local scope is entered; it should return a reference to a local trace
-   function to be used that scope, or ``None`` if the scope shouldn't be traced.
+   function to be used for the new scope, or ``None`` if the scope shouldn't be
+   traced.
 
    The local trace function should return a reference to itself (or to another
    function for further tracing in that scope), or ``None`` to turn off tracing

From b8c66779c7003071f1a330428d58bbbb34c7ae12 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
 <31488909+miss-islington@users.noreply.github.com>
Date: Thu, 5 Sep 2019 23:19:13 -0700
Subject: [PATCH 512/527] More refinements to the statistics docs (GH-15713)
 (GH-15715)

(cherry picked from commit d8c93aa5d29d3cab537357018d5806a57452a8fe)

Co-authored-by: Raymond Hettinger 
---
 Doc/library/statistics.rst | 60 +++++++++++++++++++++-----------------
 1 file changed, 33 insertions(+), 27 deletions(-)

diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst
index 2ddd393a459368..9e39828a5700c8 100644
--- a/Doc/library/statistics.rst
+++ b/Doc/library/statistics.rst
@@ -19,17 +19,21 @@
 --------------
 
 This module provides functions for calculating mathematical statistics of
-numeric (:class:`Real`-valued) data.
-
-.. note::
-
-   Unless explicitly noted otherwise, these functions support :class:`int`,
-   :class:`float`, :class:`decimal.Decimal` and :class:`fractions.Fraction`.
-   Behaviour with other types (whether in the numeric tower or not) is
-   currently unsupported.  Collections with a mix of types are also undefined
-   and implementation-dependent.  If your input data consists of mixed types,
-   you may be able to use :func:`map` to ensure a consistent result, for
-   example: ``map(float, input_data)``.
+numeric (:class:`~numbers.Real`-valued) data.
+
+The module is not intended to be a competitor to third-party libraries such
+as `NumPy `_, `SciPy `_, or
+proprietary full-featured statistics packages aimed at professional
+statisticians such as Minitab, SAS and Matlab. It is aimed at the level of
+graphing and scientific calculators.
+
+Unless explicitly noted, these functions support :class:`int`,
+:class:`float`, :class:`~decimal.Decimal` and :class:`~fractions.Fraction`.
+Behaviour with other types (whether in the numeric tower or not) is
+currently unsupported.  Collections with a mix of types are also undefined
+and implementation-dependent.  If your input data consists of mixed types,
+you may be able to use :func:`map` to ensure a consistent result, for
+example: ``map(float, input_data)``.
 
 Averages and measures of central location
 -----------------------------------------
@@ -107,7 +111,7 @@ However, for reading convenience, most of the examples show sorted sequences.
       :func:`median` and :func:`mode`.
 
       The sample mean gives an unbiased estimate of the true population mean,
-      which means that, taken on average over all the possible samples,
+      so that when taken on average over all the possible samples,
       ``mean(sample)`` converges on the true mean of the entire population.  If
       *data* represents the entire population rather than a sample, then
       ``mean(data)`` is equivalent to calculating the true population mean μ.
@@ -163,8 +167,16 @@ However, for reading convenience, most of the examples show sorted sequences.
    will be equivalent to ``3/(1/a + 1/b + 1/c)``.
 
    The harmonic mean is a type of average, a measure of the central
-   location of the data.  It is often appropriate when averaging quantities
-   which are rates or ratios, for example speeds. For example:
+   location of the data.  It is often appropriate when averaging
+   rates or ratios, for example speeds.
+
+   Suppose a car travels 10 km at 40 km/hr, then another 10 km at 60 km/hr.
+   What is the average speed?
+
+   .. doctest::
+
+      >>> harmonic_mean([40, 60])
+      48.0
 
    Suppose an investor purchases an equal value of shares in each of
    three companies, with P/E (price/earning) ratios of 2.5, 3 and 10.
@@ -175,9 +187,6 @@ However, for reading convenience, most of the examples show sorted sequences.
       >>> harmonic_mean([2.5, 3, 10])  # For an equal investment portfolio.
       3.6
 
-   Using the arithmetic mean would give an average of about 5.167, which
-   is well over the aggregate P/E ratio.
-
    :exc:`StatisticsError` is raised if *data* is empty, or any element
    is less than zero.
 
@@ -190,9 +199,9 @@ However, for reading convenience, most of the examples show sorted sequences.
    middle two" method.  If *data* is empty, :exc:`StatisticsError` is raised.
    *data* can be a sequence or iterator.
 
-   The median is a robust measure of central location, and is less affected by
-   the presence of outliers in your data.  When the number of data points is
-   odd, the middle data point is returned:
+   The median is a robust measure of central location and is less affected by
+   the presence of outliers.  When the number of data points is odd, the
+   middle data point is returned:
 
    .. doctest::
 
@@ -210,13 +219,10 @@ However, for reading convenience, most of the examples show sorted sequences.
    This is suited for when your data is discrete, and you don't mind that the
    median may not be an actual data point.
 
-   If your data is ordinal (supports order operations) but not numeric (doesn't
-   support addition), you should use :func:`median_low` or :func:`median_high`
+   If the data is ordinal (supports order operations) but not numeric (doesn't
+   support addition), consider using :func:`median_low` or :func:`median_high`
    instead.
 
-   .. seealso:: :func:`median_low`, :func:`median_high`, :func:`median_grouped`
-
-
 .. function:: median_low(data)
 
    Return the low median of numeric data.  If *data* is empty,
@@ -319,7 +325,7 @@ However, for reading convenience, most of the examples show sorted sequences.
    desired instead, use ``min(multimode(data))`` or ``max(multimode(data))``.
    If the input *data* is empty, :exc:`StatisticsError` is raised.
 
-   ``mode`` assumes discrete data, and returns a single value. This is the
+   ``mode`` assumes discrete data and returns a single value. This is the
    standard treatment of the mode as commonly taught in schools:
 
    .. doctest::
@@ -522,7 +528,7 @@ However, for reading convenience, most of the examples show sorted sequences.
    cut-point will evaluate to ``104``.
 
    The *method* for computing quantiles can be varied depending on
-   whether the data in *data* includes or excludes the lowest and
+   whether the *data* includes or excludes the lowest and
    highest possible values from the population.
 
    The default *method* is "exclusive" and is used for data sampled from

From 4d1abedce9422473af2ac78047e55cde73208208 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
 <31488909+miss-islington@users.noreply.github.com>
Date: Fri, 6 Sep 2019 02:14:31 -0700
Subject: [PATCH 513/527] bpo-37380: subprocess: don't use _active on win
 (GH-14360) (GH-15707)

As noted by @eryksun in [1] and [2], using _cleanup and _active(in
__del__) is not necessary on Windows, since:

> Unlike Unix, a process in Windows doesn't have to be waited on by
> its parent to avoid a zombie. Keeping the handle open will actually
> create a zombie until the next _cleanup() call, which may be never
> if Popen() isn't called again.

This patch simply defines `subprocess._active` as `None`, for which we already
have the proper logic in place in `subprocess.Popen.__del__`, that prevents it
from trying to append the process to the `_active`. This patch also defines
`subprocess._cleanup` as a noop for Windows.

[1] https://bugs.python.org/issue37380GH-msg346333
[2] https://bugs.python.org/issue36067GH-msg336262

Signed-off-by: Ruslan Kuprieiev 
(cherry picked from commit 042821ae3cf537e01963c9ec85d1a454d921e826)

Co-authored-by: Ruslan Kuprieiev 
---
 Lib/subprocess.py                             | 48 ++++++++++++-------
 Lib/test/test_subprocess.py                   | 34 +++++++++----
 .../2019-06-25-04-15-22.bpo-37380.tPxjuz.rst  |  2 +
 3 files changed, 59 insertions(+), 25 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Windows/2019-06-25-04-15-22.bpo-37380.tPxjuz.rst

diff --git a/Lib/subprocess.py b/Lib/subprocess.py
index c0bda96cbc0339..5bbeba47a37432 100644
--- a/Lib/subprocess.py
+++ b/Lib/subprocess.py
@@ -218,22 +218,38 @@ def __repr__(self):
         _PopenSelector = selectors.SelectSelector
 
 
-# This lists holds Popen instances for which the underlying process had not
-# exited at the time its __del__ method got called: those processes are wait()ed
-# for synchronously from _cleanup() when a new Popen object is created, to avoid
-# zombie processes.
-_active = []
-
-def _cleanup():
-    for inst in _active[:]:
-        res = inst._internal_poll(_deadstate=sys.maxsize)
-        if res is not None:
-            try:
-                _active.remove(inst)
-            except ValueError:
-                # This can happen if two threads create a new Popen instance.
-                # It's harmless that it was already removed, so ignore.
-                pass
+if _mswindows:
+    # On Windows we just need to close `Popen._handle` when we no longer need
+    # it, so that the kernel can free it. `Popen._handle` gets closed
+    # implicitly when the `Popen` instance is finalized (see `Handle.__del__`,
+    # which is calling `CloseHandle` as requested in [1]), so there is nothing
+    # for `_cleanup` to do.
+    #
+    # [1] https://docs.microsoft.com/en-us/windows/desktop/ProcThread/
+    # creating-processes
+    _active = None
+
+    def _cleanup():
+        pass
+else:
+    # This lists holds Popen instances for which the underlying process had not
+    # exited at the time its __del__ method got called: those processes are
+    # wait()ed for synchronously from _cleanup() when a new Popen object is
+    # created, to avoid zombie processes.
+    _active = []
+
+    def _cleanup():
+        if _active is None:
+            return
+        for inst in _active[:]:
+            res = inst._internal_poll(_deadstate=sys.maxsize)
+            if res is not None:
+                try:
+                    _active.remove(inst)
+                except ValueError:
+                    # This can happen if two threads create a new Popen instance.
+                    # It's harmless that it was already removed, so ignore.
+                    pass
 
 PIPE = -1
 STDOUT = -2
diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py
index 9bfc21123cc016..e58d0925df3bac 100644
--- a/Lib/test/test_subprocess.py
+++ b/Lib/test/test_subprocess.py
@@ -52,10 +52,14 @@ def setUp(self):
         support.reap_children()
 
     def tearDown(self):
-        for inst in subprocess._active:
-            inst.wait()
-        subprocess._cleanup()
-        self.assertFalse(subprocess._active, "subprocess._active not empty")
+        if not mswindows:
+            # subprocess._active is not used on Windows and is set to None.
+            for inst in subprocess._active:
+                inst.wait()
+            subprocess._cleanup()
+            self.assertFalse(
+                subprocess._active, "subprocess._active not empty"
+            )
         self.doCleanups()
         support.reap_children()
 
@@ -2672,8 +2676,12 @@ def test_zombie_fast_process_del(self):
         with support.check_warnings(('', ResourceWarning)):
             p = None
 
-        # check that p is in the active processes list
-        self.assertIn(ident, [id(o) for o in subprocess._active])
+        if mswindows:
+            # subprocess._active is not used on Windows and is set to None.
+            self.assertIsNone(subprocess._active)
+        else:
+            # check that p is in the active processes list
+            self.assertIn(ident, [id(o) for o in subprocess._active])
 
     def test_leak_fast_process_del_killed(self):
         # Issue #12650: on Unix, if Popen.__del__() was called before the
@@ -2694,8 +2702,12 @@ def test_leak_fast_process_del_killed(self):
             p = None
 
         os.kill(pid, signal.SIGKILL)
-        # check that p is in the active processes list
-        self.assertIn(ident, [id(o) for o in subprocess._active])
+        if mswindows:
+            # subprocess._active is not used on Windows and is set to None.
+            self.assertIsNone(subprocess._active)
+        else:
+            # check that p is in the active processes list
+            self.assertIn(ident, [id(o) for o in subprocess._active])
 
         # let some time for the process to exit, and create a new Popen: this
         # should trigger the wait() of p
@@ -2707,7 +2719,11 @@ def test_leak_fast_process_del_killed(self):
                 pass
         # p should have been wait()ed on, and removed from the _active list
         self.assertRaises(OSError, os.waitpid, pid, 0)
-        self.assertNotIn(ident, [id(o) for o in subprocess._active])
+        if mswindows:
+            # subprocess._active is not used on Windows and is set to None.
+            self.assertIsNone(subprocess._active)
+        else:
+            self.assertNotIn(ident, [id(o) for o in subprocess._active])
 
     def test_close_fds_after_preexec(self):
         fd_status = support.findfile("fd_status.py", subdir="subprocessdata")
diff --git a/Misc/NEWS.d/next/Windows/2019-06-25-04-15-22.bpo-37380.tPxjuz.rst b/Misc/NEWS.d/next/Windows/2019-06-25-04-15-22.bpo-37380.tPxjuz.rst
new file mode 100644
index 00000000000000..facce27954a633
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-06-25-04-15-22.bpo-37380.tPxjuz.rst
@@ -0,0 +1,2 @@
+Don't collect unfinished processes with ``subprocess._active`` on Windows to
+cleanup later. Patch by Ruslan Kuprieiev.

From 4009a8522d774c36918005527dfb0975f389c8c2 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
 <31488909+miss-islington@users.noreply.github.com>
Date: Fri, 6 Sep 2019 11:14:49 -0700
Subject: [PATCH 514/527] bpo-38041: Refine IDLE Shell restart lines. 
 (GH-15709)

 Restart lines now always start with '=' and never end with ' ' and fill the width of the window unless that would require ending with ' ', which could be wrapped by itself and possible confusing the user.
(cherry picked from commit 38da805d563422cf1bb9cd9be24c73806840fe30)

Co-authored-by: Terry Jan Reedy 
---
 Lib/idlelib/NEWS.txt                          |  4 ++++
 Lib/idlelib/idle_test/test_pyshell.py         | 22 +++++++++++++++++++
 Lib/idlelib/pyshell.py                        | 18 ++++++++++++---
 .../2019-09-05-23-12-13.bpo-38041.nxmGGK.rst  |  3 +++
 4 files changed, 44 insertions(+), 3 deletions(-)
 create mode 100644 Misc/NEWS.d/next/IDLE/2019-09-05-23-12-13.bpo-38041.nxmGGK.rst

diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt
index c9e846a6fba667..559ffd0cf4f30b 100644
--- a/Lib/idlelib/NEWS.txt
+++ b/Lib/idlelib/NEWS.txt
@@ -3,6 +3,10 @@ Released on 2019-10-20?
 ======================================
 
 
+bpo-38401: Shell restart lines now fill the window width, always start
+with '=', and avoid wrapping unnecessarily. The line will still wrap
+if the included file name is long relative to the width.
+
 bpo-37092: Add mousewheel scrolling for IDLE module, path, and stack
 browsers.  Patch by George Zhang.
 
diff --git a/Lib/idlelib/idle_test/test_pyshell.py b/Lib/idlelib/idle_test/test_pyshell.py
index 581444ca5ef21f..4a096676f25796 100644
--- a/Lib/idlelib/idle_test/test_pyshell.py
+++ b/Lib/idlelib/idle_test/test_pyshell.py
@@ -7,6 +7,28 @@
 from tkinter import Tk
 
 
+class FunctionTest(unittest.TestCase):
+    # Test stand-alone module level non-gui functions.
+
+    def test_restart_line_wide(self):
+        eq = self.assertEqual
+        for file, mul, extra in (('', 22, ''), ('finame', 21, '=')):
+            width = 60
+            bar = mul * '='
+            with self.subTest(file=file, bar=bar):
+                file = file or 'Shell'
+                line = pyshell.restart_line(width, file)
+                eq(len(line), width)
+                eq(line, f"{bar+extra} RESTART: {file} {bar}")
+
+    def test_restart_line_narrow(self):
+        expect, taglen = "= RESTART: Shell", 16
+        for width in (taglen-1, taglen, taglen+1):
+            with self.subTest(width=width):
+                self.assertEqual(pyshell.restart_line(width, ''), expect)
+        self.assertEqual(pyshell.restart_line(taglen+2, ''), expect+' =')
+
+
 class PyShellFileListTest(unittest.TestCase):
 
     @classmethod
diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py
index ea9465568bd93f..08716a9733b759 100755
--- a/Lib/idlelib/pyshell.py
+++ b/Lib/idlelib/pyshell.py
@@ -387,6 +387,19 @@ def handle_EOF(self):
         "Override the base class - just re-raise EOFError"
         raise EOFError
 
+def restart_line(width, filename):  # See bpo-38141.
+    """Return width long restart line formatted with filename.
+
+    Fill line with balanced '='s, with any extras and at least one at
+    the beginning.  Do not end with a trailing space.
+    """
+    tag = f"= RESTART: {filename or 'Shell'} ="
+    if width >= len(tag):
+        div, mod = divmod((width -len(tag)), 2)
+        return f"{(div+mod)*'='}{tag}{div*'='}"
+    else:
+        return tag[:-2]  # Remove ' ='.
+
 
 class ModifiedInterpreter(InteractiveInterpreter):
 
@@ -491,9 +504,8 @@ def restart_subprocess(self, with_cwd=False, filename=''):
         console.stop_readline()
         # annotate restart in shell window and mark it
         console.text.delete("iomark", "end-1c")
-        tag = 'RESTART: ' + (filename if filename else 'Shell')
-        halfbar = ((int(console.width) -len(tag) - 4) // 2) * '='
-        console.write("\n{0} {1} {0}".format(halfbar, tag))
+        console.write('\n')
+        console.write(restart_line(console.width, filename))
         console.text.mark_set("restart", "end-1c")
         console.text.mark_gravity("restart", "left")
         if not filename:
diff --git a/Misc/NEWS.d/next/IDLE/2019-09-05-23-12-13.bpo-38041.nxmGGK.rst b/Misc/NEWS.d/next/IDLE/2019-09-05-23-12-13.bpo-38041.nxmGGK.rst
new file mode 100644
index 00000000000000..0aa254e8ca70f0
--- /dev/null
+++ b/Misc/NEWS.d/next/IDLE/2019-09-05-23-12-13.bpo-38041.nxmGGK.rst
@@ -0,0 +1,3 @@
+Shell restart lines now fill the window width, always start with '=',
+and avoid wrapping unnecessarily. The line will still wrap if the
+included file name is long relative to the width.

From cc51a6d7c7b6b06fb537860428347d88776d802b Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
 <31488909+miss-islington@users.noreply.github.com>
Date: Sat, 7 Sep 2019 00:12:34 -0700
Subject: [PATCH 515/527] bpo-20806: Reference both times(2) and times(3) and
 link to MSDN. (GH-15479)

(cherry picked from commit 3ccdbc33385a849c60a268def578cb06b8d41be6)

Co-authored-by: Joannah Nanjekye <33177550+nanjekyejoannah@users.noreply.github.com>
---
 Doc/library/os.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Doc/library/os.rst b/Doc/library/os.rst
index 6cbfb743284157..67fe36b54e2f87 100644
--- a/Doc/library/os.rst
+++ b/Doc/library/os.rst
@@ -3879,7 +3879,9 @@ written in Python, such as a mail server's external command delivery program.
    :attr:`children_system`, and :attr:`elapsed` in that order.
 
    See the Unix manual page
-   :manpage:`times(2)` or the corresponding Windows Platform API documentation.
+   :manpage:`times(2)` and :manpage:`times(3)` manual page on Unix or `the GetProcessTimes MSDN
+   `
+   _ on Windows.
    On Windows, only :attr:`user` and :attr:`system` are known; the other
    attributes are zero.
 

From 3be4b107490be27470cb9a101a8dfecf446fd992 Mon Sep 17 00:00:00 2001
From: Serhiy Storchaka 
Date: Sun, 8 Sep 2019 13:17:24 +0300
Subject: [PATCH 516/527] [3.8] Correct Roman-numeral example in Unicode HOWTO.
 (GH-15541). (GH-15728)

(cherry picked from commit 32a960f8e1015b64b4b955b3d62920c5903d4c6f)

Co-authored-by: Greg Price 
---
 Doc/howto/unicode.rst | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst
index 24c3235e4add94..51bd64bfc232ca 100644
--- a/Doc/howto/unicode.rst
+++ b/Doc/howto/unicode.rst
@@ -57,14 +57,14 @@ their corresponding code points:
    ...
    007B    '{'; LEFT CURLY BRACKET
    ...
-   2167    'Ⅶ': ROMAN NUMERAL EIGHT
-   2168    'Ⅸ': ROMAN NUMERAL NINE
+   2167    'Ⅷ'; ROMAN NUMERAL EIGHT
+   2168    'Ⅸ'; ROMAN NUMERAL NINE
    ...
-   265E    '♞': BLACK CHESS KNIGHT
-   265F    '♟': BLACK CHESS PAWN
+   265E    '♞'; BLACK CHESS KNIGHT
+   265F    '♟'; BLACK CHESS PAWN
    ...
-   1F600   '😀': GRINNING FACE
-   1F609   '😉': WINKING FACE
+   1F600   '😀'; GRINNING FACE
+   1F609   '😉'; WINKING FACE
    ...
 
 Strictly, these definitions imply that it's meaningless to say 'this is

From cc1bdf91d53b1a4751be84ef607e24e69a327a9b Mon Sep 17 00:00:00 2001
From: Raymond Hettinger 
Date: Sun, 8 Sep 2019 18:40:06 -0700
Subject: [PATCH 517/527] [3.8] bpo-36018: Address more reviewer feedback
 (GH-15733) (GH-15734)

---
 Doc/library/statistics.rst  | 41 ++++++++++++++++++++++++-------------
 Lib/statistics.py           | 32 ++++++++++++++++++++++++-----
 Lib/test/test_statistics.py | 35 ++++++++++++++-----------------
 3 files changed, 69 insertions(+), 39 deletions(-)

diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst
index 9e39828a5700c8..bdd706d0a93e3c 100644
--- a/Doc/library/statistics.rst
+++ b/Doc/library/statistics.rst
@@ -514,15 +514,14 @@ However, for reading convenience, most of the examples show sorted sequences.
 
    Set *n* to 4 for quartiles (the default).  Set *n* to 10 for deciles.  Set
    *n* to 100 for percentiles which gives the 99 cuts points that separate
-   *data* in to 100 equal sized groups.  Raises :exc:`StatisticsError` if *n*
+   *data* into 100 equal sized groups.  Raises :exc:`StatisticsError` if *n*
    is not least 1.
 
-   The *data* can be any iterable containing sample data or it can be an
-   instance of a class that defines an :meth:`~inv_cdf` method.  For meaningful
+   The *data* can be any iterable containing sample data.  For meaningful
    results, the number of data points in *data* should be larger than *n*.
    Raises :exc:`StatisticsError` if there are not at least two data points.
 
-   For sample data, the cut points are linearly interpolated from the
+   The cut points are linearly interpolated from the
    two nearest data points.  For example, if a cut point falls one-third
    of the distance between two sample values, ``100`` and ``112``, the
    cut-point will evaluate to ``104``.
@@ -547,9 +546,6 @@ However, for reading convenience, most of the examples show sorted sequences.
    values, the method sorts them and assigns the following percentiles:
    0%, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%.
 
-   If *data* is an instance of a class that defines an
-   :meth:`~inv_cdf` method, setting *method* has no effect.
-
    .. doctest::
 
         # Decile cut points for empirically sampled data
@@ -561,11 +557,6 @@ However, for reading convenience, most of the examples show sorted sequences.
         >>> [round(q, 1) for q in quantiles(data, n=10)]
         [81.0, 86.2, 89.0, 99.4, 102.5, 103.6, 106.0, 109.8, 111.0]
 
-        >>> # Quartile cut points for the standard normal distibution
-        >>> Z = NormalDist()
-        >>> [round(q, 4) for q in quantiles(Z, n=4)]
-        [-0.6745, 0.0, 0.6745]
-
    .. versionadded:: 3.8
 
 
@@ -607,6 +598,18 @@ of applications in statistics.
        `_ of a normal
        distribution.
 
+    .. attribute:: median
+
+       A read-only property for the `median
+       `_ of a normal
+       distribution.
+
+    .. attribute:: mode
+
+       A read-only property for the `mode
+       `_ of a normal
+       distribution.
+
     .. attribute:: stdev
 
        A read-only property for the `standard deviation
@@ -678,6 +681,16 @@ of applications in statistics.
        the two probability density functions
        `_.
 
+    .. method:: NormalDist.quantiles()
+
+        Divide the normal distribution into *n* continuous intervals with
+        equal probability.  Returns a list of (n - 1) cut points separating
+        the intervals.
+
+        Set *n* to 4 for quartiles (the default).  Set *n* to 10 for deciles.
+        Set *n* to 100 for percentiles which gives the 99 cuts points that
+        separate the normal distribution into 100 equal sized groups.
+
     Instances of :class:`NormalDist` support addition, subtraction,
     multiplication and division by a constant.  These operations
     are used for translation and scaling.  For example:
@@ -733,9 +746,9 @@ Find the `quartiles `_ and `deciles
 
 .. doctest::
 
-    >>> list(map(round, quantiles(sat)))
+    >>> list(map(round, sat.quantiles()))
     [928, 1060, 1192]
-    >>> list(map(round, quantiles(sat, n=10)))
+    >>> list(map(round, sat.quantiles(n=10)))
     [810, 896, 958, 1011, 1060, 1109, 1162, 1224, 1310]
 
 To estimate the distribution for a model than isn't easy to solve
diff --git a/Lib/statistics.py b/Lib/statistics.py
index 4b172662770fb0..70c48d605d1969 100644
--- a/Lib/statistics.py
+++ b/Lib/statistics.py
@@ -624,9 +624,8 @@ def quantiles(data, /, *, n=4, method='exclusive'):
     Set *n* to 100 for percentiles which gives the 99 cuts points that
     separate *data* in to 100 equal sized groups.
 
-    The *data* can be any iterable containing sample data or it can be
-    an instance of a class that defines an inv_cdf() method.  For sample
-    data, the cut points are linearly interpolated between data points.
+    The *data* can be any iterable containing sample.
+    The cut points are linearly interpolated between data points.
 
     If *method* is set to *inclusive*, *data* is treated as population
     data.  The minimum value is treated as the 0th percentile and the
@@ -634,8 +633,6 @@ def quantiles(data, /, *, n=4, method='exclusive'):
     """
     if n < 1:
         raise StatisticsError('n must be at least 1')
-    if hasattr(data, 'inv_cdf'):
-        return [data.inv_cdf(i / n) for i in range(1, n)]
     data = sorted(data)
     ld = len(data)
     if ld < 2:
@@ -955,6 +952,17 @@ def inv_cdf(self, p):
             raise StatisticsError('cdf() not defined when sigma at or below zero')
         return _normal_dist_inv_cdf(p, self._mu, self._sigma)
 
+    def quantiles(self, n=4):
+        """Divide into *n* continuous intervals with equal probability.
+
+        Returns a list of (n - 1) cut points separating the intervals.
+
+        Set *n* to 4 for quartiles (the default).  Set *n* to 10 for deciles.
+        Set *n* to 100 for percentiles which gives the 99 cuts points that
+        separate the normal distribution in to 100 equal sized groups.
+        """
+        return [self.inv_cdf(i / n) for i in range(1, n)]
+
     def overlap(self, other):
         """Compute the overlapping coefficient (OVL) between two normal distributions.
 
@@ -994,6 +1002,20 @@ def mean(self):
         "Arithmetic mean of the normal distribution."
         return self._mu
 
+    @property
+    def median(self):
+        "Return the median of the normal distribution"
+        return self._mu
+
+    @property
+    def mode(self):
+        """Return the mode of the normal distribution
+
+        The mode is the value x where which the probability density
+        function (pdf) takes its maximum value.
+        """
+        return self._mu
+
     @property
     def stdev(self):
         "Standard deviation of the normal distribution."
diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py
index 01b317c3281eff..af26473e8fdfc3 100644
--- a/Lib/test/test_statistics.py
+++ b/Lib/test/test_statistics.py
@@ -2198,16 +2198,6 @@ def f(x):
             exp = list(map(f, expected))
             act = quantiles(map(f, data), n=n)
             self.assertTrue(all(math.isclose(e, a) for e, a in zip(exp, act)))
-        # Quartiles of a standard normal distribution
-        for n, expected in [
-            (1, []),
-            (2, [0.0]),
-            (3, [-0.4307, 0.4307]),
-            (4 ,[-0.6745, 0.0, 0.6745]),
-                ]:
-            actual = quantiles(statistics.NormalDist(), n=n)
-            self.assertTrue(all(math.isclose(e, a, abs_tol=0.0001)
-                            for e, a in zip(expected, actual)))
         # Q2 agrees with median()
         for k in range(2, 60):
             data = random.choices(range(100), k=k)
@@ -2248,16 +2238,6 @@ def f(x):
             exp = list(map(f, expected))
             act = quantiles(map(f, data), n=n, method="inclusive")
             self.assertTrue(all(math.isclose(e, a) for e, a in zip(exp, act)))
-        # Quartiles of a standard normal distribution
-        for n, expected in [
-            (1, []),
-            (2, [0.0]),
-            (3, [-0.4307, 0.4307]),
-            (4 ,[-0.6745, 0.0, 0.6745]),
-                ]:
-            actual = quantiles(statistics.NormalDist(), n=n, method="inclusive")
-            self.assertTrue(all(math.isclose(e, a, abs_tol=0.0001)
-                            for e, a in zip(expected, actual)))
         # Natural deciles
         self.assertEqual(quantiles([0, 100], n=10, method='inclusive'),
                          [10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0])
@@ -2546,6 +2526,19 @@ def test_inv_cdf(self):
         # Special values
         self.assertTrue(math.isnan(Z.inv_cdf(float('NaN'))))
 
+    def test_quantiles(self):
+        # Quartiles of a standard normal distribution
+        Z = self.module.NormalDist()
+        for n, expected in [
+            (1, []),
+            (2, [0.0]),
+            (3, [-0.4307, 0.4307]),
+            (4 ,[-0.6745, 0.0, 0.6745]),
+                ]:
+            actual = Z.quantiles(n=n)
+            self.assertTrue(all(math.isclose(e, a, abs_tol=0.0001)
+                            for e, a in zip(expected, actual)))
+
     def test_overlap(self):
         NormalDist = self.module.NormalDist
 
@@ -2612,6 +2605,8 @@ def overlap_numeric(X, Y, *, steps=8_192, z=5):
     def test_properties(self):
         X = self.module.NormalDist(100, 15)
         self.assertEqual(X.mean, 100)
+        self.assertEqual(X.median, 100)
+        self.assertEqual(X.mode, 100)
         self.assertEqual(X.stdev, 15)
         self.assertEqual(X.variance, 225)
 

From 6e3809c7ce9fbee11c3a3f89dd7e89829b7581ac Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
 <31488909+miss-islington@users.noreply.github.com>
Date: Mon, 9 Sep 2019 02:07:51 -0700
Subject: [PATCH 518/527] bpo-34410: Fix a crash in the tee iterator when
 re-enter it. (GH-15625)

RuntimeError is now raised in this case.
(cherry picked from commit 526a01467b3277f9fcf7f91e66c23321caa1245d)

Co-authored-by: Serhiy Storchaka 
---
 Doc/library/itertools.rst                     |  4 ++
 Lib/test/test_itertools.py                    | 37 +++++++++++++++++++
 .../2019-08-31-01-52-59.bpo-34410.7KbWZQ.rst  |  2 +
 Modules/itertoolsmodule.c                     |  9 +++++
 4 files changed, 52 insertions(+)
 create mode 100644 Misc/NEWS.d/next/Library/2019-08-31-01-52-59.bpo-34410.7KbWZQ.rst

diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst
index a3f403a5b40b9c..8d134d43806593 100644
--- a/Doc/library/itertools.rst
+++ b/Doc/library/itertools.rst
@@ -645,6 +645,10 @@ loops that truncate the stream.
    used anywhere else; otherwise, the *iterable* could get advanced without
    the tee objects being informed.
 
+   ``tee`` iterators are not threadsafe. A :exc:`RuntimeError` may be
+   raised when using simultaneously iterators returned by the same :func:`tee`
+   call, even if the original *iterable* is threadsafe.
+
    This itertool may require significant auxiliary storage (depending on how
    much temporary data needs to be stored). In general, if one iterator uses
    most or all of the data before another iterator starts, it is faster to use
diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py
index 98b8c83731899a..eaa6197bec395c 100644
--- a/Lib/test/test_itertools.py
+++ b/Lib/test/test_itertools.py
@@ -11,6 +11,7 @@
 from functools import reduce
 import sys
 import struct
+import threading
 maxsize = support.MAX_Py_ssize_t
 minsize = -maxsize-1
 
@@ -1494,6 +1495,42 @@ def test_tee_del_backward(self):
             del forward, backward
             raise
 
+    def test_tee_reenter(self):
+        class I:
+            first = True
+            def __iter__(self):
+                return self
+            def __next__(self):
+                first = self.first
+                self.first = False
+                if first:
+                    return next(b)
+
+        a, b = tee(I())
+        with self.assertRaisesRegex(RuntimeError, "tee"):
+            next(a)
+
+    def test_tee_concurrent(self):
+        start = threading.Event()
+        finish = threading.Event()
+        class I:
+            def __iter__(self):
+                return self
+            def __next__(self):
+                start.set()
+                finish.wait()
+
+        a, b = tee(I())
+        thread = threading.Thread(target=next, args=[a])
+        thread.start()
+        try:
+            start.wait()
+            with self.assertRaisesRegex(RuntimeError, "tee"):
+                next(b)
+        finally:
+            finish.set()
+            thread.join()
+
     def test_StopIteration(self):
         self.assertRaises(StopIteration, next, zip())
 
diff --git a/Misc/NEWS.d/next/Library/2019-08-31-01-52-59.bpo-34410.7KbWZQ.rst b/Misc/NEWS.d/next/Library/2019-08-31-01-52-59.bpo-34410.7KbWZQ.rst
new file mode 100644
index 00000000000000..64e778ee0913c2
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-08-31-01-52-59.bpo-34410.7KbWZQ.rst
@@ -0,0 +1,2 @@
+Fixed a crash in the :func:`tee` iterator when re-enter it. RuntimeError is
+now raised in this case.
diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c
index 22c04f29353e26..eba59ba1b88034 100644
--- a/Modules/itertoolsmodule.c
+++ b/Modules/itertoolsmodule.c
@@ -443,6 +443,7 @@ typedef struct {
     PyObject_HEAD
     PyObject *it;
     int numread;                /* 0 <= numread <= LINKCELLS */
+    int running;
     PyObject *nextlink;
     PyObject *(values[LINKCELLS]);
 } teedataobject;
@@ -465,6 +466,7 @@ teedataobject_newinternal(PyObject *it)
     if (tdo == NULL)
         return NULL;
 
+    tdo->running = 0;
     tdo->numread = 0;
     tdo->nextlink = NULL;
     Py_INCREF(it);
@@ -493,7 +495,14 @@ teedataobject_getitem(teedataobject *tdo, int i)
     else {
         /* this is the lead iterator, so fetch more data */
         assert(i == tdo->numread);
+        if (tdo->running) {
+            PyErr_SetString(PyExc_RuntimeError,
+                            "cannot re-enter the tee iterator");
+            return NULL;
+        }
+        tdo->running = 1;
         value = PyIter_Next(tdo->it);
+        tdo->running = 0;
         if (value == NULL)
             return NULL;
         tdo->numread++;

From e103732f5df13a97f610a8b80883895f7a273573 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
 <31488909+miss-islington@users.noreply.github.com>
Date: Mon, 9 Sep 2019 02:50:30 -0700
Subject: [PATCH 519/527] bpo-37445: Include FORMAT_MESSAGE_IGNORE_INSERTS in
 FormatMessageW() calls (GH-14462)

If FormatMessageW() is passed the FORMAT_MESSAGE_FROM_SYSTEM flag without FORMAT_MESSAGE_IGNORE_INSERTS, it will fail if there are insert sequences in the message definition.
(cherry picked from commit a6563650c835d50f7302971a5b145e94f9d0dc68)

Co-authored-by: Zackery Spytz 
---
 .../next/Windows/2019-06-28-18-10-29.bpo-37445.LsdYO6.rst     | 2 ++
 Modules/_ctypes/callproc.c                                    | 4 +++-
 Modules/overlapped.c                                          | 3 ++-
 PC/bdist_wininst/install.c                                    | 3 ++-
 4 files changed, 9 insertions(+), 3 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Windows/2019-06-28-18-10-29.bpo-37445.LsdYO6.rst

diff --git a/Misc/NEWS.d/next/Windows/2019-06-28-18-10-29.bpo-37445.LsdYO6.rst b/Misc/NEWS.d/next/Windows/2019-06-28-18-10-29.bpo-37445.LsdYO6.rst
new file mode 100644
index 00000000000000..e4805b4e02fb12
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-06-28-18-10-29.bpo-37445.LsdYO6.rst
@@ -0,0 +1,2 @@
+Include the ``FORMAT_MESSAGE_IGNORE_INSERTS`` flag in ``FormatMessageW()``
+calls.
diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c
index 1de86e48d719b5..d9bdd9881333d9 100644
--- a/Modules/_ctypes/callproc.c
+++ b/Modules/_ctypes/callproc.c
@@ -242,7 +242,9 @@ static WCHAR *FormatError(DWORD code)
 {
     WCHAR *lpMsgBuf;
     DWORD n;
-    n = FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM,
+    n = FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER |
+                       FORMAT_MESSAGE_FROM_SYSTEM |
+                       FORMAT_MESSAGE_IGNORE_INSERTS,
                        NULL,
                        code,
                        MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), /* Default language */
diff --git a/Modules/overlapped.c b/Modules/overlapped.c
index 44a0a5a834633f..52ed0bc284bcc0 100644
--- a/Modules/overlapped.c
+++ b/Modules/overlapped.c
@@ -500,7 +500,8 @@ overlapped_FormatMessage(PyObject *ignore, PyObject *args)
         return NULL;
 
     n = FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER |
-                       FORMAT_MESSAGE_FROM_SYSTEM,
+                       FORMAT_MESSAGE_FROM_SYSTEM |
+                       FORMAT_MESSAGE_IGNORE_INSERTS,
                        NULL,
                        code,
                        MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
diff --git a/PC/bdist_wininst/install.c b/PC/bdist_wininst/install.c
index 6d01ad5c2d50e7..72d9837fe4f7f0 100644
--- a/PC/bdist_wininst/install.c
+++ b/PC/bdist_wininst/install.c
@@ -938,7 +938,8 @@ static BOOL SystemError(int error, char *msg)
         LPVOID lpMsgBuf;
         FormatMessage(
             FORMAT_MESSAGE_ALLOCATE_BUFFER |
-            FORMAT_MESSAGE_FROM_SYSTEM,
+            FORMAT_MESSAGE_FROM_SYSTEM |
+            FORMAT_MESSAGE_IGNORE_INSERTS,
             NULL,
             error,
             MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),

From 14f7de72b62ec8e73a8a57b25ad8fef230dc6a3c Mon Sep 17 00:00:00 2001
From: Zachary Ware 
Date: Mon, 9 Sep 2019 04:56:38 -0500
Subject: [PATCH 520/527] [3.8] bpo-15817: gdbinit: Document commands after
 defining them (GH-15021) (#15744)

The gdb manual[1] says the following for "document":

  The command commandname must already be defined.

[1] https://sourceware.org/gdb/current/onlinedocs/gdb/Define.html

And indeed when trying to use the gdbinit file with gdb 8.3, I get:

  .../cpython/Misc/gdbinit:17: Error in sourced command file:
  Undefined command: "pyo".  Try "help".

Fix this by moving all documentation blocks after the define blocks.

This was introduced in GH-6384.
(cherry picked from commit 1f86fdcfc57270ee569cc58269a4e08afe7608ec)

Authored-by: Florian Bruhin 
---
 Misc/gdbinit | 46 +++++++++++++++++++++++-----------------------
 1 file changed, 23 insertions(+), 23 deletions(-)

diff --git a/Misc/gdbinit b/Misc/gdbinit
index afefe0818e4c77..45e79fcf6f4682 100644
--- a/Misc/gdbinit
+++ b/Misc/gdbinit
@@ -14,30 +14,27 @@
 # with embedded macros that you may find superior to what is in here.
 # See Tools/gdb/libpython.py and http://bugs.python.org/issue8032.
 
-document pyo
-  Prints a representation of the object to stderr, along with the
-  number of reference counts it currently has and the hex address the
-  object is allocated at.  The argument must be a PyObject*
-end
 define pyo
     # side effect of calling _PyObject_Dump is to dump the object's
     # info - assigning just prevents gdb from printing the
     # NULL return value
     set $_unused_void = _PyObject_Dump($arg0)
 end
-
-document pyg
+document pyo
   Prints a representation of the object to stderr, along with the
   number of reference counts it currently has and the hex address the
-  object is allocated at.  The argument must be a PyGC_Head*
+  object is allocated at.  The argument must be a PyObject*
 end
+
 define pyg
     print _PyGC_Dump($arg0)
 end
-
-document pylocals
-  Print the local variables of the current frame.
+document pyg
+  Prints a representation of the object to stderr, along with the
+  number of reference counts it currently has and the hex address the
+  object is allocated at.  The argument must be a PyGC_Head*
 end
+
 define pylocals
     set $_i = 0
     while $_i < f->f_code->co_nlocals
@@ -50,6 +47,9 @@ define pylocals
         set $_i = $_i + 1
     end
 end
+document pylocals
+  Print the local variables of the current frame.
+end
 
 # A rewrite of the Python interpreter's line number calculator in GDB's
 # command language
@@ -75,13 +75,13 @@ define lineno
     printf "%d", $__li
 end
 
-document pyframev
-  Print the current frame - verbose
-end
 define pyframev
     pyframe
     pylocals
 end
+document pyframev
+  Print the current frame - verbose
+end
 
 define pyframe
     set $__fn = PyUnicode_AsUTF8(f->f_code->co_filename)
@@ -134,9 +134,6 @@ end
 # the interpreter you may will have to change the functions you compare with
 # $pc.
 
-document pystack
-  Print the entire Python call stack
-end
 define pystack
     while $pc < Py_Main || $pc > Py_GetArgcArgv
         if $pc > PyEval_EvalFrameEx && $pc < _PyEval_EvalFrameDefault
@@ -146,10 +143,10 @@ define pystack
     end
     select-frame 0
 end
-
-document pystackv
-  Print the entire Python call stack - verbose mode
+document pystack
+  Print the entire Python call stack
 end
+
 define pystackv
     while $pc < Py_Main || $pc > Py_GetArgcArgv
         if $pc > PyEval_EvalFrameEx && $pc < _PyEval_EvalFrameDefault
@@ -159,10 +156,10 @@ define pystackv
     end
     select-frame 0
 end
-
-document pu
-  Generally useful macro to print a Unicode string
+document pystackv
+  Print the entire Python call stack - verbose mode
 end
+
 def pu
   set $uni = $arg0
   set $i = 0
@@ -174,3 +171,6 @@ def pu
     end
   end
 end
+document pu
+  Generally useful macro to print a Unicode string
+end

From ebca7eb093f31052ff9f245b306d38941c28a1ad Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
 <31488909+miss-islington@users.noreply.github.com>
Date: Mon, 9 Sep 2019 03:11:00 -0700
Subject: [PATCH 521/527] bpo-32587: Make winreg.REG_MULTI_SZ support
 zero-length strings (GH-13239)

* bpo-32587: Make winreg.REG_MULTI_SZ support PendingFileRenameOperations

* Address review comments.
(cherry picked from commit e223ba13d8d871ee58570dfca4e82a591189cc2f)

Co-authored-by: Zackery Spytz 
---
 Lib/test/test_winreg.py                       |  1 +
 .../2019-05-10-15-25-44.bpo-32587.-0g2O3.rst  |  1 +
 PC/winreg.c                                   | 41 +++++++++++--------
 3 files changed, 27 insertions(+), 16 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Windows/2019-05-10-15-25-44.bpo-32587.-0g2O3.rst

diff --git a/Lib/test/test_winreg.py b/Lib/test/test_winreg.py
index 91a2bbc066b155..5c25ec8f7ec674 100644
--- a/Lib/test/test_winreg.py
+++ b/Lib/test/test_winreg.py
@@ -41,6 +41,7 @@
     ("String Val",    "A string value",                        REG_SZ),
     ("StringExpand",  "The path is %path%",                    REG_EXPAND_SZ),
     ("Multi-string",  ["Lots", "of", "string", "values"],      REG_MULTI_SZ),
+    ("Multi-nul",     ["", "", "", ""],                        REG_MULTI_SZ),
     ("Raw Data",      b"binary\x00data",                       REG_BINARY),
     ("Big String",    "x"*(2**14-1),                           REG_SZ),
     ("Big Binary",    b"x"*(2**14),                            REG_BINARY),
diff --git a/Misc/NEWS.d/next/Windows/2019-05-10-15-25-44.bpo-32587.-0g2O3.rst b/Misc/NEWS.d/next/Windows/2019-05-10-15-25-44.bpo-32587.-0g2O3.rst
new file mode 100644
index 00000000000000..41483aa8b74a9f
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-05-10-15-25-44.bpo-32587.-0g2O3.rst
@@ -0,0 +1 @@
+Make :data:`winreg.REG_MULTI_SZ` support zero-length strings.
diff --git a/PC/winreg.c b/PC/winreg.c
index d0df7ef0ad47f4..37bc2c72dcdee1 100644
--- a/PC/winreg.c
+++ b/PC/winreg.c
@@ -518,11 +518,18 @@ fixupMultiSZ(wchar_t **str, wchar_t *data, int len)
     int i;
     wchar_t *Q;
 
-    Q = data + len;
-    for (P = data, i = 0; P < Q && *P != '\0'; P++, i++) {
+    if (len > 0 && data[len - 1] == '\0') {
+        Q = data + len - 1;
+    }
+    else {
+        Q = data + len;
+    }
+
+    for (P = data, i = 0; P < Q; P++, i++) {
         str[i] = P;
-        for (; P < Q && *P != '\0'; P++)
+        for (; P < Q && *P != '\0'; P++) {
             ;
+        }
     }
 }
 
@@ -530,12 +537,20 @@ static int
 countStrings(wchar_t *data, int len)
 {
     int strings;
-    wchar_t *P;
-    wchar_t *Q = data + len;
+    wchar_t *P, *Q;
+
+    if (len > 0 && data[len - 1] == '\0') {
+        Q = data + len - 1;
+    }
+    else {
+        Q = data + len;
+    }
 
-    for (P = data, strings = 0; P < Q && *P != '\0'; P++, strings++)
-        for (; P < Q && *P != '\0'; P++)
+    for (P = data, strings = 0; P < Q; P++, strings++) {
+        for (; P < Q && *P != '\0'; P++) {
             ;
+        }
+    }
     return strings;
 }
 
@@ -749,21 +764,15 @@ Reg2Py(BYTE *retDataBuf, DWORD retDataSize, DWORD typ)
                 }
                 for (index = 0; index < s; index++)
                 {
-                    size_t len = wcslen(str[index]);
-                    if (len > INT_MAX) {
-                        PyErr_SetString(PyExc_OverflowError,
-                            "registry string is too long for a Python string");
-                        Py_DECREF(obData);
-                        PyMem_Free(str);
-                        return NULL;
-                    }
-                    PyObject *uni = PyUnicode_FromWideChar(str[index], len);
+                    size_t slen = wcsnlen(str[index], len);
+                    PyObject *uni = PyUnicode_FromWideChar(str[index], slen);
                     if (uni == NULL) {
                         Py_DECREF(obData);
                         PyMem_Free(str);
                         return NULL;
                     }
                     PyList_SET_ITEM(obData, index, uni);
+                    len -= slen + 1;
                 }
                 PyMem_Free(str);
 

From c837ad408e85eed9d20ba8331751df15e14f6aef Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
 <31488909+miss-islington@users.noreply.github.com>
Date: Mon, 9 Sep 2019 03:25:04 -0700
Subject: [PATCH 522/527] bpo-37936: Avoid ignoring files that we actually do
 track. (GH-15451)

There were about 14 files that are actually in the repo but that are
covered by the rules in .gitignore.

Git itself takes no notice of what .gitignore says about files that
it's already tracking... but the discrepancy can be confusing to a
human that adds a new file unexpectedly covered by these rules, as
well as to non-Git software that looks at .gitignore but doesn't
implement this wrinkle in its semantics.  (E.g., `rg`.)

Several of these are from rules that apply more broadly than
intended: for example, `Makefile` applies to `Doc/Makefile` and
`Tools/freeze/test/Makefile`, whereas `/Makefile` means only the
`Makefile` at the repo's root.

And the `Modules/Setup` rule simply wasn't updated after 961d54c5c.

https://bugs.python.org/issue37936
(cherry picked from commit 5e5e9515029f70836003a8cfb30433166fcc8db7)

Co-authored-by: Greg Price 
---
 .gitignore                                                 | 7 ++++---
 .../next/Build/2019-08-24-00-29-40.bpo-37936.QrORqA.rst    | 2 ++
 2 files changed, 6 insertions(+), 3 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Build/2019-08-24-00-29-40.bpo-37936.QrORqA.rst

diff --git a/.gitignore b/.gitignore
index 9c0c2ef07a1dc8..648f07e765b122 100644
--- a/.gitignore
+++ b/.gitignore
@@ -27,12 +27,12 @@ Include/pydtrace_probes.h
 Lib/distutils/command/*.pdb
 Lib/lib2to3/*.pickle
 Lib/test/data/*
-Makefile
+!Lib/test/data/README
+/Makefile
 Makefile.pre
 Misc/python.pc
 Misc/python-embed.pc
 Misc/python-config.sh
-Modules/Setup
 Modules/Setup.config
 Modules/Setup.local
 Modules/config.c
@@ -78,6 +78,7 @@ config.log
 config.status
 config.status.lineno
 core
+!Tools/msi/core/
 db_home
 .hg/
 .idea/
@@ -88,7 +89,7 @@ libpython*.dylib
 libpython*.dll
 platform
 pybuilddir.txt
-pyconfig.h
+/pyconfig.h
 python-config
 python-config.py
 python.bat
diff --git a/Misc/NEWS.d/next/Build/2019-08-24-00-29-40.bpo-37936.QrORqA.rst b/Misc/NEWS.d/next/Build/2019-08-24-00-29-40.bpo-37936.QrORqA.rst
new file mode 100644
index 00000000000000..4c6486103881af
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2019-08-24-00-29-40.bpo-37936.QrORqA.rst
@@ -0,0 +1,2 @@
+The :file:`.gitignore` file no longer applies to any files that are in fact
+tracked in the Git repository.  Patch by Greg Price.

From eb02196bd95ea12fcccff3960f36601596811570 Mon Sep 17 00:00:00 2001
From: Steve Dower 
Date: Mon, 9 Sep 2019 03:36:04 -0700
Subject: [PATCH 523/527] bpo-11953: Extend table of Windows WSA* error codes
 (GH-15004)

---
 Lib/socket.py                                 | 82 ++++++++++++++++++-
 .../2019-07-29-21-39-45.bpo-11953.4Hpwf9.rst  |  1 +
 2 files changed, 82 insertions(+), 1 deletion(-)
 create mode 100644 Misc/NEWS.d/next/Library/2019-07-29-21-39-45.bpo-11953.4Hpwf9.rst

diff --git a/Lib/socket.py b/Lib/socket.py
index 0dd8ec70e168aa..af2ed0e76a4981 100644
--- a/Lib/socket.py
+++ b/Lib/socket.py
@@ -109,21 +109,101 @@ def _intenum_converter(value, enum_klass):
 # WSA error codes
 if sys.platform.lower().startswith("win"):
     errorTab = {}
+    errorTab[6] = "Specified event object handle is invalid."
+    errorTab[8] = "Insufficient memory available."
+    errorTab[87] = "One or more parameters are invalid."
+    errorTab[995] = "Overlapped operation aborted."
+    errorTab[996] = "Overlapped I/O event object not in signaled state."
+    errorTab[997] = "Overlapped operation will complete later."
     errorTab[10004] = "The operation was interrupted."
     errorTab[10009] = "A bad file handle was passed."
     errorTab[10013] = "Permission denied."
-    errorTab[10014] = "A fault occurred on the network??" # WSAEFAULT
+    errorTab[10014] = "A fault occurred on the network??"  # WSAEFAULT
     errorTab[10022] = "An invalid operation was attempted."
+    errorTab[10024] = "Too many open files."
     errorTab[10035] = "The socket operation would block"
     errorTab[10036] = "A blocking operation is already in progress."
+    errorTab[10037] = "Operation already in progress."
+    errorTab[10038] = "Socket operation on nonsocket."
+    errorTab[10039] = "Destination address required."
+    errorTab[10040] = "Message too long."
+    errorTab[10041] = "Protocol wrong type for socket."
+    errorTab[10042] = "Bad protocol option."
+    errorTab[10043] = "Protocol not supported."
+    errorTab[10044] = "Socket type not supported."
+    errorTab[10045] = "Operation not supported."
+    errorTab[10046] = "Protocol family not supported."
+    errorTab[10047] = "Address family not supported by protocol family."
     errorTab[10048] = "The network address is in use."
+    errorTab[10049] = "Cannot assign requested address."
+    errorTab[10050] = "Network is down."
+    errorTab[10051] = "Network is unreachable."
+    errorTab[10052] = "Network dropped connection on reset."
+    errorTab[10053] = "Software caused connection abort."
     errorTab[10054] = "The connection has been reset."
+    errorTab[10055] = "No buffer space available."
+    errorTab[10056] = "Socket is already connected."
+    errorTab[10057] = "Socket is not connected."
     errorTab[10058] = "The network has been shut down."
+    errorTab[10059] = "Too many references."
     errorTab[10060] = "The operation timed out."
     errorTab[10061] = "Connection refused."
+    errorTab[10062] = "Cannot translate name."
     errorTab[10063] = "The name is too long."
     errorTab[10064] = "The host is down."
     errorTab[10065] = "The host is unreachable."
+    errorTab[10066] = "Directory not empty."
+    errorTab[10067] = "Too many processes."
+    errorTab[10068] = "User quota exceeded."
+    errorTab[10069] = "Disk quota exceeded."
+    errorTab[10070] = "Stale file handle reference."
+    errorTab[10071] = "Item is remote."
+    errorTab[10091] = "Network subsystem is unavailable."
+    errorTab[10092] = "Winsock.dll version out of range."
+    errorTab[10093] = "Successful WSAStartup not yet performed."
+    errorTab[10101] = "Graceful shutdown in progress."
+    errorTab[10102] = "No more results from WSALookupServiceNext."
+    errorTab[10103] = "Call has been canceled."
+    errorTab[10104] = "Procedure call table is invalid."
+    errorTab[10105] = "Service provider is invalid."
+    errorTab[10106] = "Service provider failed to initialize."
+    errorTab[10107] = "System call failure."
+    errorTab[10108] = "Service not found."
+    errorTab[10109] = "Class type not found."
+    errorTab[10110] = "No more results from WSALookupServiceNext."
+    errorTab[10111] = "Call was canceled."
+    errorTab[10112] = "Database query was refused."
+    errorTab[11001] = "Host not found."
+    errorTab[11002] = "Nonauthoritative host not found."
+    errorTab[11003] = "This is a nonrecoverable error."
+    errorTab[11004] = "Valid name, no data record requested type."
+    errorTab[11005] = "QoS receivers."
+    errorTab[11006] = "QoS senders."
+    errorTab[11007] = "No QoS senders."
+    errorTab[11008] = "QoS no receivers."
+    errorTab[11009] = "QoS request confirmed."
+    errorTab[11010] = "QoS admission error."
+    errorTab[11011] = "QoS policy failure."
+    errorTab[11012] = "QoS bad style."
+    errorTab[11013] = "QoS bad object."
+    errorTab[11014] = "QoS traffic control error."
+    errorTab[11015] = "QoS generic error."
+    errorTab[11016] = "QoS service type error."
+    errorTab[11017] = "QoS flowspec error."
+    errorTab[11018] = "Invalid QoS provider buffer."
+    errorTab[11019] = "Invalid QoS filter style."
+    errorTab[11020] = "Invalid QoS filter style."
+    errorTab[11021] = "Incorrect QoS filter count."
+    errorTab[11022] = "Invalid QoS object length."
+    errorTab[11023] = "Incorrect QoS flow count."
+    errorTab[11024] = "Unrecognized QoS object."
+    errorTab[11025] = "Invalid QoS policy object."
+    errorTab[11026] = "Invalid QoS flow descriptor."
+    errorTab[11027] = "Invalid QoS provider-specific flowspec."
+    errorTab[11028] = "Invalid QoS provider-specific filterspec."
+    errorTab[11029] = "Invalid QoS shape discard mode object."
+    errorTab[11030] = "Invalid QoS shaping rate object."
+    errorTab[11031] = "Reserved policy QoS element type."
     __all__.append("errorTab")
 
 
diff --git a/Misc/NEWS.d/next/Library/2019-07-29-21-39-45.bpo-11953.4Hpwf9.rst b/Misc/NEWS.d/next/Library/2019-07-29-21-39-45.bpo-11953.4Hpwf9.rst
new file mode 100644
index 00000000000000..62f3c44c284ea0
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-07-29-21-39-45.bpo-11953.4Hpwf9.rst
@@ -0,0 +1 @@
+Completing WSA* error codes in :mod:`socket`.

From 68e401fa0b1a3407bce395ad893535f65107ee6e Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
 <31488909+miss-islington@users.noreply.github.com>
Date: Mon, 9 Sep 2019 03:37:55 -0700
Subject: [PATCH 524/527] bpo-37705: Improve the implementation of
 winerror_to_errno() (GH-15623)

winerror_to_errno() is no longer automatically generated.
Do not rely on the old _dosmapperr() function.
Add ERROR_NO_UNICODE_TRANSLATION (1113) -> EILSEQ.
(cherry picked from commit 19052a11314e7be7ba003fd6cdbb5400a5d77d96)

Co-authored-by: Zackery Spytz 
---
 .../2019-08-30-15-15-22.bpo-37705.2o4NWW.rst  |   1 +
 PC/errmap.h                                   | 214 +++++++++++-------
 PC/generrmap.c                                |  32 ---
 3 files changed, 138 insertions(+), 109 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Windows/2019-08-30-15-15-22.bpo-37705.2o4NWW.rst
 delete mode 100644 PC/generrmap.c

diff --git a/Misc/NEWS.d/next/Windows/2019-08-30-15-15-22.bpo-37705.2o4NWW.rst b/Misc/NEWS.d/next/Windows/2019-08-30-15-15-22.bpo-37705.2o4NWW.rst
new file mode 100644
index 00000000000000..a374c3a2965a6c
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2019-08-30-15-15-22.bpo-37705.2o4NWW.rst
@@ -0,0 +1 @@
+Improve the implementation of ``winerror_to_errno()``.
diff --git a/PC/errmap.h b/PC/errmap.h
index 985f673a4649c8..a7489ab75c6561 100644
--- a/PC/errmap.h
+++ b/PC/errmap.h
@@ -1,80 +1,140 @@
-/* Generated file. Do not edit. */
-int winerror_to_errno(int winerror)
+int
+winerror_to_errno(int winerror)
 {
-    switch(winerror) {
-        case 2: return 2;
-        case 3: return 2;
-        case 4: return 24;
-        case 5: return 13;
-        case 6: return 9;
-        case 7: return 12;
-        case 8: return 12;
-        case 9: return 12;
-        case 10: return 7;
-        case 11: return 8;
-        case 15: return 2;
-        case 16: return 13;
-        case 17: return 18;
-        case 18: return 2;
-        case 19: return 13;
-        case 20: return 13;
-        case 21: return 13;
-        case 22: return 13;
-        case 23: return 13;
-        case 24: return 13;
-        case 25: return 13;
-        case 26: return 13;
-        case 27: return 13;
-        case 28: return 13;
-        case 29: return 13;
-        case 30: return 13;
-        case 31: return 13;
-        case 32: return 13;
-        case 33: return 13;
-        case 34: return 13;
-        case 35: return 13;
-        case 36: return 13;
-        case 53: return 2;
-        case 65: return 13;
-        case 67: return 2;
-        case 80: return 17;
-        case 82: return 13;
-        case 83: return 13;
-        case 89: return 11;
-        case 108: return 13;
-        case 109: return 32;
-        case 112: return 28;
-        case 114: return 9;
-        case 128: return 10;
-        case 129: return 10;
-        case 130: return 9;
-        case 132: return 13;
-        case 145: return 41;
-        case 158: return 13;
-        case 161: return 2;
-        case 164: return 11;
-        case 167: return 13;
-        case 183: return 17;
-        case 188: return 8;
-        case 189: return 8;
-        case 190: return 8;
-        case 191: return 8;
-        case 192: return 8;
-        case 193: return 8;
-        case 194: return 8;
-        case 195: return 8;
-        case 196: return 8;
-        case 197: return 8;
-        case 198: return 8;
-        case 199: return 8;
-        case 200: return 8;
-        case 201: return 8;
-        case 202: return 8;
-        case 206: return 2;
-        case 215: return 11;
-        case 232: return 32;
-        case 267: return 20;
-        case 1816: return 12;
-        default: return EINVAL;
+    // Unwrap FACILITY_WIN32 HRESULT errors.
+    if ((winerror & 0xFFFF0000) == 0x80070000) {
+        winerror &= 0x0000FFFF;
+    }
+
+    // Winsock error codes (10000-11999) are errno values.
+    if (winerror >= 10000 && winerror < 12000) {
+        switch (winerror) {
+        case WSAEINTR:
+        case WSAEBADF:
+        case WSAEACCES:
+        case WSAEFAULT:
+        case WSAEINVAL:
+        case WSAEMFILE:
+            // Winsock definitions of errno values. See WinSock2.h
+            return winerror - 10000;
+        default:
+            return winerror;
+        }
+    }
+
+    switch (winerror) {
+    case ERROR_FILE_NOT_FOUND:            //    2
+    case ERROR_PATH_NOT_FOUND:            //    3
+    case ERROR_INVALID_DRIVE:             //   15
+    case ERROR_NO_MORE_FILES:             //   18
+    case ERROR_BAD_NETPATH:               //   53
+    case ERROR_BAD_NET_NAME:              //   67
+    case ERROR_BAD_PATHNAME:              //  161
+    case ERROR_FILENAME_EXCED_RANGE:      //  206
+        return ENOENT;
+
+    case ERROR_BAD_ENVIRONMENT:           //   10
+        return E2BIG;
+
+    case ERROR_BAD_FORMAT:                //   11
+    case ERROR_INVALID_STARTING_CODESEG:  //  188
+    case ERROR_INVALID_STACKSEG:          //  189
+    case ERROR_INVALID_MODULETYPE:        //  190
+    case ERROR_INVALID_EXE_SIGNATURE:     //  191
+    case ERROR_EXE_MARKED_INVALID:        //  192
+    case ERROR_BAD_EXE_FORMAT:            //  193
+    case ERROR_ITERATED_DATA_EXCEEDS_64k: //  194
+    case ERROR_INVALID_MINALLOCSIZE:      //  195
+    case ERROR_DYNLINK_FROM_INVALID_RING: //  196
+    case ERROR_IOPL_NOT_ENABLED:          //  197
+    case ERROR_INVALID_SEGDPL:            //  198
+    case ERROR_AUTODATASEG_EXCEEDS_64k:   //  199
+    case ERROR_RING2SEG_MUST_BE_MOVABLE:  //  200
+    case ERROR_RELOC_CHAIN_XEEDS_SEGLIM:  //  201
+    case ERROR_INFLOOP_IN_RELOC_CHAIN:    //  202
+        return ENOEXEC;
+
+    case ERROR_INVALID_HANDLE:            //    6
+    case ERROR_INVALID_TARGET_HANDLE:     //  114
+    case ERROR_DIRECT_ACCESS_HANDLE:      //  130
+        return EBADF;
+
+    case ERROR_WAIT_NO_CHILDREN:          //  128
+    case ERROR_CHILD_NOT_COMPLETE:        //  129
+        return ECHILD;
+
+    case ERROR_NO_PROC_SLOTS:             //   89
+    case ERROR_MAX_THRDS_REACHED:         //  164
+    case ERROR_NESTING_NOT_ALLOWED:       //  215
+        return EAGAIN;
+
+    case ERROR_ARENA_TRASHED:             //    7
+    case ERROR_NOT_ENOUGH_MEMORY:         //    8
+    case ERROR_INVALID_BLOCK:             //    9
+    case ERROR_NOT_ENOUGH_QUOTA:          // 1816
+        return ENOMEM;
+
+    case ERROR_ACCESS_DENIED:             //    5
+    case ERROR_CURRENT_DIRECTORY:         //   16
+    case ERROR_WRITE_PROTECT:             //   19
+    case ERROR_BAD_UNIT:                  //   20
+    case ERROR_NOT_READY:                 //   21
+    case ERROR_BAD_COMMAND:               //   22
+    case ERROR_CRC:                       //   23
+    case ERROR_BAD_LENGTH:                //   24
+    case ERROR_SEEK:                      //   25
+    case ERROR_NOT_DOS_DISK:              //   26
+    case ERROR_SECTOR_NOT_FOUND:          //   27
+    case ERROR_OUT_OF_PAPER:              //   28
+    case ERROR_WRITE_FAULT:               //   29
+    case ERROR_READ_FAULT:                //   30
+    case ERROR_GEN_FAILURE:               //   31
+    case ERROR_SHARING_VIOLATION:         //   32
+    case ERROR_LOCK_VIOLATION:            //   33
+    case ERROR_WRONG_DISK:                //   34
+    case ERROR_SHARING_BUFFER_EXCEEDED:   //   36
+    case ERROR_NETWORK_ACCESS_DENIED:     //   65
+    case ERROR_CANNOT_MAKE:               //   82
+    case ERROR_FAIL_I24:                  //   83
+    case ERROR_DRIVE_LOCKED:              //  108
+    case ERROR_SEEK_ON_DEVICE:            //  132
+    case ERROR_NOT_LOCKED:                //  158
+    case ERROR_LOCK_FAILED:               //  167
+    case 35:                              //   35 (undefined)
+        return EACCES;
+
+    case ERROR_FILE_EXISTS:               //   80
+    case ERROR_ALREADY_EXISTS:            //  183
+        return EEXIST;
+
+    case ERROR_NOT_SAME_DEVICE:           //   17
+        return EXDEV;
+
+    case ERROR_DIRECTORY:                 //  267 (bpo-12802)
+        return ENOTDIR;
+
+    case ERROR_TOO_MANY_OPEN_FILES:       //    4
+        return EMFILE;
+
+    case ERROR_DISK_FULL:                 //  112
+        return ENOSPC;
+
+    case ERROR_BROKEN_PIPE:               //  109
+    case ERROR_NO_DATA:                   //  232 (bpo-13063)
+        return EPIPE;
+
+    case ERROR_DIR_NOT_EMPTY:             //  145
+        return ENOTEMPTY;
+
+    case ERROR_NO_UNICODE_TRANSLATION:    // 1113
+        return EILSEQ;
+
+    case ERROR_INVALID_FUNCTION:          //    1
+    case ERROR_INVALID_ACCESS:            //   12
+    case ERROR_INVALID_DATA:              //   13
+    case ERROR_INVALID_PARAMETER:         //   87
+    case ERROR_NEGATIVE_SEEK:             //  131
+    default:
+        return EINVAL;
     }
 }
diff --git a/PC/generrmap.c b/PC/generrmap.c
deleted file mode 100644
index 953344c0d79597..00000000000000
--- a/PC/generrmap.c
+++ /dev/null
@@ -1,32 +0,0 @@
-#include 
-#include 
-#include 
-#include 
-#include 
-
-/* Extract the mapping of Win32 error codes to errno */
-
-int main()
-{
-    int i;
-    _setmode(fileno(stdout), O_BINARY);
-    printf("/* Generated file. Do not edit. */\n");
-    printf("int winerror_to_errno(int winerror)\n");
-    printf("{\n    switch(winerror) {\n");
-    for(i=1; i < 65000; i++) {
-        _dosmaperr(i);
-        if (errno == EINVAL) {
-            /* Issue #12802 */
-            if (i == ERROR_DIRECTORY)
-                errno = ENOTDIR;
-            /* Issue #13063 */
-            else if (i == ERROR_NO_DATA)
-                errno = EPIPE;
-            else
-                continue;
-        }
-        printf("        case %d: return %d;\n", i, errno);
-    }
-    printf("        default: return EINVAL;\n");
-    printf("    }\n}\n");
-}

From 87a5a331eab5a99538d60a6dab25bdf299a68e3e Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
 <31488909+miss-islington@users.noreply.github.com>
Date: Mon, 9 Sep 2019 04:25:21 -0700
Subject: [PATCH 525/527] bpo-36250: ignore ValueError from signal in non-main
 thread (GH-12251)

Authored-By: blueyed 
(cherry picked from commit 8d64bfafdffd9f866bb6ac2e5b4c4bdfcb16aea0)

Co-authored-by: Daniel Hahler 
---
 Lib/pdb.py                                    |  8 +++--
 Lib/test/test_pdb.py                          | 29 +++++++++++++++++++
 .../2019-03-09-16-04-12.bpo-36250.tSK4N1.rst  |  2 ++
 3 files changed, 37 insertions(+), 2 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Library/2019-03-09-16-04-12.bpo-36250.tSK4N1.rst

diff --git a/Lib/pdb.py b/Lib/pdb.py
index 69fd8bd6efb0e5..8c1c96163ed913 100755
--- a/Lib/pdb.py
+++ b/Lib/pdb.py
@@ -340,8 +340,12 @@ def preloop(self):
     def interaction(self, frame, traceback):
         # Restore the previous signal handler at the Pdb prompt.
         if Pdb._previous_sigint_handler:
-            signal.signal(signal.SIGINT, Pdb._previous_sigint_handler)
-            Pdb._previous_sigint_handler = None
+            try:
+                signal.signal(signal.SIGINT, Pdb._previous_sigint_handler)
+            except ValueError:  # ValueError: signal only works in main thread
+                pass
+            else:
+                Pdb._previous_sigint_handler = None
         if self.setup(frame, traceback):
             # no interaction desired at this time (happens if .pdbrc contains
             # a command like "continue")
diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py
index 16d245a5602aba..4c38e919a83b78 100644
--- a/Lib/test/test_pdb.py
+++ b/Lib/test/test_pdb.py
@@ -1333,6 +1333,35 @@ def start_pdb():
         self.assertNotIn('Error', stdout.decode(),
                          "Got an error running test script under PDB")
 
+    def test_issue36250(self):
+
+        with open(support.TESTFN, 'wb') as f:
+            f.write(textwrap.dedent("""
+                import threading
+                import pdb
+
+                evt = threading.Event()
+
+                def start_pdb():
+                    evt.wait()
+                    pdb.Pdb(readrc=False).set_trace()
+
+                t = threading.Thread(target=start_pdb)
+                t.start()
+                pdb.Pdb(readrc=False).set_trace()
+                evt.set()
+                t.join()""").encode('ascii'))
+        cmd = [sys.executable, '-u', support.TESTFN]
+        proc = subprocess.Popen(cmd,
+            stdout=subprocess.PIPE,
+            stdin=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            )
+        self.addCleanup(proc.stdout.close)
+        stdout, stderr = proc.communicate(b'cont\ncont\n')
+        self.assertNotIn('Error', stdout.decode(),
+                         "Got an error running test script under PDB")
+
     def test_issue16180(self):
         # A syntax error in the debuggee.
         script = "def f: pass\n"
diff --git a/Misc/NEWS.d/next/Library/2019-03-09-16-04-12.bpo-36250.tSK4N1.rst b/Misc/NEWS.d/next/Library/2019-03-09-16-04-12.bpo-36250.tSK4N1.rst
new file mode 100644
index 00000000000000..8d9fbcbb1cba59
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-03-09-16-04-12.bpo-36250.tSK4N1.rst
@@ -0,0 +1,2 @@
+Ignore ``ValueError`` from ``signal`` with ``interaction`` in non-main
+thread.

From 252267925d3e74cfaf5216ecb0839c89c2a1baa8 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
 <31488909+miss-islington@users.noreply.github.com>
Date: Mon, 9 Sep 2019 04:29:10 -0700
Subject: [PATCH 526/527] [3.8] Doc: Fix PDF build (NoUri). (GH-15739)
 (GH-15754)

(cherry picked from commit 63c98ed2d21d22b46f3517fd7dfd88f0c1521299)


Co-authored-by: Julien Palard 

Automerge-Triggered-By: @JulienPalard
---
 Doc/tools/extensions/pyspecific.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py
index 28b8bda31146a5..f41077b0761521 100644
--- a/Doc/tools/extensions/pyspecific.py
+++ b/Doc/tools/extensions/pyspecific.py
@@ -22,6 +22,10 @@
 
 from sphinx import addnodes
 from sphinx.builders import Builder
+try:
+    from sphinx.errors import NoUri
+except ImportError:
+    from sphinx.environment import NoUri
 from sphinx.locale import translators
 from sphinx.util import status_iterator, logging
 from sphinx.util.nodes import split_explicit_title
@@ -569,10 +573,13 @@ def process_audit_events(app, doctree, fromdocname):
         for i, (doc, label) in backlinks:
             if isinstance(label, str):
                 ref = nodes.reference("", nodes.Text("[{}]".format(i)), internal=True)
-                ref['refuri'] = "{}#{}".format(
-                    app.builder.get_relative_uri(fromdocname, doc),
-                    label,
-                )
+                try:
+                    ref['refuri'] = "{}#{}".format(
+                        app.builder.get_relative_uri(fromdocname, doc),
+                        label,
+                    )
+                except NoUri:
+                    continue
                 node += ref
         row += nodes.entry('', node)
 

From bee8bfe5f440c2dde7f5af189febdbf81b27abd5 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
 <31488909+miss-islington@users.noreply.github.com>
Date: Mon, 9 Sep 2019 04:42:43 -0700
Subject: [PATCH 527/527] bpo-37212: Preserve keyword argument order in
 unittest.mock.call and error messages (GH-14310)

(cherry picked from commit 9d607061c9c888913ae2c18543775cf360d55f27)

Co-authored-by: Xtreak 
---
 Lib/unittest/mock.py                                        | 2 +-
 Lib/unittest/test/testmock/testmock.py                      | 6 +++---
 .../next/Library/2019-06-22-22-00-35.bpo-37212.Zhv-tq.rst   | 2 ++
 3 files changed, 6 insertions(+), 4 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Library/2019-06-22-22-00-35.bpo-37212.Zhv-tq.rst

diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py
index 4c76f53f3870c0..7a4fcf4e3aa97f 100644
--- a/Lib/unittest/mock.py
+++ b/Lib/unittest/mock.py
@@ -2320,7 +2320,7 @@ def _format_call_signature(name, args, kwargs):
     formatted_args = ''
     args_string = ', '.join([repr(arg) for arg in args])
     kwargs_string = ', '.join([
-        '%s=%r' % (key, value) for key, value in sorted(kwargs.items())
+        '%s=%r' % (key, value) for key, value in kwargs.items()
     ])
     if args_string:
         formatted_args = args_string
diff --git a/Lib/unittest/test/testmock/testmock.py b/Lib/unittest/test/testmock/testmock.py
index d3a1e89da81a1e..413ee689510193 100644
--- a/Lib/unittest/test/testmock/testmock.py
+++ b/Lib/unittest/test/testmock/testmock.py
@@ -1563,11 +1563,11 @@ def test_assert_called_once_message_not_called(self):
             m.assert_called_once()
         self.assertNotIn("Calls:", str(e.exception))
 
-    #Issue21256 printout of keyword args should be in deterministic order
-    def test_sorted_call_signature(self):
+    #Issue37212 printout of keyword args now preserves the original order
+    def test_ordered_call_signature(self):
         m = Mock()
         m.hello(name='hello', daddy='hero')
-        text = "call(daddy='hero', name='hello')"
+        text = "call(name='hello', daddy='hero')"
         self.assertEqual(repr(m.hello.call_args), text)
 
     #Issue21270 overrides tuple methods for mock.call objects
diff --git a/Misc/NEWS.d/next/Library/2019-06-22-22-00-35.bpo-37212.Zhv-tq.rst b/Misc/NEWS.d/next/Library/2019-06-22-22-00-35.bpo-37212.Zhv-tq.rst
new file mode 100644
index 00000000000000..520a0229aa9d09
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-06-22-22-00-35.bpo-37212.Zhv-tq.rst
@@ -0,0 +1,2 @@
+:func:`unittest.mock.call` now preserves the order of keyword arguments in
+repr output. Patch by Karthikeyan Singaravelan.