diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 9054e7ee3181a5..a6eb86f1a0b514 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -115,3 +115,51 @@ bound into a function. the free variables. On error, ``NULL`` is returned and an exception is raised. .. versionadded:: 3.11 + +.. c:function:: int PyCode_AddWatcher(PyCode_WatchCallback callback) + + Register *callback* as a code object watcher for the current interpreter. + Return an ID which may be passed to :c:func:`PyCode_ClearWatcher`. + In case of error (e.g. no more watcher IDs available), + return ``-1`` and set an exception. + + .. versionadded:: 3.12 + +.. c:function:: int PyCode_ClearWatcher(int watcher_id) + + Clear watcher identified by *watcher_id* previously returned from + :c:func:`PyCode_AddWatcher` for the current interpreter. + Return ``0`` on success, or ``-1`` and set an exception on error + (e.g. if the given *watcher_id* was never registered.) + + .. versionadded:: 3.12 + +.. c:type:: PyCodeEvent + + Enumeration of possible code object watcher events: + - ``PY_CODE_EVENT_CREATE`` + - ``PY_CODE_EVENT_DESTROY`` + + .. versionadded:: 3.12 + +.. c:type:: int (*PyCode_WatchCallback)(PyCodeEvent event, PyCodeObject* co) + + Type of a code object watcher callback function. + + If *event* is ``PY_CODE_EVENT_CREATE``, then the callback is invoked + after `co` has been fully initialized. Otherwise, the callback is invoked + before the destruction of *co* takes place, so the prior state of *co* + can be inspected. + + Users of this API should not rely on internal runtime implementation + details. Such details may include, but are not limited to, the exact + order and timing of creation and destruction of code objects. While + changes in these details may result in differences observable by watchers + (including whether a callback is invoked or not), it does not change + the semantics of the Python code being executed. + + If the callback returns with an exception set, it must return ``-1``; this + exception will be printed as an unraisable exception using + :c:func:`PyErr_WriteUnraisable`. Otherwise it should return ``0``. + + .. versionadded:: 3.12 diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index d0a1ed2b99e55d..0bcaed5477fa91 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -33,7 +33,8 @@ an event loop: Return the running event loop in the current OS thread. - If there is no running event loop a :exc:`RuntimeError` is raised. + Raise a :exc:`RuntimeError` if there is no running event loop. + This function can only be called from a coroutine or a callback. .. versionadded:: 3.7 @@ -52,17 +53,19 @@ an event loop: :func:`get_running_loop` function is preferred to :func:`get_event_loop` in coroutines and callbacks. - Consider also using the :func:`asyncio.run` function instead of using - lower level functions to manually create and close an event loop. + As noted above, consider using the higher-level :func:`asyncio.run` function, + instead of using these lower level functions to manually create and close an + event loop. .. deprecated:: 3.10 - Deprecation warning is emitted if there is no running event loop. - In future Python releases, this function will be an alias of - :func:`get_running_loop`. + Emits a deprecation warning if there is no running event loop. + In future Python releases, this function may become an alias of + :func:`get_running_loop` and will accordingly raise a + :exc:`RuntimeError` if there is no running event loop. .. function:: set_event_loop(loop) - Set *loop* as a current event loop for the current OS thread. + Set *loop* as the current event loop for the current OS thread. .. function:: new_event_loop() diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index a02ba739146aaf..4ca3768f827c6b 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -53,11 +53,13 @@ The modern interface provides: Encode the :term:`bytes-like object` *s* using Base64 and return the encoded :class:`bytes`. - Optional *altchars* must be a :term:`bytes-like object` of at least - length 2 (additional characters are ignored) which specifies an alternative - alphabet for the ``+`` and ``/`` characters. This allows an application to e.g. - generate URL or filesystem safe Base64 strings. The default is ``None``, for - which the standard Base64 alphabet is used. + Optional *altchars* must be a :term:`bytes-like object` of length 2 which + specifies an alternative alphabet for the ``+`` and ``/`` characters. + This allows an application to e.g. generate URL or filesystem safe Base64 + strings. The default is ``None``, for which the standard Base64 alphabet is used. + + May assert or raise a a :exc:`ValueError` if the length of *altchars* is not 2. Raises a + :exc:`TypeError` if *altchars* is not a :term:`bytes-like object`. .. function:: b64decode(s, altchars=None, validate=False) @@ -65,9 +67,9 @@ The modern interface provides: Decode the Base64 encoded :term:`bytes-like object` or ASCII string *s* and return the decoded :class:`bytes`. - Optional *altchars* must be a :term:`bytes-like object` or ASCII string of - at least length 2 (additional characters are ignored) which specifies the - alternative alphabet used instead of the ``+`` and ``/`` characters. + Optional *altchars* must be a :term:`bytes-like object` or ASCII string + of length 2 which specifies the alternative alphabet used instead of the + ``+`` and ``/`` characters. A :exc:`binascii.Error` exception is raised if *s* is incorrectly padded. @@ -80,6 +82,7 @@ The modern interface provides: For more information about the strict base64 check, see :func:`binascii.a2b_base64` + May assert or raise a :exc:`ValueError` if the length of *altchars* is not 2. .. function:: standard_b64encode(s) diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 847299649d1efd..32c524a7348719 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -79,7 +79,8 @@ Module contents class C: ... - @dataclass(init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False) + @dataclass(init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, + match_args=True, kw_only=False, slots=False, weakref_slot=False) class C: ... diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 74d9e67327629e..208aecf11c809b 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -194,7 +194,7 @@ Data Types .. method:: EnumType.__getitem__(cls, name) - Returns the Enum member in *cls* matching *name*, or raises an :exc:`KeyError`:: + Returns the Enum member in *cls* matching *name*, or raises a :exc:`KeyError`:: >>> Color['BLUE'] @@ -241,7 +241,7 @@ Data Types .. note:: Enum member values - Member values can be anything: :class:`int`, :class:`str`, etc.. If + Member values can be anything: :class:`int`, :class:`str`, etc. If the exact value is unimportant you may use :class:`auto` instances and an appropriate value will be chosen for you. See :class:`auto` for the details. @@ -255,7 +255,7 @@ Data Types names will also be removed from the completed enumeration. See :ref:`TimePeriod ` for an example. - .. method:: Enum.__call__(cls, value, names=None, \*, module=None, qualname=None, type=None, start=1, boundary=None) + .. method:: Enum.__call__(cls, value, names=None, *, module=None, qualname=None, type=None, start=1, boundary=None) This method is called in two different ways: @@ -272,8 +272,8 @@ Data Types :module: The name of the module the new Enum is created in. :qualname: The actual location in the module where this Enum can be found. :type: A mix-in type for the new Enum. - :start: The first integer value for the Enum (used by :class:`auto`) - :boundary: How to handle out-of-range values from bit operations (:class:`Flag` only) + :start: The first integer value for the Enum (used by :class:`auto`). + :boundary: How to handle out-of-range values from bit operations (:class:`Flag` only). .. method:: Enum.__dir__(self) @@ -315,7 +315,7 @@ Data Types >>> PowersOfThree.SECOND.value 6 - .. method:: Enum.__init_subclass__(cls, \**kwds) + .. method:: Enum.__init_subclass__(cls, **kwds) A *classmethod* that is used to further configure subsequent subclasses. By default, does nothing. @@ -373,7 +373,7 @@ Data Types .. method:: Enum.__format__(self) Returns the string used for *format()* and *f-string* calls. By default, - returns :meth:`__str__` returns, but can be overridden:: + returns :meth:`__str__` return value, but can be overridden:: >>> class OtherStyle(Enum): ... ALTERNATE = auto() @@ -552,11 +552,11 @@ Data Types Using :class:`auto` with :class:`Flag` results in integers that are powers of two, starting with ``1``. - .. versionchanged:: 3.11 The *repr()* of zero-valued flags has changed. It + .. versionchanged:: 3.11 The *repr()* of zero-valued flags has changed. It is now:: - >>> Color(0) # doctest: +SKIP - + >>> Color(0) # doctest: +SKIP + .. class:: IntFlag @@ -600,7 +600,7 @@ Data Types *replacement of existing constants* use-case. :meth:`~object.__format__` was already :meth:`!int.__format__` for that same reason. - Inversion of a :class:`!IntFlag` now returns a positive value that is the + Inversion of an :class:`!IntFlag` now returns a positive value that is the union of all flags not in the given flag, rather than a negative value. This matches the existing :class:`Flag` behavior. @@ -612,7 +612,7 @@ Data Types * :meth:`!int.__str__` for :class:`IntEnum` and :class:`IntFlag` * :meth:`!str.__str__` for :class:`StrEnum` - Inherit from :class:`!ReprEnum` to keep the :class:`str() / :func:`format` + Inherit from :class:`!ReprEnum` to keep the :class:`str() ` / :func:`format` of the mixed-in data type instead of using the :class:`Enum`-default :meth:`str() `. @@ -658,7 +658,7 @@ Data Types .. attribute:: NAMED_FLAGS Ensure that any flag groups/masks contain only named flags -- useful when - values are specified instead of being generated by :func:`auto` + values are specified instead of being generated by :func:`auto`:: >>> from enum import Flag, verify, NAMED_FLAGS >>> @verify(NAMED_FLAGS) @@ -804,6 +804,11 @@ Utilities and Decorators * ``THREE = [auto(), -3]`` will *not* work (``, -3`` is used to create the ``THREE`` enum member) + .. versionchanged:: 3.11.1 + + In prior versions, ``auto()`` had to be the only thing + on the assignment line to work properly. + ``_generate_next_value_`` can be overridden to customize the values used by *auto*. @@ -885,23 +890,23 @@ Notes :class:`IntEnum`, :class:`StrEnum`, and :class:`IntFlag` - These three enum types are designed to be drop-in replacements for existing - integer- and string-based values; as such, they have extra limitations: + These three enum types are designed to be drop-in replacements for existing + integer- and string-based values; as such, they have extra limitations: - - ``__str__`` uses the value and not the name of the enum member + - ``__str__`` uses the value and not the name of the enum member - - ``__format__``, because it uses ``__str__``, will also use the value of - the enum member instead of its name + - ``__format__``, because it uses ``__str__``, will also use the value of + the enum member instead of its name - If you do not need/want those limitations, you can either create your own - base class by mixing in the ``int`` or ``str`` type yourself:: + If you do not need/want those limitations, you can either create your own + base class by mixing in the ``int`` or ``str`` type yourself:: - >>> from enum import Enum - >>> class MyIntEnum(int, Enum): - ... pass + >>> from enum import Enum + >>> class MyIntEnum(int, Enum): + ... pass or you can reassign the appropriate :meth:`str`, etc., in your enum:: - >>> from enum import IntEnum - >>> class MyIntEnum(IntEnum): - ... __str__ = IntEnum.__str__ + >>> from enum import IntEnum + >>> class MyIntEnum(IntEnum): + ... __str__ = IntEnum.__str__ diff --git a/Doc/library/fnmatch.rst b/Doc/library/fnmatch.rst index 9163da57c7b999..46bf0fc2848058 100644 --- a/Doc/library/fnmatch.rst +++ b/Doc/library/fnmatch.rst @@ -48,7 +48,7 @@ patterns. Also note that :func:`functools.lru_cache` with the *maxsize* of 32768 is used to cache the compiled regex patterns in the following functions: :func:`fnmatch`, -:func:`fnmatchcase`, :func:`filter`. +:func:`fnmatchcase`, :func:`.filter`. .. function:: fnmatch(filename, pattern) diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 81b6bf5373b495..154f3f3c39049f 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -512,3 +512,10 @@ Security Considerations :class:`SimpleHTTPRequestHandler` will follow symbolic links when handling requests, this makes it possible for files outside of the specified directory to be served. + +Earlier versions of Python did not scrub control characters from the +log messages emitted to stderr from ``python -m http.server`` or the +default :class:`BaseHTTPRequestHandler` ``.log_message`` +implementation. This could allow to remote clients connecting to your +server to send nefarious control codes to your terminal. + diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 0034b46fb1ced2..e6e242320fd878 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -1565,16 +1565,22 @@ search() vs. match() .. sectionauthor:: Fred L. Drake, Jr. -Python offers two different primitive operations based on regular expressions: -:func:`re.match` checks for a match only at the beginning of the string, while -:func:`re.search` checks for a match anywhere in the string (this is what Perl -does by default). +Python offers different primitive operations based on regular expressions: + ++ :func:`re.match` checks for a match only at the beginning of the string ++ :func:`re.search` checks for a match anywhere in the string + (this is what Perl does by default) ++ :func:`re.fullmatch` checks for entire string to be a match + For example:: >>> re.match("c", "abcdef") # No match >>> re.search("c", "abcdef") # Match + >>> re.fullmatch("p.*n", "python") # Match + + >>> re.fullmatch("r.*n", "python") # No match Regular expressions beginning with ``'^'`` can be used with :func:`search` to restrict the match at the beginning of the string:: @@ -1588,8 +1594,8 @@ Note however that in :const:`MULTILINE` mode :func:`match` only matches at the beginning of the string, whereas using :func:`search` with a regular expression beginning with ``'^'`` will match at the beginning of each line. :: - >>> re.match('X', 'A\nB\nX', re.MULTILINE) # No match - >>> re.search('^X', 'A\nB\nX', re.MULTILINE) # Match + >>> re.match("X", "A\nB\nX", re.MULTILINE) # No match + >>> re.search("^X", "A\nB\nX", re.MULTILINE) # Match diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst index 26785395ec0312..ceb962e860042d 100644 --- a/Doc/library/socketserver.rst +++ b/Doc/library/socketserver.rst @@ -96,8 +96,7 @@ synchronous servers of four types:: Note that :class:`UnixDatagramServer` derives from :class:`UDPServer`, not from :class:`UnixStreamServer` --- the only difference between an IP and a Unix -stream server is the address family, which is simply repeated in both Unix -server classes. +server is the address family. .. class:: ForkingMixIn @@ -431,11 +430,8 @@ Request Handler Objects The :attr:`self.rfile` and :attr:`self.wfile` attributes can be read or written, respectively, to get the request data or return data to the client. - - The :attr:`rfile` attributes of both classes support the - :class:`io.BufferedIOBase` readable interface, and - :attr:`DatagramRequestHandler.wfile` supports the - :class:`io.BufferedIOBase` writable interface. + The :attr:`!rfile` attributes support the :class:`io.BufferedIOBase` readable interface, + and :attr:`!wfile` attributes support the :class:`!io.BufferedIOBase` writable interface. .. versionchanged:: 3.6 :attr:`StreamRequestHandler.wfile` also supports the diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 7e2235b285b814..960f2966afe1f2 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -292,6 +292,7 @@ Module functions By default (``0``), type detection is disabled. :param isolation_level: + Control legacy transaction handling behaviour. See :attr:`Connection.isolation_level` and :ref:`sqlite3-transaction-control-isolation-level` for more information. Can be ``"DEFERRED"`` (default), ``"EXCLUSIVE"`` or ``"IMMEDIATE"``; @@ -325,6 +326,7 @@ Module functions enabling various :ref:`sqlite3-uri-tricks`. :param autocommit: + Control :pep:`249` transaction handling behaviour. See :attr:`Connection.autocommit` and :ref:`sqlite3-transaction-control-autocommit` for more information. *autocommit* currently defaults to @@ -2468,9 +2470,9 @@ which implies :pep:`249`-compliant transaction control. This means: * :mod:`!sqlite3` ensures that a transaction is always open, - so :meth:`Connection.commit` and :meth:`Connection.rollback` - will implicitly open a new transaction immediately after closing - the pending one. + so :func:`connect`, :meth:`Connection.commit`, and :meth:`Connection.rollback` + will implicitly open a new transaction + (immediately after closing the pending one, for the latter two). :mod:`!sqlite3` uses ``BEGIN DEFERRED`` statements when opening transactions. * Transactions should be committed explicitly using :meth:`!commit`. * Transactions should be rolled back explicitly using :meth:`!rollback`. diff --git a/Doc/library/syslog.rst b/Doc/library/syslog.rst index 766ff57cc66d69..f29ef03267b1ba 100644 --- a/Doc/library/syslog.rst +++ b/Doc/library/syslog.rst @@ -40,6 +40,13 @@ The module defines the following functions: it wasn't called prior to the call to :func:`syslog`, deferring to the syslog implementation to call ``openlog()``. + .. versionchanged:: 3.12 + This function is restricted in subinterpreters. + (Only code that runs in multiple interpreters is affected and + the restriction is not relevant for most users.) + :func:`openlog` must be called in the main interpreter before :func:`syslog` may be used + in a subinterpreter. Otherwise it will raise :exc:`RuntimeError`. + .. function:: openlog([ident[, logoption[, facility]]]) @@ -60,6 +67,13 @@ The module defines the following functions: In previous versions, keyword arguments were not allowed, and *ident* was required. + .. versionchanged:: 3.12 + This function is restricted in subinterpreters. + (Only code that runs in multiple interpreters is affected and + the restriction is not relevant for most users.) + This may only be called in the main interpreter. + It will raise :exc:`RuntimeError` if called in a subinterpreter. + .. function:: closelog() @@ -72,6 +86,13 @@ The module defines the following functions: .. audit-event:: syslog.closelog "" syslog.closelog + .. versionchanged:: 3.12 + This function is restricted in subinterpreters. + (Only code that runs in multiple interpreters is affected and + the restriction is not relevant for most users.) + This may only be called in the main interpreter. + It will raise :exc:`RuntimeError` if called in a subinterpreter. + .. function:: setlogmask(maskpri) diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index a1e542b1e927e4..73e7b21ae405d2 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -143,7 +143,7 @@ See :ref:`__slots__ documentation ` for details. ``ProxyType`` or ``CallableProxyType``, depending on whether *object* is callable. Proxy objects are not :term:`hashable` regardless of the referent; this avoids a number of problems related to their fundamentally mutable nature, and - prevent their use as dictionary keys. *callback* is the same as the parameter + prevents their use as dictionary keys. *callback* is the same as the parameter of the same name to the :func:`ref` function. Accessing an attribute of the proxy object after the referent is @@ -212,7 +212,7 @@ objects. discarded when no strong reference to it exists any more. -.. class:: WeakMethod(method) +.. class:: WeakMethod(method[, callback]) A custom :class:`ref` subclass which simulates a weak reference to a bound method (i.e., a method defined on a class and looked up on an instance). @@ -238,6 +238,8 @@ objects. >>> r() >>> + *callback* is the same as the parameter of the same name to the :func:`ref` function. + .. versionadded:: 3.4 .. class:: finalize(obj, func, /, *args, **kwargs) diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 0922972f9bf1dc..3df6ff4873b72e 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -2,6 +2,46 @@ Configure Python **************** +Build Requirements +================== + +Features required to build CPython: + +* A `C11 `_ compiler. `Optional C11 + features + `_ + are not required. + +* Support for `IEEE 754 `_ floating + point numbers and `floating point Not-a-Number (NaN) + `_. + +* Support for threads. + +* OpenSSL 1.1.1 or newer for the :mod:`ssl` and :mod:`hashlib` modules. + +* On Windows, Microsoft Visual Studio 2017 or later is required. + +.. versionchanged:: 3.11 + C11 compiler, IEEE 754 and NaN support are now required. + +.. versionchanged:: 3.10 + OpenSSL 1.1.1 is now required. + +.. versionchanged:: 3.7 + Thread support and OpenSSL 1.0.2 are now required. + +.. versionchanged:: 3.6 + Selected C99 features are now required, like ```` and ``static + inline`` functions. + +.. versionchanged:: 3.5 + On Windows, Visual Studio 2015 or later is required. + +See also :pep:`7` "Style Guide for C Code" and :pep:`11` "CPython platform +support". + + .. _configure-options: Configure Options @@ -93,6 +133,12 @@ General Options See :envvar:`PYTHONCOERCECLOCALE` and the :pep:`538`. +.. cmdoption:: --without-freelists + + Disable all freelists except the empty tuple singleton. + + .. versionadded:: 3.11 + .. cmdoption:: --with-platlibdir=DIRNAME Python library directory name (default is ``lib``). diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 2389b1a63b1ca2..1c21caf355f082 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -77,8 +77,9 @@ Interpreter improvements: New typing features: * :pep:`604`, Allow writing union types as X | Y -* :pep:`613`, Explicit Type Aliases * :pep:`612`, Parameter Specification Variables +* :pep:`613`, Explicit Type Aliases +* :pep:`647`, User-Defined Type Guards Important deprecations, removals or restrictions: diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 6eb90df89cac0e..7931988ed0b04d 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -2131,7 +2131,7 @@ Build Changes (Contributed by Dong-hee Na and Brett Holman in :issue:`44340`.) * Freelists for object structs can now be disabled. A new :program:`configure` - option :option:`!--without-freelists` can be used to disable all freelists + option :option:`--without-freelists` can be used to disable all freelists except empty tuple singleton. (Contributed by Christian Heimes in :issue:`45522`.) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index dff4de621b4c49..6f5ce818d961a5 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -178,6 +178,10 @@ Other Language Changes In a future Python version they will be eventually a :exc:`SyntaxError`. (Contributed by Victor Stinner in :gh:`98401`.) +* All builtin and extension callables expecting boolean parameters now accept + arguments of any type instead of just :class:`bool` and :class:`int`. + (Contributed by Serhiy Storchaka in :gh:`60203`.) + New Modules =========== @@ -673,6 +677,15 @@ Changes in the Python API :class:`bytes` type is accepted for bytes strings. (Contributed by Victor Stinner in :gh:`98393`.) +* :func:`syslog.openlog` and :func:`syslog.closelog` now fail if used in subinterpreters. + :func:`syslog.syslog` may still be used in subinterpreters, + but now only if :func:`syslog.openlog` has already been called in the main interpreter. + These new restrictions do not apply to the main interpreter, + so only a very small set of users might be affected. + This change helps with interpreter isolation. Furthermore, :mod:`syslog` is a wrapper + around process-global resources, which are best managed from the main interpreter. + (Contributed by Dong-hee Na in :gh:`99127`.) + Build Changes ============= @@ -764,6 +777,10 @@ New Features callbacks to receive notification on changes to a type. (Contributed by Carl Meyer in :gh:`91051`.) +* Added :c:func:`PyCode_AddWatcher` and :c:func:`PyCode_ClearWatcher` + APIs to register callbacks to receive notification on creation and + destruction of code objects. + (Contributed by Itamar Ostricher in :gh:`91054`.) * Add :c:func:`PyFrame_GetVar` and :c:func:`PyFrame_GetVarString` functions to get a frame variable by its name. diff --git a/Include/cpython/code.h b/Include/cpython/code.h index fd57e0035bc09a..f11d099e0379ef 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -181,6 +181,41 @@ PyAPI_FUNC(int) PyCode_Addr2Line(PyCodeObject *, int); PyAPI_FUNC(int) PyCode_Addr2Location(PyCodeObject *, int, int *, int *, int *, int *); +typedef enum PyCodeEvent { + PY_CODE_EVENT_CREATE, + PY_CODE_EVENT_DESTROY +} PyCodeEvent; + + +/* + * A callback that is invoked for different events in a code object's lifecycle. + * + * The callback is invoked with a borrowed reference to co, after it is + * created and before it is destroyed. + * + * If the callback returns with an exception set, it must return -1. Otherwise + * it should return 0. + */ +typedef int (*PyCode_WatchCallback)( + PyCodeEvent event, + PyCodeObject* co); + +/* + * Register a per-interpreter callback that will be invoked for code object + * lifecycle events. + * + * Returns a handle that may be passed to PyCode_ClearWatcher on success, + * or -1 and sets an error if no more handles are available. + */ +PyAPI_FUNC(int) PyCode_AddWatcher(PyCode_WatchCallback callback); + +/* + * Clear the watcher associated with the watcher_id handle. + * + * Returns 0 on success or -1 if no watcher exists for the provided id. + */ +PyAPI_FUNC(int) PyCode_ClearWatcher(int watcher_id); + /* for internal use only */ struct _opaque { int computed_line; diff --git a/Include/cpython/objimpl.h b/Include/cpython/objimpl.h index d7c76eab5c7312..0b038d31080be9 100644 --- a/Include/cpython/objimpl.h +++ b/Include/cpython/objimpl.h @@ -2,7 +2,9 @@ # error "this header file must not be included directly" #endif -#define _PyObject_SIZE(typeobj) ( (typeobj)->tp_basicsize ) +static inline size_t _PyObject_SIZE(PyTypeObject *type) { + return _Py_STATIC_CAST(size_t, type->tp_basicsize); +} /* _PyObject_VAR_SIZE returns the number of bytes (as size_t) allocated for a vrbl-size object with nitems items, exclusive of gc overhead (if any). The @@ -18,10 +20,11 @@ # error "_PyObject_VAR_SIZE requires SIZEOF_VOID_P be a power of 2" #endif -#define _PyObject_VAR_SIZE(typeobj, nitems) \ - _Py_SIZE_ROUND_UP((typeobj)->tp_basicsize + \ - (nitems)*(typeobj)->tp_itemsize, \ - SIZEOF_VOID_P) +static inline size_t _PyObject_VAR_SIZE(PyTypeObject *type, Py_ssize_t nitems) { + size_t size = _Py_STATIC_CAST(size_t, type->tp_basicsize); + size += _Py_STATIC_CAST(size_t, nitems) * _Py_STATIC_CAST(size_t, type->tp_itemsize); + return _Py_SIZE_ROUND_UP(size, SIZEOF_VOID_P); +} /* This example code implements an object constructor with a custom diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index c51542bcc895cb..0117c23f518cdb 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -353,6 +353,9 @@ PyAPI_FUNC(const PyConfig*) _Py_GetConfig(void); // is necessary to pass safely between interpreters in the same process. typedef struct _xid _PyCrossInterpreterData; +typedef PyObject *(*xid_newobjectfunc)(_PyCrossInterpreterData *); +typedef void (*xid_freefunc)(void *); + struct _xid { // data is the cross-interpreter-safe derivation of a Python object // (see _PyObject_GetCrossInterpreterData). It will be NULL if the @@ -379,7 +382,7 @@ struct _xid { // interpreter given the data. The resulting object (a new // reference) will be equivalent to the original object. This field // is required. - PyObject *(*new_object)(_PyCrossInterpreterData *); + xid_newobjectfunc new_object; // free is called when the data is released. If it is NULL then // nothing will be done to free the data. For some types this is // okay (e.g. bytes) and for those types this field should be set @@ -389,18 +392,31 @@ struct _xid { // leak. In that case, at the very least this field should be set // to PyMem_RawFree (the default if not explicitly set to NULL). // The call will happen with the original interpreter activated. - void (*free)(void *); + xid_freefunc free; }; +PyAPI_FUNC(void) _PyCrossInterpreterData_Init( + _PyCrossInterpreterData *data, + PyInterpreterState *interp, void *shared, PyObject *obj, + xid_newobjectfunc new_object); +PyAPI_FUNC(int) _PyCrossInterpreterData_InitWithSize( + _PyCrossInterpreterData *, + PyInterpreterState *interp, const size_t, PyObject *, + xid_newobjectfunc); +PyAPI_FUNC(void) _PyCrossInterpreterData_Clear( + PyInterpreterState *, _PyCrossInterpreterData *); + PyAPI_FUNC(int) _PyObject_GetCrossInterpreterData(PyObject *, _PyCrossInterpreterData *); PyAPI_FUNC(PyObject *) _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *); -PyAPI_FUNC(void) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *); +PyAPI_FUNC(int) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *); PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *); /* cross-interpreter data registry */ -typedef int (*crossinterpdatafunc)(PyObject *, _PyCrossInterpreterData *); +typedef int (*crossinterpdatafunc)(PyThreadState *tstate, PyObject *, + _PyCrossInterpreterData *); PyAPI_FUNC(int) _PyCrossInterpreterData_RegisterClass(PyTypeObject *, crossinterpdatafunc); +PyAPI_FUNC(int) _PyCrossInterpreterData_UnregisterClass(PyTypeObject *); PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *); diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 80c1bfb6c9afa2..357fc85a95cf15 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -4,6 +4,8 @@ extern "C" { #endif +#define CODE_MAX_WATCHERS 8 + /* PEP 659 * Specialization and quickening structs and helper functions */ diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 2b3b56b343ad99..c74a3437713039 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -39,7 +39,7 @@ extern PyObject *_PyDict_FromKeys(PyObject *, PyObject *, PyObject *); * Returns the version number, or zero if it was not possible to get a version number. */ extern uint32_t _PyDictKeys_GetVersionForCurrentState(PyDictKeysObject *dictkeys); -extern Py_ssize_t _PyDict_KeysSize(PyDictKeysObject *keys); +extern size_t _PyDict_KeysSize(PyDictKeysObject *keys); /* _Py_dict_lookup() returns index of entry which can be used like DK_ENTRIES(dk)[index]. * -1 when no entry found, -3 when compare raises error. diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 532b28499080f2..b5c46773c90fd3 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -191,6 +191,9 @@ struct _is { PyObject *audit_hooks; PyType_WatchCallback type_watchers[TYPE_MAX_WATCHERS]; + PyCode_WatchCallback code_watchers[CODE_MAX_WATCHERS]; + // One bit is set for each non-NULL entry in code_watchers + uint8_t active_code_watchers; struct _Py_unicode_state unicode; struct _Py_float_state float_state; @@ -246,9 +249,10 @@ extern void _PyInterpreterState_Clear(PyThreadState *tstate); struct _xidregitem; struct _xidregitem { - PyTypeObject *cls; - crossinterpdatafunc getdata; + struct _xidregitem *prev; struct _xidregitem *next; + PyObject *cls; // weakref to a PyTypeObject + crossinterpdatafunc getdata; }; PyAPI_FUNC(PyInterpreterState*) _PyInterpreterState_LookUpID(int64_t); diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 33c8c0b75ea742..8796dfe2f6b8cf 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -373,7 +373,7 @@ PyAPI_FUNC(PyObject *) _PyObject_LookupSpecial(PyObject *, PyObject *); * match. * * Third party code unintentionally rely on problematic fpcasts. The call - * trampoline mitigates common occurences of bad fpcasts on Emscripten. + * trampoline mitigates common occurrences of bad fpcasts on Emscripten. */ #if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) #define _PyCFunction_TrampolineCall(meth, self, args) \ diff --git a/Include/internal/pycore_range.h b/Include/internal/pycore_range.h index 809e89a1e01b60..bf045ec4fd8332 100644 --- a/Include/internal/pycore_range.h +++ b/Include/internal/pycore_range.h @@ -10,7 +10,6 @@ extern "C" { typedef struct { PyObject_HEAD - long index; long start; long step; long len; diff --git a/Lib/http/server.py b/Lib/http/server.py index 8aee31bac2752a..3b5bd9eb012972 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -93,6 +93,7 @@ import html import http.client import io +import itertools import mimetypes import os import posixpath @@ -562,6 +563,10 @@ def log_error(self, format, *args): self.log_message(format, *args) + # https://en.wikipedia.org/wiki/List_of_Unicode_characters#Control_codes + _control_char_table = str.maketrans( + {c: fr'\x{c:02x}' for c in itertools.chain(range(0x20), range(0x7f,0xa0))}) + def log_message(self, format, *args): """Log an arbitrary message. @@ -577,12 +582,16 @@ def log_message(self, format, *args): The client ip and current date/time are prefixed to every message. + Unicode control characters are replaced with escaped hex + before writing the output to stderr. + """ + message = format % args sys.stderr.write("%s - - [%s] %s\n" % (self.address_string(), self.log_date_time_string(), - format%args)) + message.translate(self._control_char_table))) def version_string(self): """Return the server software version string.""" diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index 3f15601e700d68..1cb71d8032e173 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -1077,15 +1077,16 @@ def is_link_local(self): @property def is_private(self): - """Test if this address is allocated for private networks. + """Test if this network belongs to a private range. Returns: - A boolean, True if the address is reserved per + A boolean, True if the network is reserved per iana-ipv4-special-registry or iana-ipv6-special-registry. """ - return (self.network_address.is_private and - self.broadcast_address.is_private) + return any(self.network_address in priv_network and + self.broadcast_address in priv_network + for priv_network in self._constants._private_networks) @property def is_global(self): @@ -1122,6 +1123,15 @@ def is_loopback(self): return (self.network_address.is_loopback and self.broadcast_address.is_loopback) + +class _BaseConstants: + + _private_networks = [] + + +_BaseNetwork._constants = _BaseConstants + + class _BaseV4: """Base IPv4 object. @@ -1561,6 +1571,7 @@ class _IPv4Constants: IPv4Address._constants = _IPv4Constants +IPv4Network._constants = _IPv4Constants class _BaseV6: @@ -2285,3 +2296,4 @@ class _IPv6Constants: IPv6Address._constants = _IPv6Constants +IPv6Network._constants = _IPv6Constants diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index 86e1efe6e653ac..9241d73d0fd03c 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -511,7 +511,7 @@ def __init__(self, *args, **kwargs): def usesTime(self): fmt = self._fmt - return fmt.find('$asctime') >= 0 or fmt.find(self.asctime_format) >= 0 + return fmt.find('$asctime') >= 0 or fmt.find(self.asctime_search) >= 0 def validate(self): pattern = Template.pattern diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py index 961d7e5991847a..4f5d88cb975cb7 100644 --- a/Lib/multiprocessing/pool.py +++ b/Lib/multiprocessing/pool.py @@ -696,7 +696,7 @@ def _terminate_pool(cls, taskqueue, inqueue, outqueue, pool, change_notifier, if (not result_handler.is_alive()) and (len(cache) != 0): raise AssertionError( - "Cannot have cache with result_hander not alive") + "Cannot have cache with result_handler not alive") result_handler._state = TERMINATE change_notifier.put(None) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 487bdc5770bb92..43dbb862a3a377 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -663,12 +663,15 @@ def _getfinalpathname_nonstrict(path): # 21: ERROR_NOT_READY (implies drive with no media) # 32: ERROR_SHARING_VIOLATION (probably an NTFS paging file) # 50: ERROR_NOT_SUPPORTED + # 53: ERROR_BAD_NETPATH + # 65: ERROR_NETWORK_ACCESS_DENIED # 67: ERROR_BAD_NET_NAME (implies remote server unavailable) # 87: ERROR_INVALID_PARAMETER # 123: ERROR_INVALID_NAME + # 161: ERROR_BAD_PATHNAME # 1920: ERROR_CANT_ACCESS_FILE # 1921: ERROR_CANT_RESOLVE_FILENAME (implies unfollowable symlink) - allowed_winerror = 1, 2, 3, 5, 21, 32, 50, 67, 87, 123, 1920, 1921 + allowed_winerror = 1, 2, 3, 5, 21, 32, 50, 53, 65, 67, 87, 123, 161, 1920, 1921 # Non-strict algorithm is to find as much of the target directory # as we can and join the rest. diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index a66f4f5b897cd3..2fa75eb4d11311 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -6042,5 +6042,5 @@ def test_semlock_subclass(self): class SemLock(_multiprocessing.SemLock): pass name = f'test_semlock_subclass-{os.getpid()}' - s = SemLock(1, 0, 10, name, 0) + s = SemLock(1, 0, 10, name, False) _multiprocessing.sem_unlink(name) diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py index 66f29b95af10c3..18900bb9f7162c 100644 --- a/Lib/test/test__xxsubinterpreters.py +++ b/Lib/test/test__xxsubinterpreters.py @@ -295,8 +295,8 @@ def clean_up_channels(): class TestBase(unittest.TestCase): def tearDown(self): - clean_up_interpreters() clean_up_channels() + clean_up_interpreters() ################################## @@ -386,7 +386,6 @@ def test_types(self): self._assert_values([ b'spam', 9999, - self.cid, ]) def test_bytes(self): @@ -412,6 +411,15 @@ def test_non_shareable_int(self): interpreters.channel_send(self.cid, i) +class ModuleTests(TestBase): + + def test_import_in_interpreter(self): + _run_output( + interpreters.create(), + 'import _xxsubinterpreters as _interpreters', + ) + + ################################## # interpreter tests @@ -1213,6 +1221,18 @@ def test_equality(self): self.assertFalse(cid1 != cid2) self.assertTrue(cid1 != cid3) + def test_shareable(self): + chan = interpreters.channel_create() + + obj = interpreters.channel_create() + interpreters.channel_send(chan, obj) + got = interpreters.channel_recv(chan) + + self.assertEqual(got, obj) + self.assertIs(type(got), type(obj)) + # XXX Check the following in the channel tests? + #self.assertIsNot(got, obj) + class ChannelTests(TestBase): @@ -1545,6 +1565,19 @@ def test_recv_default(self): self.assertEqual(obj5, b'eggs') self.assertIs(obj6, default) + def test_recv_sending_interp_destroyed(self): + cid = interpreters.channel_create() + interp = interpreters.create() + interpreters.run_string(interp, dedent(f""" + import _xxsubinterpreters as _interpreters + _interpreters.channel_send({cid}, b'spam') + """)) + interpreters.destroy(interp) + + with self.assertRaisesRegex(RuntimeError, + 'unrecognized interpreter ID'): + interpreters.channel_recv(cid) + def test_run_string_arg_unresolved(self): cid = interpreters.channel_create() interp = interpreters.create() diff --git a/Lib/test/test_asyncio/test_ssl.py b/Lib/test/test_asyncio/test_ssl.py index 5de9b7a14e87da..aaf3c37101f52a 100644 --- a/Lib/test/test_asyncio/test_ssl.py +++ b/Lib/test/test_asyncio/test_ssl.py @@ -1689,7 +1689,7 @@ def stop(self): def run(self): try: with self._sock: - self._sock.setblocking(0) + self._sock.setblocking(False) self._run() finally: self._s1.close() diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index 092edb215854b7..309a1cfdb4aa96 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -1907,7 +1907,6 @@ def test_fork_signal_handling(self): def child_main(): signal.signal(signal.SIGTERM, lambda *args: child_handled.set()) child_started.set() - time.sleep(1) async def main(): loop = asyncio.get_running_loop() diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index f148b5ebbc5a9b..c17528be97b484 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -559,7 +559,7 @@ def __index__(self): self.kwargs.clear() gc.collect() return 0 - x = IntWithDict(dont_inherit=IntWithDict()) + x = IntWithDict(optimize=IntWithDict()) # We test the argument handling of "compile" here, the compilation # itself is not relevant. When we pass flags=x below, x.__index__() is # called, which changes the keywords dict. diff --git a/Lib/test/test_capi/test_codecs.py b/Lib/test/test_capi/test_codecs.py new file mode 100644 index 00000000000000..e46726192aa05b --- /dev/null +++ b/Lib/test/test_capi/test_codecs.py @@ -0,0 +1,54 @@ +import unittest +from test.support import import_helper + +_testcapi = import_helper.import_module('_testcapi') + + +class CAPITest(unittest.TestCase): + + def test_decodeutf8(self): + """Test PyUnicode_DecodeUTF8()""" + decodeutf8 = _testcapi.unicode_decodeutf8 + + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: + b = s.encode('utf-8') + self.assertEqual(decodeutf8(b), s) + self.assertEqual(decodeutf8(b, 'strict'), s) + + self.assertRaises(UnicodeDecodeError, decodeutf8, b'\x80') + self.assertRaises(UnicodeDecodeError, decodeutf8, b'\xc0') + self.assertRaises(UnicodeDecodeError, decodeutf8, b'\xff') + self.assertRaises(UnicodeDecodeError, decodeutf8, b'a\xf0\x9f') + self.assertEqual(decodeutf8(b'a\xf0\x9f', 'replace'), 'a\ufffd') + self.assertEqual(decodeutf8(b'a\xf0\x9fb', 'replace'), 'a\ufffdb') + + self.assertRaises(LookupError, decodeutf8, b'a\x80', 'foo') + # TODO: Test PyUnicode_DecodeUTF8() with NULL as data and + # negative size. + + def test_decodeutf8stateful(self): + """Test PyUnicode_DecodeUTF8Stateful()""" + decodeutf8stateful = _testcapi.unicode_decodeutf8stateful + + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: + b = s.encode('utf-8') + self.assertEqual(decodeutf8stateful(b), (s, len(b))) + self.assertEqual(decodeutf8stateful(b, 'strict'), (s, len(b))) + + self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\x80') + self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\xc0') + self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\xff') + self.assertEqual(decodeutf8stateful(b'a\xf0\x9f'), ('a', 1)) + self.assertEqual(decodeutf8stateful(b'a\xf0\x9f', 'replace'), ('a', 1)) + self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'a\xf0\x9fb') + self.assertEqual(decodeutf8stateful(b'a\xf0\x9fb', 'replace'), ('a\ufffdb', 4)) + + self.assertRaises(LookupError, decodeutf8stateful, b'a\x80', 'foo') + # TODO: Test PyUnicode_DecodeUTF8Stateful() with NULL as data and + # negative size. + # TODO: Test PyUnicode_DecodeUTF8Stateful() with NULL as the address of + # "consumed". + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 1d30adaee9218f..06a51aa3cc219a 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -138,8 +138,9 @@ def test_seq_bytes_to_charp_array(self): class Z(object): def __len__(self): return 1 - self.assertRaises(TypeError, _posixsubprocess.fork_exec, - 1,Z(),3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23) + with self.assertRaisesRegex(TypeError, 'indexing'): + _posixsubprocess.fork_exec( + 1,Z(),True,(1, 2),5,6,7,8,9,10,11,12,13,14,True,True,17,False,19,20,21,22,False) # Issue #15736: overflow in _PySequence_BytesToCharpArray() class Z(object): def __len__(self): @@ -147,7 +148,7 @@ def __len__(self): def __getitem__(self, i): return b'x' self.assertRaises(MemoryError, _posixsubprocess.fork_exec, - 1,Z(),3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23) + 1,Z(),True,(1, 2),5,6,7,8,9,10,11,12,13,14,True,True,17,False,19,20,21,22,False) @unittest.skipUnless(_posixsubprocess, '_posixsubprocess required for this test.') def test_subprocess_fork_exec(self): @@ -157,7 +158,7 @@ def __len__(self): # Issue #15738: crash in subprocess_fork_exec() self.assertRaises(TypeError, _posixsubprocess.fork_exec, - Z(),[b'1'],3,(1, 2),5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23) + Z(),[b'1'],True,(1, 2),5,6,7,8,9,10,11,12,13,14,True,True,17,False,19,20,21,22,False) @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index 07b77d3e04bbe0..857579f758386f 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -9,10 +9,35 @@ _testcapi = None +NULL = None + +class Str(str): + pass + + class CAPITest(unittest.TestCase): - # Test PyUnicode_FromFormat() + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_fromobject(self): + """Test PyUnicode_FromObject()""" + from _testcapi import unicode_fromobject as fromobject + + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', + 'a\ud800b\udfffc', '\ud834\udd1e']: + self.assertEqual(fromobject(s), s) + o = Str(s) + s2 = fromobject(o) + self.assertEqual(s2, s) + self.assertIs(type(s2), str) + self.assertIsNot(s2, s) + + self.assertRaises(TypeError, fromobject, b'abc') + self.assertRaises(TypeError, fromobject, []) + # CRASHES fromobject(NULL) + def test_from_format(self): + """Test PyUnicode_FromFormat()""" import_helper.import_module('ctypes') from ctypes import ( c_char_p, @@ -268,10 +293,10 @@ def check_format(expected, format, *args): self.assertRaisesRegex(SystemError, 'invalid format string', PyUnicode_FromFormat, b'%+i', c_int(10)) - # Test PyUnicode_AsWideChar() @support.cpython_only @unittest.skipIf(_testcapi is None, 'need _testcapi module') def test_aswidechar(self): + """Test PyUnicode_AsWideChar()""" from _testcapi import unicode_aswidechar import_helper.import_module('ctypes') from ctypes import c_wchar, sizeof @@ -307,10 +332,10 @@ def test_aswidechar(self): self.assertEqual(size, nchar) self.assertEqual(wchar, nonbmp + '\0') - # Test PyUnicode_AsWideCharString() @support.cpython_only @unittest.skipIf(_testcapi is None, 'need _testcapi module') def test_aswidecharstring(self): + """Test PyUnicode_AsWideCharString()""" from _testcapi import unicode_aswidecharstring import_helper.import_module('ctypes') from ctypes import c_wchar, sizeof @@ -332,10 +357,10 @@ def test_aswidecharstring(self): self.assertEqual(size, nchar) self.assertEqual(wchar, nonbmp + '\0') - # Test PyUnicode_AsUCS4() @support.cpython_only @unittest.skipIf(_testcapi is None, 'need _testcapi module') def test_asucs4(self): + """Test PyUnicode_AsUCS4()""" from _testcapi import unicode_asucs4 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', 'a\ud800b\udfffc', '\ud834\udd1e']: @@ -350,10 +375,10 @@ def test_asucs4(self): self.assertEqual(unicode_asucs4(s, len(s), True), s+'\0') self.assertEqual(unicode_asucs4(s, len(s), False), s+'\uffff') - # Test PyUnicode_AsUTF8() @support.cpython_only @unittest.skipIf(_testcapi is None, 'need _testcapi module') def test_asutf8(self): + """Test PyUnicode_AsUTF8()""" from _testcapi import unicode_asutf8 bmp = '\u0100' @@ -365,10 +390,10 @@ def test_asutf8(self): self.assertEqual(unicode_asutf8(nonbmp), b'\xf4\x8f\xbf\xbf') self.assertRaises(UnicodeEncodeError, unicode_asutf8, 'a\ud800b\udfffc') - # Test PyUnicode_AsUTF8AndSize() @support.cpython_only @unittest.skipIf(_testcapi is None, 'need _testcapi module') def test_asutf8andsize(self): + """Test PyUnicode_AsUTF8AndSize()""" from _testcapi import unicode_asutf8andsize bmp = '\u0100' @@ -380,54 +405,275 @@ def test_asutf8andsize(self): self.assertEqual(unicode_asutf8andsize(nonbmp), (b'\xf4\x8f\xbf\xbf', 4)) self.assertRaises(UnicodeEncodeError, unicode_asutf8andsize, 'a\ud800b\udfffc') - # Test PyUnicode_Count() + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_concat(self): + """Test PyUnicode_Concat()""" + from _testcapi import unicode_concat as concat + + self.assertEqual(concat('abc', 'def'), 'abcdef') + self.assertEqual(concat('abc', 'где'), 'abcгде') + self.assertEqual(concat('абв', 'def'), 'абвdef') + self.assertEqual(concat('абв', 'где'), 'абвгде') + self.assertEqual(concat('a\0b', 'c\0d'), 'a\0bc\0d') + + self.assertRaises(TypeError, concat, b'abc', 'def') + self.assertRaises(TypeError, concat, 'abc', b'def') + self.assertRaises(TypeError, concat, b'abc', b'def') + self.assertRaises(TypeError, concat, [], 'def') + self.assertRaises(TypeError, concat, 'abc', []) + self.assertRaises(TypeError, concat, [], []) + # CRASHES concat(NULL, 'def') + # CRASHES concat('abc', NULL) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_split(self): + """Test PyUnicode_Split()""" + from _testcapi import unicode_split as split + + self.assertEqual(split('a|b|c|d', '|'), ['a', 'b', 'c', 'd']) + self.assertEqual(split('a|b|c|d', '|', 2), ['a', 'b', 'c|d']) + self.assertEqual(split('a|b|c|d', '\u20ac'), ['a|b|c|d']) + self.assertEqual(split('a||b|c||d', '||'), ['a', 'b|c', 'd']) + self.assertEqual(split('а|б|в|г', '|'), ['а', 'б', 'в', 'г']) + self.assertEqual(split('абабагаламага', 'а'), + ['', 'б', 'б', 'г', 'л', 'м', 'г', '']) + self.assertEqual(split(' a\tb\nc\rd\ve\f', NULL), + ['a', 'b', 'c', 'd', 'e']) + self.assertEqual(split('a\x85b\xa0c\u1680d\u2000e', NULL), + ['a', 'b', 'c', 'd', 'e']) + + self.assertRaises(ValueError, split, 'a|b|c|d', '') + self.assertRaises(TypeError, split, 'a|b|c|d', ord('|')) + self.assertRaises(TypeError, split, [], '|') + # CRASHES split(NULL, '|') + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_rsplit(self): + """Test PyUnicode_RSplit()""" + from _testcapi import unicode_rsplit as rsplit + + self.assertEqual(rsplit('a|b|c|d', '|'), ['a', 'b', 'c', 'd']) + self.assertEqual(rsplit('a|b|c|d', '|', 2), ['a|b', 'c', 'd']) + self.assertEqual(rsplit('a|b|c|d', '\u20ac'), ['a|b|c|d']) + self.assertEqual(rsplit('a||b|c||d', '||'), ['a', 'b|c', 'd']) + self.assertEqual(rsplit('а|б|в|г', '|'), ['а', 'б', 'в', 'г']) + self.assertEqual(rsplit('абабагаламага', 'а'), + ['', 'б', 'б', 'г', 'л', 'м', 'г', '']) + self.assertEqual(rsplit('aжbжcжd', 'ж'), ['a', 'b', 'c', 'd']) + self.assertEqual(rsplit(' a\tb\nc\rd\ve\f', NULL), + ['a', 'b', 'c', 'd', 'e']) + self.assertEqual(rsplit('a\x85b\xa0c\u1680d\u2000e', NULL), + ['a', 'b', 'c', 'd', 'e']) + + self.assertRaises(ValueError, rsplit, 'a|b|c|d', '') + self.assertRaises(TypeError, rsplit, 'a|b|c|d', ord('|')) + self.assertRaises(TypeError, rsplit, [], '|') + # CRASHES rsplit(NULL, '|') + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_partition(self): + """Test PyUnicode_Partition()""" + from _testcapi import unicode_partition as partition + + self.assertEqual(partition('a|b|c', '|'), ('a', '|', 'b|c')) + self.assertEqual(partition('a||b||c', '||'), ('a', '||', 'b||c')) + self.assertEqual(partition('а|б|в', '|'), ('а', '|', 'б|в')) + self.assertEqual(partition('кабан', 'а'), ('к', 'а', 'бан')) + self.assertEqual(partition('aжbжc', 'ж'), ('a', 'ж', 'bжc')) + + self.assertRaises(ValueError, partition, 'a|b|c', '') + self.assertRaises(TypeError, partition, b'a|b|c', '|') + self.assertRaises(TypeError, partition, 'a|b|c', b'|') + self.assertRaises(TypeError, partition, 'a|b|c', ord('|')) + self.assertRaises(TypeError, partition, [], '|') + # CRASHES partition(NULL, '|') + # CRASHES partition('a|b|c', NULL) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_rpartition(self): + """Test PyUnicode_RPartition()""" + from _testcapi import unicode_rpartition as rpartition + + self.assertEqual(rpartition('a|b|c', '|'), ('a|b', '|', 'c')) + self.assertEqual(rpartition('a||b||c', '||'), ('a||b', '||', 'c')) + self.assertEqual(rpartition('а|б|в', '|'), ('а|б', '|', 'в')) + self.assertEqual(rpartition('кабан', 'а'), ('каб', 'а', 'н')) + self.assertEqual(rpartition('aжbжc', 'ж'), ('aжb', 'ж', 'c')) + + self.assertRaises(ValueError, rpartition, 'a|b|c', '') + self.assertRaises(TypeError, rpartition, b'a|b|c', '|') + self.assertRaises(TypeError, rpartition, 'a|b|c', b'|') + self.assertRaises(TypeError, rpartition, 'a|b|c', ord('|')) + self.assertRaises(TypeError, rpartition, [], '|') + # CRASHES rpartition(NULL, '|') + # CRASHES rpartition('a|b|c', NULL) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_splitlines(self): + """Test PyUnicode_SplitLines()""" + from _testcapi import unicode_splitlines as splitlines + + self.assertEqual(splitlines('a\nb\rc\r\nd'), ['a', 'b', 'c', 'd']) + self.assertEqual(splitlines('a\nb\rc\r\nd', True), + ['a\n', 'b\r', 'c\r\n', 'd']) + self.assertEqual(splitlines('a\x85b\u2028c\u2029d'), + ['a', 'b', 'c', 'd']) + self.assertEqual(splitlines('a\x85b\u2028c\u2029d', True), + ['a\x85', 'b\u2028', 'c\u2029', 'd']) + self.assertEqual(splitlines('а\nб\rв\r\nг'), ['а', 'б', 'в', 'г']) + + self.assertRaises(TypeError, splitlines, b'a\nb\rc\r\nd') + # CRASHES splitlines(NULL) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_translate(self): + """Test PyUnicode_Translate()""" + from _testcapi import unicode_translate as translate + + self.assertEqual(translate('abcd', {ord('a'): 'A', ord('b'): ord('B'), ord('c'): '<>'}), 'AB<>d') + self.assertEqual(translate('абвг', {ord('а'): 'А', ord('б'): ord('Б'), ord('в'): '<>'}), 'АБ<>г') + self.assertEqual(translate('abc', {}), 'abc') + self.assertEqual(translate('abc', []), 'abc') + self.assertRaises(UnicodeTranslateError, translate, 'abc', {ord('b'): None}) + self.assertRaises(UnicodeTranslateError, translate, 'abc', {ord('b'): None}, 'strict') + self.assertRaises(LookupError, translate, 'abc', {ord('b'): None}, 'foo') + self.assertEqual(translate('abc', {ord('b'): None}, 'ignore'), 'ac') + self.assertEqual(translate('abc', {ord('b'): None}, 'replace'), 'a\ufffdc') + self.assertEqual(translate('abc', {ord('b'): None}, 'backslashreplace'), r'a\x62c') + # XXX Other error handlers do not support UnicodeTranslateError + self.assertRaises(TypeError, translate, b'abc', []) + self.assertRaises(TypeError, translate, 123, []) + self.assertRaises(TypeError, translate, 'abc', {ord('a'): b'A'}) + self.assertRaises(TypeError, translate, 'abc', 123) + self.assertRaises(TypeError, translate, 'abc', NULL) + self.assertRaises(LookupError, translate, 'abc', {ord('b'): None}, 'foo') + # CRASHES translate(NULL, []) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_join(self): + """Test PyUnicode_Join()""" + from _testcapi import unicode_join as join + self.assertEqual(join('|', ['a', 'b', 'c']), 'a|b|c') + self.assertEqual(join('|', ['a', '', 'c']), 'a||c') + self.assertEqual(join('', ['a', 'b', 'c']), 'abc') + self.assertEqual(join(NULL, ['a', 'b', 'c']), 'a b c') + self.assertEqual(join('|', ['а', 'б', 'в']), 'а|б|в') + self.assertEqual(join('ж', ['а', 'б', 'в']), 'ажбжв') + self.assertRaises(TypeError, join, b'|', ['a', 'b', 'c']) + self.assertRaises(TypeError, join, '|', [b'a', b'b', b'c']) + self.assertRaises(TypeError, join, NULL, [b'a', b'b', b'c']) + self.assertRaises(TypeError, join, '|', b'123') + self.assertRaises(TypeError, join, '|', 123) + self.assertRaises(SystemError, join, '|', NULL) + @support.cpython_only @unittest.skipIf(_testcapi is None, 'need _testcapi module') def test_count(self): + """Test PyUnicode_Count()""" from _testcapi import unicode_count - st = 'abcabd' - self.assertEqual(unicode_count(st, 'a', 0, len(st)), 2) - self.assertEqual(unicode_count(st, 'ab', 0, len(st)), 2) - self.assertEqual(unicode_count(st, 'abc', 0, len(st)), 1) - self.assertEqual(unicode_count(st, 'а', 0, len(st)), 0) # cyrillic "a" + for str in "\xa1", "\u8000\u8080", "\ud800\udc02", "\U0001f100\U0001f1f1": + for i, ch in enumerate(str): + self.assertEqual(unicode_count(str, ch, 0, len(str)), 1) + + str = "!>_= end + self.assertEqual(unicode_count(str, '!', 0, 0), 0) + self.assertEqual(unicode_count(str, '!', len(str), 0), 0) + # negative + self.assertEqual(unicode_count(str, '!', -len(str), -1), 1) + # bad arguments + self.assertRaises(TypeError, unicode_count, str, b'!', 0, len(str)) + self.assertRaises(TypeError, unicode_count, b"!>__= end - self.assertEqual(unicode_count(st, 'abc', 0, 0), 0) - self.assertEqual(unicode_count(st, 'a', 3, 2), 0) - self.assertEqual(unicode_count(st, 'a', sys.maxsize, 5), 0) + self.assertEqual(find(str, '!', 0, 0, 1), -1) + self.assertEqual(find(str, '!', len(str), 0, 1), -1) # negative - self.assertEqual(unicode_count(st, 'ab', -len(st), -1), 2) - self.assertEqual(unicode_count(st, 'a', -len(st), -3), 1) - # wrong args - self.assertRaises(TypeError, unicode_count, 'a', 'a') - self.assertRaises(TypeError, unicode_count, 'a', 'a', 1) - self.assertRaises(TypeError, unicode_count, 1, 'a', 0, 1) - self.assertRaises(TypeError, unicode_count, 'a', 1, 0, 1) - # empty string - self.assertEqual(unicode_count('abc', '', 0, 3), 4) - self.assertEqual(unicode_count('abc', '', 1, 3), 3) - self.assertEqual(unicode_count('', '', 0, 1), 1) - self.assertEqual(unicode_count('', 'a', 0, 1), 0) - # different unicode kinds - for uni in "\xa1", "\u8000\u8080", "\ud800\udc02", "\U0001f100\U0001f1f1": - for ch in uni: - self.assertEqual(unicode_count(uni, ch, 0, len(uni)), 1) - self.assertEqual(unicode_count(st, ch, 0, len(st)), 0) - - # subclasses should still work - class MyStr(str): - pass - - self.assertEqual(unicode_count(MyStr('aab'), 'a', 0, 3), 2) - - # Test PyUnicode_FindChar() + self.assertEqual(find(str, '!', -len(str), -1, 1), 0) + self.assertEqual(find(str, '!', -len(str), -1, -1), 0) + # bad arguments + self.assertRaises(TypeError, find, str, b'!', 0, len(str), 1) + self.assertRaises(TypeError, find, b"!>__'), '<>br<>c<>d<>br<>') + self.assertEqual(replace(str, 'abra', '='), '=cad=') + self.assertEqual(replace(str, 'a', '=', 2), '=br=cadabra') + self.assertEqual(replace(str, 'a', '=', 0), str) + self.assertEqual(replace(str, 'a', '=', sys.maxsize), '=br=c=d=br=') + self.assertEqual(replace(str, 'z', '='), str) + self.assertEqual(replace(str, '', '='), '=a=b=r=a=c=a=d=a=b=r=a=') + self.assertEqual(replace(str, 'a', 'ж'), 'жbrжcжdжbrж') + self.assertEqual(replace('абабагаламага', 'а', '='), '=б=б=г=л=м=г=') + self.assertEqual(replace('Баден-Баден', 'Баден', 'Baden'), 'Baden-Baden') + # bad arguments + self.assertRaises(TypeError, replace, 'a', 'a', b'=') + self.assertRaises(TypeError, replace, 'a', b'a', '=') + self.assertRaises(TypeError, replace, b'a', 'a', '=') + self.assertRaises(TypeError, replace, 'a', 'a', ord('=')) + self.assertRaises(TypeError, replace, 'a', ord('a'), '=') + self.assertRaises(TypeError, replace, [], 'a', '=') + # CRASHES replace('a', 'a', NULL) + # CRASHES replace('a', NULL, '=') + # CRASHES replace(NULL, 'a', '=') + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_compare(self): + """Test PyUnicode_Compare()""" + from _testcapi import unicode_compare as compare + + self.assertEqual(compare('abc', 'abc'), 0) + self.assertEqual(compare('abc', 'def'), -1) + self.assertEqual(compare('def', 'abc'), 1) + self.assertEqual(compare('abc', 'abc\0def'), -1) + self.assertEqual(compare('abc\0def', 'abc\0def'), 0) + self.assertEqual(compare('абв', 'abc'), 1) + + self.assertRaises(TypeError, compare, b'abc', 'abc') + self.assertRaises(TypeError, compare, 'abc', b'abc') + self.assertRaises(TypeError, compare, b'abc', b'abc') + self.assertRaises(TypeError, compare, [], 'abc') + self.assertRaises(TypeError, compare, 'abc', []) + self.assertRaises(TypeError, compare, [], []) + # CRASHES compare(NULL, 'abc') + # CRASHES compare('abc', NULL) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_comparewithasciistring(self): + """Test PyUnicode_CompareWithASCIIString()""" + from _testcapi import unicode_comparewithasciistring as comparewithasciistring + + self.assertEqual(comparewithasciistring('abc', b'abc'), 0) + self.assertEqual(comparewithasciistring('abc', b'def'), -1) + self.assertEqual(comparewithasciistring('def', b'abc'), 1) + self.assertEqual(comparewithasciistring('abc', b'abc\0def'), 0) + self.assertEqual(comparewithasciistring('abc\0def', b'abc\0def'), 1) + self.assertEqual(comparewithasciistring('абв', b'abc'), 1) + + # CRASHES comparewithasciistring(b'abc', b'abc') + # CRASHES comparewithasciistring([], b'abc') + # CRASHES comparewithasciistring(NULL, b'abc') + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_richcompare(self): + """Test PyUnicode_RichCompare()""" + from _testcapi import unicode_richcompare as richcompare + + LT, LE, EQ, NE, GT, GE = range(6) + strings = ('abc', 'абв', '\U0001f600', 'abc\0') + for s1 in strings: + for s2 in strings: + self.assertIs(richcompare(s1, s2, LT), s1 < s2) + self.assertIs(richcompare(s1, s2, LE), s1 <= s2) + self.assertIs(richcompare(s1, s2, EQ), s1 == s2) + self.assertIs(richcompare(s1, s2, NE), s1 != s2) + self.assertIs(richcompare(s1, s2, GT), s1 > s2) + self.assertIs(richcompare(s1, s2, GE), s1 >= s2) + + for op in LT, LE, EQ, NE, GT, GE: + self.assertIs(richcompare(b'abc', 'abc', op), NotImplemented) + self.assertIs(richcompare('abc', b'abc', op), NotImplemented) + self.assertIs(richcompare(b'abc', b'abc', op), NotImplemented) + self.assertIs(richcompare([], 'abc', op), NotImplemented) + self.assertIs(richcompare('abc', [], op), NotImplemented) + self.assertIs(richcompare([], [], op), NotImplemented) + + # CRASHES richcompare(NULL, 'abc', op) + # CRASHES richcompare('abc', NULL, op) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_format(self): + """Test PyUnicode_Format()""" + from _testcapi import unicode_format as format + + self.assertEqual(format('x=%d!', 42), 'x=42!') + self.assertEqual(format('x=%d!', (42,)), 'x=42!') + self.assertEqual(format('x=%d y=%s!', (42, [])), 'x=42 y=[]!') + + self.assertRaises(SystemError, format, 'x=%d!', NULL) + self.assertRaises(SystemError, format, NULL, 42) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_contains(self): + """Test PyUnicode_Contains()""" + from _testcapi import unicode_contains as contains + + self.assertEqual(contains('abcd', ''), 1) + self.assertEqual(contains('abcd', 'b'), 1) + self.assertEqual(contains('abcd', 'x'), 0) + self.assertEqual(contains('abcd', 'ж'), 0) + self.assertEqual(contains('abcd', '\0'), 0) + self.assertEqual(contains('abc\0def', '\0'), 1) + self.assertEqual(contains('abcd', 'bc'), 1) + + self.assertRaises(TypeError, contains, b'abcd', 'b') + self.assertRaises(TypeError, contains, 'abcd', b'b') + self.assertRaises(TypeError, contains, b'abcd', b'b') + self.assertRaises(TypeError, contains, [], 'b') + self.assertRaises(TypeError, contains, 'abcd', ord('b')) + # CRASHES contains(NULL, 'b') + # CRASHES contains('abcd', NULL) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_isidentifier(self): + """Test PyUnicode_IsIdentifier()""" + from _testcapi import unicode_isidentifier as isidentifier + + self.assertEqual(isidentifier("a"), 1) + self.assertEqual(isidentifier("b0"), 1) + self.assertEqual(isidentifier("µ"), 1) + self.assertEqual(isidentifier("𝔘𝔫𝔦𝔠𝔬𝔡𝔢"), 1) + + self.assertEqual(isidentifier(""), 0) + self.assertEqual(isidentifier(" "), 0) + self.assertEqual(isidentifier("["), 0) + self.assertEqual(isidentifier("©"), 0) + self.assertEqual(isidentifier("0"), 0) + self.assertEqual(isidentifier("32M"), 0) + + # CRASHES isidentifier(b"a") + # CRASHES isidentifier([]) + # CRASHES isidentifier(NULL) - # Test PyUnicode_CopyCharacters() @support.cpython_only @unittest.skipIf(_testcapi is None, 'need _testcapi module') def test_copycharacters(self): + """Test PyUnicode_CopyCharacters()""" from _testcapi import unicode_copycharacters strings = [ diff --git a/Lib/test/test_capi/test_watchers.py b/Lib/test/test_capi/test_watchers.py index 5e4f42a86006bd..1922614ef60558 100644 --- a/Lib/test/test_capi/test_watchers.py +++ b/Lib/test/test_capi/test_watchers.py @@ -336,6 +336,74 @@ def test_no_more_ids_available(self): self.add_watcher() +class TestCodeObjectWatchers(unittest.TestCase): + @contextmanager + def code_watcher(self, which_watcher): + wid = _testcapi.add_code_watcher(which_watcher) + try: + yield wid + finally: + _testcapi.clear_code_watcher(wid) + + def assert_event_counts(self, exp_created_0, exp_destroyed_0, + exp_created_1, exp_destroyed_1): + self.assertEqual( + exp_created_0, _testcapi.get_code_watcher_num_created_events(0)) + self.assertEqual( + exp_destroyed_0, _testcapi.get_code_watcher_num_destroyed_events(0)) + self.assertEqual( + exp_created_1, _testcapi.get_code_watcher_num_created_events(1)) + self.assertEqual( + exp_destroyed_1, _testcapi.get_code_watcher_num_destroyed_events(1)) + + def test_code_object_events_dispatched(self): + # verify that all counts are zero before any watchers are registered + self.assert_event_counts(0, 0, 0, 0) + + # verify that all counts remain zero when a code object is + # created and destroyed with no watchers registered + co1 = _testcapi.code_newempty("test_watchers", "dummy1", 0) + self.assert_event_counts(0, 0, 0, 0) + del co1 + self.assert_event_counts(0, 0, 0, 0) + + # verify counts are as expected when first watcher is registered + with self.code_watcher(0): + self.assert_event_counts(0, 0, 0, 0) + co2 = _testcapi.code_newempty("test_watchers", "dummy2", 0) + self.assert_event_counts(1, 0, 0, 0) + del co2 + self.assert_event_counts(1, 1, 0, 0) + + # again with second watcher registered + with self.code_watcher(1): + self.assert_event_counts(1, 1, 0, 0) + co3 = _testcapi.code_newempty("test_watchers", "dummy3", 0) + self.assert_event_counts(2, 1, 1, 0) + del co3 + self.assert_event_counts(2, 2, 1, 1) + + # verify counts are reset and don't change after both watchers are cleared + co4 = _testcapi.code_newempty("test_watchers", "dummy4", 0) + self.assert_event_counts(0, 0, 0, 0) + del co4 + self.assert_event_counts(0, 0, 0, 0) + + def test_clear_out_of_range_watcher_id(self): + with self.assertRaisesRegex(ValueError, r"Invalid code watcher ID -1"): + _testcapi.clear_code_watcher(-1) + with self.assertRaisesRegex(ValueError, r"Invalid code watcher ID 8"): + _testcapi.clear_code_watcher(8) # CODE_MAX_WATCHERS = 8 + + def test_clear_unassigned_watcher_id(self): + with self.assertRaisesRegex(ValueError, r"No code watcher set for ID 1"): + _testcapi.clear_code_watcher(1) + + def test_allocate_too_many_watchers(self): + with self.assertRaisesRegex(RuntimeError, r"no more code watcher IDs available"): + _testcapi.allocate_too_many_code_watchers() + + class TestFuncWatchers(unittest.TestCase): @contextmanager def add_watcher(self, func): diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index c838e95ad554e3..f10d72ea5547ee 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -753,6 +753,9 @@ def test_nonexisting_script(self): self.assertNotEqual(proc.returncode, 0) @unittest.skipUnless(os.path.exists('/dev/fd/0'), 'requires /dev/fd platform') + @unittest.skipIf(sys.platform.startswith("freebsd") and + os.stat("/dev").st_dev == os.stat("/dev/fd").st_dev, + "Requires fdescfs mounted on /dev/fd on FreeBSD") def test_script_as_dev_fd(self): # GH-87235: On macOS passing a non-trivial script to /dev/fd/N can cause # problems because all open /dev/fd/N file descriptors share the same diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index a937258069ed89..b0e2d713e3de77 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -26,7 +26,7 @@ import datetime import threading from unittest import mock -from io import BytesIO +from io import BytesIO, StringIO import unittest from test import support @@ -990,6 +990,25 @@ def verify_http_server_response(self, response): match = self.HTTPResponseMatch.search(response) self.assertIsNotNone(match) + def test_unprintable_not_logged(self): + # We call the method from the class directly as our Socketless + # Handler subclass overrode it... nice for everything BUT this test. + self.handler.client_address = ('127.0.0.1', 1337) + log_message = BaseHTTPRequestHandler.log_message + with mock.patch.object(sys, 'stderr', StringIO()) as fake_stderr: + log_message(self.handler, '/foo') + log_message(self.handler, '/\033bar\000\033') + log_message(self.handler, '/spam %s.', 'a') + log_message(self.handler, '/spam %s.', '\033\x7f\x9f\xa0beans') + stderr = fake_stderr.getvalue() + self.assertNotIn('\033', stderr) # non-printable chars are caught. + self.assertNotIn('\000', stderr) # non-printable chars are caught. + lines = stderr.splitlines() + self.assertIn('/foo', lines[0]) + self.assertIn(r'/\x1bbar\x00\x1b', lines[1]) + self.assertIn('/spam a.', lines[2]) + self.assertIn('/spam \\x1b\\x7f\\x9f\xa0beans.', lines[3]) + def test_http_1_1(self): result = self.send_typical_request(b'GET / HTTP/1.1\r\n\r\n') self.verify_http_server_response(result[0]) diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index 5c656c49e2e75f..a5388b2e5debd8 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -2277,6 +2277,39 @@ def testReservedIpv4(self): self.assertEqual(False, ipaddress.ip_address('128.0.0.0').is_loopback) self.assertEqual(True, ipaddress.ip_network('0.0.0.0').is_unspecified) + def testPrivateNetworks(self): + self.assertEqual(False, ipaddress.ip_network("0.0.0.0/0").is_private) + self.assertEqual(False, ipaddress.ip_network("1.0.0.0/8").is_private) + + self.assertEqual(True, ipaddress.ip_network("0.0.0.0/8").is_private) + self.assertEqual(True, ipaddress.ip_network("10.0.0.0/8").is_private) + self.assertEqual(True, ipaddress.ip_network("127.0.0.0/8").is_private) + self.assertEqual(True, ipaddress.ip_network("169.254.0.0/16").is_private) + self.assertEqual(True, ipaddress.ip_network("172.16.0.0/12").is_private) + self.assertEqual(True, ipaddress.ip_network("192.0.0.0/29").is_private) + self.assertEqual(True, ipaddress.ip_network("192.0.0.170/31").is_private) + self.assertEqual(True, ipaddress.ip_network("192.0.2.0/24").is_private) + self.assertEqual(True, ipaddress.ip_network("192.168.0.0/16").is_private) + self.assertEqual(True, ipaddress.ip_network("198.18.0.0/15").is_private) + self.assertEqual(True, ipaddress.ip_network("198.51.100.0/24").is_private) + self.assertEqual(True, ipaddress.ip_network("203.0.113.0/24").is_private) + self.assertEqual(True, ipaddress.ip_network("240.0.0.0/4").is_private) + self.assertEqual(True, ipaddress.ip_network("255.255.255.255/32").is_private) + + self.assertEqual(False, ipaddress.ip_network("::/0").is_private) + self.assertEqual(False, ipaddress.ip_network("::ff/128").is_private) + + self.assertEqual(True, ipaddress.ip_network("::1/128").is_private) + self.assertEqual(True, ipaddress.ip_network("::/128").is_private) + self.assertEqual(True, ipaddress.ip_network("::ffff:0:0/96").is_private) + self.assertEqual(True, ipaddress.ip_network("100::/64").is_private) + self.assertEqual(True, ipaddress.ip_network("2001::/23").is_private) + self.assertEqual(True, ipaddress.ip_network("2001:2::/48").is_private) + self.assertEqual(True, ipaddress.ip_network("2001:db8::/32").is_private) + self.assertEqual(True, ipaddress.ip_network("2001:10::/28").is_private) + self.assertEqual(True, ipaddress.ip_network("fc00::/7").is_private) + self.assertEqual(True, ipaddress.ip_network("fe80::/10").is_private) + def testReservedIpv6(self): self.assertEqual(True, ipaddress.ip_network('ffff::').is_multicast) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index a0a740fba8e8e3..5f5bcbc7cfb8d7 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -675,6 +675,7 @@ def test_cycle(self): self.assertRaises(TypeError, cycle, 5) self.assertEqual(list(islice(cycle(gen3()),10)), [0,1,2,0,1,2,0,1,2,0]) + def test_cycle_copy_pickle(self): # check copy, deepcopy, pickle c = cycle('abc') self.assertEqual(next(c), 'a') @@ -710,6 +711,37 @@ def test_cycle(self): d = pickle.loads(p) # rebuild the cycle object self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) + def test_cycle_unpickle_compat(self): + testcases = [ + b'citertools\ncycle\n(c__builtin__\niter\n((lI1\naI2\naI3\natRI1\nbtR((lI1\naI0\ntb.', + b'citertools\ncycle\n(c__builtin__\niter\n(](K\x01K\x02K\x03etRK\x01btR(]K\x01aK\x00tb.', + b'\x80\x02citertools\ncycle\nc__builtin__\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01aK\x00\x86b.', + b'\x80\x03citertools\ncycle\ncbuiltins\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01aK\x00\x86b.', + b'\x80\x04\x95=\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01aK\x00\x86b.', + + b'citertools\ncycle\n(c__builtin__\niter\n((lp0\nI1\naI2\naI3\natRI1\nbtR(g0\nI1\ntb.', + b'citertools\ncycle\n(c__builtin__\niter\n(]q\x00(K\x01K\x02K\x03etRK\x01btR(h\x00K\x01tb.', + b'\x80\x02citertools\ncycle\nc__builtin__\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00K\x01\x86b.', + b'\x80\x03citertools\ncycle\ncbuiltins\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00K\x01\x86b.', + b'\x80\x04\x95<\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93]\x94(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00K\x01\x86b.', + + b'citertools\ncycle\n(c__builtin__\niter\n((lI1\naI2\naI3\natRI1\nbtR((lI1\naI00\ntb.', + b'citertools\ncycle\n(c__builtin__\niter\n(](K\x01K\x02K\x03etRK\x01btR(]K\x01aI00\ntb.', + b'\x80\x02citertools\ncycle\nc__builtin__\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01a\x89\x86b.', + b'\x80\x03citertools\ncycle\ncbuiltins\niter\n](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01a\x89\x86b.', + b'\x80\x04\x95<\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93](K\x01K\x02K\x03e\x85RK\x01b\x85R]K\x01a\x89\x86b.', + + b'citertools\ncycle\n(c__builtin__\niter\n((lp0\nI1\naI2\naI3\natRI1\nbtR(g0\nI01\ntb.', + b'citertools\ncycle\n(c__builtin__\niter\n(]q\x00(K\x01K\x02K\x03etRK\x01btR(h\x00I01\ntb.', + b'\x80\x02citertools\ncycle\nc__builtin__\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00\x88\x86b.', + b'\x80\x03citertools\ncycle\ncbuiltins\niter\n]q\x00(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00\x88\x86b.', + b'\x80\x04\x95;\x00\x00\x00\x00\x00\x00\x00\x8c\titertools\x8c\x05cycle\x93\x8c\x08builtins\x8c\x04iter\x93]\x94(K\x01K\x02K\x03e\x85RK\x01b\x85Rh\x00\x88\x86b.', + ] + assert len(testcases) == 20 + for t in testcases: + it = pickle.loads(t) + self.assertEqual(take(10, it), [2, 3, 1, 2, 3, 1, 2, 3, 1, 2]) + def test_cycle_setstate(self): # Verify both modes for restoring state diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index 54c5a324897d23..3d9d6d5d0aca34 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -355,7 +355,7 @@ def test_deterministic_sets(self): for elements in ( "float('nan'), b'a', b'b', b'c', 'x', 'y', 'z'", # Also test for bad interactions with backreferencing: - "('Spam', 0), ('Spam', 1), ('Spam', 2)", + "('Spam', 0), ('Spam', 1), ('Spam', 2), ('Spam', 3), ('Spam', 4), ('Spam', 5)", ): s = f"{kind}([{elements}])" with self.subTest(s): diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 442dec8b28065b..77f42f7f9c937b 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -1646,12 +1646,6 @@ def test_resetids(self): ) support.wait_process(pid, exitcode=0) - def test_resetids_wrong_type(self): - with self.assertRaises(TypeError): - self.spawn_func(sys.executable, - [sys.executable, "-c", "pass"], - os.environ, resetids=None) - def test_setpgroup(self): pid = self.spawn_func( sys.executable, diff --git a/Lib/test/test_range.py b/Lib/test/test_range.py index 851ad5b7c2f485..7be76b32ac2935 100644 --- a/Lib/test/test_range.py +++ b/Lib/test/test_range.py @@ -407,11 +407,7 @@ def test_iterator_pickling_overflowing_index(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): with self.subTest(proto=proto): it = iter(range(2**32 + 2)) - _, _, idx = it.__reduce__() - self.assertEqual(idx, 0) - it.__setstate__(2**32 + 1) # undocumented way to set r->index - _, _, idx = it.__reduce__() - self.assertEqual(idx, 2**32 + 1) + it.__setstate__(2**32 + 1) # undocumented way to advance an iterator d = pickle.dumps(it, proto) it = pickle.loads(d) self.assertEqual(next(it), 2**32 + 1) @@ -442,6 +438,38 @@ def test_large_exhausted_iterator_pickling(self): self.assertEqual(list(i), []) self.assertEqual(list(i2), []) + def test_iterator_unpickle_compat(self): + testcases = [ + b'c__builtin__\niter\n(c__builtin__\nxrange\n(I10\nI20\nI2\ntRtRI2\nb.', + b'c__builtin__\niter\n(c__builtin__\nxrange\n(K\nK\x14K\x02tRtRK\x02b.', + b'\x80\x02c__builtin__\niter\nc__builtin__\nxrange\nK\nK\x14K\x02\x87R\x85RK\x02b.', + b'\x80\x03cbuiltins\niter\ncbuiltins\nrange\nK\nK\x14K\x02\x87R\x85RK\x02b.', + b'\x80\x04\x951\x00\x00\x00\x00\x00\x00\x00\x8c\x08builtins\x8c\x04iter\x93\x8c\x08builtins\x8c\x05range\x93K\nK\x14K\x02\x87R\x85RK\x02b.', + + b'c__builtin__\niter\n(c__builtin__\nxrange\n(L-36893488147419103232L\nI20\nI2\ntRtRL18446744073709551623L\nb.', + b'c__builtin__\niter\n(c__builtin__\nxrange\n(L-36893488147419103232L\nK\x14K\x02tRtRL18446744073709551623L\nb.', + b'\x80\x02c__builtin__\niter\nc__builtin__\nxrange\n\x8a\t\x00\x00\x00\x00\x00\x00\x00\x00\xfeK\x14K\x02\x87R\x85R\x8a\t\x07\x00\x00\x00\x00\x00\x00\x00\x01b.', + b'\x80\x03cbuiltins\niter\ncbuiltins\nrange\n\x8a\t\x00\x00\x00\x00\x00\x00\x00\x00\xfeK\x14K\x02\x87R\x85R\x8a\t\x07\x00\x00\x00\x00\x00\x00\x00\x01b.', + b'\x80\x04\x95C\x00\x00\x00\x00\x00\x00\x00\x8c\x08builtins\x8c\x04iter\x93\x8c\x08builtins\x8c\x05range\x93\x8a\t\x00\x00\x00\x00\x00\x00\x00\x00\xfeK\x14K\x02\x87R\x85R\x8a\t\x07\x00\x00\x00\x00\x00\x00\x00\x01b.', + ] + for t in testcases: + it = pickle.loads(t) + self.assertEqual(list(it), [14, 16, 18]) + + def test_iterator_setstate(self): + it = iter(range(10, 20, 2)) + it.__setstate__(2) + self.assertEqual(list(it), [14, 16, 18]) + it = reversed(range(10, 20, 2)) + it.__setstate__(3) + self.assertEqual(list(it), [12, 10]) + it = iter(range(-2**65, 20, 2)) + it.__setstate__(2**64 + 7) + self.assertEqual(list(it), [14, 16, 18]) + it = reversed(range(10, 2**65, 2)) + it.__setstate__(2**64 - 7) + self.assertEqual(list(it), [12, 10]) + def test_odd_bug(self): # This used to raise a "SystemError: NULL result without error" # because the range validation step was eating the exception diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py index cfc4b13f18f330..b05173ad00d442 100644 --- a/Lib/test/test_source_encoding.py +++ b/Lib/test/test_source_encoding.py @@ -160,6 +160,18 @@ def test_file_parse_error_multiline(self): finally: os.unlink(TESTFN) + def test_tokenizer_fstring_warning_in_first_line(self): + source = "0b1and 2" + with open(TESTFN, "w") as fd: + fd.write("{}".format(source)) + try: + retcode, stdout, stderr = script_helper.assert_python_ok(TESTFN) + self.assertIn(b"SyntaxWarning: invalid binary litera", stderr) + self.assertEqual(stderr.count(source.encode()), 1) + finally: + os.unlink(TESTFN) + + class AbstractSourceEncodingTest: def test_default_coding(self): diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py index b0f11af1a7892e..6b1f22f66fd157 100644 --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -723,23 +723,56 @@ def test_issue35714(self): struct.calcsize(s) @support.cpython_only - def test_issue45034_unsigned(self): - _testcapi = import_helper.import_module('_testcapi') - error_msg = f'ushort format requires 0 <= number <= {_testcapi.USHRT_MAX}' - with self.assertRaisesRegex(struct.error, error_msg): - struct.pack('H', 70000) # too large - with self.assertRaisesRegex(struct.error, error_msg): - struct.pack('H', -1) # too small + def test_issue98248(self): + def test_error_msg(prefix, int_type, is_unsigned): + fmt_str = prefix + int_type + size = struct.calcsize(fmt_str) + if is_unsigned: + max_ = 2 ** (size * 8) - 1 + min_ = 0 + else: + max_ = 2 ** (size * 8 - 1) - 1 + min_ = -2 ** (size * 8 - 1) + error_msg = f"'{int_type}' format requires {min_} <= number <= {max_}" + for number in [int(-1e50), min_ - 1, max_ + 1, int(1e50)]: + with self.subTest(format_str=fmt_str, number=number): + with self.assertRaisesRegex(struct.error, error_msg): + struct.pack(fmt_str, number) + error_msg = "required argument is not an integer" + not_number = "" + with self.subTest(format_str=fmt_str, number=not_number): + with self.assertRaisesRegex(struct.error, error_msg): + struct.pack(fmt_str, not_number) + + for prefix in '@=<>': + for int_type in 'BHILQ': + test_error_msg(prefix, int_type, True) + for int_type in 'bhilq': + test_error_msg(prefix, int_type, False) + + int_type = 'N' + test_error_msg('@', int_type, True) + + int_type = 'n' + test_error_msg('@', int_type, False) @support.cpython_only - def test_issue45034_signed(self): - _testcapi = import_helper.import_module('_testcapi') - error_msg = f'short format requires {_testcapi.SHRT_MIN} <= number <= {_testcapi.SHRT_MAX}' - with self.assertRaisesRegex(struct.error, error_msg): - struct.pack('h', 70000) # too large - with self.assertRaisesRegex(struct.error, error_msg): - struct.pack('h', -70000) # too small - + def test_issue98248_error_propagation(self): + class Div0: + def __index__(self): + 1 / 0 + + def test_error_propagation(fmt_str): + with self.subTest(format_str=fmt_str, exception="ZeroDivisionError"): + with self.assertRaises(ZeroDivisionError): + struct.pack(fmt_str, Div0()) + + for prefix in '@=<>': + for int_type in 'BHILQbhilq': + test_error_propagation(prefix + int_type) + + test_error_propagation('N') + test_error_propagation('n') class UnpackIteratorTest(unittest.TestCase): """ diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 424a4a93b6f972..abd0dd8b25699b 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -2832,7 +2832,7 @@ def test_close_fds(self): @unittest.skipIf(sys.platform.startswith("freebsd") and os.stat("/dev").st_dev == os.stat("/dev/fd").st_dev, - "Requires fdescfs mounted on /dev/fd on FreeBSD.") + "Requires fdescfs mounted on /dev/fd on FreeBSD") def test_close_fds_when_max_fd_is_lowered(self): """Confirm that issue21618 is fixed (may fail under valgrind).""" fd_status = support.findfile("fd_status.py", subdir="subprocessdata") @@ -3209,7 +3209,7 @@ def __int__(self): 1, 2, 3, 4, True, True, 0, None, None, None, -1, - None, "no vfork") + None, True) self.assertIn('fds_to_keep', str(c.exception)) finally: if not gc_enabled: diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 2403c7c815f2c0..17a5026e2571e1 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1484,7 +1484,8 @@ def delx(self): del self.__x # PyCapsule # XXX # rangeiterator - check(iter(range(1)), size('4l')) + check(iter(range(1)), size('3l')) + check(iter(range(2**65)), size('3P')) # reverse check(reversed(''), size('nP')) # range diff --git a/Lib/test/test_syslog.py b/Lib/test/test_syslog.py index 2125ec58d87e03..54db80fa9df1af 100644 --- a/Lib/test/test_syslog.py +++ b/Lib/test/test_syslog.py @@ -5,6 +5,7 @@ import threading import time import unittest +from textwrap import dedent # XXX(nnorwitz): This test sucks. I don't know of a platform independent way # to verify that the messages were really logged. @@ -78,6 +79,69 @@ def logger(): finally: sys.setswitchinterval(orig_si) + def test_subinterpreter_syslog(self): + # syslog.syslog() is not allowed in subinterpreters, but only if + # syslog.openlog() hasn't been called in the main interpreter yet. + with self.subTest('before openlog()'): + code = dedent(''' + import syslog + caught_error = False + try: + syslog.syslog('foo') + except RuntimeError: + caught_error = True + assert(caught_error) + ''') + res = support.run_in_subinterp(code) + self.assertEqual(res, 0) + + syslog.openlog() + try: + with self.subTest('after openlog()'): + code = dedent(''' + import syslog + syslog.syslog('foo') + ''') + res = support.run_in_subinterp(code) + self.assertEqual(res, 0) + finally: + syslog.closelog() + + def test_subinterpreter_openlog(self): + try: + code = dedent(''' + import syslog + caught_error = False + try: + syslog.openlog() + except RuntimeError: + caught_error = True + + assert(caught_error) + ''') + res = support.run_in_subinterp(code) + self.assertEqual(res, 0) + finally: + syslog.closelog() + + def test_subinterpreter_closelog(self): + syslog.openlog('python') + try: + code = dedent(''' + import syslog + caught_error = False + try: + syslog.closelog() + except RuntimeError: + caught_error = True + + assert(caught_error) + ''') + res = support.run_in_subinterp(code) + self.assertEqual(res, 0) + finally: + syslog.closelog() + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index c17bbb48b65b2d..95b1bae4f60850 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -2978,9 +2978,9 @@ class MyClass: for name in ("b", "v", "m", "py"): with self.subTest(name=name): actual = self.get_suggestion(MyClass, name) - self.assertNotIn("you mean", actual) - self.assertNotIn("vvv", actual) - self.assertNotIn("mom", actual) + self.assertNotIn("Did you mean", actual) + self.assertNotIn("'vvv", actual) + self.assertNotIn("'mom'", actual) self.assertNotIn("'id'", actual) self.assertNotIn("'w'", actual) self.assertNotIn("'pytho'", actual) @@ -3168,9 +3168,9 @@ def test_import_from_error_bad_suggestions_do_not_trigger_for_small_names(self): for name in ("b", "v", "m", "py"): with self.subTest(name=name): actual = self.get_import_from_suggestion(code, name) - self.assertNotIn("you mean", actual) - self.assertNotIn("vvv", actual) - self.assertNotIn("mom", actual) + self.assertNotIn("Did you mean", actual) + self.assertNotIn("'vvv'", actual) + self.assertNotIn("'mom'", actual) self.assertNotIn("'id'", actual) self.assertNotIn("'w'", actual) self.assertNotIn("'pytho'", actual) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 6ff7f612867e62..da602b0199d52c 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -772,20 +772,42 @@ class C(Generic[*Ts]): pass ('generic[*Ts]', '[*Ts]', 'generic[*Ts]'), ('generic[*Ts]', '[T, *Ts]', 'generic[T, *Ts]'), ('generic[*Ts]', '[*Ts, T]', 'generic[*Ts, T]'), + ('generic[T, *Ts]', '[()]', 'TypeError'), ('generic[T, *Ts]', '[int]', 'generic[int]'), ('generic[T, *Ts]', '[int, str]', 'generic[int, str]'), ('generic[T, *Ts]', '[int, str, bool]', 'generic[int, str, bool]'), + ('generic[list[T], *Ts]', '[()]', 'TypeError'), ('generic[list[T], *Ts]', '[int]', 'generic[list[int]]'), ('generic[list[T], *Ts]', '[int, str]', 'generic[list[int], str]'), ('generic[list[T], *Ts]', '[int, str, bool]', 'generic[list[int], str, bool]'), + ('generic[*Ts, T]', '[()]', 'TypeError'), ('generic[*Ts, T]', '[int]', 'generic[int]'), ('generic[*Ts, T]', '[int, str]', 'generic[int, str]'), ('generic[*Ts, T]', '[int, str, bool]', 'generic[int, str, bool]'), + ('generic[*Ts, list[T]]', '[()]', 'TypeError'), ('generic[*Ts, list[T]]', '[int]', 'generic[list[int]]'), ('generic[*Ts, list[T]]', '[int, str]', 'generic[int, list[str]]'), ('generic[*Ts, list[T]]', '[int, str, bool]', 'generic[int, str, list[bool]]'), + ('generic[T1, T2, *Ts]', '[()]', 'TypeError'), + ('generic[T1, T2, *Ts]', '[int]', 'TypeError'), + ('generic[T1, T2, *Ts]', '[int, str]', 'generic[int, str]'), + ('generic[T1, T2, *Ts]', '[int, str, bool]', 'generic[int, str, bool]'), + ('generic[T1, T2, *Ts]', '[int, str, bool, bytes]', 'generic[int, str, bool, bytes]'), + + ('generic[*Ts, T1, T2]', '[()]', 'TypeError'), + ('generic[*Ts, T1, T2]', '[int]', 'TypeError'), + ('generic[*Ts, T1, T2]', '[int, str]', 'generic[int, str]'), + ('generic[*Ts, T1, T2]', '[int, str, bool]', 'generic[int, str, bool]'), + ('generic[*Ts, T1, T2]', '[int, str, bool, bytes]', 'generic[int, str, bool, bytes]'), + + ('generic[T1, *Ts, T2]', '[()]', 'TypeError'), + ('generic[T1, *Ts, T2]', '[int]', 'TypeError'), + ('generic[T1, *Ts, T2]', '[int, str]', 'generic[int, str]'), + ('generic[T1, *Ts, T2]', '[int, str, bool]', 'generic[int, str, bool]'), + ('generic[T1, *Ts, T2]', '[int, str, bool, bytes]', 'generic[int, str, bool, bytes]'), + ('generic[T, *Ts]', '[*tuple_type[int, ...]]', 'generic[int, *tuple_type[int, ...]]'), ('generic[T, *Ts]', '[str, *tuple_type[int, ...]]', 'generic[str, *tuple_type[int, ...]]'), ('generic[T, *Ts]', '[*tuple_type[int, ...], str]', 'generic[int, *tuple_type[int, ...], str]'), @@ -7241,6 +7263,65 @@ class X(Generic[P, P2]): self.assertEqual(G1.__args__, ((int, str), (bytes,))) self.assertEqual(G2.__args__, ((int,), (str, bytes))) + def test_typevartuple_and_paramspecs_in_user_generics(self): + Ts = TypeVarTuple("Ts") + P = ParamSpec("P") + + class X(Generic[*Ts, P]): + f: Callable[P, int] + g: Tuple[*Ts] + + G1 = X[int, [bytes]] + self.assertEqual(G1.__args__, (int, (bytes,))) + G2 = X[int, str, [bytes]] + self.assertEqual(G2.__args__, (int, str, (bytes,))) + G3 = X[[bytes]] + self.assertEqual(G3.__args__, ((bytes,),)) + G4 = X[[]] + self.assertEqual(G4.__args__, ((),)) + with self.assertRaises(TypeError): + X[()] + + class Y(Generic[P, *Ts]): + f: Callable[P, int] + g: Tuple[*Ts] + + G1 = Y[[bytes], int] + self.assertEqual(G1.__args__, ((bytes,), int)) + G2 = Y[[bytes], int, str] + self.assertEqual(G2.__args__, ((bytes,), int, str)) + G3 = Y[[bytes]] + self.assertEqual(G3.__args__, ((bytes,),)) + G4 = Y[[]] + self.assertEqual(G4.__args__, ((),)) + with self.assertRaises(TypeError): + Y[()] + + def test_typevartuple_and_paramspecs_in_generic_aliases(self): + P = ParamSpec('P') + T = TypeVar('T') + Ts = TypeVarTuple('Ts') + + for C in Callable, collections.abc.Callable: + with self.subTest(generic=C): + A = C[P, Tuple[*Ts]] + B = A[[int, str], bytes, float] + self.assertEqual(B.__args__, (int, str, Tuple[bytes, float])) + + class X(Generic[T, P]): + pass + + A = X[Tuple[*Ts], P] + B = A[bytes, float, [int, str]] + self.assertEqual(B.__args__, (Tuple[bytes, float], (int, str,))) + + class Y(Generic[P, T]): + pass + + A = Y[P, Tuple[*Ts]] + B = A[[int, str], bytes, float] + self.assertEqual(B.__args__, ((int, str,), Tuple[bytes, float])) + def test_var_substitution(self): T = TypeVar("T") P = ParamSpec("P") diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index a85bda3144bc35..74503c89e559a0 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -12,7 +12,8 @@ import unicodedata import unittest from test.support import (open_urlresource, requires_resource, script_helper, - cpython_only, check_disallow_instantiation) + cpython_only, check_disallow_instantiation, + ResourceDenied) class UnicodeMethodsTest(unittest.TestCase): @@ -364,8 +365,8 @@ def test_normalization(self): except PermissionError: self.skipTest(f"Permission error when downloading {TESTDATAURL} " f"into the test data directory") - except (OSError, HTTPException): - self.fail(f"Could not retrieve {TESTDATAURL}") + except (OSError, HTTPException) as exc: + self.skipTest(f"Failed to download {TESTDATAURL}: {exc}") with testdata: self.run_normalization_tests(testdata) diff --git a/Lib/test/test_unittest/test_result.py b/Lib/test/test_unittest/test_result.py index e71d114751d94d..efd9c902350506 100644 --- a/Lib/test/test_unittest/test_result.py +++ b/Lib/test/test_unittest/test_result.py @@ -275,6 +275,62 @@ def get_exc_info(): self.assertEqual(len(dropped), 1) self.assertIn("raise self.failureException(msg)", dropped[0]) + def test_addFailure_filter_traceback_frames_chained_exception_self_loop(self): + class Foo(unittest.TestCase): + def test_1(self): + pass + + def get_exc_info(): + try: + loop = Exception("Loop") + loop.__cause__ = loop + loop.__context__ = loop + raise loop + except: + return sys.exc_info() + + exc_info_tuple = get_exc_info() + + test = Foo('test_1') + result = unittest.TestResult() + result.startTest(test) + result.addFailure(test, exc_info_tuple) + result.stopTest(test) + + formatted_exc = result.failures[0][1] + self.assertEqual(formatted_exc.count("Exception: Loop\n"), 1) + + def test_addFailure_filter_traceback_frames_chained_exception_cycle(self): + class Foo(unittest.TestCase): + def test_1(self): + pass + + def get_exc_info(): + try: + # Create two directionally opposed cycles + # __cause__ in one direction, __context__ in the other + A, B, C = Exception("A"), Exception("B"), Exception("C") + edges = [(C, B), (B, A), (A, C)] + for ex1, ex2 in edges: + ex1.__cause__ = ex2 + ex2.__context__ = ex1 + raise C + except: + return sys.exc_info() + + exc_info_tuple = get_exc_info() + + test = Foo('test_1') + result = unittest.TestResult() + result.startTest(test) + result.addFailure(test, exc_info_tuple) + result.stopTest(test) + + formatted_exc = result.failures[0][1] + self.assertEqual(formatted_exc.count("Exception: A\n"), 1) + self.assertEqual(formatted_exc.count("Exception: B\n"), 1) + self.assertEqual(formatted_exc.count("Exception: C\n"), 1) + # "addError(test, err)" # ... # "Called when the test case test raises an unexpected exception err diff --git a/Lib/typing.py b/Lib/typing.py index 233941598f76a3..38e227e3c55d59 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -284,25 +284,6 @@ def _unpack_args(args): newargs.append(arg) return newargs -def _prepare_paramspec_params(cls, params): - """Prepares the parameters for a Generic containing ParamSpec - variables (internal helper). - """ - # Special case where Z[[int, str, bool]] == Z[int, str, bool] in PEP 612. - if (len(cls.__parameters__) == 1 - and params and not _is_param_expr(params[0])): - assert isinstance(cls.__parameters__[0], ParamSpec) - return (params,) - else: - _check_generic(cls, params, len(cls.__parameters__)) - _params = [] - # Convert lists to tuples to help other libraries cache the results. - for p, tvar in zip(params, cls.__parameters__): - if isinstance(tvar, ParamSpec) and isinstance(p, list): - p = tuple(p) - _params.append(p) - return tuple(_params) - def _deduplicate(params): # Weed out strict duplicates, preserving the first of each occurrence. all_params = set(params) @@ -344,6 +325,7 @@ def _flatten_literal_params(parameters): _cleanups = [] +_caches = {} def _tp_cache(func=None, /, *, typed=False): @@ -351,13 +333,20 @@ def _tp_cache(func=None, /, *, typed=False): original function for non-hashable arguments. """ def decorator(func): - cached = functools.lru_cache(typed=typed)(func) - _cleanups.append(cached.cache_clear) + # The callback 'inner' references the newly created lru_cache + # indirectly by performing a lookup in the global '_caches' dictionary. + # This breaks a reference that can be problematic when combined with + # C API extensions that leak references to types. See GH-98253. + + cache = functools.lru_cache(typed=typed)(func) + _caches[func] = cache + _cleanups.append(cache.cache_clear) + del cache @functools.wraps(func) def inner(*args, **kwds): try: - return cached(*args, **kwds) + return _caches[func](*args, **kwds) except TypeError: pass # All real errors (not unhashable args) are raised below. return func(*args, **kwds) @@ -1238,7 +1227,18 @@ def __typing_subst__(self, arg): return arg def __typing_prepare_subst__(self, alias, args): - return _prepare_paramspec_params(alias, args) + params = alias.__parameters__ + i = params.index(self) + if i >= len(args): + raise TypeError(f"Too few arguments for {alias}") + # Special case where Z[[int, str, bool]] == Z[int, str, bool] in PEP 612. + if len(params) == 1 and not _is_param_expr(args[0]): + assert i == 0 + args = (args,) + # Convert lists to tuples to help other libraries cache the results. + elif isinstance(args[i], list): + args = (*args[:i], tuple(args[i]), *args[i+1:]) + return args def _is_dunder(attr): return attr.startswith('__') and attr.endswith('__') @@ -1801,23 +1801,13 @@ def __class_getitem__(cls, params): if not isinstance(params, tuple): params = (params,) - if not params: - # We're only ok with `params` being empty if the class's only type - # parameter is a `TypeVarTuple` (which can contain zero types). - class_params = getattr(cls, "__parameters__", None) - only_class_parameter_is_typevartuple = ( - class_params is not None - and len(class_params) == 1 - and isinstance(class_params[0], TypeVarTuple) - ) - if not only_class_parameter_is_typevartuple: - raise TypeError( - f"Parameter list to {cls.__qualname__}[...] cannot be empty" - ) - params = tuple(_type_convert(p) for p in params) if cls in (Generic, Protocol): # Generic and Protocol can only be subscripted with unique type variables. + if not params: + raise TypeError( + f"Parameter list to {cls.__qualname__}[...] cannot be empty" + ) if not all(_is_typevar_like(p) for p in params): raise TypeError( f"Parameters to {cls.__name__}[...] must all be type variables " @@ -1827,13 +1817,20 @@ def __class_getitem__(cls, params): f"Parameters to {cls.__name__}[...] must all be unique") else: # Subscripting a regular Generic subclass. - if any(isinstance(t, ParamSpec) for t in cls.__parameters__): - params = _prepare_paramspec_params(cls, params) - elif not any(isinstance(p, TypeVarTuple) for p in cls.__parameters__): - # We only run this if there are no TypeVarTuples, because we - # don't check variadic generic arity at runtime (to reduce - # complexity of typing.py). - _check_generic(cls, params, len(cls.__parameters__)) + for param in cls.__parameters__: + prepare = getattr(param, '__typing_prepare_subst__', None) + if prepare is not None: + params = prepare(cls, params) + _check_generic(cls, params, len(cls.__parameters__)) + + new_args = [] + for param, new_arg in zip(cls.__parameters__, params): + if isinstance(param, TypeVarTuple): + new_args.extend(new_arg) + else: + new_args.append(new_arg) + params = tuple(new_args) + return _GenericAlias(cls, params, _paramspec_tvars=True) diff --git a/Lib/unittest/result.py b/Lib/unittest/result.py index 3da7005e603f4a..5ca4c23238b419 100644 --- a/Lib/unittest/result.py +++ b/Lib/unittest/result.py @@ -196,6 +196,7 @@ def _clean_tracebacks(self, exctype, value, tb, test): ret = None first = True excs = [(exctype, value, tb)] + seen = {id(value)} # Detect loops in chained exceptions. while excs: (exctype, value, tb) = excs.pop() # Skip test runner traceback levels @@ -214,8 +215,9 @@ def _clean_tracebacks(self, exctype, value, tb, test): if value is not None: for c in (value.__cause__, value.__context__): - if c is not None: + if c is not None and id(c) not in seen: excs.append((type(c), c, c.__traceback__)) + seen.add(id(c)) return ret def _is_relevant_tb_level(self, tb): diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index 8f834267b28c2e..e1833dd1772d56 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -1202,10 +1202,10 @@ def close(self): if not self._zip64: if self._file_size > ZIP64_LIMIT: raise RuntimeError( - 'File size unexpectedly exceeded ZIP64 limit') + 'File size too large, try using force_zip64') if self._compress_size > ZIP64_LIMIT: raise RuntimeError( - 'Compressed size unexpectedly exceeded ZIP64 limit') + 'Compressed size too large, try using force_zip64') # Seek backwards and write file header (which will now include # correct CRC and file sizes) diff --git a/Misc/ACKS b/Misc/ACKS index 5d97067b85d3d4..d50cb3c2d1ee4f 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1320,6 +1320,7 @@ Michele Orrù Tomáš Orsava Oleg Oshmyan Denis Osipov +Itamar Ostricher Denis S. Otkidach Peter Otten Michael Otteneder @@ -1627,6 +1628,7 @@ Silas Sewell Ian Seyer Dmitry Shachnev Anish Shah +Jaineel Shah Daniel Shahaf Hui Shang Geoff Shannon diff --git a/Misc/NEWS.d/next/C API/2022-11-20-09-52-50.gh-issue-99612.eBHksg.rst b/Misc/NEWS.d/next/C API/2022-11-20-09-52-50.gh-issue-99612.eBHksg.rst new file mode 100644 index 00000000000000..40e3c8db5403c7 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-11-20-09-52-50.gh-issue-99612.eBHksg.rst @@ -0,0 +1,2 @@ +Fix :c:func:`PyUnicode_DecodeUTF8Stateful` for ASCII-only data: +``*consumed`` was not set. diff --git a/Misc/NEWS.d/next/C API/2022-12-05-17-30-13.gh-issue-98680.FiMCxZ.rst b/Misc/NEWS.d/next/C API/2022-12-05-17-30-13.gh-issue-98680.FiMCxZ.rst new file mode 100644 index 00000000000000..a87090168e065b --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-12-05-17-30-13.gh-issue-98680.FiMCxZ.rst @@ -0,0 +1,3 @@ +``PyBUF_*`` constants were marked as part of Limited API of Python 3.11+. +These were available in 3.11.0 with :c:macro:`Py_LIMITED_API` defined for +3.11, and are necessary to use the buffer API. diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-29-15-55-19.bpo-45026.z7nTA3.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-29-15-55-19.bpo-45026.z7nTA3.rst new file mode 100644 index 00000000000000..481ab53e4f5197 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-08-29-15-55-19.bpo-45026.z7nTA3.rst @@ -0,0 +1,3 @@ +Optimize the :class:`range` object iterator. It is now smaller, faster +iteration of ranges containing large numbers. Smaller pickles, faster +unpickling. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-05-22-26-35.gh-issue-99127.Btk7ih.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-05-22-26-35.gh-issue-99127.Btk7ih.rst new file mode 100644 index 00000000000000..e93ae4e7b127d1 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-11-05-22-26-35.gh-issue-99127.Btk7ih.rst @@ -0,0 +1 @@ +Allow some features of :mod:`syslog` to the main interpreter only. Patch by Dong-hee Na. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-27-13-50-13.gh-issue-91054.oox_kW.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-27-13-50-13.gh-issue-91054.oox_kW.rst new file mode 100644 index 00000000000000..c46459c15b9e65 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-11-27-13-50-13.gh-issue-91054.oox_kW.rst @@ -0,0 +1,3 @@ +Add :c:func:`PyCode_AddWatcher` and :c:func:`PyCode_ClearWatcher` APIs to +register callbacks to receive notification on creation and destruction of +code objects. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst new file mode 100644 index 00000000000000..20cd361affeaa5 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst @@ -0,0 +1,3 @@ +Fix a bug in the tokenizer that could cause infinite recursion when showing +syntax warnings that happen in the first line of the source. Patch by Pablo +Galindo diff --git a/Misc/NEWS.d/next/Library/2019-08-30-10-48-53.bpo-15999.QqsRRi.rst b/Misc/NEWS.d/next/Library/2019-08-30-10-48-53.bpo-15999.QqsRRi.rst new file mode 100644 index 00000000000000..9650c694cc674a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-08-30-10-48-53.bpo-15999.QqsRRi.rst @@ -0,0 +1,2 @@ +All built-in functions now accept arguments of any type instead of just +``bool`` and ``int`` for boolean parameters. diff --git a/Misc/NEWS.d/next/Library/2021-08-03-05-31-00.bpo-44817.wOW_Qn.rst b/Misc/NEWS.d/next/Library/2021-08-03-05-31-00.bpo-44817.wOW_Qn.rst new file mode 100644 index 00000000000000..79f8c506b54f37 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-08-03-05-31-00.bpo-44817.wOW_Qn.rst @@ -0,0 +1,2 @@ +Ignore WinError 53 (ERROR_BAD_NETPATH), 65 (ERROR_NETWORK_ACCESS_DENIED) +and 161 (ERROR_BAD_PATHNAME) when using ntpath.realpath(). diff --git a/Misc/NEWS.d/next/Library/2022-04-04-22-54-11.bpo-47220.L9jYu4.rst b/Misc/NEWS.d/next/Library/2022-04-04-22-54-11.bpo-47220.L9jYu4.rst new file mode 100644 index 00000000000000..6e2af088640b55 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-04-04-22-54-11.bpo-47220.L9jYu4.rst @@ -0,0 +1,2 @@ +Document the optional *callback* parameter of :class:`WeakMethod`. Patch by +Géry Ogam. diff --git a/Misc/NEWS.d/next/Library/2022-10-02-12-38-22.gh-issue-82836.OvYLmC.rst b/Misc/NEWS.d/next/Library/2022-10-02-12-38-22.gh-issue-82836.OvYLmC.rst new file mode 100644 index 00000000000000..dcbea66d66bf7c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-02-12-38-22.gh-issue-82836.OvYLmC.rst @@ -0,0 +1 @@ +Fix :attr:`~ipaddress.IPv4Address.is_private` properties in the :mod:`ipaddress` module. Previously non-private networks (0.0.0.0/0) would return True from this method; now they correctly return False. diff --git a/Misc/NEWS.d/next/Library/2022-10-13-22-13-54.gh-issue-98248.lwyygy.rst b/Misc/NEWS.d/next/Library/2022-10-13-22-13-54.gh-issue-98248.lwyygy.rst new file mode 100644 index 00000000000000..347f6e160335e2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-13-22-13-54.gh-issue-98248.lwyygy.rst @@ -0,0 +1 @@ +Provide informative error messages in :func:`struct.pack` when its integral arguments are not in range. diff --git a/Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst b/Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst new file mode 100644 index 00000000000000..f74195cc8e7dc6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-19-18-31-53.gh-issue-98458.vwyq7O.rst @@ -0,0 +1 @@ +Fix infinite loop in unittest when a self-referencing chained exception is raised diff --git a/Misc/NEWS.d/next/Library/2022-10-24-11-01-05.gh-issue-98253.HVd5v4.rst b/Misc/NEWS.d/next/Library/2022-10-24-11-01-05.gh-issue-98253.HVd5v4.rst new file mode 100644 index 00000000000000..00df0070f3b9c1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-24-11-01-05.gh-issue-98253.HVd5v4.rst @@ -0,0 +1,10 @@ +The implementation of the typing module is now more resilient to reference +leaks in binary extension modules. + +Previously, a reference leak in a typed C API-based extension module could leak +internals of the typing module, which could in turn introduce leaks in +essentially any other package with typed function signatures. Although the +typing package is not the original source of the problem, such non-local +dependences exacerbate debugging of large-scale projects, and the +implementation was therefore changed to reduce harm by providing better +isolation. diff --git a/Misc/NEWS.d/next/Library/2022-11-12-12-08-34.gh-issue-99344.7M_u8G.rst b/Misc/NEWS.d/next/Library/2022-11-12-12-08-34.gh-issue-99344.7M_u8G.rst new file mode 100644 index 00000000000000..412c8c793435af --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-12-12-08-34.gh-issue-99344.7M_u8G.rst @@ -0,0 +1,2 @@ +Fix substitution of :class:`~typing.TypeVarTuple` and +:class:`~typing.ParamSpec` together in user generics. diff --git a/Misc/NEWS.d/next/Library/2022-11-12-12-10-23.gh-issue-99379.bcGhxF.rst b/Misc/NEWS.d/next/Library/2022-11-12-12-10-23.gh-issue-99379.bcGhxF.rst new file mode 100644 index 00000000000000..1950680b1df86c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-12-12-10-23.gh-issue-99379.bcGhxF.rst @@ -0,0 +1,2 @@ +Fix substitution of :class:`~typing.ParamSpec` followed by +:class:`~typing.TypeVarTuple` in generic aliases. diff --git a/Misc/NEWS.d/next/Library/2022-11-12-12-15-30.gh-issue-99382.dKg_rW.rst b/Misc/NEWS.d/next/Library/2022-11-12-12-15-30.gh-issue-99382.dKg_rW.rst new file mode 100644 index 00000000000000..f153f2fceac844 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-12-12-15-30.gh-issue-99382.dKg_rW.rst @@ -0,0 +1,2 @@ +Check the number of arguments in substitution in user generics containing a +:class:`~typing.TypeVarTuple` and one or more :class:`~typing.TypeVar`. diff --git a/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst b/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst new file mode 100644 index 00000000000000..a396e95cd83f82 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2022-12-05-01-39-10.gh-issue-100001.uD05Fc.rst @@ -0,0 +1,6 @@ +``python -m http.server`` no longer allows terminal control characters sent +within a garbage request to be printed to the stderr server log. + +This is done by changing the :mod:`http.server` :class:`BaseHTTPRequestHandler` +``.log_message`` method to replace control characters with a ``\xHH`` hex escape +before printing. diff --git a/Misc/NEWS.d/next/Tests/2022-11-19-13-34-28.gh-issue-99593.8ZfCkj.rst b/Misc/NEWS.d/next/Tests/2022-11-19-13-34-28.gh-issue-99593.8ZfCkj.rst new file mode 100644 index 00000000000000..ec4cda2080323f --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-11-19-13-34-28.gh-issue-99593.8ZfCkj.rst @@ -0,0 +1 @@ +Cover the Unicode C API with tests. diff --git a/Misc/NEWS.d/next/Tests/2022-11-23-18-32-16.gh-issue-99741.q4R7NH.rst b/Misc/NEWS.d/next/Tests/2022-11-23-18-32-16.gh-issue-99741.q4R7NH.rst new file mode 100644 index 00000000000000..791726b52bfbe1 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-11-23-18-32-16.gh-issue-99741.q4R7NH.rst @@ -0,0 +1,2 @@ +We've implemented multi-phase init (PEP 489/630/687) +for the internal (for testing) _xxsubinterpreters module. diff --git a/Misc/NEWS.d/next/Tests/2022-12-01-18-55-18.gh-issue-99934.Ox3Fqf.rst b/Misc/NEWS.d/next/Tests/2022-12-01-18-55-18.gh-issue-99934.Ox3Fqf.rst new file mode 100644 index 00000000000000..1b1287c33dbd94 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-12-01-18-55-18.gh-issue-99934.Ox3Fqf.rst @@ -0,0 +1 @@ +Correct test_marsh on (32 bit) x86: test_deterministic sets was failing. diff --git a/Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst b/Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst new file mode 100644 index 00000000000000..eded0361fbebbb --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-12-05-16-12-56.gh-issue-99892.sz_eW8.rst @@ -0,0 +1,2 @@ +Skip test_normalization() of test_unicodedata if it fails to download +NormalizationTest.txt file from pythontest.net. Patch by Victor Stinner. diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index aa12bcc85cebc7..c716f403d638ac 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2271,6 +2271,50 @@ [function.PyMemoryView_FromBuffer] added = '3.11' +# Constants for Py_buffer API added to this list in Python 3.11.1 (https://github.com/python/cpython/issues/98680) +# (they were available with 3.11.0) +[const.PyBUF_MAX_NDIM] + added = '3.11' +[const.PyBUF_SIMPLE] + added = '3.11' +[const.PyBUF_WRITABLE] + added = '3.11' +[const.PyBUF_FORMAT] + added = '3.11' +[const.PyBUF_ND] + added = '3.11' +[const.PyBUF_STRIDES] + added = '3.11' +[const.PyBUF_C_CONTIGUOUS] + added = '3.11' +[const.PyBUF_F_CONTIGUOUS] + added = '3.11' +[const.PyBUF_ANY_CONTIGUOUS] + added = '3.11' +[const.PyBUF_INDIRECT] + added = '3.11' +[const.PyBUF_CONTIG] + added = '3.11' +[const.PyBUF_CONTIG_RO] + added = '3.11' +[const.PyBUF_STRIDED] + added = '3.11' +[const.PyBUF_STRIDED_RO] + added = '3.11' +[const.PyBUF_RECORDS] + added = '3.11' +[const.PyBUF_RECORDS_RO] + added = '3.11' +[const.PyBUF_FULL] + added = '3.11' +[const.PyBUF_FULL_RO] + added = '3.11' +[const.PyBUF_READ] + added = '3.11' +[const.PyBUF_WRITE] + added = '3.11' + + # (Detailed comments aren't really needed for further entries: from here on # we can use version control logs.) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 3edd8a9b2dbf91..cabcaec94bec7e 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -3,11 +3,12 @@ #endif #include "Python.h" -#include "pycore_pyerrors.h" // _PyErr_ClearExcState() -#include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_runtime.h" // _PyRuntime -#include "pycore_global_objects.h" // _Py_ID() -#include // offsetof() +#include "pycore_pyerrors.h" // _PyErr_ClearExcState() +#include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_runtime_init.h" // _Py_ID() +#include "pycore_moduleobject.h" // _PyModule_GetState() +#include "structmember.h" // PyMemberDef +#include // offsetof() /*[clinic input] @@ -17,35 +18,80 @@ module _asyncio /* State of the _asyncio module */ -static PyObject *asyncio_mod; -static PyObject *traceback_extract_stack; -static PyObject *asyncio_get_event_loop_policy; -static PyObject *asyncio_future_repr_func; -static PyObject *asyncio_iscoroutine_func; -static PyObject *asyncio_task_get_stack_func; -static PyObject *asyncio_task_print_stack_func; -static PyObject *asyncio_task_repr_func; -static PyObject *asyncio_InvalidStateError; -static PyObject *asyncio_CancelledError; -static PyObject *context_kwname; -static int module_initialized; +typedef struct { + PyTypeObject *FutureIterType; + PyTypeObject *TaskStepMethWrapper_Type; + PyTypeObject *FutureType; + PyTypeObject *TaskType; + PyTypeObject *PyRunningLoopHolder_Type; + + PyObject *asyncio_mod; + PyObject *context_kwname; + + /* Dictionary containing tasks that are currently active in + all running event loops. {EventLoop: Task} */ + PyObject *current_tasks; + + /* WeakSet containing all alive tasks. */ + PyObject *all_tasks; + + /* An isinstance type cache for the 'is_coroutine()' function. */ + PyObject *iscoroutine_typecache; -static PyObject *cached_running_holder; -static volatile uint64_t cached_running_holder_tsid; + /* Imports from asyncio.events. */ + PyObject *asyncio_get_event_loop_policy; -/* Counter for autogenerated Task names */ -static uint64_t task_name_counter = 0; + /* Imports from asyncio.base_futures. */ + PyObject *asyncio_future_repr_func; -/* WeakSet containing all alive tasks. */ -static PyObject *all_tasks; + /* Imports from asyncio.exceptions. */ + PyObject *asyncio_CancelledError; + PyObject *asyncio_InvalidStateError; -/* Dictionary containing tasks that are currently active in - all running event loops. {EventLoop: Task} */ -static PyObject *current_tasks; + /* Imports from asyncio.base_tasks. */ + PyObject *asyncio_task_get_stack_func; + PyObject *asyncio_task_print_stack_func; + PyObject *asyncio_task_repr_func; -/* An isinstance type cache for the 'is_coroutine()' function. */ -static PyObject *iscoroutine_typecache; + /* Imports from asyncio.coroutines. */ + PyObject *asyncio_iscoroutine_func; + /* Imports from traceback. */ + PyObject *traceback_extract_stack; + + PyObject *cached_running_holder; // Borrowed ref. + volatile uint64_t cached_running_holder_tsid; + + /* Counter for autogenerated Task names */ + uint64_t task_name_counter; +} asyncio_state; + +static inline asyncio_state * +get_asyncio_state(PyObject *mod) +{ + asyncio_state *state = _PyModule_GetState(mod); + assert(state != NULL); + return state; +} + +static inline asyncio_state * +get_asyncio_state_by_cls(PyTypeObject *cls) +{ + asyncio_state *state = (asyncio_state *)PyType_GetModuleState(cls); + assert(state != NULL); + return state; +} + +static struct PyModuleDef _asynciomodule; + +static inline asyncio_state * +get_asyncio_state_by_def(PyObject *self) +{ + PyTypeObject *tp = Py_TYPE(self); + PyObject *mod = PyType_GetModuleByDef(tp, &_asynciomodule); + assert(mod != NULL); + return get_asyncio_state(mod); +} typedef enum { STATE_PENDING, @@ -101,16 +147,11 @@ typedef struct { } PyRunningLoopHolder; -static PyTypeObject FutureType; -static PyTypeObject TaskType; -static PyTypeObject PyRunningLoopHolder_Type; +#define Future_CheckExact(state, obj) Py_IS_TYPE(obj, state->FutureType) +#define Task_CheckExact(state, obj) Py_IS_TYPE(obj, state->TaskType) - -#define Future_CheckExact(obj) Py_IS_TYPE(obj, &FutureType) -#define Task_CheckExact(obj) Py_IS_TYPE(obj, &TaskType) - -#define Future_Check(obj) PyObject_TypeCheck(obj, &FutureType) -#define Task_Check(obj) PyObject_TypeCheck(obj, &TaskType) +#define Future_Check(state, obj) PyObject_TypeCheck(obj, state->FutureType) +#define Task_Check(state, obj) PyObject_TypeCheck(obj, state->TaskType) #include "clinic/_asynciomodule.c.h" @@ -124,11 +165,11 @@ class _asyncio.Future "FutureObj *" "&Future_Type" /* Get FutureIter from Future */ static PyObject * future_new_iter(PyObject *); -static PyRunningLoopHolder * new_running_loop_holder(PyObject *); +static PyRunningLoopHolder * new_running_loop_holder(asyncio_state *, PyObject *); static int -_is_coroutine(PyObject *coro) +_is_coroutine(asyncio_state *state, PyObject *coro) { /* 'coro' is not a native coroutine, call asyncio.iscoroutine() to check if it's another coroutine flavour. @@ -136,7 +177,7 @@ _is_coroutine(PyObject *coro) Do this check after 'future_init()'; in case we need to raise an error, __del__ needs a properly initialized object. */ - PyObject *res = PyObject_CallOneArg(asyncio_iscoroutine_func, coro); + PyObject *res = PyObject_CallOneArg(state->asyncio_iscoroutine_func, coro); if (res == NULL) { return -1; } @@ -147,12 +188,12 @@ _is_coroutine(PyObject *coro) return is_res_true; } - if (PySet_GET_SIZE(iscoroutine_typecache) < 100) { + if (PySet_GET_SIZE(state->iscoroutine_typecache) < 100) { /* Just in case we don't want to cache more than 100 positive types. That shouldn't ever happen, unless someone stressing the system on purpose. */ - if (PySet_Add(iscoroutine_typecache, (PyObject*) Py_TYPE(coro))) { + if (PySet_Add(state->iscoroutine_typecache, (PyObject*) Py_TYPE(coro))) { return -1; } } @@ -162,7 +203,7 @@ _is_coroutine(PyObject *coro) static inline int -is_coroutine(PyObject *coro) +is_coroutine(asyncio_state *state, PyObject *coro) { if (PyCoro_CheckExact(coro)) { return 1; @@ -178,10 +219,10 @@ is_coroutine(PyObject *coro) a pure-Python function in 99.9% cases. */ int has_it = PySet_Contains( - iscoroutine_typecache, (PyObject*) Py_TYPE(coro)); + state->iscoroutine_typecache, (PyObject*) Py_TYPE(coro)); if (has_it == 0) { /* type(coro) is not in iscoroutine_typecache */ - return _is_coroutine(coro); + return _is_coroutine(state, coro); } /* either an error has occurred or @@ -192,13 +233,13 @@ is_coroutine(PyObject *coro) static PyObject * -get_future_loop(PyObject *fut) +get_future_loop(asyncio_state *state, PyObject *fut) { /* Implementation of `asyncio.futures._get_loop` */ PyObject *getloop; - if (Future_CheckExact(fut) || Task_CheckExact(fut)) { + if (Future_CheckExact(state, fut) || Task_CheckExact(state, fut)) { PyObject *loop = ((FutureObj *)fut)->fut_loop; return Py_NewRef(loop); } @@ -217,15 +258,17 @@ get_future_loop(PyObject *fut) static int -get_running_loop(PyObject **loop) +get_running_loop(asyncio_state *state, PyObject **loop) { PyObject *rl; PyThreadState *ts = _PyThreadState_GET(); uint64_t ts_id = PyThreadState_GetID(ts); - if (ts_id == cached_running_holder_tsid && cached_running_holder != NULL) { + if (state->cached_running_holder_tsid == ts_id && + state->cached_running_holder != NULL) + { // Fast path, check the cache. - rl = cached_running_holder; // borrowed + rl = state->cached_running_holder; // borrowed } else { PyObject *ts_dict = _PyThreadState_GetDict(ts); // borrowed @@ -244,11 +287,11 @@ get_running_loop(PyObject **loop) } } - cached_running_holder = rl; // borrowed - cached_running_holder_tsid = ts_id; + state->cached_running_holder = rl; // borrowed + state->cached_running_holder_tsid = ts_id; } - assert(Py_IS_TYPE(rl, &PyRunningLoopHolder_Type)); + assert(Py_IS_TYPE(rl, state->PyRunningLoopHolder_Type)); PyObject *running_loop = ((PyRunningLoopHolder *)rl)->rl_loop; if (running_loop == Py_None) { @@ -278,7 +321,7 @@ get_running_loop(PyObject **loop) static int -set_running_loop(PyObject *loop) +set_running_loop(asyncio_state *state, PyObject *loop) { PyObject *ts_dict = NULL; @@ -293,7 +336,7 @@ set_running_loop(PyObject *loop) return -1; } - PyRunningLoopHolder *rl = new_running_loop_holder(loop); + PyRunningLoopHolder *rl = new_running_loop_holder(state, loop); if (rl == NULL) { return -1; } @@ -306,20 +349,20 @@ set_running_loop(PyObject *loop) } Py_DECREF(rl); - cached_running_holder = (PyObject *)rl; - cached_running_holder_tsid = PyThreadState_GetID(tstate); + state->cached_running_holder = (PyObject *)rl; + state->cached_running_holder_tsid = PyThreadState_GetID(tstate); return 0; } static PyObject * -get_event_loop(int stacklevel) +get_event_loop(asyncio_state *state, int stacklevel) { PyObject *loop; PyObject *policy; - if (get_running_loop(&loop)) { + if (get_running_loop(state, &loop)) { return NULL; } if (loop != NULL) { @@ -333,7 +376,7 @@ get_event_loop(int stacklevel) return NULL; } - policy = PyObject_CallNoArgs(asyncio_get_event_loop_policy); + policy = PyObject_CallNoArgs(state->asyncio_get_event_loop_policy); if (policy == NULL) { return NULL; } @@ -345,7 +388,8 @@ get_event_loop(int stacklevel) static int -call_soon(PyObject *loop, PyObject *func, PyObject *arg, PyObject *ctx) +call_soon(asyncio_state *state, PyObject *loop, PyObject *func, PyObject *arg, + PyObject *ctx) { PyObject *handle; PyObject *stack[3]; @@ -372,7 +416,8 @@ call_soon(PyObject *loop, PyObject *func, PyObject *arg, PyObject *ctx) } stack[nargs] = (PyObject *)ctx; EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_API, callable); - handle = PyObject_Vectorcall(callable, stack, nargs, context_kwname); + handle = PyObject_Vectorcall(callable, stack, nargs, + state->context_kwname); Py_DECREF(callable); } @@ -403,17 +448,18 @@ future_ensure_alive(FutureObj *fut) } -#define ENSURE_FUTURE_ALIVE(fut) \ - do { \ - assert(Future_Check(fut) || Task_Check(fut)); \ - if (future_ensure_alive((FutureObj*)fut)) { \ - return NULL; \ - } \ +#define ENSURE_FUTURE_ALIVE(state, fut) \ + do { \ + assert(Future_Check(state, fut) || Task_Check(state, fut)); \ + (void)state; \ + if (future_ensure_alive((FutureObj*)fut)) { \ + return NULL; \ + } \ } while(0); static int -future_schedule_callbacks(FutureObj *fut) +future_schedule_callbacks(asyncio_state *state, FutureObj *fut) { Py_ssize_t len; Py_ssize_t i; @@ -421,7 +467,7 @@ future_schedule_callbacks(FutureObj *fut) if (fut->fut_callback0 != NULL) { /* There's a 1st callback */ - int ret = call_soon( + int ret = call_soon(state, fut->fut_loop, fut->fut_callback0, (PyObject *)fut, fut->fut_context0); @@ -455,7 +501,7 @@ future_schedule_callbacks(FutureObj *fut) PyObject *cb = PyTuple_GET_ITEM(cb_tup, 0); PyObject *ctx = PyTuple_GET_ITEM(cb_tup, 1); - if (call_soon(fut->fut_loop, cb, (PyObject *)fut, ctx)) { + if (call_soon(state, fut->fut_loop, cb, (PyObject *)fut, ctx)) { /* If an error occurs in pure-Python implementation, all callbacks are cleared. */ Py_CLEAR(fut->fut_callbacks); @@ -491,7 +537,8 @@ future_init(FutureObj *fut, PyObject *loop) fut->fut_blocking = 0; if (loop == Py_None) { - loop = get_event_loop(1); + asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); + loop = get_event_loop(state, 1); if (loop == NULL) { return -1; } @@ -517,7 +564,8 @@ future_init(FutureObj *fut, PyObject *loop) method, which is called during the interpreter shutdown and the traceback module is already unloaded. */ - fut->fut_source_tb = PyObject_CallNoArgs(traceback_extract_stack); + asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); + fut->fut_source_tb = PyObject_CallNoArgs(state->traceback_extract_stack); if (fut->fut_source_tb == NULL) { return -1; } @@ -527,14 +575,14 @@ future_init(FutureObj *fut, PyObject *loop) } static PyObject * -future_set_result(FutureObj *fut, PyObject *res) +future_set_result(asyncio_state *state, FutureObj *fut, PyObject *res) { if (future_ensure_alive(fut)) { return NULL; } if (fut->fut_state != STATE_PENDING) { - PyErr_SetString(asyncio_InvalidStateError, "invalid state"); + PyErr_SetString(state->asyncio_InvalidStateError, "invalid state"); return NULL; } @@ -542,19 +590,19 @@ future_set_result(FutureObj *fut, PyObject *res) fut->fut_result = Py_NewRef(res); fut->fut_state = STATE_FINISHED; - if (future_schedule_callbacks(fut) == -1) { + if (future_schedule_callbacks(state, fut) == -1) { return NULL; } Py_RETURN_NONE; } static PyObject * -future_set_exception(FutureObj *fut, PyObject *exc) +future_set_exception(asyncio_state *state, FutureObj *fut, PyObject *exc) { PyObject *exc_val = NULL; if (fut->fut_state != STATE_PENDING) { - PyErr_SetString(asyncio_InvalidStateError, "invalid state"); + PyErr_SetString(state->asyncio_InvalidStateError, "invalid state"); return NULL; } @@ -565,7 +613,7 @@ future_set_exception(FutureObj *fut, PyObject *exc) } if (fut->fut_state != STATE_PENDING) { Py_DECREF(exc_val); - PyErr_SetString(asyncio_InvalidStateError, "invalid state"); + PyErr_SetString(state->asyncio_InvalidStateError, "invalid state"); return NULL; } } @@ -591,7 +639,7 @@ future_set_exception(FutureObj *fut, PyObject *exc) fut->fut_exception_tb = PyException_GetTraceback(exc_val); fut->fut_state = STATE_FINISHED; - if (future_schedule_callbacks(fut) == -1) { + if (future_schedule_callbacks(state, fut) == -1) { return NULL; } @@ -600,7 +648,7 @@ future_set_exception(FutureObj *fut, PyObject *exc) } static PyObject * -create_cancelled_error(FutureObj *fut) +create_cancelled_error(asyncio_state *state, FutureObj *fut) { PyObject *exc; if (fut->fut_cancelled_exc != NULL) { @@ -611,34 +659,35 @@ create_cancelled_error(FutureObj *fut) } PyObject *msg = fut->fut_cancel_msg; if (msg == NULL || msg == Py_None) { - exc = PyObject_CallNoArgs(asyncio_CancelledError); + exc = PyObject_CallNoArgs(state->asyncio_CancelledError); } else { - exc = PyObject_CallOneArg(asyncio_CancelledError, msg); + exc = PyObject_CallOneArg(state->asyncio_CancelledError, msg); } return exc; } static void -future_set_cancelled_error(FutureObj *fut) +future_set_cancelled_error(asyncio_state *state, FutureObj *fut) { - PyObject *exc = create_cancelled_error(fut); + PyObject *exc = create_cancelled_error(state, fut); if (exc == NULL) { return; } - PyErr_SetObject(asyncio_CancelledError, exc); + PyErr_SetObject(state->asyncio_CancelledError, exc); Py_DECREF(exc); } static int -future_get_result(FutureObj *fut, PyObject **result) +future_get_result(asyncio_state *state, FutureObj *fut, PyObject **result) { if (fut->fut_state == STATE_CANCELLED) { - future_set_cancelled_error(fut); + future_set_cancelled_error(state, fut); return -1; } if (fut->fut_state != STATE_FINISHED) { - PyErr_SetString(asyncio_InvalidStateError, "Result is not set."); + PyErr_SetString(state->asyncio_InvalidStateError, + "Result is not set."); return -1; } @@ -661,7 +710,8 @@ future_get_result(FutureObj *fut, PyObject **result) } static PyObject * -future_add_done_callback(FutureObj *fut, PyObject *arg, PyObject *ctx) +future_add_done_callback(asyncio_state *state, FutureObj *fut, PyObject *arg, + PyObject *ctx) { if (!future_is_alive(fut)) { PyErr_SetString(PyExc_RuntimeError, "uninitialized Future object"); @@ -671,7 +721,7 @@ future_add_done_callback(FutureObj *fut, PyObject *arg, PyObject *ctx) if (fut->fut_state != STATE_PENDING) { /* The future is done/cancelled, so schedule the callback right away. */ - if (call_soon(fut->fut_loop, arg, (PyObject*) fut, ctx)) { + if (call_soon(state, fut->fut_loop, arg, (PyObject*) fut, ctx)) { return NULL; } } @@ -735,7 +785,7 @@ future_add_done_callback(FutureObj *fut, PyObject *arg, PyObject *ctx) } static PyObject * -future_cancel(FutureObj *fut, PyObject *msg) +future_cancel(asyncio_state *state, FutureObj *fut, PyObject *msg) { fut->fut_log_tb = 0; @@ -747,7 +797,7 @@ future_cancel(FutureObj *fut, PyObject *msg) Py_XINCREF(msg); Py_XSETREF(fut->fut_cancel_msg, msg); - if (future_schedule_callbacks(fut) == -1) { + if (future_schedule_callbacks(state, fut) == -1) { return NULL; } @@ -802,6 +852,7 @@ FutureObj_clear(FutureObj *fut) static int FutureObj_traverse(FutureObj *fut, visitproc visit, void *arg) { + Py_VISIT(Py_TYPE(fut)); Py_VISIT(fut->fut_loop); Py_VISIT(fut->fut_callback0); Py_VISIT(fut->fut_context0); @@ -830,15 +881,16 @@ static PyObject * _asyncio_Future_result_impl(FutureObj *self) /*[clinic end generated code: output=f35f940936a4b1e5 input=49ecf9cf5ec50dc5]*/ { + asyncio_state *state = get_asyncio_state_by_def((PyObject *)self); PyObject *result; if (!future_is_alive(self)) { - PyErr_SetString(asyncio_InvalidStateError, + PyErr_SetString(state->asyncio_InvalidStateError, "Future object is not initialized."); return NULL; } - int res = future_get_result(self, &result); + int res = future_get_result(state, self, &result); if (res == -1) { return NULL; @@ -858,6 +910,9 @@ _asyncio_Future_result_impl(FutureObj *self) /*[clinic input] _asyncio.Future.exception + cls: defining_class + / + Return the exception that was set on this future. The exception (or None if no exception was set) is returned only if @@ -867,22 +922,26 @@ InvalidStateError. [clinic start generated code]*/ static PyObject * -_asyncio_Future_exception_impl(FutureObj *self) -/*[clinic end generated code: output=88b20d4f855e0710 input=733547a70c841c68]*/ +_asyncio_Future_exception_impl(FutureObj *self, PyTypeObject *cls) +/*[clinic end generated code: output=ce75576b187c905b input=3faf15c22acdb60d]*/ { if (!future_is_alive(self)) { - PyErr_SetString(asyncio_InvalidStateError, + asyncio_state *state = get_asyncio_state_by_cls(cls); + PyErr_SetString(state->asyncio_InvalidStateError, "Future object is not initialized."); return NULL; } if (self->fut_state == STATE_CANCELLED) { - future_set_cancelled_error(self); + asyncio_state *state = get_asyncio_state_by_cls(cls); + future_set_cancelled_error(state, self); return NULL; } if (self->fut_state != STATE_FINISHED) { - PyErr_SetString(asyncio_InvalidStateError, "Exception is not set."); + asyncio_state *state = get_asyncio_state_by_cls(cls); + PyErr_SetString(state->asyncio_InvalidStateError, + "Exception is not set."); return NULL; } @@ -897,6 +956,7 @@ _asyncio_Future_exception_impl(FutureObj *self) /*[clinic input] _asyncio.Future.set_result + cls: defining_class result: object / @@ -907,16 +967,19 @@ InvalidStateError. [clinic start generated code]*/ static PyObject * -_asyncio_Future_set_result(FutureObj *self, PyObject *result) -/*[clinic end generated code: output=1ec2e6bcccd6f2ce input=8b75172c2a7b05f1]*/ +_asyncio_Future_set_result_impl(FutureObj *self, PyTypeObject *cls, + PyObject *result) +/*[clinic end generated code: output=99afbbe78f99c32d input=d5a41c1e353acc2e]*/ { - ENSURE_FUTURE_ALIVE(self) - return future_set_result(self, result); + asyncio_state *state = get_asyncio_state_by_cls(cls); + ENSURE_FUTURE_ALIVE(state, self) + return future_set_result(state, self, result); } /*[clinic input] _asyncio.Future.set_exception + cls: defining_class exception: object / @@ -927,16 +990,19 @@ InvalidStateError. [clinic start generated code]*/ static PyObject * -_asyncio_Future_set_exception(FutureObj *self, PyObject *exception) -/*[clinic end generated code: output=f1c1b0cd321be360 input=e45b7d7aa71cc66d]*/ +_asyncio_Future_set_exception_impl(FutureObj *self, PyTypeObject *cls, + PyObject *exception) +/*[clinic end generated code: output=0a5e8b5a52f058d6 input=a245cd49d3df939b]*/ { - ENSURE_FUTURE_ALIVE(self) - return future_set_exception(self, exception); + asyncio_state *state = get_asyncio_state_by_cls(cls); + ENSURE_FUTURE_ALIVE(state, self) + return future_set_exception(state, self, exception); } /*[clinic input] _asyncio.Future.add_done_callback + cls: defining_class fn: object / * @@ -950,25 +1016,27 @@ scheduled with call_soon. [clinic start generated code]*/ static PyObject * -_asyncio_Future_add_done_callback_impl(FutureObj *self, PyObject *fn, - PyObject *context) -/*[clinic end generated code: output=7ce635bbc9554c1e input=15ab0693a96e9533]*/ +_asyncio_Future_add_done_callback_impl(FutureObj *self, PyTypeObject *cls, + PyObject *fn, PyObject *context) +/*[clinic end generated code: output=922e9a4cbd601167 input=599261c521458cc2]*/ { + asyncio_state *state = get_asyncio_state_by_cls(cls); if (context == NULL) { context = PyContext_CopyCurrent(); if (context == NULL) { return NULL; } - PyObject *res = future_add_done_callback(self, fn, context); + PyObject *res = future_add_done_callback(state, self, fn, context); Py_DECREF(context); return res; } - return future_add_done_callback(self, fn, context); + return future_add_done_callback(state, self, fn, context); } /*[clinic input] _asyncio.Future.remove_done_callback + cls: defining_class fn: object / @@ -978,14 +1046,16 @@ Returns the number of callbacks removed. [clinic start generated code]*/ static PyObject * -_asyncio_Future_remove_done_callback(FutureObj *self, PyObject *fn) -/*[clinic end generated code: output=5ab1fb52b24ef31f input=0a43280a149d505b]*/ +_asyncio_Future_remove_done_callback_impl(FutureObj *self, PyTypeObject *cls, + PyObject *fn) +/*[clinic end generated code: output=2da35ccabfe41b98 input=c7518709b86fc747]*/ { PyObject *newlist; Py_ssize_t len, i, j=0; Py_ssize_t cleared_callback0 = 0; - ENSURE_FUTURE_ALIVE(self) + asyncio_state *state = get_asyncio_state_by_cls(cls); + ENSURE_FUTURE_ALIVE(state, self) if (self->fut_callback0 != NULL) { int cmp = PyObject_RichCompareBool(self->fut_callback0, fn, Py_EQ); @@ -1082,6 +1152,8 @@ _asyncio_Future_remove_done_callback(FutureObj *self, PyObject *fn) /*[clinic input] _asyncio.Future.cancel + cls: defining_class + / msg: object = None Cancel the future and schedule callbacks. @@ -1092,11 +1164,13 @@ return True. [clinic start generated code]*/ static PyObject * -_asyncio_Future_cancel_impl(FutureObj *self, PyObject *msg) -/*[clinic end generated code: output=3edebbc668e5aba3 input=925eb545251f2c5a]*/ +_asyncio_Future_cancel_impl(FutureObj *self, PyTypeObject *cls, + PyObject *msg) +/*[clinic end generated code: output=074956f35904b034 input=bba8f8b786941a94]*/ { - ENSURE_FUTURE_ALIVE(self) - return future_cancel(self, msg); + asyncio_state *state = get_asyncio_state_by_cls(cls); + ENSURE_FUTURE_ALIVE(state, self) + return future_cancel(state, self, msg); } /*[clinic input] @@ -1141,14 +1215,18 @@ _asyncio_Future_done_impl(FutureObj *self) /*[clinic input] _asyncio.Future.get_loop + cls: defining_class + / + Return the event loop the Future is bound to. [clinic start generated code]*/ static PyObject * -_asyncio_Future_get_loop_impl(FutureObj *self) -/*[clinic end generated code: output=119b6ea0c9816c3f input=cba48c2136c79d1f]*/ +_asyncio_Future_get_loop_impl(FutureObj *self, PyTypeObject *cls) +/*[clinic end generated code: output=f50ea6c374d9ee97 input=163c2c498b45a1f0]*/ { - ENSURE_FUTURE_ALIVE(self) + asyncio_state *state = get_asyncio_state_by_cls(cls); + ENSURE_FUTURE_ALIVE(state, self) return Py_NewRef(self->fut_loop); } @@ -1185,7 +1263,8 @@ FutureObj_set_blocking(FutureObj *fut, PyObject *val, void *Py_UNUSED(ignored)) static PyObject * FutureObj_get_log_traceback(FutureObj *fut, void *Py_UNUSED(ignored)) { - ENSURE_FUTURE_ALIVE(fut) + asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); + ENSURE_FUTURE_ALIVE(state, fut) if (fut->fut_log_tb) { Py_RETURN_TRUE; } @@ -1226,9 +1305,10 @@ FutureObj_get_loop(FutureObj *fut, void *Py_UNUSED(ignored)) static PyObject * FutureObj_get_callbacks(FutureObj *fut, void *Py_UNUSED(ignored)) { + asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); Py_ssize_t i; - ENSURE_FUTURE_ALIVE(fut) + ENSURE_FUTURE_ALIVE(state, fut) if (fut->fut_callback0 == NULL) { if (fut->fut_callbacks == NULL) { @@ -1277,7 +1357,8 @@ FutureObj_get_callbacks(FutureObj *fut, void *Py_UNUSED(ignored)) static PyObject * FutureObj_get_result(FutureObj *fut, void *Py_UNUSED(ignored)) { - ENSURE_FUTURE_ALIVE(fut) + asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); + ENSURE_FUTURE_ALIVE(state, fut) if (fut->fut_result == NULL) { Py_RETURN_NONE; } @@ -1287,7 +1368,8 @@ FutureObj_get_result(FutureObj *fut, void *Py_UNUSED(ignored)) static PyObject * FutureObj_get_exception(FutureObj *fut, void *Py_UNUSED(ignored)) { - ENSURE_FUTURE_ALIVE(fut) + asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); + ENSURE_FUTURE_ALIVE(state, fut) if (fut->fut_exception == NULL) { Py_RETURN_NONE; } @@ -1328,9 +1410,10 @@ FutureObj_set_cancel_message(FutureObj *fut, PyObject *msg, static PyObject * FutureObj_get_state(FutureObj *fut, void *Py_UNUSED(ignored)) { + asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); PyObject *ret = NULL; - ENSURE_FUTURE_ALIVE(fut) + ENSURE_FUTURE_ALIVE(state, fut) switch (fut->fut_state) { case STATE_PENDING: @@ -1351,8 +1434,9 @@ FutureObj_get_state(FutureObj *fut, void *Py_UNUSED(ignored)) static PyObject * FutureObj_repr(FutureObj *fut) { - ENSURE_FUTURE_ALIVE(fut) - return PyObject_CallOneArg(asyncio_future_repr_func, (PyObject *)fut); + asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); + ENSURE_FUTURE_ALIVE(state, fut) + return PyObject_CallOneArg(state->asyncio_future_repr_func, (PyObject *)fut); } /*[clinic input] @@ -1368,7 +1452,8 @@ static PyObject * _asyncio_Future__make_cancelled_error_impl(FutureObj *self) /*[clinic end generated code: output=a5df276f6c1213de input=ac6effe4ba795ecc]*/ { - return create_cancelled_error(self); + asyncio_state *state = get_asyncio_state_by_def((PyObject *)self); + return create_cancelled_error(state, self); } static void @@ -1431,13 +1516,6 @@ FutureObj_finalize(FutureObj *fut) PyErr_Restore(error_type, error_value, error_traceback); } -static PyAsyncMethods FutureType_as_async = { - (unaryfunc)future_new_iter, /* am_await */ - 0, /* am_aiter */ - 0, /* am_anext */ - 0, /* am_send */ -}; - static PyMethodDef FutureType_methods[] = { _ASYNCIO_FUTURE_RESULT_METHODDEF _ASYNCIO_FUTURE_EXCEPTION_METHODDEF @@ -1454,6 +1532,12 @@ static PyMethodDef FutureType_methods[] = { {NULL, NULL} /* Sentinel */ }; +static PyMemberDef FutureType_members[] = { + {"__weaklistoffset__", T_PYSSIZET, offsetof(FutureObj, fut_weakreflist), READONLY}, + {"__dictoffset__", T_PYSSIZET, offsetof(FutureObj, dict), READONLY}, + {NULL}, +}; + #define FUTURE_COMMON_GETSETLIST \ {"_state", (getter)FutureObj_get_state, NULL, NULL}, \ {"_asyncio_future_blocking", (getter)FutureObj_get_blocking, \ @@ -1476,25 +1560,31 @@ static PyGetSetDef FutureType_getsetlist[] = { static void FutureObj_dealloc(PyObject *self); -static PyTypeObject FutureType = { - PyVarObject_HEAD_INIT(NULL, 0) - "_asyncio.Future", - sizeof(FutureObj), /* tp_basicsize */ - .tp_dealloc = FutureObj_dealloc, - .tp_as_async = &FutureType_as_async, - .tp_repr = (reprfunc)FutureObj_repr, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, - .tp_doc = _asyncio_Future___init____doc__, - .tp_traverse = (traverseproc)FutureObj_traverse, - .tp_clear = (inquiry)FutureObj_clear, - .tp_weaklistoffset = offsetof(FutureObj, fut_weakreflist), - .tp_iter = (getiterfunc)future_new_iter, - .tp_methods = FutureType_methods, - .tp_getset = FutureType_getsetlist, - .tp_dictoffset = offsetof(FutureObj, dict), - .tp_init = (initproc)_asyncio_Future___init__, - .tp_new = PyType_GenericNew, - .tp_finalize = (destructor)FutureObj_finalize, +static PyType_Slot Future_slots[] = { + {Py_tp_dealloc, FutureObj_dealloc}, + {Py_tp_repr, (reprfunc)FutureObj_repr}, + {Py_tp_doc, (void *)_asyncio_Future___init____doc__}, + {Py_tp_traverse, (traverseproc)FutureObj_traverse}, + {Py_tp_clear, (inquiry)FutureObj_clear}, + {Py_tp_iter, (getiterfunc)future_new_iter}, + {Py_tp_methods, FutureType_methods}, + {Py_tp_members, FutureType_members}, + {Py_tp_getset, FutureType_getsetlist}, + {Py_tp_init, (initproc)_asyncio_Future___init__}, + {Py_tp_new, PyType_GenericNew}, + {Py_tp_finalize, (destructor)FutureObj_finalize}, + + // async slots + {Py_am_await, (unaryfunc)future_new_iter}, + {0, NULL}, +}; + +static PyType_Spec Future_spec = { + .name = "_asyncio.Future", + .basicsize = sizeof(FutureObj), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = Future_slots, }; static void @@ -1502,16 +1592,12 @@ FutureObj_dealloc(PyObject *self) { FutureObj *fut = (FutureObj *)self; - if (Future_CheckExact(fut)) { - /* When fut is subclass of Future, finalizer is called from - * subtype_dealloc. - */ - if (PyObject_CallFinalizerFromDealloc(self) < 0) { - // resurrected. - return; - } + if (PyObject_CallFinalizerFromDealloc(self) < 0) { + // resurrected. + return; } + PyTypeObject *tp = Py_TYPE(fut); PyObject_GC_UnTrack(self); if (fut->fut_weakreflist != NULL) { @@ -1519,7 +1605,8 @@ FutureObj_dealloc(PyObject *self) } (void)FutureObj_clear(fut); - Py_TYPE(fut)->tp_free(fut); + tp->tp_free(fut); + Py_DECREF(tp); } @@ -1539,8 +1626,9 @@ static Py_ssize_t fi_freelist_len = 0; static void FutureIter_dealloc(futureiterobject *it) { + PyTypeObject *tp = Py_TYPE(it); PyObject_GC_UnTrack(it); - Py_CLEAR(it->future); + tp->tp_clear((PyObject *)it); if (fi_freelist_len < FI_FREELIST_MAXLEN) { fi_freelist_len++; @@ -1549,6 +1637,7 @@ FutureIter_dealloc(futureiterobject *it) } else { PyObject_GC_Del(it); + Py_DECREF(tp); } } @@ -1686,16 +1775,24 @@ FutureIter_throw(futureiterobject *self, PyObject *const *args, Py_ssize_t nargs return NULL; } +static int +FutureIter_clear(futureiterobject *it) +{ + Py_CLEAR(it->future); + return 0; +} + static PyObject * FutureIter_close(futureiterobject *self, PyObject *arg) { - Py_CLEAR(self->future); + (void)FutureIter_clear(self); Py_RETURN_NONE; } static int FutureIter_traverse(futureiterobject *it, visitproc visit, void *arg) { + Py_VISIT(Py_TYPE(it)); Py_VISIT(it->future); return 0; } @@ -1707,27 +1804,26 @@ static PyMethodDef FutureIter_methods[] = { {NULL, NULL} /* Sentinel */ }; -static PyAsyncMethods FutureIterType_as_async = { - 0, /* am_await */ - 0, /* am_aiter */ - 0, /* am_anext */ - (sendfunc)FutureIter_am_send, /* am_send */ +static PyType_Slot FutureIter_slots[] = { + {Py_tp_dealloc, (destructor)FutureIter_dealloc}, + {Py_tp_getattro, PyObject_GenericGetAttr}, + {Py_tp_traverse, (traverseproc)FutureIter_traverse}, + {Py_tp_clear, FutureIter_clear}, + {Py_tp_iter, PyObject_SelfIter}, + {Py_tp_iternext, (iternextfunc)FutureIter_iternext}, + {Py_tp_methods, FutureIter_methods}, + + // async methods + {Py_am_send, (sendfunc)FutureIter_am_send}, + {0, NULL}, }; - -static PyTypeObject FutureIterType = { - PyVarObject_HEAD_INIT(NULL, 0) - "_asyncio.FutureIter", - .tp_basicsize = sizeof(futureiterobject), - .tp_itemsize = 0, - .tp_dealloc = (destructor)FutureIter_dealloc, - .tp_as_async = &FutureIterType_as_async, - .tp_getattro = PyObject_GenericGetAttr, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)FutureIter_traverse, - .tp_iter = PyObject_SelfIter, - .tp_iternext = (iternextfunc)FutureIter_iternext, - .tp_methods = FutureIter_methods, +static PyType_Spec FutureIter_spec = { + .name = "_asyncio.FutureIter", + .basicsize = sizeof(futureiterobject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = FutureIter_slots, }; static PyObject * @@ -1735,12 +1831,13 @@ future_new_iter(PyObject *fut) { futureiterobject *it; - if (!PyObject_TypeCheck(fut, &FutureType)) { + asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); + if (!Future_Check(state, fut)) { PyErr_BadInternalCall(); return NULL; } - ENSURE_FUTURE_ALIVE(fut) + ENSURE_FUTURE_ALIVE(state, fut) if (fi_freelist_len) { fi_freelist_len--; @@ -1750,7 +1847,7 @@ future_new_iter(PyObject *fut) _Py_NewReference((PyObject*) it); } else { - it = PyObject_GC_New(futureiterobject, &FutureIterType); + it = PyObject_GC_New(futureiterobject, state->FutureIterType); if (it == NULL) { return NULL; } @@ -1770,9 +1867,9 @@ class _asyncio.Task "TaskObj *" "&Task_Type" [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=719dcef0fcc03b37]*/ -static int task_call_step_soon(TaskObj *, PyObject *); +static int task_call_step_soon(asyncio_state *state, TaskObj *, PyObject *); static PyObject * task_wakeup(TaskObj *, PyObject *); -static PyObject * task_step(TaskObj *, PyObject *); +static PyObject * task_step(asyncio_state *, TaskObj *, PyObject *); /* ----- Task._step wrapper */ @@ -1787,9 +1884,11 @@ TaskStepMethWrapper_clear(TaskStepMethWrapper *o) static void TaskStepMethWrapper_dealloc(TaskStepMethWrapper *o) { + PyTypeObject *tp = Py_TYPE(o); PyObject_GC_UnTrack(o); (void)TaskStepMethWrapper_clear(o); Py_TYPE(o)->tp_free(o); + Py_DECREF(tp); } static PyObject * @@ -1804,13 +1903,15 @@ TaskStepMethWrapper_call(TaskStepMethWrapper *o, PyErr_SetString(PyExc_TypeError, "function takes no positional arguments"); return NULL; } - return task_step(o->sw_task, o->sw_arg); + asyncio_state *state = get_asyncio_state_by_def((PyObject *)o); + return task_step(state, o->sw_task, o->sw_arg); } static int TaskStepMethWrapper_traverse(TaskStepMethWrapper *o, visitproc visit, void *arg) { + Py_VISIT(Py_TYPE(o)); Py_VISIT(o->sw_task); Py_VISIT(o->sw_arg); return 0; @@ -1830,25 +1931,30 @@ static PyGetSetDef TaskStepMethWrapper_getsetlist[] = { {NULL} /* Sentinel */ }; -static PyTypeObject TaskStepMethWrapper_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "TaskStepMethWrapper", - .tp_basicsize = sizeof(TaskStepMethWrapper), - .tp_itemsize = 0, - .tp_getset = TaskStepMethWrapper_getsetlist, - .tp_dealloc = (destructor)TaskStepMethWrapper_dealloc, - .tp_call = (ternaryfunc)TaskStepMethWrapper_call, - .tp_getattro = PyObject_GenericGetAttr, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)TaskStepMethWrapper_traverse, - .tp_clear = (inquiry)TaskStepMethWrapper_clear, +static PyType_Slot TaskStepMethWrapper_slots[] = { + {Py_tp_getset, TaskStepMethWrapper_getsetlist}, + {Py_tp_dealloc, (destructor)TaskStepMethWrapper_dealloc}, + {Py_tp_call, (ternaryfunc)TaskStepMethWrapper_call}, + {Py_tp_getattro, PyObject_GenericGetAttr}, + {Py_tp_traverse, (traverseproc)TaskStepMethWrapper_traverse}, + {Py_tp_clear, (inquiry)TaskStepMethWrapper_clear}, + {0, NULL}, +}; + +static PyType_Spec TaskStepMethWrapper_spec = { + .name = "_asyncio.TaskStepMethWrapper", + .basicsize = sizeof(TaskStepMethWrapper), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = TaskStepMethWrapper_slots, }; static PyObject * TaskStepMethWrapper_new(TaskObj *task, PyObject *arg) { + asyncio_state *state = get_asyncio_state_by_def((PyObject *)task); TaskStepMethWrapper *o; - o = PyObject_GC_New(TaskStepMethWrapper, &TaskStepMethWrapper_Type); + o = PyObject_GC_New(TaskStepMethWrapper, state->TaskStepMethWrapper_Type); if (o == NULL) { return NULL; } @@ -1872,10 +1978,10 @@ static PyMethodDef TaskWakeupDef = { /* ----- Task introspection helpers */ static int -register_task(PyObject *task) +register_task(asyncio_state *state, PyObject *task) { - PyObject *res = PyObject_CallMethodOneArg(all_tasks, - &_Py_ID(add), task); + PyObject *res = PyObject_CallMethodOneArg(state->all_tasks, + &_Py_ID(add), task); if (res == NULL) { return -1; } @@ -1885,9 +1991,9 @@ register_task(PyObject *task) static int -unregister_task(PyObject *task) +unregister_task(asyncio_state *state, PyObject *task) { - PyObject *res = PyObject_CallMethodOneArg(all_tasks, + PyObject *res = PyObject_CallMethodOneArg(state->all_tasks, &_Py_ID(discard), task); if (res == NULL) { return -1; @@ -1898,7 +2004,7 @@ unregister_task(PyObject *task) static int -enter_task(PyObject *loop, PyObject *task) +enter_task(asyncio_state *state, PyObject *loop, PyObject *task) { PyObject *item; Py_hash_t hash; @@ -1906,7 +2012,7 @@ enter_task(PyObject *loop, PyObject *task) if (hash == -1) { return -1; } - item = _PyDict_GetItem_KnownHash(current_tasks, loop, hash); + item = _PyDict_GetItem_KnownHash(state->current_tasks, loop, hash); if (item != NULL) { Py_INCREF(item); PyErr_Format( @@ -1920,12 +2026,12 @@ enter_task(PyObject *loop, PyObject *task) if (PyErr_Occurred()) { return -1; } - return _PyDict_SetItem_KnownHash(current_tasks, loop, task, hash); + return _PyDict_SetItem_KnownHash(state->current_tasks, loop, task, hash); } static int -leave_task(PyObject *loop, PyObject *task) +leave_task(asyncio_state *state, PyObject *loop, PyObject *task) /*[clinic end generated code: output=0ebf6db4b858fb41 input=51296a46313d1ad8]*/ { PyObject *item; @@ -1934,7 +2040,7 @@ leave_task(PyObject *loop, PyObject *task) if (hash == -1) { return -1; } - item = _PyDict_GetItem_KnownHash(current_tasks, loop, hash); + item = _PyDict_GetItem_KnownHash(state->current_tasks, loop, hash); if (item != task) { if (item == NULL) { /* Not entered, replace with None */ @@ -1946,7 +2052,7 @@ leave_task(PyObject *loop, PyObject *task) task, item, NULL); return -1; } - return _PyDict_DelItem_KnownHash(current_tasks, loop, hash); + return _PyDict_DelItem_KnownHash(state->current_tasks, loop, hash); } /* ----- Task */ @@ -1973,7 +2079,8 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, return -1; } - int is_coro = is_coroutine(coro); + asyncio_state *state = get_asyncio_state_by_def((PyObject *)self); + int is_coro = is_coroutine(state, coro); if (is_coro == -1) { return -1; } @@ -2002,7 +2109,8 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, Py_XSETREF(self->task_coro, coro); if (name == Py_None) { - name = PyUnicode_FromFormat("Task-%" PRIu64, ++task_name_counter); + name = PyUnicode_FromFormat("Task-%" PRIu64, + ++state->task_name_counter); } else if (!PyUnicode_CheckExact(name)) { name = PyObject_Str(name); } else { @@ -2013,10 +2121,10 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, return -1; } - if (task_call_step_soon(self, NULL)) { + if (task_call_step_soon(state, self, NULL)) { return -1; } - return register_task((PyObject*)self); + return register_task(state, (PyObject*)self); } static int @@ -2033,11 +2141,23 @@ TaskObj_clear(TaskObj *task) static int TaskObj_traverse(TaskObj *task, visitproc visit, void *arg) { + Py_VISIT(Py_TYPE(task)); Py_VISIT(task->task_context); Py_VISIT(task->task_coro); Py_VISIT(task->task_name); Py_VISIT(task->task_fut_waiter); - (void)FutureObj_traverse((FutureObj*) task, visit, arg); + FutureObj *fut = (FutureObj *)task; + Py_VISIT(fut->fut_loop); + Py_VISIT(fut->fut_callback0); + Py_VISIT(fut->fut_context0); + Py_VISIT(fut->fut_callbacks); + Py_VISIT(fut->fut_result); + Py_VISIT(fut->fut_exception); + Py_VISIT(fut->fut_exception_tb); + Py_VISIT(fut->fut_source_tb); + Py_VISIT(fut->fut_cancel_msg); + Py_VISIT(fut->fut_cancelled_exc); + Py_VISIT(fut->dict); return 0; } @@ -2101,7 +2221,9 @@ TaskObj_get_fut_waiter(TaskObj *task, void *Py_UNUSED(ignored)) static PyObject * TaskObj_repr(TaskObj *task) { - return PyObject_CallOneArg(asyncio_task_repr_func, (PyObject *)task); + asyncio_state *state = get_asyncio_state_by_def((PyObject *)task); + return PyObject_CallOneArg(state->asyncio_task_repr_func, + (PyObject *)task); } @@ -2238,6 +2360,8 @@ _asyncio_Task_uncancel_impl(TaskObj *self) /*[clinic input] _asyncio.Task.get_stack + cls: defining_class + / * limit: object = None @@ -2263,16 +2387,20 @@ returned for a suspended coroutine. [clinic start generated code]*/ static PyObject * -_asyncio_Task_get_stack_impl(TaskObj *self, PyObject *limit) -/*[clinic end generated code: output=c9aeeeebd1e18118 input=05b323d42b809b90]*/ +_asyncio_Task_get_stack_impl(TaskObj *self, PyTypeObject *cls, + PyObject *limit) +/*[clinic end generated code: output=6774dfc10d3857fa input=8e01c9b2618ae953]*/ { + asyncio_state *state = get_asyncio_state_by_cls(cls); return PyObject_CallFunctionObjArgs( - asyncio_task_get_stack_func, self, limit, NULL); + state->asyncio_task_get_stack_func, self, limit, NULL); } /*[clinic input] _asyncio.Task.print_stack + cls: defining_class + / * limit: object = None file: object = None @@ -2287,12 +2415,13 @@ to sys.stderr. [clinic start generated code]*/ static PyObject * -_asyncio_Task_print_stack_impl(TaskObj *self, PyObject *limit, - PyObject *file) -/*[clinic end generated code: output=7339e10314cd3f4d input=1a0352913b7fcd92]*/ +_asyncio_Task_print_stack_impl(TaskObj *self, PyTypeObject *cls, + PyObject *limit, PyObject *file) +/*[clinic end generated code: output=b38affe9289ec826 input=150b35ba2d3a7dee]*/ { + asyncio_state *state = get_asyncio_state_by_cls(cls); return PyObject_CallFunctionObjArgs( - asyncio_task_print_stack_func, self, limit, file, NULL); + state->asyncio_task_print_stack_func, self, limit, file, NULL); } /*[clinic input] @@ -2475,6 +2604,12 @@ static PyMethodDef TaskType_methods[] = { {NULL, NULL} /* Sentinel */ }; +static PyMemberDef TaskType_members[] = { + {"__weaklistoffset__", T_PYSSIZET, offsetof(TaskObj, task_weakreflist), READONLY}, + {"__dictoffset__", T_PYSSIZET, offsetof(TaskObj, dict), READONLY}, + {NULL}, +}; + static PyGetSetDef TaskType_getsetlist[] = { FUTURE_COMMON_GETSETLIST {"_log_destroy_pending", (getter)TaskObj_get_log_destroy_pending, @@ -2485,26 +2620,31 @@ static PyGetSetDef TaskType_getsetlist[] = { {NULL} /* Sentinel */ }; -static PyTypeObject TaskType = { - PyVarObject_HEAD_INIT(NULL, 0) - "_asyncio.Task", - sizeof(TaskObj), /* tp_basicsize */ - .tp_base = &FutureType, - .tp_dealloc = TaskObj_dealloc, - .tp_as_async = &FutureType_as_async, - .tp_repr = (reprfunc)TaskObj_repr, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, - .tp_doc = _asyncio_Task___init____doc__, - .tp_traverse = (traverseproc)TaskObj_traverse, - .tp_clear = (inquiry)TaskObj_clear, - .tp_weaklistoffset = offsetof(TaskObj, task_weakreflist), - .tp_iter = (getiterfunc)future_new_iter, - .tp_methods = TaskType_methods, - .tp_getset = TaskType_getsetlist, - .tp_dictoffset = offsetof(TaskObj, dict), - .tp_init = (initproc)_asyncio_Task___init__, - .tp_new = PyType_GenericNew, - .tp_finalize = (destructor)TaskObj_finalize, +static PyType_Slot Task_slots[] = { + {Py_tp_dealloc, TaskObj_dealloc}, + {Py_tp_repr, (reprfunc)TaskObj_repr}, + {Py_tp_doc, (void *)_asyncio_Task___init____doc__}, + {Py_tp_traverse, (traverseproc)TaskObj_traverse}, + {Py_tp_clear, (inquiry)TaskObj_clear}, + {Py_tp_iter, (getiterfunc)future_new_iter}, + {Py_tp_methods, TaskType_methods}, + {Py_tp_members, TaskType_members}, + {Py_tp_getset, TaskType_getsetlist}, + {Py_tp_init, (initproc)_asyncio_Task___init__}, + {Py_tp_new, PyType_GenericNew}, + {Py_tp_finalize, (destructor)TaskObj_finalize}, + + // async slots + {Py_am_await, (unaryfunc)future_new_iter}, + {0, NULL}, +}; + +static PyType_Spec Task_spec = { + .name = "_asyncio.Task", + .basicsize = sizeof(TaskObj), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = Task_slots, }; static void @@ -2512,16 +2652,12 @@ TaskObj_dealloc(PyObject *self) { TaskObj *task = (TaskObj *)self; - if (Task_CheckExact(self)) { - /* When fut is subclass of Task, finalizer is called from - * subtype_dealloc. - */ - if (PyObject_CallFinalizerFromDealloc(self) < 0) { - // resurrected. - return; - } + if (PyObject_CallFinalizerFromDealloc(self) < 0) { + // resurrected. + return; } + PyTypeObject *tp = Py_TYPE(task); PyObject_GC_UnTrack(self); if (task->task_weakreflist != NULL) { @@ -2529,24 +2665,26 @@ TaskObj_dealloc(PyObject *self) } (void)TaskObj_clear(task); - Py_TYPE(task)->tp_free(task); + tp->tp_free(task); + Py_DECREF(tp); } static int -task_call_step_soon(TaskObj *task, PyObject *arg) +task_call_step_soon(asyncio_state *state, TaskObj *task, PyObject *arg) { PyObject *cb = TaskStepMethWrapper_new(task, arg); if (cb == NULL) { return -1; } - int ret = call_soon(task->task_loop, cb, NULL, task->task_context); + int ret = call_soon(state, task->task_loop, cb, NULL, task->task_context); Py_DECREF(cb); return ret; } static PyObject * -task_set_error_soon(TaskObj *task, PyObject *et, const char *format, ...) +task_set_error_soon(asyncio_state *state, TaskObj *task, PyObject *et, + const char *format, ...) { PyObject* msg; @@ -2565,7 +2703,7 @@ task_set_error_soon(TaskObj *task, PyObject *et, const char *format, ...) return NULL; } - if (task_call_step_soon(task, e) == -1) { + if (task_call_step_soon(state, task, e) == -1) { Py_DECREF(e); return NULL; } @@ -2589,7 +2727,7 @@ gen_status_from_result(PyObject **result) } static PyObject * -task_step_impl(TaskObj *task, PyObject *exc) +task_step_impl(asyncio_state *state, TaskObj *task, PyObject *exc) { int res; int clear_exc = 0; @@ -2598,7 +2736,7 @@ task_step_impl(TaskObj *task, PyObject *exc) PyObject *o; if (task->task_state != STATE_PENDING) { - PyErr_Format(asyncio_InvalidStateError, + PyErr_Format(state->asyncio_InvalidStateError, "_step(): already done: %R %R", task, exc ? exc : Py_None); @@ -2610,7 +2748,7 @@ task_step_impl(TaskObj *task, PyObject *exc) if (exc) { /* Check if exc is a CancelledError */ - res = PyObject_IsInstance(exc, asyncio_CancelledError); + res = PyObject_IsInstance(exc, state->asyncio_CancelledError); if (res == -1) { /* An error occurred, abort */ goto fail; @@ -2623,7 +2761,7 @@ task_step_impl(TaskObj *task, PyObject *exc) if (!exc) { /* exc was not a CancelledError */ - exc = create_cancelled_error((FutureObj*)task); + exc = create_cancelled_error(state, (FutureObj*)task); if (!exc) { goto fail; @@ -2670,10 +2808,11 @@ task_step_impl(TaskObj *task, PyObject *exc) if (task->task_must_cancel) { // Task is cancelled right before coro stops. task->task_must_cancel = 0; - tmp = future_cancel((FutureObj*)task, task->task_cancel_msg); + tmp = future_cancel(state, (FutureObj*)task, + task->task_cancel_msg); } else { - tmp = future_set_result((FutureObj*)task, result); + tmp = future_set_result(state, (FutureObj*)task, result); } Py_DECREF(result); @@ -2685,7 +2824,7 @@ task_step_impl(TaskObj *task, PyObject *exc) Py_RETURN_NONE; } - if (PyErr_ExceptionMatches(asyncio_CancelledError)) { + if (PyErr_ExceptionMatches(state->asyncio_CancelledError)) { /* CancelledError */ PyErr_Fetch(&et, &ev, &tb); assert(et); @@ -2700,7 +2839,7 @@ task_step_impl(TaskObj *task, PyObject *exc) /* transfer ownership */ fut->fut_cancelled_exc = ev; - return future_cancel(fut, NULL); + return future_cancel(state, fut, NULL); } /* Some other exception; pop it and call Task.set_exception() */ @@ -2711,7 +2850,7 @@ task_step_impl(TaskObj *task, PyObject *exc) PyException_SetTraceback(ev, tb); } - o = future_set_exception((FutureObj*)task, ev); + o = future_set_exception(state, (FutureObj*)task, ev); if (!o) { /* An exception in Task.set_exception() */ Py_DECREF(et); @@ -2743,7 +2882,7 @@ task_step_impl(TaskObj *task, PyObject *exc) } /* Check if `result` is FutureObj or TaskObj (and not a subclass) */ - if (Future_CheckExact(result) || Task_CheckExact(result)) { + if (Future_CheckExact(state, result) || Task_CheckExact(state, result)) { PyObject *wrapper; PyObject *tmp; FutureObj *fut = (FutureObj*)result; @@ -2764,7 +2903,7 @@ task_step_impl(TaskObj *task, PyObject *exc) if (wrapper == NULL) { goto fail; } - tmp = future_add_done_callback( + tmp = future_add_done_callback(state, (FutureObj*)result, wrapper, task->task_context); Py_DECREF(wrapper); if (tmp == NULL) { @@ -2799,7 +2938,7 @@ task_step_impl(TaskObj *task, PyObject *exc) /* Check if `result` is None */ if (result == Py_None) { /* Bare yield relinquishes control for one event loop iteration. */ - if (task_call_step_soon(task, NULL)) { + if (task_call_step_soon(state, task, NULL)) { goto fail; } return result; @@ -2821,7 +2960,7 @@ task_step_impl(TaskObj *task, PyObject *exc) } /* Check if `result` future is attached to a different loop */ - PyObject *oloop = get_future_loop(result); + PyObject *oloop = get_future_loop(state, result); if (oloop == NULL) { goto fail; } @@ -2857,7 +2996,7 @@ task_step_impl(TaskObj *task, PyObject *exc) stack[0] = wrapper; stack[1] = (PyObject *)task->task_context; EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_API, add_cb); - tmp = PyObject_Vectorcall(add_cb, stack, 1, context_kwname); + tmp = PyObject_Vectorcall(add_cb, stack, 1, state->context_kwname); Py_DECREF(add_cb); Py_DECREF(wrapper); if (tmp == NULL) { @@ -2898,7 +3037,7 @@ task_step_impl(TaskObj *task, PyObject *exc) if (res) { /* `result` is a generator */ o = task_set_error_soon( - task, PyExc_RuntimeError, + state, task, PyExc_RuntimeError, "yield was used instead of yield from for " "generator in task %R with %R", task, result); Py_DECREF(result); @@ -2907,20 +3046,20 @@ task_step_impl(TaskObj *task, PyObject *exc) /* The `result` is none of the above */ o = task_set_error_soon( - task, PyExc_RuntimeError, "Task got bad yield: %R", result); + state, task, PyExc_RuntimeError, "Task got bad yield: %R", result); Py_DECREF(result); return o; self_await: o = task_set_error_soon( - task, PyExc_RuntimeError, + state, task, PyExc_RuntimeError, "Task cannot await on itself: %R", task); Py_DECREF(result); return o; yield_insteadof_yf: o = task_set_error_soon( - task, PyExc_RuntimeError, + state, task, PyExc_RuntimeError, "yield was used instead of yield from " "in task %R with %R", task, result); @@ -2929,7 +3068,7 @@ task_step_impl(TaskObj *task, PyObject *exc) different_loop: o = task_set_error_soon( - task, PyExc_RuntimeError, + state, task, PyExc_RuntimeError, "Task %R got Future %R attached to a different loop", task, result); Py_DECREF(result); @@ -2941,25 +3080,25 @@ task_step_impl(TaskObj *task, PyObject *exc) } static PyObject * -task_step(TaskObj *task, PyObject *exc) +task_step(asyncio_state *state, TaskObj *task, PyObject *exc) { PyObject *res; - if (enter_task(task->task_loop, (PyObject*)task) < 0) { + if (enter_task(state, task->task_loop, (PyObject*)task) < 0) { return NULL; } - res = task_step_impl(task, exc); + res = task_step_impl(state, task, exc); if (res == NULL) { PyObject *et, *ev, *tb; PyErr_Fetch(&et, &ev, &tb); - leave_task(task->task_loop, (PyObject*)task); + leave_task(state, task->task_loop, (PyObject*)task); _PyErr_ChainExceptions(et, ev, tb); /* Normalizes (et, ev, tb) */ return NULL; } else { - if (leave_task(task->task_loop, (PyObject*)task) < 0) { + if (leave_task(state, task->task_loop, (PyObject*)task) < 0) { Py_DECREF(res); return NULL; } @@ -2976,9 +3115,10 @@ task_wakeup(TaskObj *task, PyObject *o) PyObject *result; assert(o); - if (Future_CheckExact(o) || Task_CheckExact(o)) { + asyncio_state *state = get_asyncio_state_by_def((PyObject *)task); + if (Future_CheckExact(state, o) || Task_CheckExact(state, o)) { PyObject *fut_result = NULL; - int res = future_get_result((FutureObj*)o, &fut_result); + int res = future_get_result(state, (FutureObj*)o, &fut_result); switch(res) { case -1: @@ -2986,10 +3126,10 @@ task_wakeup(TaskObj *task, PyObject *o) break; /* exception raised */ case 0: Py_DECREF(fut_result); - return task_step(task, NULL); + return task_step(state, task, NULL); default: assert(res == 1); - result = task_step(task, fut_result); + result = task_step(state, task, fut_result); Py_DECREF(fut_result); return result; } @@ -2998,7 +3138,7 @@ task_wakeup(TaskObj *task, PyObject *o) PyObject *fut_result = PyObject_CallMethod(o, "result", NULL); if (fut_result != NULL) { Py_DECREF(fut_result); - return task_step(task, NULL); + return task_step(state, task, NULL); } /* exception raised */ } @@ -3010,7 +3150,7 @@ task_wakeup(TaskObj *task, PyObject *o) PyException_SetTraceback(ev, tb); } - result = task_step(task, ev); + result = task_step(state, task, ev); Py_DECREF(et); Py_XDECREF(tb); @@ -3038,7 +3178,8 @@ _asyncio__get_running_loop_impl(PyObject *module) /*[clinic end generated code: output=b4390af721411a0a input=0a21627e25a4bd43]*/ { PyObject *loop; - if (get_running_loop(&loop)) { + asyncio_state *state = get_asyncio_state(module); + if (get_running_loop(state, &loop)) { return NULL; } if (loop == NULL) { @@ -3063,7 +3204,8 @@ static PyObject * _asyncio__set_running_loop(PyObject *module, PyObject *loop) /*[clinic end generated code: output=ae56bf7a28ca189a input=4c9720233d606604]*/ { - if (set_running_loop(loop)) { + asyncio_state *state = get_asyncio_state(module); + if (set_running_loop(state, loop)) { return NULL; } Py_RETURN_NONE; @@ -3086,7 +3228,8 @@ static PyObject * _asyncio_get_event_loop_impl(PyObject *module) /*[clinic end generated code: output=2a2d8b2f824c648b input=9364bf2916c8655d]*/ { - return get_event_loop(1); + asyncio_state *state = get_asyncio_state(module); + return get_event_loop(state, 1); } /*[clinic input] @@ -3098,7 +3241,8 @@ static PyObject * _asyncio__get_event_loop_impl(PyObject *module, int stacklevel) /*[clinic end generated code: output=9c1d6d3c802e67c9 input=d17aebbd686f711d]*/ { - return get_event_loop(stacklevel-1); + asyncio_state *state = get_asyncio_state(module); + return get_event_loop(state, stacklevel-1); } /*[clinic input] @@ -3114,7 +3258,8 @@ _asyncio_get_running_loop_impl(PyObject *module) /*[clinic end generated code: output=c247b5f9e529530e input=2a3bf02ba39f173d]*/ { PyObject *loop; - if (get_running_loop(&loop)) { + asyncio_state *state = get_asyncio_state(module); + if (get_running_loop(state, &loop)) { return NULL; } if (loop == NULL) { @@ -3139,7 +3284,8 @@ static PyObject * _asyncio__register_task_impl(PyObject *module, PyObject *task) /*[clinic end generated code: output=8672dadd69a7d4e2 input=21075aaea14dfbad]*/ { - if (register_task(task) < 0) { + asyncio_state *state = get_asyncio_state(module); + if (register_task(state, task) < 0) { return NULL; } Py_RETURN_NONE; @@ -3160,7 +3306,8 @@ static PyObject * _asyncio__unregister_task_impl(PyObject *module, PyObject *task) /*[clinic end generated code: output=6e5585706d568a46 input=28fb98c3975f7bdc]*/ { - if (unregister_task(task) < 0) { + asyncio_state *state = get_asyncio_state(module); + if (unregister_task(state, task) < 0) { return NULL; } Py_RETURN_NONE; @@ -3184,7 +3331,8 @@ static PyObject * _asyncio__enter_task_impl(PyObject *module, PyObject *loop, PyObject *task) /*[clinic end generated code: output=a22611c858035b73 input=de1b06dca70d8737]*/ { - if (enter_task(loop, task) < 0) { + asyncio_state *state = get_asyncio_state(module); + if (enter_task(state, loop, task) < 0) { return NULL; } Py_RETURN_NONE; @@ -3208,7 +3356,8 @@ static PyObject * _asyncio__leave_task_impl(PyObject *module, PyObject *loop, PyObject *task) /*[clinic end generated code: output=0ebf6db4b858fb41 input=51296a46313d1ad8]*/ { - if (leave_task(loop, task) < 0) { + asyncio_state *state = get_asyncio_state(module); + if (leave_task(state, loop, task) < 0) { return NULL; } Py_RETURN_NONE; @@ -3219,10 +3368,10 @@ _asyncio__leave_task_impl(PyObject *module, PyObject *loop, PyObject *task) static PyRunningLoopHolder * -new_running_loop_holder(PyObject *loop) +new_running_loop_holder(asyncio_state *state, PyObject *loop) { - PyRunningLoopHolder *rl = PyObject_New( - PyRunningLoopHolder, &PyRunningLoopHolder_Type); + PyRunningLoopHolder *rl = PyObject_GC_New( + PyRunningLoopHolder, state->PyRunningLoopHolder_Type); if (rl == NULL) { return NULL; } @@ -3232,28 +3381,59 @@ new_running_loop_holder(PyObject *loop) #endif rl->rl_loop = Py_NewRef(loop); + PyObject_GC_Track(rl); return rl; } +static int +PyRunningLoopHolder_clear(PyRunningLoopHolder *rl) +{ + Py_CLEAR(rl->rl_loop); + return 0; +} + + +static int +PyRunningLoopHolder_traverse(PyRunningLoopHolder *rl, visitproc visit, + void *arg) +{ + Py_VISIT(Py_TYPE(rl)); + Py_VISIT(rl->rl_loop); + return 0; +} + + static void PyRunningLoopHolder_tp_dealloc(PyRunningLoopHolder *rl) { - if (cached_running_holder == (PyObject *)rl) { - cached_running_holder = NULL; + asyncio_state *state = get_asyncio_state_by_def((PyObject *)rl); + if (state->cached_running_holder == (PyObject *)rl) { + state->cached_running_holder = NULL; } - Py_CLEAR(rl->rl_loop); - PyObject_Free(rl); + PyTypeObject *tp = Py_TYPE(rl); + PyObject_GC_UnTrack(rl); + PyRunningLoopHolder_clear(rl); + PyObject_GC_Del(rl); + Py_DECREF(tp); } -static PyTypeObject PyRunningLoopHolder_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_RunningLoopHolder", - sizeof(PyRunningLoopHolder), - .tp_getattro = PyObject_GenericGetAttr, - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_dealloc = (destructor)PyRunningLoopHolder_tp_dealloc, +static PyType_Slot PyRunningLoopHolder_slots[] = { + {Py_tp_getattro, PyObject_GenericGetAttr}, + {Py_tp_dealloc, (destructor)PyRunningLoopHolder_tp_dealloc}, + {Py_tp_traverse, (traverseproc)PyRunningLoopHolder_traverse}, + {Py_tp_clear, PyRunningLoopHolder_clear}, + {0, NULL}, +}; + + +static PyType_Spec PyRunningLoopHolder_spec = { + .name = "_asyncio._RunningLoopHolder", + .basicsize = sizeof(PyRunningLoopHolder), + .slots = PyRunningLoopHolder_slots, + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), }; @@ -3279,58 +3459,106 @@ module_free_freelists(void) fi_freelist = NULL; } +static int +module_traverse(PyObject *mod, visitproc visit, void *arg) +{ + asyncio_state *state = get_asyncio_state(mod); + + Py_VISIT(state->FutureIterType); + Py_VISIT(state->TaskStepMethWrapper_Type); + Py_VISIT(state->FutureType); + Py_VISIT(state->TaskType); + Py_VISIT(state->PyRunningLoopHolder_Type); + + Py_VISIT(state->asyncio_mod); + Py_VISIT(state->traceback_extract_stack); + Py_VISIT(state->asyncio_future_repr_func); + Py_VISIT(state->asyncio_get_event_loop_policy); + Py_VISIT(state->asyncio_iscoroutine_func); + Py_VISIT(state->asyncio_task_get_stack_func); + Py_VISIT(state->asyncio_task_print_stack_func); + Py_VISIT(state->asyncio_task_repr_func); + Py_VISIT(state->asyncio_InvalidStateError); + Py_VISIT(state->asyncio_CancelledError); + + Py_VISIT(state->all_tasks); + Py_VISIT(state->current_tasks); + Py_VISIT(state->iscoroutine_typecache); + + Py_VISIT(state->context_kwname); + + // Visit freelist. + PyObject *next = (PyObject*) fi_freelist; + while (next != NULL) { + PyObject *current = next; + Py_VISIT(current); + next = (PyObject*) ((futureiterobject*) current)->future; + } + return 0; +} -static void -module_free(void *m) +static int +module_clear(PyObject *mod) { - Py_CLEAR(asyncio_mod); - Py_CLEAR(traceback_extract_stack); - Py_CLEAR(asyncio_future_repr_func); - Py_CLEAR(asyncio_get_event_loop_policy); - Py_CLEAR(asyncio_iscoroutine_func); - Py_CLEAR(asyncio_task_get_stack_func); - Py_CLEAR(asyncio_task_print_stack_func); - Py_CLEAR(asyncio_task_repr_func); - Py_CLEAR(asyncio_InvalidStateError); - Py_CLEAR(asyncio_CancelledError); + asyncio_state *state = get_asyncio_state(mod); + + Py_CLEAR(state->FutureIterType); + Py_CLEAR(state->TaskStepMethWrapper_Type); + Py_CLEAR(state->FutureType); + Py_CLEAR(state->TaskType); + Py_CLEAR(state->PyRunningLoopHolder_Type); - Py_CLEAR(all_tasks); - Py_CLEAR(current_tasks); - Py_CLEAR(iscoroutine_typecache); + Py_CLEAR(state->asyncio_mod); + Py_CLEAR(state->traceback_extract_stack); + Py_CLEAR(state->asyncio_future_repr_func); + Py_CLEAR(state->asyncio_get_event_loop_policy); + Py_CLEAR(state->asyncio_iscoroutine_func); + Py_CLEAR(state->asyncio_task_get_stack_func); + Py_CLEAR(state->asyncio_task_print_stack_func); + Py_CLEAR(state->asyncio_task_repr_func); + Py_CLEAR(state->asyncio_InvalidStateError); + Py_CLEAR(state->asyncio_CancelledError); - Py_CLEAR(context_kwname); + Py_CLEAR(state->all_tasks); + Py_CLEAR(state->current_tasks); + Py_CLEAR(state->iscoroutine_typecache); + + Py_CLEAR(state->context_kwname); module_free_freelists(); - module_initialized = 0; + return 0; +} + +static void +module_free(void *mod) +{ + (void)module_clear((PyObject *)mod); } static int -module_init(void) +module_init(asyncio_state *state) { PyObject *module = NULL; - if (module_initialized) { - return 0; - } - asyncio_mod = PyImport_ImportModule("asyncio"); - if (asyncio_mod == NULL) { + state->asyncio_mod = PyImport_ImportModule("asyncio"); + if (state->asyncio_mod == NULL) { goto fail; } - current_tasks = PyDict_New(); - if (current_tasks == NULL) { + state->current_tasks = PyDict_New(); + if (state->current_tasks == NULL) { goto fail; } - iscoroutine_typecache = PySet_New(NULL); - if (iscoroutine_typecache == NULL) { + state->iscoroutine_typecache = PySet_New(NULL); + if (state->iscoroutine_typecache == NULL) { goto fail; } - context_kwname = Py_BuildValue("(s)", "context"); - if (context_kwname == NULL) { + state->context_kwname = Py_BuildValue("(s)", "context"); + if (state->context_kwname == NULL) { goto fail; } @@ -3348,42 +3576,40 @@ module_init(void) } WITH_MOD("asyncio.events") - GET_MOD_ATTR(asyncio_get_event_loop_policy, "get_event_loop_policy") + GET_MOD_ATTR(state->asyncio_get_event_loop_policy, "get_event_loop_policy") WITH_MOD("asyncio.base_futures") - GET_MOD_ATTR(asyncio_future_repr_func, "_future_repr") + GET_MOD_ATTR(state->asyncio_future_repr_func, "_future_repr") WITH_MOD("asyncio.exceptions") - GET_MOD_ATTR(asyncio_InvalidStateError, "InvalidStateError") - GET_MOD_ATTR(asyncio_CancelledError, "CancelledError") + GET_MOD_ATTR(state->asyncio_InvalidStateError, "InvalidStateError") + GET_MOD_ATTR(state->asyncio_CancelledError, "CancelledError") WITH_MOD("asyncio.base_tasks") - GET_MOD_ATTR(asyncio_task_repr_func, "_task_repr") - GET_MOD_ATTR(asyncio_task_get_stack_func, "_task_get_stack") - GET_MOD_ATTR(asyncio_task_print_stack_func, "_task_print_stack") + GET_MOD_ATTR(state->asyncio_task_repr_func, "_task_repr") + GET_MOD_ATTR(state->asyncio_task_get_stack_func, "_task_get_stack") + GET_MOD_ATTR(state->asyncio_task_print_stack_func, "_task_print_stack") WITH_MOD("asyncio.coroutines") - GET_MOD_ATTR(asyncio_iscoroutine_func, "iscoroutine") + GET_MOD_ATTR(state->asyncio_iscoroutine_func, "iscoroutine") WITH_MOD("traceback") - GET_MOD_ATTR(traceback_extract_stack, "extract_stack") + GET_MOD_ATTR(state->traceback_extract_stack, "extract_stack") PyObject *weak_set; WITH_MOD("weakref") GET_MOD_ATTR(weak_set, "WeakSet"); - all_tasks = PyObject_CallNoArgs(weak_set); + state->all_tasks = PyObject_CallNoArgs(weak_set); Py_CLEAR(weak_set); - if (all_tasks == NULL) { + if (state->all_tasks == NULL) { goto fail; } - module_initialized = 1; Py_DECREF(module); return 0; fail: Py_CLEAR(module); - module_free(NULL); return -1; #undef WITH_MOD @@ -3405,64 +3631,71 @@ static PyMethodDef asyncio_methods[] = { {NULL, NULL} }; -static struct PyModuleDef _asynciomodule = { - PyModuleDef_HEAD_INIT, /* m_base */ - "_asyncio", /* m_name */ - module_doc, /* m_doc */ - -1, /* m_size */ - asyncio_methods, /* m_methods */ - NULL, /* m_slots */ - NULL, /* m_traverse */ - NULL, /* m_clear */ - (freefunc)module_free /* m_free */ -}; +static int +module_exec(PyObject *mod) +{ + asyncio_state *state = get_asyncio_state(mod); +#define CREATE_TYPE(m, tp, spec, base) \ + do { \ + tp = (PyTypeObject *)PyType_FromMetaclass(NULL, m, spec, \ + (PyObject *)base); \ + if (tp == NULL) { \ + return -1; \ + } \ + } while (0) -PyMODINIT_FUNC -PyInit__asyncio(void) -{ - if (module_init() < 0) { - return NULL; - } - if (PyType_Ready(&FutureIterType) < 0) { - return NULL; + CREATE_TYPE(mod, state->TaskStepMethWrapper_Type, &TaskStepMethWrapper_spec, NULL); + CREATE_TYPE(mod, state->PyRunningLoopHolder_Type, &PyRunningLoopHolder_spec, NULL); + CREATE_TYPE(mod, state->FutureIterType, &FutureIter_spec, NULL); + CREATE_TYPE(mod, state->FutureType, &Future_spec, NULL); + CREATE_TYPE(mod, state->TaskType, &Task_spec, state->FutureType); + +#undef CREATE_TYPE + + if (PyModule_AddType(mod, state->FutureType) < 0) { + return -1; } - if (PyType_Ready(&TaskStepMethWrapper_Type) < 0) { - return NULL; + + if (PyModule_AddType(mod, state->TaskType) < 0) { + return -1; } - if (PyType_Ready(&PyRunningLoopHolder_Type) < 0) { - return NULL; + // Must be done after types are added to avoid a circular dependency + if (module_init(state) < 0) { + return -1; } - PyObject *m = PyModule_Create(&_asynciomodule); - if (m == NULL) { - return NULL; + if (PyModule_AddObjectRef(mod, "_all_tasks", state->all_tasks) < 0) { + return -1; } - /* FutureType and TaskType are made ready by PyModule_AddType() calls below. */ - if (PyModule_AddType(m, &FutureType) < 0) { - Py_DECREF(m); - return NULL; + if (PyModule_AddObjectRef(mod, "_current_tasks", state->current_tasks) < 0) { + return -1; } - if (PyModule_AddType(m, &TaskType) < 0) { - Py_DECREF(m); - return NULL; - } - Py_INCREF(all_tasks); - if (PyModule_AddObject(m, "_all_tasks", all_tasks) < 0) { - Py_DECREF(all_tasks); - Py_DECREF(m); - return NULL; - } + return 0; +} - Py_INCREF(current_tasks); - if (PyModule_AddObject(m, "_current_tasks", current_tasks) < 0) { - Py_DECREF(current_tasks); - Py_DECREF(m); - return NULL; - } +static struct PyModuleDef_Slot module_slots[] = { + {Py_mod_exec, module_exec}, + {0, NULL}, +}; + +static struct PyModuleDef _asynciomodule = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "_asyncio", + .m_doc = module_doc, + .m_size = sizeof(asyncio_state), + .m_methods = asyncio_methods, + .m_slots = module_slots, + .m_traverse = module_traverse, + .m_clear = module_clear, + .m_free = (freefunc)module_free, +}; - return m; +PyMODINIT_FUNC +PyInit__asyncio(void) +{ + return PyModuleDef_Init(&_asynciomodule); } diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c index 8a0df4266e8354..d5035d20600ae2 100644 --- a/Modules/_codecsmodule.c +++ b/Modules/_codecsmodule.c @@ -256,14 +256,14 @@ _codecs_escape_encode_impl(PyObject *module, PyObject *data, _codecs.utf_7_decode data: Py_buffer errors: str(accept={str, NoneType}) = None - final: bool(accept={int}) = False + final: bool = False / [clinic start generated code]*/ static PyObject * _codecs_utf_7_decode_impl(PyObject *module, Py_buffer *data, const char *errors, int final) -/*[clinic end generated code: output=0cd3a944a32a4089 input=22c395d357815d26]*/ +/*[clinic end generated code: output=0cd3a944a32a4089 input=dbf8c8998102dc7d]*/ { Py_ssize_t consumed = data->len; PyObject *decoded = PyUnicode_DecodeUTF7Stateful(data->buf, data->len, @@ -276,14 +276,14 @@ _codecs_utf_7_decode_impl(PyObject *module, Py_buffer *data, _codecs.utf_8_decode data: Py_buffer errors: str(accept={str, NoneType}) = None - final: bool(accept={int}) = False + final: bool = False / [clinic start generated code]*/ static PyObject * _codecs_utf_8_decode_impl(PyObject *module, Py_buffer *data, const char *errors, int final) -/*[clinic end generated code: output=10f74dec8d9bb8bf input=f611b3867352ba59]*/ +/*[clinic end generated code: output=10f74dec8d9bb8bf input=ca06bc8a9c970e25]*/ { Py_ssize_t consumed = data->len; PyObject *decoded = PyUnicode_DecodeUTF8Stateful(data->buf, data->len, @@ -296,14 +296,14 @@ _codecs_utf_8_decode_impl(PyObject *module, Py_buffer *data, _codecs.utf_16_decode data: Py_buffer errors: str(accept={str, NoneType}) = None - final: bool(accept={int}) = False + final: bool = False / [clinic start generated code]*/ static PyObject * _codecs_utf_16_decode_impl(PyObject *module, Py_buffer *data, const char *errors, int final) -/*[clinic end generated code: output=783b442abcbcc2d0 input=191d360bd7309180]*/ +/*[clinic end generated code: output=783b442abcbcc2d0 input=5b0f52071ba6cadc]*/ { int byteorder = 0; /* This is overwritten unless final is true. */ @@ -318,14 +318,14 @@ _codecs_utf_16_decode_impl(PyObject *module, Py_buffer *data, _codecs.utf_16_le_decode data: Py_buffer errors: str(accept={str, NoneType}) = None - final: bool(accept={int}) = False + final: bool = False / [clinic start generated code]*/ static PyObject * _codecs_utf_16_le_decode_impl(PyObject *module, Py_buffer *data, const char *errors, int final) -/*[clinic end generated code: output=899b9e6364379dcd input=c6904fdc27fb4724]*/ +/*[clinic end generated code: output=899b9e6364379dcd input=115bd8c7b783d0bf]*/ { int byteorder = -1; /* This is overwritten unless final is true. */ @@ -340,14 +340,14 @@ _codecs_utf_16_le_decode_impl(PyObject *module, Py_buffer *data, _codecs.utf_16_be_decode data: Py_buffer errors: str(accept={str, NoneType}) = None - final: bool(accept={int}) = False + final: bool = False / [clinic start generated code]*/ static PyObject * _codecs_utf_16_be_decode_impl(PyObject *module, Py_buffer *data, const char *errors, int final) -/*[clinic end generated code: output=49f6465ea07669c8 input=e49012400974649b]*/ +/*[clinic end generated code: output=49f6465ea07669c8 input=63131422b01f9cb4]*/ { int byteorder = 1; /* This is overwritten unless final is true. */ @@ -370,14 +370,14 @@ _codecs.utf_16_ex_decode data: Py_buffer errors: str(accept={str, NoneType}) = None byteorder: int = 0 - final: bool(accept={int}) = False + final: bool = False / [clinic start generated code]*/ static PyObject * _codecs_utf_16_ex_decode_impl(PyObject *module, Py_buffer *data, const char *errors, int byteorder, int final) -/*[clinic end generated code: output=0f385f251ecc1988 input=5a9c19f2e6b6cf0e]*/ +/*[clinic end generated code: output=0f385f251ecc1988 input=f368a51cf384bf4c]*/ { /* This is overwritten unless final is true. */ Py_ssize_t consumed = data->len; @@ -394,14 +394,14 @@ _codecs_utf_16_ex_decode_impl(PyObject *module, Py_buffer *data, _codecs.utf_32_decode data: Py_buffer errors: str(accept={str, NoneType}) = None - final: bool(accept={int}) = False + final: bool = False / [clinic start generated code]*/ static PyObject * _codecs_utf_32_decode_impl(PyObject *module, Py_buffer *data, const char *errors, int final) -/*[clinic end generated code: output=2fc961807f7b145f input=fd7193965627eb58]*/ +/*[clinic end generated code: output=2fc961807f7b145f input=fcdf3658c5e9b5f3]*/ { int byteorder = 0; /* This is overwritten unless final is true. */ @@ -416,14 +416,14 @@ _codecs_utf_32_decode_impl(PyObject *module, Py_buffer *data, _codecs.utf_32_le_decode data: Py_buffer errors: str(accept={str, NoneType}) = None - final: bool(accept={int}) = False + final: bool = False / [clinic start generated code]*/ static PyObject * _codecs_utf_32_le_decode_impl(PyObject *module, Py_buffer *data, const char *errors, int final) -/*[clinic end generated code: output=ec8f46b67a94f3e6 input=9078ec70acfe7613]*/ +/*[clinic end generated code: output=ec8f46b67a94f3e6 input=12220556e885f817]*/ { int byteorder = -1; /* This is overwritten unless final is true. */ @@ -438,14 +438,14 @@ _codecs_utf_32_le_decode_impl(PyObject *module, Py_buffer *data, _codecs.utf_32_be_decode data: Py_buffer errors: str(accept={str, NoneType}) = None - final: bool(accept={int}) = False + final: bool = False / [clinic start generated code]*/ static PyObject * _codecs_utf_32_be_decode_impl(PyObject *module, Py_buffer *data, const char *errors, int final) -/*[clinic end generated code: output=ff82bae862c92c4e input=f1ae1bbbb86648ff]*/ +/*[clinic end generated code: output=ff82bae862c92c4e input=2bc669b4781598db]*/ { int byteorder = 1; /* This is overwritten unless final is true. */ @@ -468,14 +468,14 @@ _codecs.utf_32_ex_decode data: Py_buffer errors: str(accept={str, NoneType}) = None byteorder: int = 0 - final: bool(accept={int}) = False + final: bool = False / [clinic start generated code]*/ static PyObject * _codecs_utf_32_ex_decode_impl(PyObject *module, Py_buffer *data, const char *errors, int byteorder, int final) -/*[clinic end generated code: output=6bfb177dceaf4848 input=e46a73bc859d0bd0]*/ +/*[clinic end generated code: output=6bfb177dceaf4848 input=4a2323d0013620df]*/ { Py_ssize_t consumed = data->len; PyObject *decoded = PyUnicode_DecodeUTF32Stateful(data->buf, data->len, @@ -490,14 +490,14 @@ _codecs_utf_32_ex_decode_impl(PyObject *module, Py_buffer *data, _codecs.unicode_escape_decode data: Py_buffer(accept={str, buffer}) errors: str(accept={str, NoneType}) = None - final: bool(accept={int}) = True + final: bool = True / [clinic start generated code]*/ static PyObject * _codecs_unicode_escape_decode_impl(PyObject *module, Py_buffer *data, const char *errors, int final) -/*[clinic end generated code: output=b284f97b12c635ee input=6154f039a9f7c639]*/ +/*[clinic end generated code: output=b284f97b12c635ee input=15019f081ffe272b]*/ { Py_ssize_t consumed = data->len; PyObject *decoded = _PyUnicode_DecodeUnicodeEscapeStateful(data->buf, data->len, @@ -510,14 +510,14 @@ _codecs_unicode_escape_decode_impl(PyObject *module, Py_buffer *data, _codecs.raw_unicode_escape_decode data: Py_buffer(accept={str, buffer}) errors: str(accept={str, NoneType}) = None - final: bool(accept={int}) = True + final: bool = True / [clinic start generated code]*/ static PyObject * _codecs_raw_unicode_escape_decode_impl(PyObject *module, Py_buffer *data, const char *errors, int final) -/*[clinic end generated code: output=11dbd96301e2879e input=2d166191beb3235a]*/ +/*[clinic end generated code: output=11dbd96301e2879e input=b93f823aa8c343ad]*/ { Py_ssize_t consumed = data->len; PyObject *decoded = _PyUnicode_DecodeRawUnicodeEscapeStateful(data->buf, data->len, @@ -586,14 +586,14 @@ _codecs_charmap_decode_impl(PyObject *module, Py_buffer *data, _codecs.mbcs_decode data: Py_buffer errors: str(accept={str, NoneType}) = None - final: bool(accept={int}) = False + final: bool = False / [clinic start generated code]*/ static PyObject * _codecs_mbcs_decode_impl(PyObject *module, Py_buffer *data, const char *errors, int final) -/*[clinic end generated code: output=39b65b8598938c4b input=1c1d50f08fa53789]*/ +/*[clinic end generated code: output=39b65b8598938c4b input=f144ad1ed6d8f5a6]*/ { Py_ssize_t consumed = data->len; PyObject *decoded = PyUnicode_DecodeMBCSStateful(data->buf, data->len, @@ -605,14 +605,14 @@ _codecs_mbcs_decode_impl(PyObject *module, Py_buffer *data, _codecs.oem_decode data: Py_buffer errors: str(accept={str, NoneType}) = None - final: bool(accept={int}) = False + final: bool = False / [clinic start generated code]*/ static PyObject * _codecs_oem_decode_impl(PyObject *module, Py_buffer *data, const char *errors, int final) -/*[clinic end generated code: output=da1617612f3fcad8 input=81b67cba811022e5]*/ +/*[clinic end generated code: output=da1617612f3fcad8 input=629bf87376d211b4]*/ { Py_ssize_t consumed = data->len; PyObject *decoded = PyUnicode_DecodeCodePageStateful(CP_OEMCP, @@ -625,14 +625,14 @@ _codecs.code_page_decode codepage: int data: Py_buffer errors: str(accept={str, NoneType}) = None - final: bool(accept={int}) = False + final: bool = False / [clinic start generated code]*/ static PyObject * _codecs_code_page_decode_impl(PyObject *module, int codepage, Py_buffer *data, const char *errors, int final) -/*[clinic end generated code: output=53008ea967da3fff input=c5f58d036cb63575]*/ +/*[clinic end generated code: output=53008ea967da3fff input=6a32589b0658c277]*/ { Py_ssize_t consumed = data->len; PyObject *decoded = PyUnicode_DecodeCodePageStateful(codepage, diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 5fa583821889f3..68131f3b54d2ea 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1508,15 +1508,13 @@ deque_init(dequeobject *deque, PyObject *args, PyObject *kwdargs) static PyObject * deque_sizeof(dequeobject *deque, void *unused) { - Py_ssize_t res; - Py_ssize_t blocks; - - res = _PyObject_SIZE(Py_TYPE(deque)); + size_t res = _PyObject_SIZE(Py_TYPE(deque)); + size_t blocks; blocks = (size_t)(deque->leftindex + Py_SIZE(deque) + BLOCKLEN - 1) / BLOCKLEN; - assert(deque->leftindex + Py_SIZE(deque) - 1 == - (blocks - 1) * BLOCKLEN + deque->rightindex); + assert(((size_t)deque->leftindex + (size_t)Py_SIZE(deque) - 1) == + ((blocks - 1) * BLOCKLEN + (size_t)deque->rightindex)); res += blocks * sizeof(block); - return PyLong_FromSsize_t(res); + return PyLong_FromSize_t(res); } PyDoc_STRVAR(sizeof_doc, diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index 5146b4004a141a..5691a419a32f8e 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -2371,7 +2371,7 @@ _curses.window.touchline start: int count: int [ - changed: bool(accept={int}) = True + changed: bool = True ] / @@ -2384,7 +2384,7 @@ as having been changed (changed=True) or unchanged (changed=False). static PyObject * _curses_window_touchline_impl(PyCursesWindowObject *self, int start, int count, int group_right_1, int changed) -/*[clinic end generated code: output=65d05b3f7438c61d input=918ad1cbdadf93ea]*/ +/*[clinic end generated code: output=65d05b3f7438c61d input=a98aa4f79b6be845]*/ { if (!group_right_1) { return PyCursesCheckERR(touchline(self->win, start, count), "touchline"); @@ -2706,7 +2706,7 @@ NoArgTrueFalseFunctionBody(can_change_color) /*[clinic input] _curses.cbreak - flag: bool(accept={int}) = True + flag: bool = True If false, the effect is the same as calling nocbreak(). / @@ -2721,7 +2721,7 @@ Calling first raw() then cbreak() leaves the terminal in cbreak mode. static PyObject * _curses_cbreak_impl(PyObject *module, int flag) -/*[clinic end generated code: output=9f9dee9664769751 input=150be619eb1f1458]*/ +/*[clinic end generated code: output=9f9dee9664769751 input=c7d0bddda93016c1]*/ NoArgOrFlagNoReturnFunctionBody(cbreak, flag) /*[clinic input] @@ -2870,7 +2870,7 @@ NoArgNoReturnFunctionBody(doupdate) /*[clinic input] _curses.echo - flag: bool(accept={int}) = True + flag: bool = True If false, the effect is the same as calling noecho(). / @@ -2881,7 +2881,7 @@ In echo mode, each character input is echoed to the screen as it is entered. static PyObject * _curses_echo_impl(PyObject *module, int flag) -/*[clinic end generated code: output=03acb2ddfa6c8729 input=2e9e891d637eac5d]*/ +/*[clinic end generated code: output=03acb2ddfa6c8729 input=86cd4d5bb1d569c0]*/ NoArgOrFlagNoReturnFunctionBody(echo, flag) /*[clinic input] @@ -3496,14 +3496,14 @@ _curses_set_tabsize_impl(PyObject *module, int size) /*[clinic input] _curses.intrflush - flag: bool(accept={int}) + flag: bool / [clinic start generated code]*/ static PyObject * _curses_intrflush_impl(PyObject *module, int flag) -/*[clinic end generated code: output=c1986df35e999a0f input=fcba57bb28dfd795]*/ +/*[clinic end generated code: output=c1986df35e999a0f input=c65fe2ef973fe40a]*/ { PyCursesInitialised; @@ -3605,7 +3605,7 @@ NoArgReturnStringFunctionBody(longname) /*[clinic input] _curses.meta - yes: bool(accept={int}) + yes: bool / Enable/disable meta keys. @@ -3616,7 +3616,7 @@ allow only 7-bit characters. static PyObject * _curses_meta_impl(PyObject *module, int yes) -/*[clinic end generated code: output=22f5abda46a605d8 input=af9892e3a74f35db]*/ +/*[clinic end generated code: output=22f5abda46a605d8 input=cfe7da79f51d0e30]*/ { PyCursesInitialised; @@ -3766,7 +3766,7 @@ _curses_newwin_impl(PyObject *module, int nlines, int ncols, /*[clinic input] _curses.nl - flag: bool(accept={int}) = True + flag: bool = True If false, the effect is the same as calling nonl(). / @@ -3778,7 +3778,7 @@ newline into return and line-feed on output. Newline mode is initially on. static PyObject * _curses_nl_impl(PyObject *module, int flag) -/*[clinic end generated code: output=b39cc0ffc9015003 input=cf36a63f7b86e28a]*/ +/*[clinic end generated code: output=b39cc0ffc9015003 input=18e3e9c6e8cfcf6f]*/ NoArgOrFlagNoReturnFunctionBody(nl, flag) /*[clinic input] @@ -3925,7 +3925,7 @@ _curses_putp_impl(PyObject *module, const char *string) /*[clinic input] _curses.qiflush - flag: bool(accept={int}) = True + flag: bool = True If false, the effect is the same as calling noqiflush(). / @@ -3937,7 +3937,7 @@ will be flushed when the INTR, QUIT and SUSP characters are read. static PyObject * _curses_qiflush_impl(PyObject *module, int flag) -/*[clinic end generated code: output=9167e862f760ea30 input=e9e4a389946a0dbc]*/ +/*[clinic end generated code: output=9167e862f760ea30 input=6ec8b3e2b717ec40]*/ { PyCursesInitialised; @@ -4018,7 +4018,7 @@ _curses_update_lines_cols_impl(PyObject *module) /*[clinic input] _curses.raw - flag: bool(accept={int}) = True + flag: bool = True If false, the effect is the same as calling noraw(). / @@ -4031,7 +4031,7 @@ curses input functions one by one. static PyObject * _curses_raw_impl(PyObject *module, int flag) -/*[clinic end generated code: output=a750e4b342be015b input=e36d8db27832b848]*/ +/*[clinic end generated code: output=a750e4b342be015b input=4b447701389fb4df]*/ NoArgOrFlagNoReturnFunctionBody(raw, flag) /*[clinic input] @@ -4503,7 +4503,7 @@ _curses_unget_wch(PyObject *module, PyObject *ch) /*[clinic input] _curses.use_env - flag: bool(accept={int}) + flag: bool / Use environment variables LINES and COLUMNS. @@ -4520,7 +4520,7 @@ not set). static PyObject * _curses_use_env_impl(PyObject *module, int flag) -/*[clinic end generated code: output=b2c445e435c0b164 input=1778eb1e9151ea37]*/ +/*[clinic end generated code: output=b2c445e435c0b164 input=06ac30948f2d78e4]*/ { use_env(flag); Py_RETURN_NONE; diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index 2d6e4e4d62b42c..bc97615ffb4b72 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -4796,13 +4796,11 @@ dec_reduce(PyObject *self, PyObject *dummy UNUSED) static PyObject * dec_sizeof(PyObject *v, PyObject *dummy UNUSED) { - Py_ssize_t res; - - res = _PyObject_SIZE(Py_TYPE(v)); + size_t res = _PyObject_SIZE(Py_TYPE(v)); if (mpd_isdynamic_data(MPD(v))) { - res += MPD(v)->alloc * sizeof(mpd_uint_t); + res += (size_t)MPD(v)->alloc * sizeof(mpd_uint_t); } - return PyLong_FromSsize_t(res); + return PyLong_FromSize_t(res); } /* __trunc__ */ diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 0c68ede42ca61d..555c22f88b36d5 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -876,19 +876,20 @@ deepcopy(PyObject *object, PyObject *memo) /*[clinic input] -_elementtree.Element.__sizeof__ -> Py_ssize_t +_elementtree.Element.__sizeof__ -> size_t [clinic start generated code]*/ -static Py_ssize_t +static size_t _elementtree_Element___sizeof___impl(ElementObject *self) -/*[clinic end generated code: output=bf73867721008000 input=70f4b323d55a17c1]*/ +/*[clinic end generated code: output=baae4e7ae9fe04ec input=54e298c501f3e0d0]*/ { - Py_ssize_t result = _PyObject_SIZE(Py_TYPE(self)); + size_t result = _PyObject_SIZE(Py_TYPE(self)); if (self->extra) { result += sizeof(ElementObjectExtra); - if (self->extra->children != self->extra->_children) - result += sizeof(PyObject*) * self->extra->allocated; + if (self->extra->children != self->extra->_children) { + result += (size_t)self->extra->allocated * sizeof(PyObject*); + } } return result; } diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 121d9617e1883b..af5950cf66c178 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -59,7 +59,7 @@ PyDoc_STRVAR(module_doc, " I/O classes. open() uses the file's blksize (as obtained by os.stat) if\n" " possible.\n" ); - + /* * The main open() function @@ -74,7 +74,7 @@ _io.open encoding: str(accept={str, NoneType}) = None errors: str(accept={str, NoneType}) = None newline: str(accept={str, NoneType}) = None - closefd: bool(accept={int}) = True + closefd: bool = True opener: object = None Open file and return a stream. Raise OSError upon failure. @@ -196,7 +196,7 @@ static PyObject * _io_open_impl(PyObject *module, PyObject *file, const char *mode, int buffering, const char *encoding, const char *errors, const char *newline, int closefd, PyObject *opener) -/*[clinic end generated code: output=aefafc4ce2b46dc0 input=5bb37f174cb2fb11]*/ +/*[clinic end generated code: output=aefafc4ce2b46dc0 input=cd034e7cdfbf4e78]*/ { unsigned i; @@ -204,8 +204,7 @@ _io_open_impl(PyObject *module, PyObject *file, const char *mode, int text = 0, binary = 0; char rawmode[6], *m; - int line_buffering, is_number; - long isatty = 0; + int line_buffering, is_number, isatty = 0; PyObject *raw, *modeobj = NULL, *buffer, *wrapper, *result = NULL, *path_or_fd = NULL; @@ -345,9 +344,9 @@ _io_open_impl(PyObject *module, PyObject *file, const char *mode, PyObject *res = PyObject_CallMethodNoArgs(raw, &_Py_ID(isatty)); if (res == NULL) goto error; - isatty = PyLong_AsLong(res); + isatty = PyObject_IsTrue(res); Py_DECREF(res); - if (isatty == -1 && PyErr_Occurred()) + if (isatty < 0) goto error; } @@ -509,7 +508,7 @@ _io_open_code_impl(PyObject *module, PyObject *path) { return PyFile_OpenCodeObject(path); } - + /* * Private helpers for the io module. */ diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index e2610931caae07..6df55b5b8303c2 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -389,12 +389,11 @@ buffered_dealloc(buffered *self) static PyObject * buffered_sizeof(buffered *self, PyObject *Py_UNUSED(ignored)) { - Py_ssize_t res; - - res = _PyObject_SIZE(Py_TYPE(self)); - if (self->buffer) - res += self->buffer_size; - return PyLong_FromSsize_t(res); + size_t res = _PyObject_SIZE(Py_TYPE(self)); + if (self->buffer) { + res += (size_t)self->buffer_size; + } + return PyLong_FromSize_t(res); } static int diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index 41be3497506d3f..6698c60355fcc5 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -957,17 +957,15 @@ _io_BytesIO___init___impl(bytesio *self, PyObject *initvalue) static PyObject * bytesio_sizeof(bytesio *self, void *unused) { - Py_ssize_t res; - - res = _PyObject_SIZE(Py_TYPE(self)); + size_t res = _PyObject_SIZE(Py_TYPE(self)); if (self->buf && !SHARED_BUF(self)) { - Py_ssize_t s = _PySys_GetSizeOf(self->buf); - if (s == -1) { + size_t s = _PySys_GetSizeOf(self->buf); + if (s == (size_t)-1) { return NULL; } res += s; } - return PyLong_FromSsize_t(res); + return PyLong_FromSize_t(res); } static int diff --git a/Modules/_io/clinic/_iomodule.c.h b/Modules/_io/clinic/_iomodule.c.h index b38738486f6856..4d76e333b0f293 100644 --- a/Modules/_io/clinic/_iomodule.c.h +++ b/Modules/_io/clinic/_iomodule.c.h @@ -280,8 +280,8 @@ _io_open(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw } } if (args[6]) { - closefd = _PyLong_AsInt(args[6]); - if (closefd == -1 && PyErr_Occurred()) { + closefd = PyObject_IsTrue(args[6]); + if (closefd < 0) { goto exit; } if (!--noptargs) { @@ -407,4 +407,4 @@ _io_open_code(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=1f8001287a423470 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f387eba3f4c0254a input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/fileio.c.h b/Modules/_io/clinic/fileio.c.h index a925b94fe07531..b6e9bd5b65a029 100644 --- a/Modules/_io/clinic/fileio.c.h +++ b/Modules/_io/clinic/fileio.c.h @@ -116,8 +116,8 @@ _io_FileIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) } } if (fastargs[2]) { - closefd = _PyLong_AsInt(fastargs[2]); - if (closefd == -1 && PyErr_Occurred()) { + closefd = PyObject_IsTrue(fastargs[2]); + if (closefd < 0) { goto exit; } if (!--noptargs) { @@ -466,4 +466,4 @@ _io_FileIO_isatty(fileio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO_FILEIO_TRUNCATE_METHODDEF #define _IO_FILEIO_TRUNCATE_METHODDEF #endif /* !defined(_IO_FILEIO_TRUNCATE_METHODDEF) */ -/*[clinic end generated code: output=ff479a26cab0d479 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=27f883807a6c29ae input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index 038f0a5c209d49..db968e884cc805 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -68,8 +68,8 @@ _io_IncrementalNewlineDecoder___init__(PyObject *self, PyObject *args, PyObject goto exit; } decoder = fastargs[0]; - translate = _PyLong_AsInt(fastargs[1]); - if (translate == -1 && PyErr_Occurred()) { + translate = PyObject_IsTrue(fastargs[1]); + if (translate < 0) { goto exit; } if (!noptargs) { @@ -137,8 +137,8 @@ _io_IncrementalNewlineDecoder_decode(nldecoder_object *self, PyObject *const *ar if (!noptargs) { goto skip_optional_pos; } - final = _PyLong_AsInt(args[1]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[1]); + if (final < 0) { goto exit; } skip_optional_pos: @@ -331,16 +331,16 @@ _io_TextIOWrapper___init__(PyObject *self, PyObject *args, PyObject *kwargs) } } if (fastargs[4]) { - line_buffering = _PyLong_AsInt(fastargs[4]); - if (line_buffering == -1 && PyErr_Occurred()) { + line_buffering = PyObject_IsTrue(fastargs[4]); + if (line_buffering < 0) { goto exit; } if (!--noptargs) { goto skip_optional_pos; } } - write_through = _PyLong_AsInt(fastargs[5]); - if (write_through == -1 && PyErr_Occurred()) { + write_through = PyObject_IsTrue(fastargs[5]); + if (write_through < 0) { goto exit; } skip_optional_pos: @@ -769,4 +769,4 @@ _io_TextIOWrapper_close(textio *self, PyObject *Py_UNUSED(ignored)) { return _io_TextIOWrapper_close_impl(self); } -/*[clinic end generated code: output=aecd376eca3cb148 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=73f84b13c343b34b input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/winconsoleio.c.h b/Modules/_io/clinic/winconsoleio.c.h index 65820a8f2ea0b3..df834dbde40f5b 100644 --- a/Modules/_io/clinic/winconsoleio.c.h +++ b/Modules/_io/clinic/winconsoleio.c.h @@ -115,8 +115,8 @@ _io__WindowsConsoleIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) } } if (fastargs[2]) { - closefd = _PyLong_AsInt(fastargs[2]); - if (closefd == -1 && PyErr_Occurred()) { + closefd = PyObject_IsTrue(fastargs[2]); + if (closefd < 0) { goto exit; } if (!--noptargs) { @@ -407,4 +407,4 @@ _io__WindowsConsoleIO_isatty(winconsoleio *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #define _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #endif /* !defined(_IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF) */ -/*[clinic end generated code: output=08ae244e9a44da55 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4920e9068e0cf08a input=a9049054013a1b77]*/ diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 659297ef1b1d30..d1a183cedac53a 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -198,7 +198,7 @@ extern int _Py_open_cloexec_works; _io.FileIO.__init__ file as nameobj: object mode: str = "r" - closefd: bool(accept={int}) = True + closefd: bool = True opener: object = None Open a file. @@ -219,7 +219,7 @@ results in functionality similar to passing None). static int _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, int closefd, PyObject *opener) -/*[clinic end generated code: output=23413f68e6484bbd input=1596c9157a042a39]*/ +/*[clinic end generated code: output=23413f68e6484bbd input=588aac967e0ba74b]*/ { #ifdef MS_WINDOWS Py_UNICODE *widename = NULL; diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 3091f6efafccd4..32ab8a44c62151 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -212,7 +212,7 @@ typedef struct { /*[clinic input] _io.IncrementalNewlineDecoder.__init__ decoder: object - translate: int + translate: bool errors: object(c_default="NULL") = "strict" Codec used when reading a file in universal newlines mode. @@ -229,7 +229,7 @@ static int _io_IncrementalNewlineDecoder___init___impl(nldecoder_object *self, PyObject *decoder, int translate, PyObject *errors) -/*[clinic end generated code: output=fbd04d443e764ec2 input=89db6b19c6b126bf]*/ +/*[clinic end generated code: output=fbd04d443e764ec2 input=ed547aa257616b0e]*/ { if (errors == NULL) { @@ -484,13 +484,13 @@ _PyIncrementalNewlineDecoder_decode(PyObject *myself, /*[clinic input] _io.IncrementalNewlineDecoder.decode input: object - final: bool(accept={int}) = False + final: bool = False [clinic start generated code]*/ static PyObject * _io_IncrementalNewlineDecoder_decode_impl(nldecoder_object *self, PyObject *input, int final) -/*[clinic end generated code: output=0d486755bb37a66e input=a4ea97f26372d866]*/ +/*[clinic end generated code: output=0d486755bb37a66e input=90e223c70322c5cd]*/ { return _PyIncrementalNewlineDecoder_decode((PyObject *) self, input, final); } @@ -1023,8 +1023,8 @@ _io.TextIOWrapper.__init__ encoding: str(accept={str, NoneType}) = None errors: object = None newline: str(accept={str, NoneType}) = None - line_buffering: bool(accept={int}) = False - write_through: bool(accept={int}) = False + line_buffering: bool = False + write_through: bool = False Character and line based layer over a BufferedIOBase object, buffer. @@ -1061,7 +1061,7 @@ _io_TextIOWrapper___init___impl(textio *self, PyObject *buffer, const char *encoding, PyObject *errors, const char *newline, int line_buffering, int write_through) -/*[clinic end generated code: output=72267c0c01032ed2 input=72590963698f289b]*/ +/*[clinic end generated code: output=72267c0c01032ed2 input=e6cfaaaf6059d4f5]*/ { PyObject *raw, *codec_info = NULL; PyObject *res; diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 5c1a6dd86fc54f..d5de64b4ac3dfd 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -235,7 +235,7 @@ winconsoleio_new(PyTypeObject *type, PyObject *args, PyObject *kwds) _io._WindowsConsoleIO.__init__ file as nameobj: object mode: str = "r" - closefd: bool(accept={int}) = True + closefd: bool = True opener: object = None Open a console buffer by file descriptor. @@ -249,7 +249,7 @@ static int _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, const char *mode, int closefd, PyObject *opener) -/*[clinic end generated code: output=3fd9cbcdd8d95429 input=06ae4b863c63244b]*/ +/*[clinic end generated code: output=3fd9cbcdd8d95429 input=7a3eed6bbe998fd9]*/ { const char *s; wchar_t *name = NULL; diff --git a/Modules/_json.c b/Modules/_json.c index 81431aa1041c55..429b4ee0fa8d8d 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -556,7 +556,7 @@ py_scanstring(PyObject* Py_UNUSED(self), PyObject *args) Py_ssize_t end; Py_ssize_t next_end = -1; int strict = 1; - if (!PyArg_ParseTuple(args, "On|i:scanstring", &pystr, &end, &strict)) { + if (!PyArg_ParseTuple(args, "On|p:scanstring", &pystr, &end, &strict)) { return NULL; } if (PyUnicode_Check(pystr)) { diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index cb80c8d339325b..37170bbea56ad3 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -667,7 +667,7 @@ profiler_enable(ProfilerObject *self, PyObject *args, PyObject *kwds) int subcalls = -1; int builtins = -1; static char *kwlist[] = {"subcalls", "builtins", 0}; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|ii:enable", + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|pp:enable", kwlist, &subcalls, &builtins)) return NULL; if (setSubcalls(self, subcalls) < 0 || setBuiltins(self, builtins) < 0) { @@ -770,7 +770,7 @@ profiler_init(ProfilerObject *pObj, PyObject *args, PyObject *kw) static char *kwlist[] = {"timer", "timeunit", "subcalls", "builtins", 0}; - if (!PyArg_ParseTupleAndKeywords(args, kw, "|Odii:Profiler", kwlist, + if (!PyArg_ParseTupleAndKeywords(args, kw, "|Odpp:Profiler", kwlist, &timer, &timeunit, &subcalls, &builtins)) return -1; diff --git a/Modules/_multiprocessing/clinic/semaphore.c.h b/Modules/_multiprocessing/clinic/semaphore.c.h index dce0366c266f10..35347169bc1591 100644 --- a/Modules/_multiprocessing/clinic/semaphore.c.h +++ b/Modules/_multiprocessing/clinic/semaphore.c.h @@ -65,8 +65,8 @@ _multiprocessing_SemLock_acquire(SemLockObject *self, PyObject *const *args, Py_ goto skip_optional_pos; } if (args[0]) { - blocking = _PyLong_AsInt(args[0]); - if (blocking == -1 && PyErr_Occurred()) { + blocking = PyObject_IsTrue(args[0]); + if (blocking < 0) { goto exit; } if (!--noptargs) { @@ -162,8 +162,8 @@ _multiprocessing_SemLock_acquire(SemLockObject *self, PyObject *const *args, Py_ goto skip_optional_pos; } if (args[0]) { - blocking = _PyLong_AsInt(args[0]); - if (blocking == -1 && PyErr_Occurred()) { + blocking = PyObject_IsTrue(args[0]); + if (blocking < 0) { goto exit; } if (!--noptargs) { @@ -275,8 +275,8 @@ _multiprocessing_SemLock(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } - unlink = _PyLong_AsInt(fastargs[4]); - if (unlink == -1 && PyErr_Occurred()) { + unlink = PyObject_IsTrue(fastargs[4]); + if (unlink < 0) { goto exit; } return_value = _multiprocessing_SemLock_impl(type, kind, value, maxvalue, name, unlink); @@ -542,4 +542,4 @@ _multiprocessing_SemLock___exit__(SemLockObject *self, PyObject *const *args, Py #ifndef _MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF #define _MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF #endif /* !defined(_MULTIPROCESSING_SEMLOCK___EXIT___METHODDEF) */ -/*[clinic end generated code: output=720d7d0066dc0954 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=dae57a702cc01512 input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/semaphore.c b/Modules/_multiprocessing/semaphore.c index 58fb0eb96aeeed..897b8db7110a41 100644 --- a/Modules/_multiprocessing/semaphore.c +++ b/Modules/_multiprocessing/semaphore.c @@ -79,7 +79,7 @@ _GetSemaphoreValue(HANDLE handle, long *value) /*[clinic input] _multiprocessing.SemLock.acquire - block as blocking: bool(accept={int}) = True + block as blocking: bool = True timeout as timeout_obj: object = None Acquire the semaphore/lock. @@ -88,7 +88,7 @@ Acquire the semaphore/lock. static PyObject * _multiprocessing_SemLock_acquire_impl(SemLockObject *self, int blocking, PyObject *timeout_obj) -/*[clinic end generated code: output=f9998f0b6b0b0872 input=86f05662cf753eb4]*/ +/*[clinic end generated code: output=f9998f0b6b0b0872 input=e5b45f5cbb775166]*/ { double timeout; DWORD res, full_msecs, nhandles; @@ -295,7 +295,7 @@ sem_timedwait_save(sem_t *sem, struct timespec *deadline, PyThreadState *_save) /*[clinic input] _multiprocessing.SemLock.acquire - block as blocking: bool(accept={int}) = True + block as blocking: bool = True timeout as timeout_obj: object = None Acquire the semaphore/lock. @@ -304,7 +304,7 @@ Acquire the semaphore/lock. static PyObject * _multiprocessing_SemLock_acquire_impl(SemLockObject *self, int blocking, PyObject *timeout_obj) -/*[clinic end generated code: output=f9998f0b6b0b0872 input=86f05662cf753eb4]*/ +/*[clinic end generated code: output=f9998f0b6b0b0872 input=e5b45f5cbb775166]*/ { int res, err = 0; struct timespec deadline = {0}; @@ -474,14 +474,14 @@ _multiprocessing.SemLock.__new__ value: int maxvalue: int name: str - unlink: bool(accept={int}) + unlink: bool [clinic start generated code]*/ static PyObject * _multiprocessing_SemLock_impl(PyTypeObject *type, int kind, int value, int maxvalue, const char *name, int unlink) -/*[clinic end generated code: output=30727e38f5f7577a input=b378c3ee27d3a0fa]*/ +/*[clinic end generated code: output=30727e38f5f7577a input=fdaeb69814471c5b]*/ { SEM_HANDLE handle = SEM_FAILED; PyObject *result; diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 2078779663a919..1b34977806b661 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -4575,26 +4575,25 @@ _pickle_Pickler_dump(PicklerObject *self, PyObject *obj) /*[clinic input] -_pickle.Pickler.__sizeof__ -> Py_ssize_t +_pickle.Pickler.__sizeof__ -> size_t Returns size in memory, in bytes. [clinic start generated code]*/ -static Py_ssize_t +static size_t _pickle_Pickler___sizeof___impl(PicklerObject *self) -/*[clinic end generated code: output=106edb3123f332e1 input=8cbbec9bd5540d42]*/ +/*[clinic end generated code: output=23ad75658d3b59ff input=d8127c8e7012ebd7]*/ { - Py_ssize_t res, s; - - res = _PyObject_SIZE(Py_TYPE(self)); + size_t res = _PyObject_SIZE(Py_TYPE(self)); if (self->memo != NULL) { res += sizeof(PyMemoTable); res += self->memo->mt_allocated * sizeof(PyMemoEntry); } if (self->output_buffer != NULL) { - s = _PySys_GetSizeOf(self->output_buffer); - if (s == -1) + size_t s = _PySys_GetSizeOf(self->output_buffer); + if (s == (size_t)-1) { return -1; + } res += s; } return res; @@ -7079,22 +7078,20 @@ _pickle_Unpickler_find_class_impl(UnpicklerObject *self, /*[clinic input] -_pickle.Unpickler.__sizeof__ -> Py_ssize_t +_pickle.Unpickler.__sizeof__ -> size_t Returns size in memory, in bytes. [clinic start generated code]*/ -static Py_ssize_t +static size_t _pickle_Unpickler___sizeof___impl(UnpicklerObject *self) -/*[clinic end generated code: output=119d9d03ad4c7651 input=13333471fdeedf5e]*/ +/*[clinic end generated code: output=4648d84c228196df input=27180b2b6b524012]*/ { - Py_ssize_t res; - - res = _PyObject_SIZE(Py_TYPE(self)); + size_t res = _PyObject_SIZE(Py_TYPE(self)); if (self->memo != NULL) res += self->memo_size * sizeof(PyObject *); if (self->marks != NULL) - res += self->marks_size * sizeof(Py_ssize_t); + res += (size_t)self->marks_size * sizeof(Py_ssize_t); if (self->input_line != NULL) res += strlen(self->input_line) + 1; if (self->encoding != NULL) diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index 717b1cf2202105..b7563ee8250a94 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -814,7 +814,7 @@ subprocess_fork_exec(PyObject *module, PyObject *args) int allow_vfork; if (!PyArg_ParseTuple( - args, "OOpO!OOiiiiiiiiii" _Py_PARSE_PID "OOOiOp:fork_exec", + args, "OOpO!OOiiiiiiiipp" _Py_PARSE_PID "OOOiOp:fork_exec", &process_args, &executable_list, &close_fds, &PyTuple_Type, &py_fds_to_keep, &cwd_obj, &env_list, diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index 1f9841c368b313..4c3fd1bd27411b 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -100,8 +100,8 @@ pysqlite_connection_init(PyObject *self, PyObject *args, PyObject *kwargs) } } if (fastargs[4]) { - check_same_thread = _PyLong_AsInt(fastargs[4]); - if (check_same_thread == -1 && PyErr_Occurred()) { + check_same_thread = PyObject_IsTrue(fastargs[4]); + if (check_same_thread < 0) { goto exit; } if (!--noptargs) { @@ -305,8 +305,8 @@ blobopen(pysqlite_Connection *self, PyObject *const *args, Py_ssize_t nargs, PyO goto skip_optional_kwonly; } if (args[3]) { - readonly = _PyLong_AsInt(args[3]); - if (readonly == -1 && PyErr_Occurred()) { + readonly = PyObject_IsTrue(args[3]); + if (readonly < 0) { goto exit; } if (!--noptargs) { @@ -831,8 +831,8 @@ pysqlite_connection_enable_load_extension(pysqlite_Connection *self, PyObject *a PyObject *return_value = NULL; int onoff; - onoff = _PyLong_AsInt(arg); - if (onoff == -1 && PyErr_Occurred()) { + onoff = PyObject_IsTrue(arg); + if (onoff < 0) { goto exit; } return_value = pysqlite_connection_enable_load_extension_impl(self, onoff); @@ -1532,4 +1532,4 @@ getlimit(pysqlite_Connection *self, PyObject *arg) #ifndef DESERIALIZE_METHODDEF #define DESERIALIZE_METHODDEF #endif /* !defined(DESERIALIZE_METHODDEF) */ -/*[clinic end generated code: output=20e929a7a7d62a01 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f10306e10427488b input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 2854c1b5c31b2f..4c07d5e0b61f8c 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -197,7 +197,7 @@ _sqlite3.Connection.__init__ as pysqlite_connection_init timeout: double = 5.0 detect_types: int = 0 isolation_level: IsolationLevel = "" - check_same_thread: bool(accept={int}) = True + check_same_thread: bool = True factory: object(c_default='(PyObject*)clinic_state()->ConnectionType') = ConnectionType cached_statements as cache_size: int = 128 uri: bool = False @@ -212,7 +212,7 @@ pysqlite_connection_init_impl(pysqlite_Connection *self, PyObject *database, int check_same_thread, PyObject *factory, int cache_size, int uri, enum autocommit_mode autocommit) -/*[clinic end generated code: output=cba057313ea7712f input=b21abce28ebcd304]*/ +/*[clinic end generated code: output=cba057313ea7712f input=9b0ab6c12f674fa3]*/ { if (PySys_Audit("sqlite3.connect", "O", database) < 0) { return -1; @@ -485,7 +485,7 @@ _sqlite3.Connection.blobopen as blobopen Row index. / * - readonly: bool(accept={int}) = False + readonly: bool = False Open the BLOB without write permissions. name: str = "main" Database name. @@ -496,7 +496,7 @@ Open and return a BLOB object. static PyObject * blobopen_impl(pysqlite_Connection *self, const char *table, const char *col, int row, int readonly, const char *name) -/*[clinic end generated code: output=0c8e2e58516d0b5c input=1e7052516acfc94d]*/ +/*[clinic end generated code: output=0c8e2e58516d0b5c input=fa73c83aa7a7ddee]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -1564,7 +1564,7 @@ pysqlite_connection_set_trace_callback_impl(pysqlite_Connection *self, /*[clinic input] _sqlite3.Connection.enable_load_extension as pysqlite_connection_enable_load_extension - enable as onoff: bool(accept={int}) + enable as onoff: bool / Enable dynamic loading of SQLite extension modules. @@ -1573,7 +1573,7 @@ Enable dynamic loading of SQLite extension modules. static PyObject * pysqlite_connection_enable_load_extension_impl(pysqlite_Connection *self, int onoff) -/*[clinic end generated code: output=9cac37190d388baf input=5f00e93f7a9d3540]*/ +/*[clinic end generated code: output=9cac37190d388baf input=2a1e87931486380f]*/ { int rc; diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 2885774295b065..591eb91dd0f340 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -4168,7 +4168,7 @@ _ssl__SSLContext_load_dh_params(PySSLContext *self, PyObject *filepath) /*[clinic input] _ssl._SSLContext._wrap_socket sock: object(subclass_of="get_state_ctx(self)->Sock_Type") - server_side: int + server_side: bool server_hostname as hostname_obj: object = None * owner: object = None @@ -4180,7 +4180,7 @@ static PyObject * _ssl__SSLContext__wrap_socket_impl(PySSLContext *self, PyObject *sock, int server_side, PyObject *hostname_obj, PyObject *owner, PyObject *session) -/*[clinic end generated code: output=f103f238633940b4 input=f5916eadbc6eae81]*/ +/*[clinic end generated code: output=f103f238633940b4 input=700ca8fedff53994]*/ { char *hostname = NULL; PyObject *res; @@ -4205,7 +4205,7 @@ _ssl__SSLContext__wrap_socket_impl(PySSLContext *self, PyObject *sock, _ssl._SSLContext._wrap_bio incoming: object(subclass_of="get_state_ctx(self)->PySSLMemoryBIO_Type", type="PySSLMemoryBIO *") outgoing: object(subclass_of="get_state_ctx(self)->PySSLMemoryBIO_Type", type="PySSLMemoryBIO *") - server_side: int + server_side: bool server_hostname as hostname_obj: object = None * owner: object = None @@ -4218,7 +4218,7 @@ _ssl__SSLContext__wrap_bio_impl(PySSLContext *self, PySSLMemoryBIO *incoming, PySSLMemoryBIO *outgoing, int server_side, PyObject *hostname_obj, PyObject *owner, PyObject *session) -/*[clinic end generated code: output=5c5d6d9b41f99332 input=331edeec9c738382]*/ +/*[clinic end generated code: output=5c5d6d9b41f99332 input=a9205d097fd45a82]*/ { char *hostname = NULL; PyObject *res; diff --git a/Modules/_struct.c b/Modules/_struct.c index c960b81b246ece..3db7b991acd0a1 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -167,9 +167,6 @@ get_long(_structmodulestate *state, PyObject *v, long *p) x = PyLong_AsLong(v); Py_DECREF(v); if (x == (long)-1 && PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_OverflowError)) - PyErr_SetString(state->StructError, - "argument out of range"); return -1; } *p = x; @@ -191,9 +188,6 @@ get_ulong(_structmodulestate *state, PyObject *v, unsigned long *p) x = PyLong_AsUnsignedLong(v); Py_DECREF(v); if (x == (unsigned long)-1 && PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_OverflowError)) - PyErr_SetString(state->StructError, - "argument out of range"); return -1; } *p = x; @@ -214,9 +208,6 @@ get_longlong(_structmodulestate *state, PyObject *v, long long *p) x = PyLong_AsLongLong(v); Py_DECREF(v); if (x == (long long)-1 && PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_OverflowError)) - PyErr_SetString(state->StructError, - "argument out of range"); return -1; } *p = x; @@ -237,9 +228,6 @@ get_ulonglong(_structmodulestate *state, PyObject *v, unsigned long long *p) x = PyLong_AsUnsignedLongLong(v); Py_DECREF(v); if (x == (unsigned long long)-1 && PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_OverflowError)) - PyErr_SetString(state->StructError, - "argument out of range"); return -1; } *p = x; @@ -260,9 +248,6 @@ get_ssize_t(_structmodulestate *state, PyObject *v, Py_ssize_t *p) x = PyLong_AsSsize_t(v); Py_DECREF(v); if (x == (Py_ssize_t)-1 && PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_OverflowError)) - PyErr_SetString(state->StructError, - "argument out of range"); return -1; } *p = x; @@ -283,9 +268,6 @@ get_size_t(_structmodulestate *state, PyObject *v, size_t *p) x = PyLong_AsSize_t(v); Py_DECREF(v); if (x == (size_t)-1 && PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_OverflowError)) - PyErr_SetString(state->StructError, - "argument out of range"); return -1; } *p = x; @@ -293,7 +275,7 @@ get_size_t(_structmodulestate *state, PyObject *v, size_t *p) } -#define RANGE_ERROR(state, x, f, flag, mask) return _range_error(state, f, flag) +#define RANGE_ERROR(state, f, flag) return _range_error(state, f, flag) /* Floating point helpers */ @@ -545,12 +527,14 @@ static int np_byte(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) { long x; - if (get_long(state, v, &x) < 0) + if (get_long(state, v, &x) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + RANGE_ERROR(state, f, 0); + } return -1; + } if (x < -128 || x > 127) { - PyErr_SetString(state->StructError, - "byte format requires -128 <= number <= 127"); - return -1; + RANGE_ERROR(state, f, 0); } *p = (char)x; return 0; @@ -560,12 +544,14 @@ static int np_ubyte(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) { long x; - if (get_long(state, v, &x) < 0) + if (get_long(state, v, &x) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + RANGE_ERROR(state, f, 1); + } return -1; + } if (x < 0 || x > 255) { - PyErr_SetString(state->StructError, - "ubyte format requires 0 <= number <= 255"); - return -1; + RANGE_ERROR(state, f, 1); } *(unsigned char *)p = (unsigned char)x; return 0; @@ -588,13 +574,14 @@ np_short(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) { long x; short y; - if (get_long(state, v, &x) < 0) + if (get_long(state, v, &x) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + RANGE_ERROR(state, f, 0); + } return -1; + } if (x < SHRT_MIN || x > SHRT_MAX) { - PyErr_Format(state->StructError, - "short format requires %d <= number <= %d", - (int)SHRT_MIN, (int)SHRT_MAX); - return -1; + RANGE_ERROR(state, f, 0); } y = (short)x; memcpy(p, (char *)&y, sizeof y); @@ -606,13 +593,14 @@ np_ushort(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) { long x; unsigned short y; - if (get_long(state, v, &x) < 0) + if (get_long(state, v, &x) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + RANGE_ERROR(state, f, 1); + } return -1; + } if (x < 0 || x > USHRT_MAX) { - PyErr_Format(state->StructError, - "ushort format requires 0 <= number <= %u", - (unsigned int)USHRT_MAX); - return -1; + RANGE_ERROR(state, f, 1); } y = (unsigned short)x; memcpy(p, (char *)&y, sizeof y); @@ -624,11 +612,15 @@ np_int(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) { long x; int y; - if (get_long(state, v, &x) < 0) + if (get_long(state, v, &x) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + RANGE_ERROR(state, f, 0); + } return -1; + } #if (SIZEOF_LONG > SIZEOF_INT) if ((x < ((long)INT_MIN)) || (x > ((long)INT_MAX))) - RANGE_ERROR(state, x, f, 0, -1); + RANGE_ERROR(state, f, 0); #endif y = (int)x; memcpy(p, (char *)&y, sizeof y); @@ -640,12 +632,16 @@ np_uint(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) { unsigned long x; unsigned int y; - if (get_ulong(state, v, &x) < 0) + if (get_ulong(state, v, &x) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + RANGE_ERROR(state, f, 1); + } return -1; + } y = (unsigned int)x; #if (SIZEOF_LONG > SIZEOF_INT) if (x > ((unsigned long)UINT_MAX)) - RANGE_ERROR(state, y, f, 1, -1); + RANGE_ERROR(state, f, 1); #endif memcpy(p, (char *)&y, sizeof y); return 0; @@ -655,8 +651,12 @@ static int np_long(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) { long x; - if (get_long(state, v, &x) < 0) + if (get_long(state, v, &x) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + RANGE_ERROR(state, f, 0); + } return -1; + } memcpy(p, (char *)&x, sizeof x); return 0; } @@ -665,8 +665,12 @@ static int np_ulong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) { unsigned long x; - if (get_ulong(state, v, &x) < 0) + if (get_ulong(state, v, &x) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + RANGE_ERROR(state, f, 1); + } return -1; + } memcpy(p, (char *)&x, sizeof x); return 0; } @@ -675,8 +679,12 @@ static int np_ssize_t(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) { Py_ssize_t x; - if (get_ssize_t(state, v, &x) < 0) + if (get_ssize_t(state, v, &x) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + RANGE_ERROR(state, f, 0); + } return -1; + } memcpy(p, (char *)&x, sizeof x); return 0; } @@ -685,8 +693,12 @@ static int np_size_t(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) { size_t x; - if (get_size_t(state, v, &x) < 0) + if (get_size_t(state, v, &x) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + RANGE_ERROR(state, f, 1); + } return -1; + } memcpy(p, (char *)&x, sizeof x); return 0; } @@ -695,8 +707,16 @@ static int np_longlong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) { long long x; - if (get_longlong(state, v, &x) < 0) + if (get_longlong(state, v, &x) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + PyErr_Format(state->StructError, + "'%c' format requires %lld <= number <= %lld", + f->format, + LLONG_MIN, + LLONG_MAX); + } return -1; + } memcpy(p, (char *)&x, sizeof x); return 0; } @@ -705,8 +725,15 @@ static int np_ulonglong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) { unsigned long long x; - if (get_ulonglong(state, v, &x) < 0) + if (get_ulonglong(state, v, &x) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + PyErr_Format(state->StructError, + "'%c' format requires 0 <= number <= %llu", + f->format, + ULLONG_MAX); + } return -1; + } memcpy(p, (char *)&x, sizeof x); return 0; } @@ -911,15 +938,19 @@ bp_int(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) long x; Py_ssize_t i; unsigned char *q = (unsigned char *)p; - if (get_long(state, v, &x) < 0) + if (get_long(state, v, &x) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + RANGE_ERROR(state, f, 0); + } return -1; + } i = f->size; if (i != SIZEOF_LONG) { if ((i == 2) && (x < -32768 || x > 32767)) - RANGE_ERROR(state, x, f, 0, 0xffffL); + RANGE_ERROR(state, f, 0); #if (SIZEOF_LONG != 4) else if ((i == 4) && (x < -2147483648L || x > 2147483647L)) - RANGE_ERROR(state, x, f, 0, 0xffffffffL); + RANGE_ERROR(state, f, 0); #endif } do { @@ -935,14 +966,18 @@ bp_uint(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) unsigned long x; Py_ssize_t i; unsigned char *q = (unsigned char *)p; - if (get_ulong(state, v, &x) < 0) + if (get_ulong(state, v, &x) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + RANGE_ERROR(state, f, 1); + } return -1; + } i = f->size; if (i != SIZEOF_LONG) { unsigned long maxint = 1; maxint <<= (unsigned long)(i * 8); if (x >= maxint) - RANGE_ERROR(state, x, f, 1, maxint - 1); + RANGE_ERROR(state, f, 1); } do { q[--i] = (unsigned char)(x & 0xffUL); @@ -964,6 +999,14 @@ bp_longlong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) 0, /* little_endian */ 1 /* signed */); Py_DECREF(v); + if (res == -1 && PyErr_Occurred()) { + PyErr_Format(state->StructError, + "'%c' format requires %lld <= number <= %lld", + f->format, + LLONG_MIN, + LLONG_MAX); + return -1; + } return res; } @@ -980,6 +1023,13 @@ bp_ulonglong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f 0, /* little_endian */ 0 /* signed */); Py_DECREF(v); + if (res == -1 && PyErr_Occurred()) { + PyErr_Format(state->StructError, + "'%c' format requires 0 <= number <= %llu", + f->format, + ULLONG_MAX); + return -1; + } return res; } @@ -1148,15 +1198,19 @@ lp_int(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) long x; Py_ssize_t i; unsigned char *q = (unsigned char *)p; - if (get_long(state, v, &x) < 0) + if (get_long(state, v, &x) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + RANGE_ERROR(state, f, 0); + } return -1; + } i = f->size; if (i != SIZEOF_LONG) { if ((i == 2) && (x < -32768 || x > 32767)) - RANGE_ERROR(state, x, f, 0, 0xffffL); + RANGE_ERROR(state, f, 0); #if (SIZEOF_LONG != 4) else if ((i == 4) && (x < -2147483648L || x > 2147483647L)) - RANGE_ERROR(state, x, f, 0, 0xffffffffL); + RANGE_ERROR(state, f, 0); #endif } do { @@ -1172,14 +1226,18 @@ lp_uint(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) unsigned long x; Py_ssize_t i; unsigned char *q = (unsigned char *)p; - if (get_ulong(state, v, &x) < 0) + if (get_ulong(state, v, &x) < 0) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + RANGE_ERROR(state, f, 1); + } return -1; + } i = f->size; if (i != SIZEOF_LONG) { unsigned long maxint = 1; maxint <<= (unsigned long)(i * 8); if (x >= maxint) - RANGE_ERROR(state, x, f, 1, maxint - 1); + RANGE_ERROR(state, f, 1); } do { *q++ = (unsigned char)(x & 0xffUL); @@ -1201,6 +1259,14 @@ lp_longlong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) 1, /* little_endian */ 1 /* signed */); Py_DECREF(v); + if (res == -1 && PyErr_Occurred()) { + PyErr_Format(state->StructError, + "'%c' format requires %lld <= number <= %lld", + f->format, + LLONG_MIN, + LLONG_MAX); + return -1; + } return res; } @@ -1217,6 +1283,13 @@ lp_ulonglong(_structmodulestate *state, char *p, PyObject *v, const formatdef *f 1, /* little_endian */ 0 /* signed */); Py_DECREF(v); + if (res == -1 && PyErr_Occurred()) { + PyErr_Format(state->StructError, + "'%c' format requires 0 <= number <= %llu", + f->format, + ULLONG_MAX); + return -1; + } return res; } @@ -2090,13 +2163,11 @@ PyDoc_STRVAR(s_sizeof__doc__, static PyObject * s_sizeof(PyStructObject *self, void *unused) { - Py_ssize_t size; - formatcode *code; - - size = _PyObject_SIZE(Py_TYPE(self)) + sizeof(formatcode); - for (code = self->s_codes; code->fmtdef != NULL; code++) + size_t size = _PyObject_SIZE(Py_TYPE(self)) + sizeof(formatcode); + for (formatcode *code = self->s_codes; code->fmtdef != NULL; code++) { size += sizeof(formatcode); - return PyLong_FromSsize_t(size); + } + return PyLong_FromSize_t(size); } /* List of functions */ diff --git a/Modules/_testcapi/unicode.c b/Modules/_testcapi/unicode.c index d5c4a9e5b95ec6..2d23993ce420b3 100644 --- a/Modules/_testcapi/unicode.c +++ b/Modules/_testcapi/unicode.c @@ -1,3 +1,4 @@ +#define PY_SSIZE_T_CLEAN #include "parts.h" static struct PyModuleDef *_testcapimodule = NULL; // set at initialization @@ -99,6 +100,17 @@ test_widechar(PyObject *self, PyObject *Py_UNUSED(ignored)) Py_RETURN_NONE; } +#define NULLABLE(x) do { if (x == Py_None) x = NULL; } while (0); + +/* Test PyUnicode_FromObject() */ +static PyObject * +unicode_fromobject(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return PyUnicode_FromObject(arg); +} + +/* Test PyUnicode_AsWideChar() */ static PyObject * unicode_aswidechar(PyObject *self, PyObject *args) { @@ -130,6 +142,7 @@ unicode_aswidechar(PyObject *self, PyObject *args) return Py_BuildValue("(Nn)", result, size); } +/* Test PyUnicode_AsWideCharString() */ static PyObject * unicode_aswidecharstring(PyObject *self, PyObject *args) { @@ -151,6 +164,7 @@ unicode_aswidecharstring(PyObject *self, PyObject *args) return Py_BuildValue("(Nn)", result, size); } +/* Test PyUnicode_AsUCS4() */ static PyObject * unicode_asucs4(PyObject *self, PyObject *args) { @@ -181,6 +195,7 @@ unicode_asucs4(PyObject *self, PyObject *args) return result; } +/* Test PyUnicode_AsUTF8() */ static PyObject * unicode_asutf8(PyObject *self, PyObject *args) { @@ -199,6 +214,7 @@ unicode_asutf8(PyObject *self, PyObject *args) return PyBytes_FromString(buffer); } +/* Test PyUnicode_AsUTF8AndSize() */ static PyObject * unicode_asutf8andsize(PyObject *self, PyObject *args) { @@ -223,26 +239,228 @@ unicode_asutf8andsize(PyObject *self, PyObject *args) return Py_BuildValue("(Nn)", result, utf8_len); } +/* Test PyUnicode_DecodeUTF8() */ +static PyObject * +unicode_decodeutf8(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + return PyUnicode_DecodeUTF8(data, size, errors); +} + +/* Test PyUnicode_DecodeUTF8Stateful() */ +static PyObject * +unicode_decodeutf8stateful(PyObject *self, PyObject *args) +{ + const char *data; + Py_ssize_t size; + const char *errors = NULL; + Py_ssize_t consumed = 123456789; + PyObject *result; + + if (!PyArg_ParseTuple(args, "y#|z", &data, &size, &errors)) + return NULL; + + result = PyUnicode_DecodeUTF8Stateful(data, size, errors, &consumed); + if (!result) { + return NULL; + } + return Py_BuildValue("(Nn)", result, consumed); +} + +/* Test PyUnicode_Concat() */ +static PyObject * +unicode_concat(PyObject *self, PyObject *args) +{ + PyObject *left; + PyObject *right; + + if (!PyArg_ParseTuple(args, "OO", &left, &right)) + return NULL; + + NULLABLE(left); + NULLABLE(right); + return PyUnicode_Concat(left, right); +} + +/* Test PyUnicode_Split() */ +static PyObject * +unicode_split(PyObject *self, PyObject *args) +{ + PyObject *s; + PyObject *sep; + Py_ssize_t maxsplit = -1; + + if (!PyArg_ParseTuple(args, "OO|n", &s, &sep, &maxsplit)) + return NULL; + + NULLABLE(s); + NULLABLE(sep); + return PyUnicode_Split(s, sep, maxsplit); +} + +/* Test PyUnicode_RSplit() */ +static PyObject * +unicode_rsplit(PyObject *self, PyObject *args) +{ + PyObject *s; + PyObject *sep; + Py_ssize_t maxsplit = -1; + + if (!PyArg_ParseTuple(args, "OO|n", &s, &sep, &maxsplit)) + return NULL; + + NULLABLE(s); + NULLABLE(sep); + return PyUnicode_RSplit(s, sep, maxsplit); +} + +/* Test PyUnicode_Splitlines() */ +static PyObject * +unicode_splitlines(PyObject *self, PyObject *args) +{ + PyObject *s; + int keepends = 0; + + if (!PyArg_ParseTuple(args, "O|i", &s, &keepends)) + return NULL; + + NULLABLE(s); + return PyUnicode_Splitlines(s, keepends); +} + +/* Test PyUnicode_Partition() */ +static PyObject * +unicode_partition(PyObject *self, PyObject *args) +{ + PyObject *s; + PyObject *sep; + + if (!PyArg_ParseTuple(args, "OO", &s, &sep)) + return NULL; + + NULLABLE(s); + NULLABLE(sep); + return PyUnicode_Partition(s, sep); +} + +/* Test PyUnicode_RPartition() */ +static PyObject * +unicode_rpartition(PyObject *self, PyObject *args) +{ + PyObject *s; + PyObject *sep; + + if (!PyArg_ParseTuple(args, "OO", &s, &sep)) + return NULL; + + NULLABLE(s); + NULLABLE(sep); + return PyUnicode_RPartition(s, sep); +} + +/* Test PyUnicode_Translate() */ +static PyObject * +unicode_translate(PyObject *self, PyObject *args) +{ + PyObject *obj; + PyObject *table; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "OO|z", &obj, &table, &errors)) + return NULL; + + NULLABLE(obj); + NULLABLE(table); + return PyUnicode_Translate(obj, table, errors); +} + +/* Test PyUnicode_Join() */ +static PyObject * +unicode_join(PyObject *self, PyObject *args) +{ + PyObject *sep; + PyObject *seq; + + if (!PyArg_ParseTuple(args, "OO", &sep, &seq)) + return NULL; + + NULLABLE(sep); + NULLABLE(seq); + return PyUnicode_Join(sep, seq); +} + +/* Test PyUnicode_Count() */ static PyObject * unicode_count(PyObject *self, PyObject *args) { PyObject *str; PyObject *substr; + Py_ssize_t start; + Py_ssize_t end; Py_ssize_t result; - Py_ssize_t start, end; - if (!PyArg_ParseTuple(args, "UUnn:unicode_count", &str, &substr, - &start, &end)) { + if (!PyArg_ParseTuple(args, "OOnn", &str, &substr, &start, &end)) return NULL; - } + NULLABLE(str); + NULLABLE(substr); result = PyUnicode_Count(str, substr, start, end); if (result == -1) return NULL; - else - return PyLong_FromSsize_t(result); + return PyLong_FromSsize_t(result); +} + +/* Test PyUnicode_Find() */ +static PyObject * +unicode_find(PyObject *self, PyObject *args) +{ + PyObject *str; + PyObject *substr; + Py_ssize_t start; + Py_ssize_t end; + int direction; + Py_ssize_t result; + + if (!PyArg_ParseTuple(args, "OOnni", &str, &substr, &start, &end, &direction)) + return NULL; + + NULLABLE(str); + NULLABLE(substr); + result = PyUnicode_Find(str, substr, start, end, direction); + if (result == -2) + return NULL; + return PyLong_FromSsize_t(result); } +/* Test PyUnicode_Tailmatch() */ +static PyObject * +unicode_tailmatch(PyObject *self, PyObject *args) +{ + PyObject *str; + PyObject *substr; + Py_ssize_t start; + Py_ssize_t end; + int direction; + Py_ssize_t result; + + if (!PyArg_ParseTuple(args, "OOnni", &str, &substr, &start, &end, &direction)) + return NULL; + + NULLABLE(str); + NULLABLE(substr); + result = PyUnicode_Tailmatch(str, substr, start, end, direction); + if (result == -1) + return NULL; + return PyLong_FromSsize_t(result); +} + +/* Test PyUnicode_FindChar() */ static PyObject * unicode_findchar(PyObject *self, PyObject *args) { @@ -264,6 +482,130 @@ unicode_findchar(PyObject *self, PyObject *args) return PyLong_FromSsize_t(result); } +/* Test PyUnicode_Replace() */ +static PyObject * +unicode_replace(PyObject *self, PyObject *args) +{ + PyObject *str; + PyObject *substr; + PyObject *replstr; + Py_ssize_t maxcount = -1; + + if (!PyArg_ParseTuple(args, "OOO|n", &str, &substr, &replstr, &maxcount)) + return NULL; + + NULLABLE(str); + NULLABLE(substr); + NULLABLE(replstr); + return PyUnicode_Replace(str, substr, replstr, maxcount); +} + +/* Test PyUnicode_Compare() */ +static PyObject * +unicode_compare(PyObject *self, PyObject *args) +{ + PyObject *left; + PyObject *right; + int result; + + if (!PyArg_ParseTuple(args, "OO", &left, &right)) + return NULL; + + NULLABLE(left); + NULLABLE(right); + result = PyUnicode_Compare(left, right); + if (result == -1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromLong(result); +} + +/* Test PyUnicode_CompareWithASCIIString() */ +static PyObject * +unicode_comparewithasciistring(PyObject *self, PyObject *args) +{ + PyObject *left; + const char *right = NULL; + Py_ssize_t right_len; + int result; + + if (!PyArg_ParseTuple(args, "O|y#", &left, &right, &right_len)) + return NULL; + + NULLABLE(left); + result = PyUnicode_CompareWithASCIIString(left, right); + if (result == -1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromLong(result); +} + +/* Test PyUnicode_RichCompare() */ +static PyObject * +unicode_richcompare(PyObject *self, PyObject *args) +{ + PyObject *left; + PyObject *right; + int op; + + if (!PyArg_ParseTuple(args, "OOi", &left, &right, &op)) + return NULL; + + NULLABLE(left); + NULLABLE(right); + return PyUnicode_RichCompare(left, right, op); +} + +/* Test PyUnicode_Format() */ +static PyObject * +unicode_format(PyObject *self, PyObject *args) +{ + PyObject *format; + PyObject *fargs; + + if (!PyArg_ParseTuple(args, "OO", &format, &fargs)) + return NULL; + + NULLABLE(format); + NULLABLE(fargs); + return PyUnicode_Format(format, fargs); +} + +/* Test PyUnicode_Contains() */ +static PyObject * +unicode_contains(PyObject *self, PyObject *args) +{ + PyObject *container; + PyObject *element; + int result; + + if (!PyArg_ParseTuple(args, "OO", &container, &element)) + return NULL; + + NULLABLE(container); + NULLABLE(element); + result = PyUnicode_Contains(container, element); + if (result == -1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromLong(result); +} + +/* Test PyUnicode_IsIdentifier() */ +static PyObject * +unicode_isidentifier(PyObject *self, PyObject *arg) +{ + int result; + + NULLABLE(arg); + result = PyUnicode_IsIdentifier(arg); + if (result == -1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromLong(result); +} + +/* Test PyUnicode_CopyCharacters() */ static PyObject * unicode_copycharacters(PyObject *self, PyObject *args) { @@ -711,13 +1053,33 @@ static PyMethodDef TestMethods[] = { test_unicode_compare_with_ascii, METH_NOARGS}, {"test_string_from_format", test_string_from_format, METH_NOARGS}, {"test_widechar", test_widechar, METH_NOARGS}, + {"unicode_fromobject", unicode_fromobject, METH_O}, {"unicode_aswidechar", unicode_aswidechar, METH_VARARGS}, {"unicode_aswidecharstring", unicode_aswidecharstring, METH_VARARGS}, {"unicode_asucs4", unicode_asucs4, METH_VARARGS}, {"unicode_asutf8", unicode_asutf8, METH_VARARGS}, {"unicode_asutf8andsize", unicode_asutf8andsize, METH_VARARGS}, + {"unicode_decodeutf8", unicode_decodeutf8, METH_VARARGS}, + {"unicode_decodeutf8stateful",unicode_decodeutf8stateful, METH_VARARGS}, + {"unicode_concat", unicode_concat, METH_VARARGS}, + {"unicode_splitlines", unicode_splitlines, METH_VARARGS}, + {"unicode_split", unicode_split, METH_VARARGS}, + {"unicode_rsplit", unicode_rsplit, METH_VARARGS}, + {"unicode_partition", unicode_partition, METH_VARARGS}, + {"unicode_rpartition", unicode_rpartition, METH_VARARGS}, + {"unicode_translate", unicode_translate, METH_VARARGS}, + {"unicode_join", unicode_join, METH_VARARGS}, {"unicode_count", unicode_count, METH_VARARGS}, + {"unicode_tailmatch", unicode_tailmatch, METH_VARARGS}, + {"unicode_find", unicode_find, METH_VARARGS}, {"unicode_findchar", unicode_findchar, METH_VARARGS}, + {"unicode_replace", unicode_replace, METH_VARARGS}, + {"unicode_compare", unicode_compare, METH_VARARGS}, + {"unicode_comparewithasciistring",unicode_comparewithasciistring,METH_VARARGS}, + {"unicode_richcompare", unicode_richcompare, METH_VARARGS}, + {"unicode_format", unicode_format, METH_VARARGS}, + {"unicode_contains", unicode_contains, METH_VARARGS}, + {"unicode_isidentifier", unicode_isidentifier, METH_O}, {"unicode_copycharacters", unicode_copycharacters, METH_VARARGS}, {NULL}, }; diff --git a/Modules/_testcapi/watchers.c b/Modules/_testcapi/watchers.c index 608cd780d12a26..1d91c206f63092 100644 --- a/Modules/_testcapi/watchers.c +++ b/Modules/_testcapi/watchers.c @@ -2,6 +2,7 @@ #define Py_BUILD_CORE #include "pycore_function.h" // FUNC_MAX_WATCHERS +#include "pycore_code.h" // CODE_MAX_WATCHERS // Test dict watching static PyObject *g_dict_watch_events; @@ -277,6 +278,135 @@ unwatch_type(PyObject *self, PyObject *args) Py_RETURN_NONE; } + +// Test code object watching + +#define NUM_CODE_WATCHERS 2 +static int num_code_object_created_events[NUM_CODE_WATCHERS] = {0, 0}; +static int num_code_object_destroyed_events[NUM_CODE_WATCHERS] = {0, 0}; + +static int +handle_code_object_event(int which_watcher, PyCodeEvent event, PyCodeObject *co) { + if (event == PY_CODE_EVENT_CREATE) { + num_code_object_created_events[which_watcher]++; + } + else if (event == PY_CODE_EVENT_DESTROY) { + num_code_object_destroyed_events[which_watcher]++; + } + else { + return -1; + } + return 0; +} + +static int +first_code_object_callback(PyCodeEvent event, PyCodeObject *co) +{ + return handle_code_object_event(0, event, co); +} + +static int +second_code_object_callback(PyCodeEvent event, PyCodeObject *co) +{ + return handle_code_object_event(1, event, co); +} + +static int +noop_code_event_handler(PyCodeEvent event, PyCodeObject *co) +{ + return 0; +} + +static PyObject * +add_code_watcher(PyObject *self, PyObject *which_watcher) +{ + int watcher_id; + assert(PyLong_Check(which_watcher)); + long which_l = PyLong_AsLong(which_watcher); + if (which_l == 0) { + watcher_id = PyCode_AddWatcher(first_code_object_callback); + num_code_object_created_events[0] = 0; + num_code_object_destroyed_events[0] = 0; + } + else if (which_l == 1) { + watcher_id = PyCode_AddWatcher(second_code_object_callback); + num_code_object_created_events[1] = 0; + num_code_object_destroyed_events[1] = 0; + } + else { + return NULL; + } + if (watcher_id < 0) { + return NULL; + } + return PyLong_FromLong(watcher_id); +} + +static PyObject * +clear_code_watcher(PyObject *self, PyObject *watcher_id) +{ + assert(PyLong_Check(watcher_id)); + long watcher_id_l = PyLong_AsLong(watcher_id); + if (PyCode_ClearWatcher(watcher_id_l) < 0) { + return NULL; + } + // reset static events counters + if (watcher_id_l >= 0 && watcher_id_l < NUM_CODE_WATCHERS) { + num_code_object_created_events[watcher_id_l] = 0; + num_code_object_destroyed_events[watcher_id_l] = 0; + } + Py_RETURN_NONE; +} + +static PyObject * +get_code_watcher_num_created_events(PyObject *self, PyObject *watcher_id) +{ + assert(PyLong_Check(watcher_id)); + long watcher_id_l = PyLong_AsLong(watcher_id); + assert(watcher_id_l >= 0 && watcher_id_l < NUM_CODE_WATCHERS); + return PyLong_FromLong(num_code_object_created_events[watcher_id_l]); +} + +static PyObject * +get_code_watcher_num_destroyed_events(PyObject *self, PyObject *watcher_id) +{ + assert(PyLong_Check(watcher_id)); + long watcher_id_l = PyLong_AsLong(watcher_id); + assert(watcher_id_l >= 0 && watcher_id_l < NUM_CODE_WATCHERS); + return PyLong_FromLong(num_code_object_destroyed_events[watcher_id_l]); +} + +static PyObject * +allocate_too_many_code_watchers(PyObject *self, PyObject *args) +{ + int watcher_ids[CODE_MAX_WATCHERS + 1]; + int num_watchers = 0; + for (unsigned long i = 0; i < sizeof(watcher_ids) / sizeof(int); i++) { + int watcher_id = PyCode_AddWatcher(noop_code_event_handler); + if (watcher_id == -1) { + break; + } + watcher_ids[i] = watcher_id; + num_watchers++; + } + PyObject *type, *value, *traceback; + PyErr_Fetch(&type, &value, &traceback); + for (int i = 0; i < num_watchers; i++) { + if (PyCode_ClearWatcher(watcher_ids[i]) < 0) { + PyErr_WriteUnraisable(Py_None); + break; + } + } + if (type) { + PyErr_Restore(type, value, traceback); + return NULL; + } + else if (PyErr_Occurred()) { + return NULL; + } + Py_RETURN_NONE; +} + // Test function watchers #define NUM_FUNC_WATCHERS 2 @@ -509,6 +639,16 @@ static PyMethodDef test_methods[] = { {"unwatch_type", unwatch_type, METH_VARARGS, NULL}, {"get_type_modified_events", get_type_modified_events, METH_NOARGS, NULL}, + // Code object watchers. + {"add_code_watcher", add_code_watcher, METH_O, NULL}, + {"clear_code_watcher", clear_code_watcher, METH_O, NULL}, + {"get_code_watcher_num_created_events", + get_code_watcher_num_created_events, METH_O, NULL}, + {"get_code_watcher_num_destroyed_events", + get_code_watcher_num_destroyed_events, METH_O, NULL}, + {"allocate_too_many_code_watchers", + (PyCFunction) allocate_too_many_code_watchers, METH_NOARGS, NULL}, + // Function watchers. {"add_func_watcher", add_func_watcher, METH_O, NULL}, {"clear_func_watcher", clear_func_watcher, METH_O, NULL}, diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index ec8b6d881124da..d323ca83dcffa7 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -135,7 +135,7 @@ lock_acquire_parse_args(PyObject *args, PyObject *kwds, *timeout = unset_timeout ; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|iO:acquire", kwlist, + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|pO:acquire", kwlist, &blocking, &timeout_obj)) return -1; diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 93d4474f65d62c..d4a129058702a2 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -2832,7 +2832,7 @@ Tkapp_WantObjects(PyObject *self, PyObject *args) { int wantobjects = -1; - if (!PyArg_ParseTuple(args, "|i:wantobjects", &wantobjects)) + if (!PyArg_ParseTuple(args, "|p:wantobjects", &wantobjects)) return NULL; if (wantobjects == -1) return PyBool_FromLong(((TkappObject*)self)->wantobjects); @@ -2978,11 +2978,11 @@ _tkinter.create screenName: str(accept={str, NoneType}) = None baseName: str = "" className: str = "Tk" - interactive: bool(accept={int}) = False - wantobjects: bool(accept={int}) = False - wantTk: bool(accept={int}) = True + interactive: bool = False + wantobjects: bool = False + wantTk: bool = True if false, then Tk_Init() doesn't get called - sync: bool(accept={int}) = False + sync: bool = False if true, then pass -sync to wish use: str(accept={str, NoneType}) = None if not None, then pass -use to wish @@ -2995,7 +2995,7 @@ _tkinter_create_impl(PyObject *module, const char *screenName, const char *baseName, const char *className, int interactive, int wantobjects, int wantTk, int sync, const char *use) -/*[clinic end generated code: output=e3315607648e6bb4 input=da9b17ee7358d862]*/ +/*[clinic end generated code: output=e3315607648e6bb4 input=09afef9adea70a19]*/ { /* XXX baseName is not used anymore; * try getting rid of it. */ diff --git a/Modules/_winapi.c b/Modules/_winapi.c index bb4514c36bc7d0..f4d982b15d402a 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -404,13 +404,13 @@ _winapi_CloseHandle_impl(PyObject *module, HANDLE handle) _winapi.ConnectNamedPipe handle: HANDLE - overlapped as use_overlapped: bool(accept={int}) = False + overlapped as use_overlapped: bool = False [clinic start generated code]*/ static PyObject * _winapi_ConnectNamedPipe_impl(PyObject *module, HANDLE handle, int use_overlapped) -/*[clinic end generated code: output=335a0e7086800671 input=34f937c1c86e5e68]*/ +/*[clinic end generated code: output=335a0e7086800671 input=a80e56e8bd370e31]*/ { BOOL success; OverlappedObject *overlapped = NULL; @@ -1576,13 +1576,13 @@ _winapi.ReadFile handle: HANDLE size: DWORD - overlapped as use_overlapped: bool(accept={int}) = False + overlapped as use_overlapped: bool = False [clinic start generated code]*/ static PyObject * _winapi_ReadFile_impl(PyObject *module, HANDLE handle, DWORD size, int use_overlapped) -/*[clinic end generated code: output=d3d5b44a8201b944 input=08c439d03a11aac5]*/ +/*[clinic end generated code: output=d3d5b44a8201b944 input=4f82f8e909ad91ad]*/ { DWORD nread; PyObject *buf; @@ -1862,13 +1862,13 @@ _winapi.WriteFile handle: HANDLE buffer: object - overlapped as use_overlapped: bool(accept={int}) = False + overlapped as use_overlapped: bool = False [clinic start generated code]*/ static PyObject * _winapi_WriteFile_impl(PyObject *module, HANDLE handle, PyObject *buffer, int use_overlapped) -/*[clinic end generated code: output=2ca80f6bf3fa92e3 input=11eae2a03aa32731]*/ +/*[clinic end generated code: output=2ca80f6bf3fa92e3 input=2badb008c8a2e2a0]*/ { Py_buffer _buf, *buf; DWORD len, written; diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 2c9e0cda1ab048..d7d7fcaf00a81a 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -6,11 +6,15 @@ #endif #include "Python.h" +// XXX This module should not rely on internal API. #include "pycore_frame.h" #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_interpreteridobject.h" +#define MODULE_NAME "_xxsubinterpreters" + + static char * _copy_raw_string(PyObject *strobj) { @@ -28,13 +32,186 @@ _copy_raw_string(PyObject *strobj) } static PyInterpreterState * -_get_current(void) +_get_current_interp(void) { // PyInterpreterState_Get() aborts if lookup fails, so don't need // to check the result for NULL. return PyInterpreterState_Get(); } +static PyObject * +_get_current_module(void) +{ + // We ensured it was imported in _run_script(). + PyObject *name = PyUnicode_FromString(MODULE_NAME); + if (name == NULL) { + return NULL; + } + PyObject *mod = PyImport_GetModule(name); + Py_DECREF(name); + if (mod == NULL) { + return NULL; + } + assert(mod != Py_None); + return mod; +} + +static PyObject * +get_module_from_owned_type(PyTypeObject *cls) +{ + assert(cls != NULL); + return _get_current_module(); + // XXX Use the more efficient API now that we use heap types: + //return PyType_GetModule(cls); +} + +static struct PyModuleDef moduledef; + +static PyObject * +get_module_from_type(PyTypeObject *cls) +{ + assert(cls != NULL); + return _get_current_module(); + // XXX Use the more efficient API now that we use heap types: + //return PyType_GetModuleByDef(cls, &moduledef); +} + +static PyObject * +add_new_exception(PyObject *mod, const char *name, PyObject *base) +{ + assert(!PyObject_HasAttrString(mod, name)); + PyObject *exctype = PyErr_NewException(name, base, NULL); + if (exctype == NULL) { + return NULL; + } + int res = PyModule_AddType(mod, (PyTypeObject *)exctype); + if (res < 0) { + Py_DECREF(exctype); + return NULL; + } + return exctype; +} + +#define ADD_NEW_EXCEPTION(MOD, NAME, BASE) \ + add_new_exception(MOD, MODULE_NAME "." Py_STRINGIFY(NAME), BASE) + +static PyTypeObject * +add_new_type(PyObject *mod, PyType_Spec *spec, crossinterpdatafunc shared) +{ + PyTypeObject *cls = (PyTypeObject *)PyType_FromMetaclass( + NULL, mod, spec, NULL); + if (cls == NULL) { + return NULL; + } + if (PyModule_AddType(mod, cls) < 0) { + Py_DECREF(cls); + return NULL; + } + if (shared != NULL) { + if (_PyCrossInterpreterData_RegisterClass(cls, shared)) { + Py_DECREF(cls); + return NULL; + } + } + return cls; +} + +static int +_release_xid_data(_PyCrossInterpreterData *data, int ignoreexc) +{ + PyObject *exctype, *excval, *exctb; + if (ignoreexc) { + PyErr_Fetch(&exctype, &excval, &exctb); + } + int res = _PyCrossInterpreterData_Release(data); + if (res < 0) { + // XXX Fix this! + /* The owning interpreter is already destroyed. + * Ideally, this shouldn't ever happen. When an interpreter is + * about to be destroyed, we should clear out all of its objects + * from every channel associated with that interpreter. + * For now we hack around that to resolve refleaks, by decref'ing + * the released object here, even if its the wrong interpreter. + * The owning interpreter has already been destroyed + * so we should be okay, especially since the currently + * shareable types are all very basic, with no GC. + * That said, it becomes much messier once interpreters + * no longer share a GIL, so this needs to be fixed before then. */ + _PyCrossInterpreterData_Clear(NULL, data); + if (ignoreexc) { + // XXX Emit a warning? + PyErr_Clear(); + } + } + if (ignoreexc) { + PyErr_Restore(exctype, excval, exctb); + } + return res; +} + + +/* module state *************************************************************/ + +typedef struct { + PyTypeObject *ChannelIDType; + + /* interpreter exceptions */ + PyObject *RunFailedError; + + /* channel exceptions */ + PyObject *ChannelError; + PyObject *ChannelNotFoundError; + PyObject *ChannelClosedError; + PyObject *ChannelEmptyError; + PyObject *ChannelNotEmptyError; +} module_state; + +static inline module_state * +get_module_state(PyObject *mod) +{ + assert(mod != NULL); + module_state *state = PyModule_GetState(mod); + assert(state != NULL); + return state; +} + +static int +traverse_module_state(module_state *state, visitproc visit, void *arg) +{ + /* heap types */ + Py_VISIT(state->ChannelIDType); + + /* interpreter exceptions */ + Py_VISIT(state->RunFailedError); + + /* channel exceptions */ + Py_VISIT(state->ChannelError); + Py_VISIT(state->ChannelNotFoundError); + Py_VISIT(state->ChannelClosedError); + Py_VISIT(state->ChannelEmptyError); + Py_VISIT(state->ChannelNotEmptyError); + return 0; +} + +static int +clear_module_state(module_state *state) +{ + /* heap types */ + (void)_PyCrossInterpreterData_UnregisterClass(state->ChannelIDType); + Py_CLEAR(state->ChannelIDType); + + /* interpreter exceptions */ + Py_CLEAR(state->RunFailedError); + + /* channel exceptions */ + Py_CLEAR(state->ChannelError); + Py_CLEAR(state->ChannelNotFoundError); + Py_CLEAR(state->ChannelClosedError); + Py_CLEAR(state->ChannelEmptyError); + Py_CLEAR(state->ChannelNotEmptyError); + return 0; +} + /* data-sharing-specific code ***********************************************/ @@ -66,7 +243,7 @@ _sharednsitem_clear(struct _sharednsitem *item) PyMem_Free(item->name); item->name = NULL; } - _PyCrossInterpreterData_Release(&item->data); + (void)_release_xid_data(&item->data, 1); } static int @@ -121,8 +298,10 @@ _sharedns_free(_sharedns *shared) } static _sharedns * -_get_shared_ns(PyObject *shareable) +_get_shared_ns(PyObject *shareable, PyTypeObject *channelidtype, + int *needs_import) { + *needs_import = 0; if (shareable == NULL || shareable == Py_None) { return NULL; } @@ -144,6 +323,9 @@ _get_shared_ns(PyObject *shareable) if (_sharednsitem_init(&shared->items[i], key, value) != 0) { break; } + if (Py_TYPE(value) == channelidtype) { + *needs_import = 1; + } } if (PyErr_Occurred()) { _sharedns_free(shared); @@ -287,68 +469,97 @@ _sharedexception_apply(_sharedexception *exc, PyObject *wrapperclass) #define CHANNEL_BOTH 0 #define CHANNEL_RECV -1 -static PyObject *ChannelError; -static PyObject *ChannelNotFoundError; -static PyObject *ChannelClosedError; -static PyObject *ChannelEmptyError; -static PyObject *ChannelNotEmptyError; +/* channel errors */ + +#define ERR_CHANNEL_NOT_FOUND -2 +#define ERR_CHANNEL_CLOSED -3 +#define ERR_CHANNEL_INTERP_CLOSED -4 +#define ERR_CHANNEL_EMPTY -5 +#define ERR_CHANNEL_NOT_EMPTY -6 +#define ERR_CHANNEL_MUTEX_INIT -7 +#define ERR_CHANNELS_MUTEX_INIT -8 +#define ERR_NO_NEXT_CHANNEL_ID -9 static int -channel_exceptions_init(PyObject *ns) +channel_exceptions_init(PyObject *mod) { - // XXX Move the exceptions into per-module memory? - - // A channel-related operation failed. - ChannelError = PyErr_NewException("_xxsubinterpreters.ChannelError", - PyExc_RuntimeError, NULL); - if (ChannelError == NULL) { - return -1; - } - if (PyDict_SetItemString(ns, "ChannelError", ChannelError) != 0) { + module_state *state = get_module_state(mod); + if (state == NULL) { return -1; } - // An operation tried to use a channel that doesn't exist. - ChannelNotFoundError = PyErr_NewException( - "_xxsubinterpreters.ChannelNotFoundError", ChannelError, NULL); - if (ChannelNotFoundError == NULL) { - return -1; - } - if (PyDict_SetItemString(ns, "ChannelNotFoundError", ChannelNotFoundError) != 0) { - return -1; - } +#define ADD(NAME, BASE) \ + do { \ + assert(state->NAME == NULL); \ + state->NAME = ADD_NEW_EXCEPTION(mod, NAME, BASE); \ + if (state->NAME == NULL) { \ + return -1; \ + } \ + } while (0) + // A channel-related operation failed. + ADD(ChannelError, PyExc_RuntimeError); + // An operation tried to use a channel that doesn't exist. + ADD(ChannelNotFoundError, state->ChannelError); // An operation tried to use a closed channel. - ChannelClosedError = PyErr_NewException( - "_xxsubinterpreters.ChannelClosedError", ChannelError, NULL); - if (ChannelClosedError == NULL) { - return -1; - } - if (PyDict_SetItemString(ns, "ChannelClosedError", ChannelClosedError) != 0) { - return -1; - } - + ADD(ChannelClosedError, state->ChannelError); // An operation tried to pop from an empty channel. - ChannelEmptyError = PyErr_NewException( - "_xxsubinterpreters.ChannelEmptyError", ChannelError, NULL); - if (ChannelEmptyError == NULL) { - return -1; + ADD(ChannelEmptyError, state->ChannelError); + // An operation tried to close a non-empty channel. + ADD(ChannelNotEmptyError, state->ChannelError); +#undef ADD + + return 0; +} + +static int +handle_channel_error(int err, PyObject *mod, int64_t cid) +{ + if (err == 0) { + assert(!PyErr_Occurred()); + return 0; } - if (PyDict_SetItemString(ns, "ChannelEmptyError", ChannelEmptyError) != 0) { - return -1; + assert(err < 0); + module_state *state = get_module_state(mod); + assert(state != NULL); + if (err == ERR_CHANNEL_NOT_FOUND) { + PyErr_Format(state->ChannelNotFoundError, + "channel %" PRId64 " not found", cid); + } + else if (err == ERR_CHANNEL_CLOSED) { + PyErr_Format(state->ChannelClosedError, + "channel %" PRId64 " is closed", cid); + } + else if (err == ERR_CHANNEL_INTERP_CLOSED) { + PyErr_Format(state->ChannelClosedError, + "channel %" PRId64 " is already closed", cid); + } + else if (err == ERR_CHANNEL_EMPTY) { + PyErr_Format(state->ChannelEmptyError, + "channel %" PRId64 " is empty", cid); + } + else if (err == ERR_CHANNEL_NOT_EMPTY) { + PyErr_Format(state->ChannelNotEmptyError, + "channel %" PRId64 " may not be closed " + "if not empty (try force=True)", + cid); + } + else if (err == ERR_CHANNEL_MUTEX_INIT) { + PyErr_SetString(state->ChannelError, + "can't initialize mutex for new channel"); } - - // An operation tried to close a non-empty channel. - ChannelNotEmptyError = PyErr_NewException( - "_xxsubinterpreters.ChannelNotEmptyError", ChannelError, NULL); - if (ChannelNotEmptyError == NULL) { - return -1; + else if (err == ERR_CHANNELS_MUTEX_INIT) { + PyErr_SetString(state->ChannelError, + "can't initialize mutex for channel management"); } - if (PyDict_SetItemString(ns, "ChannelNotEmptyError", ChannelNotEmptyError) != 0) { - return -1; + else if (err == ERR_NO_NEXT_CHANNEL_ID) { + PyErr_SetString(state->ChannelError, + "failed to get a channel ID"); } - - return 0; + else { + assert(PyErr_Occurred()); + } + return 1; } /* the channel queue */ @@ -377,7 +588,7 @@ static void _channelitem_clear(_channelitem *item) { if (item->data != NULL) { - _PyCrossInterpreterData_Release(item->data); + (void)_release_xid_data(item->data, 1); PyMem_Free(item->data); item->data = NULL; } @@ -621,8 +832,7 @@ _channelends_associate(_channelends *ends, int64_t interp, int send) interp, &prev); if (end != NULL) { if (!end->open) { - PyErr_SetString(ChannelClosedError, "channel already closed"); - return -1; + return ERR_CHANNEL_CLOSED; } // already associated return 0; @@ -721,19 +931,13 @@ typedef struct _channel { } _PyChannelState; static _PyChannelState * -_channel_new(void) +_channel_new(PyThread_type_lock mutex) { _PyChannelState *chan = PyMem_NEW(_PyChannelState, 1); if (chan == NULL) { return NULL; } - chan->mutex = PyThread_allocate_lock(); - if (chan->mutex == NULL) { - PyMem_Free(chan); - PyErr_SetString(ChannelError, - "can't initialize mutex for new channel"); - return NULL; - } + chan->mutex = mutex; chan->queue = _channelqueue_new(); if (chan->queue == NULL) { PyMem_Free(chan); @@ -771,10 +975,11 @@ _channel_add(_PyChannelState *chan, int64_t interp, PyThread_acquire_lock(chan->mutex, WAIT_LOCK); if (!chan->open) { - PyErr_SetString(ChannelClosedError, "channel closed"); + res = ERR_CHANNEL_CLOSED; goto done; } if (_channelends_associate(chan->ends, interp, 1) != 0) { + res = ERR_CHANNEL_INTERP_CLOSED; goto done; } @@ -788,31 +993,34 @@ _channel_add(_PyChannelState *chan, int64_t interp, return res; } -static _PyCrossInterpreterData * -_channel_next(_PyChannelState *chan, int64_t interp) +static int +_channel_next(_PyChannelState *chan, int64_t interp, + _PyCrossInterpreterData **res) { - _PyCrossInterpreterData *data = NULL; + int err = 0; PyThread_acquire_lock(chan->mutex, WAIT_LOCK); if (!chan->open) { - PyErr_SetString(ChannelClosedError, "channel closed"); + err = ERR_CHANNEL_CLOSED; goto done; } if (_channelends_associate(chan->ends, interp, 0) != 0) { + err = ERR_CHANNEL_INTERP_CLOSED; goto done; } - data = _channelqueue_get(chan->queue); + _PyCrossInterpreterData *data = _channelqueue_get(chan->queue); if (data == NULL && !PyErr_Occurred() && chan->closing != NULL) { chan->open = 0; } + *res = data; done: PyThread_release_lock(chan->mutex); if (chan->queue->count == 0) { _channel_finish_closing(chan); } - return data; + return err; } static int @@ -822,7 +1030,7 @@ _channel_close_interpreter(_PyChannelState *chan, int64_t interp, int end) int res = -1; if (!chan->open) { - PyErr_SetString(ChannelClosedError, "channel already closed"); + res = ERR_CHANNEL_CLOSED; goto done; } @@ -844,13 +1052,12 @@ _channel_close_all(_PyChannelState *chan, int end, int force) PyThread_acquire_lock(chan->mutex, WAIT_LOCK); if (!chan->open) { - PyErr_SetString(ChannelClosedError, "channel already closed"); + res = ERR_CHANNEL_CLOSED; goto done; } if (!force && chan->queue->count > 0) { - PyErr_SetString(ChannelNotEmptyError, - "may not be closed if not empty (try force=True)"); + res = ERR_CHANNEL_NOT_EMPTY; goto done; } @@ -935,21 +1142,24 @@ typedef struct _channels { int64_t next_id; } _channels; -static int -_channels_init(_channels *channels) +static void +_channels_init(_channels *channels, PyThread_type_lock mutex) { - if (channels->mutex == NULL) { - channels->mutex = PyThread_allocate_lock(); - if (channels->mutex == NULL) { - PyErr_SetString(ChannelError, - "can't initialize mutex for channel management"); - return -1; - } - } + channels->mutex = mutex; channels->head = NULL; channels->numopen = 0; channels->next_id = 0; - return 0; +} + +static void +_channels_fini(_channels *channels) +{ + assert(channels->numopen == 0); + assert(channels->head == NULL); + if (channels->mutex != NULL) { + PyThread_free_lock(channels->mutex); + channels->mutex = NULL; + } } static int64_t @@ -958,17 +1168,17 @@ _channels_next_id(_channels *channels) // needs lock int64_t id = channels->next_id; if (id < 0) { /* overflow */ - PyErr_SetString(ChannelError, - "failed to get a channel ID"); return -1; } channels->next_id += 1; return id; } -static _PyChannelState * -_channels_lookup(_channels *channels, int64_t id, PyThread_type_lock *pmutex) +static int +_channels_lookup(_channels *channels, int64_t id, PyThread_type_lock *pmutex, + _PyChannelState **res) { + int err = -1; _PyChannelState *chan = NULL; PyThread_acquire_lock(channels->mutex, WAIT_LOCK); if (pmutex != NULL) { @@ -977,11 +1187,11 @@ _channels_lookup(_channels *channels, int64_t id, PyThread_type_lock *pmutex) _channelref *ref = _channelref_find(channels->head, id, NULL); if (ref == NULL) { - PyErr_Format(ChannelNotFoundError, "channel %" PRId64 " not found", id); + err = ERR_CHANNEL_NOT_FOUND; goto done; } if (ref->chan == NULL || !ref->chan->open) { - PyErr_Format(ChannelClosedError, "channel %" PRId64 " closed", id); + err = ERR_CHANNEL_CLOSED; goto done; } @@ -991,11 +1201,14 @@ _channels_lookup(_channels *channels, int64_t id, PyThread_type_lock *pmutex) } chan = ref->chan; + err = 0; + done: if (pmutex == NULL || *pmutex == NULL) { PyThread_release_lock(channels->mutex); } - return chan; + *res = chan; + return err; } static int64_t @@ -1007,6 +1220,7 @@ _channels_add(_channels *channels, _PyChannelState *chan) // Create a new ref. int64_t id = _channels_next_id(channels); if (id < 0) { + cid = ERR_NO_NEXT_CHANNEL_ID; goto done; } _channelref *ref = _channelref_new(id, chan); @@ -1041,31 +1255,32 @@ _channels_close(_channels *channels, int64_t cid, _PyChannelState **pchan, _channelref *ref = _channelref_find(channels->head, cid, NULL); if (ref == NULL) { - PyErr_Format(ChannelNotFoundError, "channel %" PRId64 " not found", cid); + res = ERR_CHANNEL_NOT_FOUND; goto done; } if (ref->chan == NULL) { - PyErr_Format(ChannelClosedError, "channel %" PRId64 " closed", cid); + res = ERR_CHANNEL_CLOSED; goto done; } else if (!force && end == CHANNEL_SEND && ref->chan->closing != NULL) { - PyErr_Format(ChannelClosedError, "channel %" PRId64 " closed", cid); + res = ERR_CHANNEL_CLOSED; goto done; } else { - if (_channel_close_all(ref->chan, end, force) != 0) { - if (end == CHANNEL_SEND && - PyErr_ExceptionMatches(ChannelNotEmptyError)) { + int err = _channel_close_all(ref->chan, end, force); + if (err != 0) { + if (end == CHANNEL_SEND && err == ERR_CHANNEL_NOT_EMPTY) { if (ref->chan->closing != NULL) { - PyErr_Format(ChannelClosedError, - "channel %" PRId64 " closed", cid); + res = ERR_CHANNEL_CLOSED; goto done; } // Mark the channel as closing and return. The channel // will be cleaned up in _channel_next(). PyErr_Clear(); - if (_channel_set_closing(ref, channels->mutex) != 0) { + int err = _channel_set_closing(ref, channels->mutex); + if (err != 0) { + res = err; goto done; } if (pchan != NULL) { @@ -1073,6 +1288,9 @@ _channels_close(_channels *channels, int64_t cid, _PyChannelState **pchan, } res = 0; } + else { + res = err; + } goto done; } if (pchan != NULL) { @@ -1121,7 +1339,7 @@ _channels_remove(_channels *channels, int64_t id, _PyChannelState **pchan) _channelref *prev = NULL; _channelref *ref = _channelref_find(channels->head, id, &prev); if (ref == NULL) { - PyErr_Format(ChannelNotFoundError, "channel %" PRId64 " not found", id); + res = ERR_CHANNEL_NOT_FOUND; goto done; } @@ -1141,7 +1359,7 @@ _channels_add_id_object(_channels *channels, int64_t id) _channelref *ref = _channelref_find(channels->head, id, NULL); if (ref == NULL) { - PyErr_Format(ChannelNotFoundError, "channel %" PRId64 " not found", id); + res = ERR_CHANNEL_NOT_FOUND; goto done; } ref->objcount += 1; @@ -1215,7 +1433,7 @@ _channel_set_closing(struct _channelref *ref, PyThread_type_lock mutex) { int res = -1; PyThread_acquire_lock(chan->mutex, WAIT_LOCK); if (chan->closing != NULL) { - PyErr_SetString(ChannelClosedError, "channel closed"); + res = ERR_CHANNEL_CLOSED; goto done; } chan->closing = PyMem_NEW(struct _channel_closing, 1); @@ -1258,14 +1476,18 @@ _channel_finish_closing(struct _channel *chan) { static int64_t _channel_create(_channels *channels) { - _PyChannelState *chan = _channel_new(); + PyThread_type_lock mutex = PyThread_allocate_lock(); + if (mutex == NULL) { + return ERR_CHANNEL_MUTEX_INIT; + } + _PyChannelState *chan = _channel_new(mutex); if (chan == NULL) { + PyThread_free_lock(mutex); return -1; } int64_t id = _channels_add(channels, chan); if (id < 0) { _channel_free(chan); - return -1; } return id; } @@ -1274,8 +1496,9 @@ static int _channel_destroy(_channels *channels, int64_t id) { _PyChannelState *chan = NULL; - if (_channels_remove(channels, id, &chan) != 0) { - return -1; + int err = _channels_remove(channels, id, &chan); + if (err != 0) { + return err; } if (chan != NULL) { _channel_free(chan); @@ -1286,23 +1509,24 @@ _channel_destroy(_channels *channels, int64_t id) static int _channel_send(_channels *channels, int64_t id, PyObject *obj) { - PyInterpreterState *interp = _get_current(); + PyInterpreterState *interp = _get_current_interp(); if (interp == NULL) { return -1; } // Look up the channel. PyThread_type_lock mutex = NULL; - _PyChannelState *chan = _channels_lookup(channels, id, &mutex); - if (chan == NULL) { - return -1; + _PyChannelState *chan = NULL; + int err = _channels_lookup(channels, id, &mutex, &chan); + if (err != 0) { + return err; } + assert(chan != NULL); // Past this point we are responsible for releasing the mutex. if (chan->closing != NULL) { - PyErr_Format(ChannelClosedError, "channel %" PRId64 " closed", id); PyThread_release_lock(mutex); - return -1; + return ERR_CHANNEL_CLOSED; } // Convert the object to cross-interpreter data. @@ -1321,61 +1545,87 @@ _channel_send(_channels *channels, int64_t id, PyObject *obj) int res = _channel_add(chan, PyInterpreterState_GetID(interp), data); PyThread_release_lock(mutex); if (res != 0) { - _PyCrossInterpreterData_Release(data); + // We may chain an exception here: + (void)_release_xid_data(data, 0); PyMem_Free(data); - return -1; + return res; } return 0; } -static PyObject * -_channel_recv(_channels *channels, int64_t id) +static int +_channel_recv(_channels *channels, int64_t id, PyObject **res) { - PyInterpreterState *interp = _get_current(); + int err; + *res = NULL; + + PyInterpreterState *interp = _get_current_interp(); if (interp == NULL) { - return NULL; + // XXX Is this always an error? + if (PyErr_Occurred()) { + return -1; + } + return 0; } // Look up the channel. PyThread_type_lock mutex = NULL; - _PyChannelState *chan = _channels_lookup(channels, id, &mutex); - if (chan == NULL) { - return NULL; + _PyChannelState *chan = NULL; + err = _channels_lookup(channels, id, &mutex, &chan); + if (err != 0) { + return err; } + assert(chan != NULL); // Past this point we are responsible for releasing the mutex. // Pop off the next item from the channel. - _PyCrossInterpreterData *data = _channel_next(chan, PyInterpreterState_GetID(interp)); + _PyCrossInterpreterData *data = NULL; + err = _channel_next(chan, PyInterpreterState_GetID(interp), &data); PyThread_release_lock(mutex); - if (data == NULL) { - return NULL; + if (err != 0) { + return err; + } + else if (data == NULL) { + assert(!PyErr_Occurred()); + return 0; } // Convert the data back to an object. PyObject *obj = _PyCrossInterpreterData_NewObject(data); - _PyCrossInterpreterData_Release(data); - PyMem_Free(data); if (obj == NULL) { - return NULL; + assert(PyErr_Occurred()); + (void)_release_xid_data(data, 1); + PyMem_Free(data); + return -1; + } + int release_res = _release_xid_data(data, 0); + PyMem_Free(data); + if (release_res < 0) { + // The source interpreter has been destroyed already. + assert(PyErr_Occurred()); + Py_DECREF(obj); + return -1; } - return obj; + *res = obj; + return 0; } static int _channel_drop(_channels *channels, int64_t id, int send, int recv) { - PyInterpreterState *interp = _get_current(); + PyInterpreterState *interp = _get_current_interp(); if (interp == NULL) { return -1; } // Look up the channel. PyThread_type_lock mutex = NULL; - _PyChannelState *chan = _channels_lookup(channels, id, &mutex); - if (chan == NULL) { - return -1; + _PyChannelState *chan = NULL; + int err = _channels_lookup(channels, id, &mutex, &chan); + if (err != 0) { + return err; } // Past this point we are responsible for releasing the mutex. @@ -1395,12 +1645,13 @@ static int _channel_is_associated(_channels *channels, int64_t cid, int64_t interp, int send) { - _PyChannelState *chan = _channels_lookup(channels, cid, NULL); - if (chan == NULL) { - return -1; - } else if (send && chan->closing != NULL) { - PyErr_Format(ChannelClosedError, "channel %" PRId64 " closed", cid); - return -1; + _PyChannelState *chan = NULL; + int err = _channels_lookup(channels, cid, NULL, &chan); + if (err != 0) { + return err; + } + else if (send && chan->closing != NULL) { + return ERR_CHANNEL_CLOSED; } _channelend *end = _channelend_find(send ? chan->ends->send : chan->ends->recv, @@ -1411,8 +1662,6 @@ _channel_is_associated(_channels *channels, int64_t cid, int64_t interp, /* ChannelID class */ -static PyTypeObject ChannelIDtype; - typedef struct channelid { PyObject_HEAD int64_t id; @@ -1421,11 +1670,19 @@ typedef struct channelid { _channels *channels; } channelid; +struct channel_id_converter_data { + PyObject *module; + int64_t cid; +}; + static int channel_id_converter(PyObject *arg, void *ptr) { int64_t cid; - if (PyObject_TypeCheck(arg, &ChannelIDtype)) { + struct channel_id_converter_data *data = ptr; + module_state *state = get_module_state(data->module); + assert(state != NULL); + if (PyObject_TypeCheck(arg, state->ChannelIDType)) { cid = ((channelid *)arg)->id; } else if (PyIndex_Check(arg)) { @@ -1445,51 +1702,62 @@ channel_id_converter(PyObject *arg, void *ptr) Py_TYPE(arg)->tp_name); return 0; } - *(int64_t *)ptr = cid; + data->cid = cid; return 1; } -static channelid * +static int newchannelid(PyTypeObject *cls, int64_t cid, int end, _channels *channels, - int force, int resolve) + int force, int resolve, channelid **res) { + *res = NULL; + channelid *self = PyObject_New(channelid, cls); if (self == NULL) { - return NULL; + return -1; } self->id = cid; self->end = end; self->resolve = resolve; self->channels = channels; - if (_channels_add_id_object(channels, cid) != 0) { - if (force && PyErr_ExceptionMatches(ChannelNotFoundError)) { - PyErr_Clear(); + int err = _channels_add_id_object(channels, cid); + if (err != 0) { + if (force && err == ERR_CHANNEL_NOT_FOUND) { + assert(!PyErr_Occurred()); } else { Py_DECREF((PyObject *)self); - return NULL; + return err; } } - return self; + *res = self; + return 0; } static _channels * _global_channels(void); static PyObject * -channelid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds) +_channelid_new(PyObject *mod, PyTypeObject *cls, + PyObject *args, PyObject *kwds) { static char *kwlist[] = {"id", "send", "recv", "force", "_resolve", NULL}; int64_t cid; + struct channel_id_converter_data cid_data = { + .module = mod, + }; int send = -1; int recv = -1; int force = 0; int resolve = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&|$pppp:ChannelID.__new__", kwlist, - channel_id_converter, &cid, &send, &recv, &force, &resolve)) + channel_id_converter, &cid_data, + &send, &recv, &force, &resolve)) { return NULL; + } + cid = cid_data.cid; // Handle "send" and "recv". if (send == 0 && recv == 0) { @@ -1508,16 +1776,33 @@ channelid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds) end = CHANNEL_RECV; } - return (PyObject *)newchannelid(cls, cid, end, _global_channels(), - force, resolve); + PyObject *id = NULL; + int err = newchannelid(cls, cid, end, _global_channels(), + force, resolve, + (channelid **)&id); + if (handle_channel_error(err, mod, cid)) { + assert(id == NULL); + return NULL; + } + assert(id != NULL); + return id; } static void -channelid_dealloc(PyObject *v) +channelid_dealloc(PyObject *self) { - int64_t cid = ((channelid *)v)->id; - _channels *channels = ((channelid *)v)->channels; - Py_TYPE(v)->tp_free(v); + int64_t cid = ((channelid *)self)->id; + _channels *channels = ((channelid *)self)->channels; + + PyTypeObject *tp = Py_TYPE(self); + tp->tp_free(self); + /* "Instances of heap-allocated types hold a reference to their type." + * See: https://docs.python.org/3.11/howto/isolating-extensions.html#garbage-collection-protocol + * See: https://docs.python.org/3.11/c-api/typeobj.html#c.PyTypeObject.tp_traverse + */ + // XXX Why don't we implement Py_TPFLAGS_HAVE_GC, e.g. Py_tp_traverse, + // like we do for _abc._abc_data? + Py_DECREF(tp); _channels_drop_id_object(channels, cid); } @@ -1556,46 +1841,6 @@ channelid_int(PyObject *self) return PyLong_FromLongLong(cid->id); } -static PyNumberMethods channelid_as_number = { - 0, /* nb_add */ - 0, /* nb_subtract */ - 0, /* nb_multiply */ - 0, /* nb_remainder */ - 0, /* nb_divmod */ - 0, /* nb_power */ - 0, /* nb_negative */ - 0, /* nb_positive */ - 0, /* nb_absolute */ - 0, /* nb_bool */ - 0, /* nb_invert */ - 0, /* nb_lshift */ - 0, /* nb_rshift */ - 0, /* nb_and */ - 0, /* nb_xor */ - 0, /* nb_or */ - (unaryfunc)channelid_int, /* nb_int */ - 0, /* nb_reserved */ - 0, /* nb_float */ - - 0, /* nb_inplace_add */ - 0, /* nb_inplace_subtract */ - 0, /* nb_inplace_multiply */ - 0, /* nb_inplace_remainder */ - 0, /* nb_inplace_power */ - 0, /* nb_inplace_lshift */ - 0, /* nb_inplace_rshift */ - 0, /* nb_inplace_and */ - 0, /* nb_inplace_xor */ - 0, /* nb_inplace_or */ - - 0, /* nb_floor_divide */ - 0, /* nb_true_divide */ - 0, /* nb_inplace_floor_divide */ - 0, /* nb_inplace_true_divide */ - - (unaryfunc)channelid_int, /* nb_index */ -}; - static Py_hash_t channelid_hash(PyObject *self) { @@ -1612,17 +1857,28 @@ channelid_hash(PyObject *self) static PyObject * channelid_richcompare(PyObject *self, PyObject *other, int op) { + PyObject *res = NULL; if (op != Py_EQ && op != Py_NE) { Py_RETURN_NOTIMPLEMENTED; } - if (!PyObject_TypeCheck(self, &ChannelIDtype)) { - Py_RETURN_NOTIMPLEMENTED; + PyObject *mod = get_module_from_type(Py_TYPE(self)); + if (mod == NULL) { + return NULL; + } + module_state *state = get_module_state(mod); + if (state == NULL) { + goto done; + } + + if (!PyObject_TypeCheck(self, state->ChannelIDType)) { + res = Py_NewRef(Py_NotImplemented); + goto done; } channelid *cid = (channelid *)self; int equal; - if (PyObject_TypeCheck(other, &ChannelIDtype)) { + if (PyObject_TypeCheck(other, state->ChannelIDType)) { channelid *othercid = (channelid *)other; equal = (cid->end == othercid->end) && (cid->id == othercid->id); } @@ -1631,27 +1887,34 @@ channelid_richcompare(PyObject *self, PyObject *other, int op) int overflow; long long othercid = PyLong_AsLongLongAndOverflow(other, &overflow); if (othercid == -1 && PyErr_Occurred()) { - return NULL; + goto done; } equal = !overflow && (othercid >= 0) && (cid->id == othercid); } else if (PyNumber_Check(other)) { PyObject *pyid = PyLong_FromLongLong(cid->id); if (pyid == NULL) { - return NULL; + goto done; } - PyObject *res = PyObject_RichCompare(pyid, other, op); + res = PyObject_RichCompare(pyid, other, op); Py_DECREF(pyid); - return res; + goto done; } else { - Py_RETURN_NOTIMPLEMENTED; + res = Py_NewRef(Py_NotImplemented); + goto done; } if ((op == Py_EQ && equal) || (op == Py_NE && !equal)) { - Py_RETURN_TRUE; + res = Py_NewRef(Py_True); } - Py_RETURN_FALSE; + else { + res = Py_NewRef(Py_False); + } + +done: + Py_DECREF(mod); + return res; } static PyObject * @@ -1690,41 +1953,63 @@ static PyObject * _channelid_from_xid(_PyCrossInterpreterData *data) { struct _channelid_xid *xid = (struct _channelid_xid *)data->data; + + PyObject *mod = _get_current_module(); + if (mod == NULL) { + return NULL; + } + module_state *state = get_module_state(mod); + if (state == NULL) { + return NULL; + } + // Note that we do not preserve the "resolve" flag. - PyObject *cid = (PyObject *)newchannelid(&ChannelIDtype, xid->id, xid->end, - _global_channels(), 0, 0); + PyObject *cid = NULL; + int err = newchannelid(state->ChannelIDType, xid->id, xid->end, + _global_channels(), 0, 0, + (channelid **)&cid); + if (err != 0) { + assert(cid == NULL); + (void)handle_channel_error(err, mod, xid->id); + goto done; + } + assert(cid != NULL); if (xid->end == 0) { - return cid; + goto done; } if (!xid->resolve) { - return cid; + goto done; } /* Try returning a high-level channel end but fall back to the ID. */ PyObject *chan = _channel_from_cid(cid, xid->end); if (chan == NULL) { PyErr_Clear(); - return cid; + goto done; } Py_DECREF(cid); - return chan; + cid = chan; + +done: + Py_DECREF(mod); + return cid; } static int -_channelid_shared(PyObject *obj, _PyCrossInterpreterData *data) -{ - struct _channelid_xid *xid = PyMem_NEW(struct _channelid_xid, 1); - if (xid == NULL) { +_channelid_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) +{ + if (_PyCrossInterpreterData_InitWithSize( + data, tstate->interp, sizeof(struct _channelid_xid), obj, + _channelid_from_xid + ) < 0) + { return -1; } + struct _channelid_xid *xid = (struct _channelid_xid *)data->data; xid->id = ((channelid *)obj)->id; xid->end = ((channelid *)obj)->end; xid->resolve = ((channelid *)obj)->resolve; - - data->data = xid; - data->obj = Py_NewRef(obj); - data->new_object = _channelid_from_xid; - data->free = PyMem_Free; return 0; } @@ -1734,8 +2019,22 @@ channelid_end(PyObject *self, void *end) int force = 1; channelid *cid = (channelid *)self; if (end != NULL) { - return (PyObject *)newchannelid(Py_TYPE(self), cid->id, *(int *)end, - cid->channels, force, cid->resolve); + PyObject *id = NULL; + int err = newchannelid(Py_TYPE(self), cid->id, *(int *)end, + cid->channels, force, cid->resolve, + (channelid **)&id); + if (err != 0) { + assert(id == NULL); + PyObject *mod = get_module_from_type(Py_TYPE(self)); + if (mod == NULL) { + return NULL; + } + (void)handle_channel_error(err, mod, cid->id); + Py_DECREF(mod); + return NULL; + } + assert(id != NULL); + return id; } if (cid->end == CHANNEL_SEND) { @@ -1763,66 +2062,52 @@ static PyGetSetDef channelid_getsets[] = { PyDoc_STRVAR(channelid_doc, "A channel ID identifies a channel and may be used as an int."); -static PyTypeObject ChannelIDtype = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - "_xxsubinterpreters.ChannelID", /* tp_name */ - sizeof(channelid), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor)channelid_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - (reprfunc)channelid_repr, /* tp_repr */ - &channelid_as_number, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - channelid_hash, /* tp_hash */ - 0, /* tp_call */ - (reprfunc)channelid_str, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - // Use Py_TPFLAGS_DISALLOW_INSTANTIATION so the type cannot be instantiated - // from Python code. We do this because there is a strong relationship - // between channel IDs and the channel lifecycle, so this limitation avoids - // related complications. Use the _channel_id() function instead. - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE - | Py_TPFLAGS_DISALLOW_INSTANTIATION, /* tp_flags */ - channelid_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - channelid_richcompare, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - channelid_getsets, /* tp_getset */ +static PyType_Slot ChannelIDType_slots[] = { + {Py_tp_dealloc, (destructor)channelid_dealloc}, + {Py_tp_doc, (void *)channelid_doc}, + {Py_tp_repr, (reprfunc)channelid_repr}, + {Py_tp_str, (reprfunc)channelid_str}, + {Py_tp_hash, channelid_hash}, + {Py_tp_richcompare, channelid_richcompare}, + {Py_tp_getset, channelid_getsets}, + // number slots + {Py_nb_int, (unaryfunc)channelid_int}, + {Py_nb_index, (unaryfunc)channelid_int}, + {0, NULL}, }; +static PyType_Spec ChannelIDType_spec = { + .name = "_xxsubinterpreters.ChannelID", + .basicsize = sizeof(channelid), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE), + .slots = ChannelIDType_slots, +}; -/* interpreter-specific code ************************************************/ -static PyObject * RunFailedError = NULL; +/* interpreter-specific code ************************************************/ static int -interp_exceptions_init(PyObject *ns) +interp_exceptions_init(PyObject *mod) { - // XXX Move the exceptions into per-module memory? - - if (RunFailedError == NULL) { - // An uncaught exception came out of interp_run_string(). - RunFailedError = PyErr_NewException("_xxsubinterpreters.RunFailedError", - PyExc_RuntimeError, NULL); - if (RunFailedError == NULL) { - return -1; - } - if (PyDict_SetItemString(ns, "RunFailedError", RunFailedError) != 0) { - return -1; - } + module_state *state = get_module_state(mod); + if (state == NULL) { + return -1; } +#define ADD(NAME, BASE) \ + do { \ + assert(state->NAME == NULL); \ + state->NAME = ADD_NEW_EXCEPTION(mod, NAME, BASE); \ + if (state->NAME == NULL) { \ + return -1; \ + } \ + } while (0) + + // An uncaught exception came out of interp_run_string(). + ADD(RunFailedError, PyExc_RuntimeError); +#undef ADD + return 0; } @@ -1860,12 +2145,24 @@ _ensure_not_running(PyInterpreterState *interp) static int _run_script(PyInterpreterState *interp, const char *codestr, - _sharedns *shared, _sharedexception **exc) + _sharedns *shared, int needs_import, + _sharedexception **exc) { PyObject *exctype = NULL; PyObject *excval = NULL; PyObject *tb = NULL; + if (needs_import) { + // It might not have been imported yet in the current interpreter. + // However, it will (almost) always have been imported already + // in the main interpreter. + PyObject *mod = PyImport_ImportModule(MODULE_NAME); + if (mod == NULL) { + goto error; + } + Py_DECREF(mod); + } + PyObject *main_mod = _PyInterpreterState_GetMainModule(interp); if (main_mod == NULL) { goto error; @@ -1918,14 +2215,17 @@ _run_script(PyInterpreterState *interp, const char *codestr, } static int -_run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, - PyObject *shareables) +_run_script_in_interpreter(PyObject *mod, PyInterpreterState *interp, + const char *codestr, PyObject *shareables) { if (_ensure_not_running(interp) < 0) { return -1; } + module_state *state = get_module_state(mod); - _sharedns *shared = _get_shared_ns(shareables); + int needs_import = 0; + _sharedns *shared = _get_shared_ns(shareables, state->ChannelIDType, + &needs_import); if (shared == NULL && PyErr_Occurred()) { return -1; } @@ -1941,7 +2241,7 @@ _run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, // Run the script. _sharedexception *exc = NULL; - int result = _run_script(interp, codestr, shared, &exc); + int result = _run_script(interp, codestr, shared, needs_import, &exc); // Switch back. if (save_tstate != NULL) { @@ -1950,7 +2250,8 @@ _run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, // Propagate any exception out to the caller. if (exc != NULL) { - _sharedexception_apply(exc, RunFailedError); + assert(state != NULL); + _sharedexception_apply(exc, state->RunFailedError); _sharedexception_free(exc); } else if (result != 0) { @@ -1972,18 +2273,41 @@ _run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, the data that we need to share between interpreters, so it cannot hold PyObject values. */ static struct globals { + int module_count; _channels channels; -} _globals = {{0}}; +} _globals = {0}; static int -_init_globals(void) +_globals_init(void) { - if (_channels_init(&_globals.channels) != 0) { - return -1; + // XXX This isn't thread-safe. + _globals.module_count++; + if (_globals.module_count > 1) { + // Already initialized. + return 0; + } + + assert(_globals.channels.mutex == NULL); + PyThread_type_lock mutex = PyThread_allocate_lock(); + if (mutex == NULL) { + return ERR_CHANNELS_MUTEX_INIT; } + _channels_init(&_globals.channels, mutex); return 0; } +static void +_globals_fini(void) +{ + // XXX This isn't thread-safe. + _globals.module_count--; + if (_globals.module_count > 0) { + return; + } + + _channels_fini(&_globals.channels); +} + static _channels * _global_channels(void) { return &_globals.channels; @@ -2052,7 +2376,7 @@ interp_destroy(PyObject *self, PyObject *args, PyObject *kwds) } // Ensure we don't try to destroy the current interpreter. - PyInterpreterState *current = _get_current(); + PyInterpreterState *current = _get_current_interp(); if (current == NULL) { return NULL; } @@ -2129,7 +2453,7 @@ Return a list containing the ID of every existing interpreter."); static PyObject * interp_get_current(PyObject *self, PyObject *Py_UNUSED(ignored)) { - PyInterpreterState *interp =_get_current(); + PyInterpreterState *interp =_get_current_interp(); if (interp == NULL) { return NULL; } @@ -2187,7 +2511,7 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) } // Run the code in the interpreter. - if (_run_script_in_interpreter(interp, codestr, shared) != 0) { + if (_run_script_in_interpreter(self, interp, codestr, shared) != 0) { return NULL; } Py_RETURN_NONE; @@ -2259,16 +2583,26 @@ channel_create(PyObject *self, PyObject *Py_UNUSED(ignored)) { int64_t cid = _channel_create(&_globals.channels); if (cid < 0) { + (void)handle_channel_error(cid, self, -1); return NULL; } - PyObject *id = (PyObject *)newchannelid(&ChannelIDtype, cid, 0, - &_globals.channels, 0, 0); - if (id == NULL) { - if (_channel_destroy(&_globals.channels, cid) != 0) { + module_state *state = get_module_state(self); + if (state == NULL) { + return NULL; + } + PyObject *id = NULL; + int err = newchannelid(state->ChannelIDType, cid, 0, + &_globals.channels, 0, 0, + (channelid **)&id); + if (handle_channel_error(err, self, cid)) { + assert(id == NULL); + err = _channel_destroy(&_globals.channels, cid); + if (handle_channel_error(err, self, cid)) { // XXX issue a warning? } return NULL; } + assert(id != NULL); assert(((channelid *)id)->channels != NULL); return id; } @@ -2283,12 +2617,17 @@ channel_destroy(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"cid", NULL}; int64_t cid; + struct channel_id_converter_data cid_data = { + .module = self, + }; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&:channel_destroy", kwlist, - channel_id_converter, &cid)) { + channel_id_converter, &cid_data)) { return NULL; } + cid = cid_data.cid; - if (_channel_destroy(&_globals.channels, cid) != 0) { + int err = _channel_destroy(&_globals.channels, cid); + if (handle_channel_error(err, self, cid)) { return NULL; } Py_RETURN_NONE; @@ -2315,14 +2654,24 @@ channel_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) if (ids == NULL) { goto finally; } + module_state *state = get_module_state(self); + if (state == NULL) { + Py_DECREF(ids); + ids = NULL; + goto finally; + } int64_t *cur = cids; for (int64_t i=0; i < count; cur++, i++) { - PyObject *id = (PyObject *)newchannelid(&ChannelIDtype, *cur, 0, - &_globals.channels, 0, 0); - if (id == NULL) { + PyObject *id = NULL; + int err = newchannelid(state->ChannelIDType, *cur, 0, + &_globals.channels, 0, 0, + (channelid **)&id); + if (handle_channel_error(err, self, *cur)) { + assert(id == NULL); Py_SETREF(ids, NULL); break; } + assert(id != NULL); PyList_SET_ITEM(ids, (Py_ssize_t)i, id); } @@ -2341,6 +2690,9 @@ channel_list_interpreters(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"cid", "send", NULL}; int64_t cid; /* Channel ID */ + struct channel_id_converter_data cid_data = { + .module = self, + }; int send = 0; /* Send or receive end? */ int64_t id; PyObject *ids, *id_obj; @@ -2348,9 +2700,10 @@ channel_list_interpreters(PyObject *self, PyObject *args, PyObject *kwds) if (!PyArg_ParseTupleAndKeywords( args, kwds, "O&$p:channel_list_interpreters", - kwlist, channel_id_converter, &cid, &send)) { + kwlist, channel_id_converter, &cid_data, &send)) { return NULL; } + cid = cid_data.cid; ids = PyList_New(0); if (ids == NULL) { @@ -2363,6 +2716,7 @@ channel_list_interpreters(PyObject *self, PyObject *args, PyObject *kwds) assert(id >= 0); int res = _channel_is_associated(&_globals.channels, cid, id, send); if (res < 0) { + (void)handle_channel_error(res, self, cid); goto except; } if (res) { @@ -2402,13 +2756,18 @@ channel_send(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"cid", "obj", NULL}; int64_t cid; + struct channel_id_converter_data cid_data = { + .module = self, + }; PyObject *obj; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&O:channel_send", kwlist, - channel_id_converter, &cid, &obj)) { + channel_id_converter, &cid_data, &obj)) { return NULL; } + cid = cid_data.cid; - if (_channel_send(&_globals.channels, cid, obj) != 0) { + int err = _channel_send(&_globals.channels, cid, obj); + if (handle_channel_error(err, self, cid)) { return NULL; } Py_RETURN_NONE; @@ -2424,26 +2783,32 @@ channel_recv(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"cid", "default", NULL}; int64_t cid; + struct channel_id_converter_data cid_data = { + .module = self, + }; PyObject *dflt = NULL; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&|O:channel_recv", kwlist, - channel_id_converter, &cid, &dflt)) { + channel_id_converter, &cid_data, &dflt)) { return NULL; } - Py_XINCREF(dflt); + cid = cid_data.cid; - PyObject *obj = _channel_recv(&_globals.channels, cid); - if (obj != NULL) { - Py_XDECREF(dflt); - return obj; - } else if (PyErr_Occurred()) { - Py_XDECREF(dflt); - return NULL; - } else if (dflt != NULL) { - return dflt; - } else { - PyErr_Format(ChannelEmptyError, "channel %" PRId64 " is empty", cid); + PyObject *obj = NULL; + int err = _channel_recv(&_globals.channels, cid, &obj); + if (handle_channel_error(err, self, cid)) { return NULL; } + Py_XINCREF(dflt); + if (obj == NULL) { + // Use the default. + if (dflt == NULL) { + (void)handle_channel_error(ERR_CHANNEL_EMPTY, self, cid); + return NULL; + } + obj = Py_NewRef(dflt); + } + Py_XDECREF(dflt); + return obj; } PyDoc_STRVAR(channel_recv_doc, @@ -2459,16 +2824,22 @@ channel_close(PyObject *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"cid", "send", "recv", "force", NULL}; int64_t cid; + struct channel_id_converter_data cid_data = { + .module = self, + }; int send = 0; int recv = 0; int force = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&|$ppp:channel_close", kwlist, - channel_id_converter, &cid, &send, &recv, &force)) { + channel_id_converter, &cid_data, + &send, &recv, &force)) { return NULL; } + cid = cid_data.cid; - if (_channel_close(&_globals.channels, cid, send-recv, force) != 0) { + int err = _channel_close(&_globals.channels, cid, send-recv, force); + if (handle_channel_error(err, self, cid)) { return NULL; } Py_RETURN_NONE; @@ -2507,14 +2878,19 @@ channel_release(PyObject *self, PyObject *args, PyObject *kwds) // Note that only the current interpreter is affected. static char *kwlist[] = {"cid", "send", "recv", "force", NULL}; int64_t cid; + struct channel_id_converter_data cid_data = { + .module = self, + }; int send = 0; int recv = 0; int force = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&|$ppp:channel_release", kwlist, - channel_id_converter, &cid, &send, &recv, &force)) { + channel_id_converter, &cid_data, + &send, &recv, &force)) { return NULL; } + cid = cid_data.cid; if (send == 0 && recv == 0) { send = 1; recv = 1; @@ -2523,7 +2899,8 @@ channel_release(PyObject *self, PyObject *args, PyObject *kwds) // XXX Handle force is True. // XXX Fix implicit release. - if (_channel_drop(&_globals.channels, cid, send, recv) != 0) { + int err = _channel_drop(&_globals.channels, cid, send, recv); + if (handle_channel_error(err, self, cid)) { return NULL; } Py_RETURN_NONE; @@ -2539,7 +2916,18 @@ ends are closed. Closing an already closed end is a noop."); static PyObject * channel__channel_id(PyObject *self, PyObject *args, PyObject *kwds) { - return channelid_new(&ChannelIDtype, args, kwds); + module_state *state = get_module_state(self); + if (state == NULL) { + return NULL; + } + PyTypeObject *cls = state->ChannelIDType; + PyObject *mod = get_module_from_owned_type(cls); + if (mod == NULL) { + return NULL; + } + PyObject *cid = _channelid_new(mod, cls, args, kwds); + Py_DECREF(mod); + return cid; } static PyMethodDef module_functions[] = { @@ -2590,59 +2978,90 @@ PyDoc_STRVAR(module_doc, "This module provides primitive operations to manage Python interpreters.\n\ The 'interpreters' module provides a more convenient interface."); -static struct PyModuleDef interpretersmodule = { - PyModuleDef_HEAD_INIT, - "_xxsubinterpreters", /* m_name */ - module_doc, /* m_doc */ - -1, /* m_size */ - module_functions, /* m_methods */ - NULL, /* m_slots */ - NULL, /* m_traverse */ - NULL, /* m_clear */ - NULL /* m_free */ -}; - - -PyMODINIT_FUNC -PyInit__xxsubinterpreters(void) +static int +module_exec(PyObject *mod) { - if (_init_globals() != 0) { - return NULL; - } - - /* Initialize types */ - if (PyType_Ready(&ChannelIDtype) != 0) { - return NULL; - } - - /* Create the module */ - PyObject *module = PyModule_Create(&interpretersmodule); - if (module == NULL) { - return NULL; + if (_globals_init() != 0) { + return -1; } /* Add exception types */ - PyObject *ns = PyModule_GetDict(module); // borrowed - if (interp_exceptions_init(ns) != 0) { - return NULL; + if (interp_exceptions_init(mod) != 0) { + goto error; } - if (channel_exceptions_init(ns) != 0) { - return NULL; + if (channel_exceptions_init(mod) != 0) { + goto error; } /* Add other types */ - if (PyDict_SetItemString(ns, "ChannelID", - Py_NewRef(&ChannelIDtype)) != 0) { - return NULL; - } - if (PyDict_SetItemString(ns, "InterpreterID", - Py_NewRef(&_PyInterpreterID_Type)) != 0) { - return NULL; + module_state *state = get_module_state(mod); + + // ChannelID + state->ChannelIDType = add_new_type( + mod, &ChannelIDType_spec, _channelid_shared); + if (state->ChannelIDType == NULL) { + goto error; } - if (_PyCrossInterpreterData_RegisterClass(&ChannelIDtype, _channelid_shared)) { - return NULL; + // PyInterpreterID + if (PyModule_AddType(mod, &_PyInterpreterID_Type) < 0) { + goto error; } - return module; + return 0; + +error: + (void)_PyCrossInterpreterData_UnregisterClass(state->ChannelIDType); + _globals_fini(); + return -1; +} + +static struct PyModuleDef_Slot module_slots[] = { + {Py_mod_exec, module_exec}, + {0, NULL}, +}; + +static int +module_traverse(PyObject *mod, visitproc visit, void *arg) +{ + module_state *state = get_module_state(mod); + assert(state != NULL); + traverse_module_state(state, visit, arg); + return 0; +} + +static int +module_clear(PyObject *mod) +{ + module_state *state = get_module_state(mod); + assert(state != NULL); + clear_module_state(state); + return 0; +} + +static void +module_free(void *mod) +{ + module_state *state = get_module_state(mod); + assert(state != NULL); + clear_module_state(state); + _globals_fini(); +} + +static struct PyModuleDef moduledef = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = MODULE_NAME, + .m_doc = module_doc, + .m_size = sizeof(module_state), + .m_methods = module_functions, + .m_slots = module_slots, + .m_traverse = module_traverse, + .m_clear = module_clear, + .m_free = (freefunc)module_free, +}; + +PyMODINIT_FUNC +PyInit__xxsubinterpreters(void) +{ + return PyModuleDef_Init(&moduledef); } diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index dcf510e9788ee5..114c69a033593c 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -1773,9 +1773,9 @@ static PyObject * array_array___sizeof___impl(arrayobject *self) /*[clinic end generated code: output=d8e1c61ebbe3eaed input=805586565bf2b3c6]*/ { - Py_ssize_t res; - res = _PyObject_SIZE(Py_TYPE(self)) + self->allocated * self->ob_descr->itemsize; - return PyLong_FromSsize_t(res); + size_t res = _PyObject_SIZE(Py_TYPE(self)); + res += (size_t)self->allocated * (size_t)self->ob_descr->itemsize; + return PyLong_FromSize_t(res); } diff --git a/Modules/binascii.c b/Modules/binascii.c index ffc2c59413613b..95ddb26988d6c9 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -303,14 +303,14 @@ binascii.b2a_uu data: Py_buffer / * - backtick: bool(accept={int}) = False + backtick: bool = False Uuencode line of data. [clinic start generated code]*/ static PyObject * binascii_b2a_uu_impl(PyObject *module, Py_buffer *data, int backtick) -/*[clinic end generated code: output=b1b99de62d9bbeb8 input=b26bc8d32b6ed2f6]*/ +/*[clinic end generated code: output=b1b99de62d9bbeb8 input=beb27822241095cd]*/ { unsigned char *ascii_data; const unsigned char *bin_data; @@ -375,7 +375,7 @@ binascii.a2b_base64 data: ascii_buffer / * - strict_mode: bool(accept={int}) = False + strict_mode: bool = False Decode a line of base64 data. @@ -386,7 +386,7 @@ Decode a line of base64 data. static PyObject * binascii_a2b_base64_impl(PyObject *module, Py_buffer *data, int strict_mode) -/*[clinic end generated code: output=5409557788d4f975 input=3a30c4e3528317c6]*/ +/*[clinic end generated code: output=5409557788d4f975 input=c0c15fd0f8f9a62d]*/ { assert(data->len >= 0); @@ -521,14 +521,14 @@ binascii.b2a_base64 data: Py_buffer / * - newline: bool(accept={int}) = True + newline: bool = True Base64-code line of data. [clinic start generated code]*/ static PyObject * binascii_b2a_base64_impl(PyObject *module, Py_buffer *data, int newline) -/*[clinic end generated code: output=4ad62c8e8485d3b3 input=6083dac5777fa45d]*/ +/*[clinic end generated code: output=4ad62c8e8485d3b3 input=0e20ff59c5f2e3e1]*/ { unsigned char *ascii_data; const unsigned char *bin_data; @@ -952,14 +952,14 @@ binascii_unhexlify_impl(PyObject *module, Py_buffer *hexstr) binascii.a2b_qp data: ascii_buffer - header: bool(accept={int}) = False + header: bool = False Decode a string of qp-encoded data. [clinic start generated code]*/ static PyObject * binascii_a2b_qp_impl(PyObject *module, Py_buffer *data, int header) -/*[clinic end generated code: output=e99f7846cfb9bc53 input=bf6766fea76cce8f]*/ +/*[clinic end generated code: output=e99f7846cfb9bc53 input=bdfb31598d4e47b9]*/ { Py_ssize_t in, out; char ch; @@ -1048,9 +1048,9 @@ to_hex (unsigned char ch, unsigned char *s) binascii.b2a_qp data: Py_buffer - quotetabs: bool(accept={int}) = False - istext: bool(accept={int}) = True - header: bool(accept={int}) = False + quotetabs: bool = False + istext: bool = True + header: bool = False Encode a string using quoted-printable encoding. @@ -1062,7 +1062,7 @@ are both encoded. When quotetabs is set, space and tabs are encoded. static PyObject * binascii_b2a_qp_impl(PyObject *module, Py_buffer *data, int quotetabs, int istext, int header) -/*[clinic end generated code: output=e9884472ebb1a94c input=21fb7eea4a184ba6]*/ +/*[clinic end generated code: output=e9884472ebb1a94c input=e9102879afb0defd]*/ { Py_ssize_t in, out; const unsigned char *databuf; diff --git a/Modules/cjkcodecs/clinic/multibytecodec.c.h b/Modules/cjkcodecs/clinic/multibytecodec.c.h index b7e340e68796fa..1b41c231eac5d8 100644 --- a/Modules/cjkcodecs/clinic/multibytecodec.c.h +++ b/Modules/cjkcodecs/clinic/multibytecodec.c.h @@ -246,8 +246,8 @@ _multibytecodec_MultibyteIncrementalEncoder_encode(MultibyteIncrementalEncoderOb if (!noptargs) { goto skip_optional_pos; } - final = _PyLong_AsInt(args[1]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[1]); + if (final < 0) { goto exit; } skip_optional_pos: @@ -381,8 +381,8 @@ _multibytecodec_MultibyteIncrementalDecoder_decode(MultibyteIncrementalDecoderOb if (!noptargs) { goto skip_optional_pos; } - final = _PyLong_AsInt(args[1]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[1]); + if (final < 0) { goto exit; } skip_optional_pos: @@ -690,4 +690,4 @@ PyDoc_STRVAR(_multibytecodec___create_codec__doc__, #define _MULTIBYTECODEC___CREATE_CODEC_METHODDEF \ {"__create_codec", (PyCFunction)_multibytecodec___create_codec, METH_O, _multibytecodec___create_codec__doc__}, -/*[clinic end generated code: output=b034ec7126c11bde input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5f0e8dacddb0ac76 input=a9049054013a1b77]*/ diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index 1d77fd33ac3b87..8564494f6262fb 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -893,14 +893,14 @@ decoder_feed_buffer(MultibyteStatefulDecoderContext *ctx, _multibytecodec.MultibyteIncrementalEncoder.encode input: object - final: bool(accept={int}) = False + final: bool = False [clinic start generated code]*/ static PyObject * _multibytecodec_MultibyteIncrementalEncoder_encode_impl(MultibyteIncrementalEncoderObject *self, PyObject *input, int final) -/*[clinic end generated code: output=123361b6c505e2c1 input=093a1ddbb2fc6721]*/ +/*[clinic end generated code: output=123361b6c505e2c1 input=bd5f7d40d43e99b0]*/ { return encoder_encode_stateful(STATEFUL_ECTX(self), input, final); } @@ -1114,14 +1114,14 @@ static PyType_Spec encoder_spec = { _multibytecodec.MultibyteIncrementalDecoder.decode input: Py_buffer - final: bool(accept={int}) = False + final: bool = False [clinic start generated code]*/ static PyObject * _multibytecodec_MultibyteIncrementalDecoder_decode_impl(MultibyteIncrementalDecoderObject *self, Py_buffer *input, int final) -/*[clinic end generated code: output=b9b9090e8a9ce2ba input=c9132b24d503eb1d]*/ +/*[clinic end generated code: output=b9b9090e8a9ce2ba input=8795fbb20860027a]*/ { MultibyteDecodeBuffer buf; char *data, *wdata = NULL; diff --git a/Modules/clinic/_asynciomodule.c.h b/Modules/clinic/_asynciomodule.c.h index ddec54c8d7c2bc..11db478a8b4827 100644 --- a/Modules/clinic/_asynciomodule.c.h +++ b/Modules/clinic/_asynciomodule.c.h @@ -112,15 +112,19 @@ PyDoc_STRVAR(_asyncio_Future_exception__doc__, "InvalidStateError."); #define _ASYNCIO_FUTURE_EXCEPTION_METHODDEF \ - {"exception", (PyCFunction)_asyncio_Future_exception, METH_NOARGS, _asyncio_Future_exception__doc__}, + {"exception", _PyCFunction_CAST(_asyncio_Future_exception), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _asyncio_Future_exception__doc__}, static PyObject * -_asyncio_Future_exception_impl(FutureObj *self); +_asyncio_Future_exception_impl(FutureObj *self, PyTypeObject *cls); static PyObject * -_asyncio_Future_exception(FutureObj *self, PyObject *Py_UNUSED(ignored)) +_asyncio_Future_exception(FutureObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - return _asyncio_Future_exception_impl(self); + if (nargs) { + PyErr_SetString(PyExc_TypeError, "exception() takes no arguments"); + return NULL; + } + return _asyncio_Future_exception_impl(self, cls); } PyDoc_STRVAR(_asyncio_Future_set_result__doc__, @@ -133,7 +137,42 @@ PyDoc_STRVAR(_asyncio_Future_set_result__doc__, "InvalidStateError."); #define _ASYNCIO_FUTURE_SET_RESULT_METHODDEF \ - {"set_result", (PyCFunction)_asyncio_Future_set_result, METH_O, _asyncio_Future_set_result__doc__}, + {"set_result", _PyCFunction_CAST(_asyncio_Future_set_result), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _asyncio_Future_set_result__doc__}, + +static PyObject * +_asyncio_Future_set_result_impl(FutureObj *self, PyTypeObject *cls, + PyObject *result); + +static PyObject * +_asyncio_Future_set_result(FutureObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "set_result", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *result; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + result = args[0]; + return_value = _asyncio_Future_set_result_impl(self, cls, result); + +exit: + return return_value; +} PyDoc_STRVAR(_asyncio_Future_set_exception__doc__, "set_exception($self, exception, /)\n" @@ -145,7 +184,42 @@ PyDoc_STRVAR(_asyncio_Future_set_exception__doc__, "InvalidStateError."); #define _ASYNCIO_FUTURE_SET_EXCEPTION_METHODDEF \ - {"set_exception", (PyCFunction)_asyncio_Future_set_exception, METH_O, _asyncio_Future_set_exception__doc__}, + {"set_exception", _PyCFunction_CAST(_asyncio_Future_set_exception), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _asyncio_Future_set_exception__doc__}, + +static PyObject * +_asyncio_Future_set_exception_impl(FutureObj *self, PyTypeObject *cls, + PyObject *exception); + +static PyObject * +_asyncio_Future_set_exception(FutureObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "set_exception", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *exception; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + exception = args[0]; + return_value = _asyncio_Future_set_exception_impl(self, cls, exception); + +exit: + return return_value; +} PyDoc_STRVAR(_asyncio_Future_add_done_callback__doc__, "add_done_callback($self, fn, /, *, context=)\n" @@ -158,14 +232,14 @@ PyDoc_STRVAR(_asyncio_Future_add_done_callback__doc__, "scheduled with call_soon."); #define _ASYNCIO_FUTURE_ADD_DONE_CALLBACK_METHODDEF \ - {"add_done_callback", _PyCFunction_CAST(_asyncio_Future_add_done_callback), METH_FASTCALL|METH_KEYWORDS, _asyncio_Future_add_done_callback__doc__}, + {"add_done_callback", _PyCFunction_CAST(_asyncio_Future_add_done_callback), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _asyncio_Future_add_done_callback__doc__}, static PyObject * -_asyncio_Future_add_done_callback_impl(FutureObj *self, PyObject *fn, - PyObject *context); +_asyncio_Future_add_done_callback_impl(FutureObj *self, PyTypeObject *cls, + PyObject *fn, PyObject *context); static PyObject * -_asyncio_Future_add_done_callback(FutureObj *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_asyncio_Future_add_done_callback(FutureObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -208,7 +282,7 @@ _asyncio_Future_add_done_callback(FutureObj *self, PyObject *const *args, Py_ssi } context = args[1]; skip_optional_kwonly: - return_value = _asyncio_Future_add_done_callback_impl(self, fn, context); + return_value = _asyncio_Future_add_done_callback_impl(self, cls, fn, context); exit: return return_value; @@ -223,7 +297,42 @@ PyDoc_STRVAR(_asyncio_Future_remove_done_callback__doc__, "Returns the number of callbacks removed."); #define _ASYNCIO_FUTURE_REMOVE_DONE_CALLBACK_METHODDEF \ - {"remove_done_callback", (PyCFunction)_asyncio_Future_remove_done_callback, METH_O, _asyncio_Future_remove_done_callback__doc__}, + {"remove_done_callback", _PyCFunction_CAST(_asyncio_Future_remove_done_callback), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _asyncio_Future_remove_done_callback__doc__}, + +static PyObject * +_asyncio_Future_remove_done_callback_impl(FutureObj *self, PyTypeObject *cls, + PyObject *fn); + +static PyObject * +_asyncio_Future_remove_done_callback(FutureObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "remove_done_callback", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *fn; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + fn = args[0]; + return_value = _asyncio_Future_remove_done_callback_impl(self, cls, fn); + +exit: + return return_value; +} PyDoc_STRVAR(_asyncio_Future_cancel__doc__, "cancel($self, /, msg=None)\n" @@ -236,13 +345,14 @@ PyDoc_STRVAR(_asyncio_Future_cancel__doc__, "return True."); #define _ASYNCIO_FUTURE_CANCEL_METHODDEF \ - {"cancel", _PyCFunction_CAST(_asyncio_Future_cancel), METH_FASTCALL|METH_KEYWORDS, _asyncio_Future_cancel__doc__}, + {"cancel", _PyCFunction_CAST(_asyncio_Future_cancel), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _asyncio_Future_cancel__doc__}, static PyObject * -_asyncio_Future_cancel_impl(FutureObj *self, PyObject *msg); +_asyncio_Future_cancel_impl(FutureObj *self, PyTypeObject *cls, + PyObject *msg); static PyObject * -_asyncio_Future_cancel(FutureObj *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_asyncio_Future_cancel(FutureObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -283,7 +393,7 @@ _asyncio_Future_cancel(FutureObj *self, PyObject *const *args, Py_ssize_t nargs, } msg = args[0]; skip_optional_pos: - return_value = _asyncio_Future_cancel_impl(self, msg); + return_value = _asyncio_Future_cancel_impl(self, cls, msg); exit: return return_value; @@ -335,15 +445,19 @@ PyDoc_STRVAR(_asyncio_Future_get_loop__doc__, "Return the event loop the Future is bound to."); #define _ASYNCIO_FUTURE_GET_LOOP_METHODDEF \ - {"get_loop", (PyCFunction)_asyncio_Future_get_loop, METH_NOARGS, _asyncio_Future_get_loop__doc__}, + {"get_loop", _PyCFunction_CAST(_asyncio_Future_get_loop), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _asyncio_Future_get_loop__doc__}, static PyObject * -_asyncio_Future_get_loop_impl(FutureObj *self); +_asyncio_Future_get_loop_impl(FutureObj *self, PyTypeObject *cls); static PyObject * -_asyncio_Future_get_loop(FutureObj *self, PyObject *Py_UNUSED(ignored)) +_asyncio_Future_get_loop(FutureObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - return _asyncio_Future_get_loop_impl(self); + if (nargs) { + PyErr_SetString(PyExc_TypeError, "get_loop() takes no arguments"); + return NULL; + } + return _asyncio_Future_get_loop_impl(self, cls); } PyDoc_STRVAR(_asyncio_Future__make_cancelled_error__doc__, @@ -612,13 +726,14 @@ PyDoc_STRVAR(_asyncio_Task_get_stack__doc__, "returned for a suspended coroutine."); #define _ASYNCIO_TASK_GET_STACK_METHODDEF \ - {"get_stack", _PyCFunction_CAST(_asyncio_Task_get_stack), METH_FASTCALL|METH_KEYWORDS, _asyncio_Task_get_stack__doc__}, + {"get_stack", _PyCFunction_CAST(_asyncio_Task_get_stack), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _asyncio_Task_get_stack__doc__}, static PyObject * -_asyncio_Task_get_stack_impl(TaskObj *self, PyObject *limit); +_asyncio_Task_get_stack_impl(TaskObj *self, PyTypeObject *cls, + PyObject *limit); static PyObject * -_asyncio_Task_get_stack(TaskObj *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_asyncio_Task_get_stack(TaskObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -659,7 +774,7 @@ _asyncio_Task_get_stack(TaskObj *self, PyObject *const *args, Py_ssize_t nargs, } limit = args[0]; skip_optional_kwonly: - return_value = _asyncio_Task_get_stack_impl(self, limit); + return_value = _asyncio_Task_get_stack_impl(self, cls, limit); exit: return return_value; @@ -678,14 +793,14 @@ PyDoc_STRVAR(_asyncio_Task_print_stack__doc__, "to sys.stderr."); #define _ASYNCIO_TASK_PRINT_STACK_METHODDEF \ - {"print_stack", _PyCFunction_CAST(_asyncio_Task_print_stack), METH_FASTCALL|METH_KEYWORDS, _asyncio_Task_print_stack__doc__}, + {"print_stack", _PyCFunction_CAST(_asyncio_Task_print_stack), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _asyncio_Task_print_stack__doc__}, static PyObject * -_asyncio_Task_print_stack_impl(TaskObj *self, PyObject *limit, - PyObject *file); +_asyncio_Task_print_stack_impl(TaskObj *self, PyTypeObject *cls, + PyObject *limit, PyObject *file); static PyObject * -_asyncio_Task_print_stack(TaskObj *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_asyncio_Task_print_stack(TaskObj *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -733,7 +848,7 @@ _asyncio_Task_print_stack(TaskObj *self, PyObject *const *args, Py_ssize_t nargs } file = args[1]; skip_optional_kwonly: - return_value = _asyncio_Task_print_stack_impl(self, limit, file); + return_value = _asyncio_Task_print_stack_impl(self, cls, limit, file); exit: return return_value; @@ -1189,4 +1304,4 @@ _asyncio__leave_task(PyObject *module, PyObject *const *args, Py_ssize_t nargs, exit: return return_value; } -/*[clinic end generated code: output=f117b2246eaf7a55 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=550bc6603df89ed9 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_codecsmodule.c.h b/Modules/clinic/_codecsmodule.c.h index 25db060cd900fa..f11bcc8815b920 100644 --- a/Modules/clinic/_codecsmodule.c.h +++ b/Modules/clinic/_codecsmodule.c.h @@ -450,8 +450,8 @@ _codecs_utf_7_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 3) { goto skip_optional; } - final = _PyLong_AsInt(args[2]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[2]); + if (final < 0) { goto exit; } skip_optional: @@ -520,8 +520,8 @@ _codecs_utf_8_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 3) { goto skip_optional; } - final = _PyLong_AsInt(args[2]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[2]); + if (final < 0) { goto exit; } skip_optional: @@ -590,8 +590,8 @@ _codecs_utf_16_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 3) { goto skip_optional; } - final = _PyLong_AsInt(args[2]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[2]); + if (final < 0) { goto exit; } skip_optional: @@ -660,8 +660,8 @@ _codecs_utf_16_le_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar if (nargs < 3) { goto skip_optional; } - final = _PyLong_AsInt(args[2]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[2]); + if (final < 0) { goto exit; } skip_optional: @@ -730,8 +730,8 @@ _codecs_utf_16_be_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar if (nargs < 3) { goto skip_optional; } - final = _PyLong_AsInt(args[2]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[2]); + if (final < 0) { goto exit; } skip_optional: @@ -809,8 +809,8 @@ _codecs_utf_16_ex_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar if (nargs < 4) { goto skip_optional; } - final = _PyLong_AsInt(args[3]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[3]); + if (final < 0) { goto exit; } skip_optional: @@ -879,8 +879,8 @@ _codecs_utf_32_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 3) { goto skip_optional; } - final = _PyLong_AsInt(args[2]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[2]); + if (final < 0) { goto exit; } skip_optional: @@ -949,8 +949,8 @@ _codecs_utf_32_le_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar if (nargs < 3) { goto skip_optional; } - final = _PyLong_AsInt(args[2]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[2]); + if (final < 0) { goto exit; } skip_optional: @@ -1019,8 +1019,8 @@ _codecs_utf_32_be_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar if (nargs < 3) { goto skip_optional; } - final = _PyLong_AsInt(args[2]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[2]); + if (final < 0) { goto exit; } skip_optional: @@ -1098,8 +1098,8 @@ _codecs_utf_32_ex_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar if (nargs < 4) { goto skip_optional; } - final = _PyLong_AsInt(args[3]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[3]); + if (final < 0) { goto exit; } skip_optional: @@ -1178,8 +1178,8 @@ _codecs_unicode_escape_decode(PyObject *module, PyObject *const *args, Py_ssize_ if (nargs < 3) { goto skip_optional; } - final = _PyLong_AsInt(args[2]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[2]); + if (final < 0) { goto exit; } skip_optional: @@ -1258,8 +1258,8 @@ _codecs_raw_unicode_escape_decode(PyObject *module, PyObject *const *args, Py_ss if (nargs < 3) { goto skip_optional; } - final = _PyLong_AsInt(args[2]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[2]); + if (final < 0) { goto exit; } skip_optional: @@ -1521,8 +1521,8 @@ _codecs_mbcs_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 3) { goto skip_optional; } - final = _PyLong_AsInt(args[2]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[2]); + if (final < 0) { goto exit; } skip_optional: @@ -1595,8 +1595,8 @@ _codecs_oem_decode(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 3) { goto skip_optional; } - final = _PyLong_AsInt(args[2]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[2]); + if (final < 0) { goto exit; } skip_optional: @@ -1674,8 +1674,8 @@ _codecs_code_page_decode(PyObject *module, PyObject *const *args, Py_ssize_t nar if (nargs < 4) { goto skip_optional; } - final = _PyLong_AsInt(args[3]); - if (final == -1 && PyErr_Occurred()) { + final = PyObject_IsTrue(args[3]); + if (final < 0) { goto exit; } skip_optional: @@ -2869,4 +2869,4 @@ _codecs_lookup_error(PyObject *module, PyObject *arg) #ifndef _CODECS_CODE_PAGE_ENCODE_METHODDEF #define _CODECS_CODE_PAGE_ENCODE_METHODDEF #endif /* !defined(_CODECS_CODE_PAGE_ENCODE_METHODDEF) */ -/*[clinic end generated code: output=e885abad241bc54d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=603da07cf8dfeb4b input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_cursesmodule.c.h b/Modules/clinic/_cursesmodule.c.h index 67fadace863970..9d99d41af5d2d9 100644 --- a/Modules/clinic/_cursesmodule.c.h +++ b/Modules/clinic/_cursesmodule.c.h @@ -1748,7 +1748,7 @@ _curses_window_touchline(PyCursesWindowObject *self, PyObject *args) } break; case 3: - if (!PyArg_ParseTuple(args, "iii:touchline", &start, &count, &changed)) { + if (!PyArg_ParseTuple(args, "iip:touchline", &start, &count, &changed)) { goto exit; } group_right_1 = 1; @@ -1941,8 +1941,8 @@ _curses_cbreak(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 1) { goto skip_optional; } - flag = _PyLong_AsInt(args[0]); - if (flag == -1 && PyErr_Occurred()) { + flag = PyObject_IsTrue(args[0]); + if (flag < 0) { goto exit; } skip_optional: @@ -2177,8 +2177,8 @@ _curses_echo(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 1) { goto skip_optional; } - flag = _PyLong_AsInt(args[0]); - if (flag == -1 && PyErr_Occurred()) { + flag = PyObject_IsTrue(args[0]); + if (flag < 0) { goto exit; } skip_optional: @@ -2900,8 +2900,8 @@ _curses_intrflush(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int flag; - flag = _PyLong_AsInt(arg); - if (flag == -1 && PyErr_Occurred()) { + flag = PyObject_IsTrue(arg); + if (flag < 0) { goto exit; } return_value = _curses_intrflush_impl(module, flag); @@ -3064,8 +3064,8 @@ _curses_meta(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int yes; - yes = _PyLong_AsInt(arg); - if (yes == -1 && PyErr_Occurred()) { + yes = PyObject_IsTrue(arg); + if (yes < 0) { goto exit; } return_value = _curses_meta_impl(module, yes); @@ -3308,8 +3308,8 @@ _curses_nl(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 1) { goto skip_optional; } - flag = _PyLong_AsInt(args[0]); - if (flag == -1 && PyErr_Occurred()) { + flag = PyObject_IsTrue(args[0]); + if (flag < 0) { goto exit; } skip_optional: @@ -3540,8 +3540,8 @@ _curses_qiflush(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 1) { goto skip_optional; } - flag = _PyLong_AsInt(args[0]); - if (flag == -1 && PyErr_Occurred()) { + flag = PyObject_IsTrue(args[0]); + if (flag < 0) { goto exit; } skip_optional: @@ -3603,8 +3603,8 @@ _curses_raw(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 1) { goto skip_optional; } - flag = _PyLong_AsInt(args[0]); - if (flag == -1 && PyErr_Occurred()) { + flag = PyObject_IsTrue(args[0]); + if (flag < 0) { goto exit; } skip_optional: @@ -4164,8 +4164,8 @@ _curses_use_env(PyObject *module, PyObject *arg) PyObject *return_value = NULL; int flag; - flag = _PyLong_AsInt(arg); - if (flag == -1 && PyErr_Occurred()) { + flag = PyObject_IsTrue(arg); + if (flag < 0) { goto exit; } return_value = _curses_use_env_impl(module, flag); @@ -4313,4 +4313,4 @@ _curses_has_extended_color_support(PyObject *module, PyObject *Py_UNUSED(ignored #ifndef _CURSES_USE_DEFAULT_COLORS_METHODDEF #define _CURSES_USE_DEFAULT_COLORS_METHODDEF #endif /* !defined(_CURSES_USE_DEFAULT_COLORS_METHODDEF) */ -/*[clinic end generated code: output=b2e71e2012f16197 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=27a2364193b503c1 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_elementtree.c.h b/Modules/clinic/_elementtree.c.h index 0a2a74e220c2d1..33ccaf7e7c7eaf 100644 --- a/Modules/clinic/_elementtree.c.h +++ b/Modules/clinic/_elementtree.c.h @@ -106,20 +106,20 @@ PyDoc_STRVAR(_elementtree_Element___sizeof____doc__, #define _ELEMENTTREE_ELEMENT___SIZEOF___METHODDEF \ {"__sizeof__", (PyCFunction)_elementtree_Element___sizeof__, METH_NOARGS, _elementtree_Element___sizeof____doc__}, -static Py_ssize_t +static size_t _elementtree_Element___sizeof___impl(ElementObject *self); static PyObject * _elementtree_Element___sizeof__(ElementObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; - Py_ssize_t _return_value; + size_t _return_value; _return_value = _elementtree_Element___sizeof___impl(self); - if ((_return_value == -1) && PyErr_Occurred()) { + if ((_return_value == (size_t)-1) && PyErr_Occurred()) { goto exit; } - return_value = PyLong_FromSsize_t(_return_value); + return_value = PyLong_FromSize_t(_return_value); exit: return return_value; @@ -1105,4 +1105,4 @@ _elementtree_XMLParser__setevents(XMLParserObject *self, PyObject *const *args, exit: return return_value; } -/*[clinic end generated code: output=67a80531eaf43815 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4ad006cadce01571 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_pickle.c.h b/Modules/clinic/_pickle.c.h index e9ff2604719abd..adb3abc5eb2372 100644 --- a/Modules/clinic/_pickle.c.h +++ b/Modules/clinic/_pickle.c.h @@ -49,20 +49,20 @@ PyDoc_STRVAR(_pickle_Pickler___sizeof____doc__, #define _PICKLE_PICKLER___SIZEOF___METHODDEF \ {"__sizeof__", (PyCFunction)_pickle_Pickler___sizeof__, METH_NOARGS, _pickle_Pickler___sizeof____doc__}, -static Py_ssize_t +static size_t _pickle_Pickler___sizeof___impl(PicklerObject *self); static PyObject * _pickle_Pickler___sizeof__(PicklerObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; - Py_ssize_t _return_value; + size_t _return_value; _return_value = _pickle_Pickler___sizeof___impl(self); - if ((_return_value == -1) && PyErr_Occurred()) { + if ((_return_value == (size_t)-1) && PyErr_Occurred()) { goto exit; } - return_value = PyLong_FromSsize_t(_return_value); + return_value = PyLong_FromSize_t(_return_value); exit: return return_value; @@ -301,20 +301,20 @@ PyDoc_STRVAR(_pickle_Unpickler___sizeof____doc__, #define _PICKLE_UNPICKLER___SIZEOF___METHODDEF \ {"__sizeof__", (PyCFunction)_pickle_Unpickler___sizeof__, METH_NOARGS, _pickle_Unpickler___sizeof____doc__}, -static Py_ssize_t +static size_t _pickle_Unpickler___sizeof___impl(UnpicklerObject *self); static PyObject * _pickle_Unpickler___sizeof__(UnpicklerObject *self, PyObject *Py_UNUSED(ignored)) { PyObject *return_value = NULL; - Py_ssize_t _return_value; + size_t _return_value; _return_value = _pickle_Unpickler___sizeof___impl(self); - if ((_return_value == -1) && PyErr_Occurred()) { + if ((_return_value == (size_t)-1) && PyErr_Occurred()) { goto exit; } - return_value = PyLong_FromSsize_t(_return_value); + return_value = PyLong_FromSize_t(_return_value); exit: return return_value; @@ -980,4 +980,4 @@ _pickle_loads(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=3321309c2157ee74 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=730dc26938561313 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index 622e321fa1d8b3..2d7c98c4f014a3 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -757,8 +757,8 @@ _ssl__SSLContext__wrap_socket(PySSLContext *self, PyObject *const *args, Py_ssiz goto exit; } sock = args[0]; - server_side = _PyLong_AsInt(args[1]); - if (server_side == -1 && PyErr_Occurred()) { + server_side = PyObject_IsTrue(args[1]); + if (server_side < 0) { goto exit; } if (!noptargs) { @@ -855,8 +855,8 @@ _ssl__SSLContext__wrap_bio(PySSLContext *self, PyObject *const *args, Py_ssize_t goto exit; } outgoing = (PySSLMemoryBIO *)args[1]; - server_side = _PyLong_AsInt(args[2]); - if (server_side == -1 && PyErr_Occurred()) { + server_side = PyObject_IsTrue(args[2]); + if (server_side < 0) { goto exit; } if (!noptargs) { @@ -1543,4 +1543,4 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #ifndef _SSL_ENUM_CRLS_METHODDEF #define _SSL_ENUM_CRLS_METHODDEF #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */ -/*[clinic end generated code: output=9f477b0c709acb28 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a3d97a19163bb044 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_tkinter.c.h b/Modules/clinic/_tkinter.c.h index a251202f9bba2c..96c6ee26f426c3 100644 --- a/Modules/clinic/_tkinter.c.h +++ b/Modules/clinic/_tkinter.c.h @@ -747,29 +747,29 @@ _tkinter_create(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 4) { goto skip_optional; } - interactive = _PyLong_AsInt(args[3]); - if (interactive == -1 && PyErr_Occurred()) { + interactive = PyObject_IsTrue(args[3]); + if (interactive < 0) { goto exit; } if (nargs < 5) { goto skip_optional; } - wantobjects = _PyLong_AsInt(args[4]); - if (wantobjects == -1 && PyErr_Occurred()) { + wantobjects = PyObject_IsTrue(args[4]); + if (wantobjects < 0) { goto exit; } if (nargs < 6) { goto skip_optional; } - wantTk = _PyLong_AsInt(args[5]); - if (wantTk == -1 && PyErr_Occurred()) { + wantTk = PyObject_IsTrue(args[5]); + if (wantTk < 0) { goto exit; } if (nargs < 7) { goto skip_optional; } - sync = _PyLong_AsInt(args[6]); - if (sync == -1 && PyErr_Occurred()) { + sync = PyObject_IsTrue(args[6]); + if (sync < 0) { goto exit; } if (nargs < 8) { @@ -865,4 +865,4 @@ _tkinter_getbusywaitinterval(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF #define _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF #endif /* !defined(_TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF) */ -/*[clinic end generated code: output=d022835d05fc8608 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2a4e3bf8448604b5 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h index 13bf8b482cd69e..891b3f851d1243 100644 --- a/Modules/clinic/_winapi.c.h +++ b/Modules/clinic/_winapi.c.h @@ -133,7 +133,7 @@ _winapi_ConnectNamedPipe(PyObject *module, PyObject *const *args, Py_ssize_t nar static const char * const _keywords[] = {"handle", "overlapped", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .format = "" F_HANDLE "|i:ConnectNamedPipe", + .format = "" F_HANDLE "|p:ConnectNamedPipe", .kwtuple = KWTUPLE, }; #undef KWTUPLE @@ -972,7 +972,7 @@ _winapi_ReadFile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb static const char * const _keywords[] = {"handle", "size", "overlapped", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .format = "" F_HANDLE "k|i:ReadFile", + .format = "" F_HANDLE "k|p:ReadFile", .kwtuple = KWTUPLE, }; #undef KWTUPLE @@ -1220,7 +1220,7 @@ _winapi_WriteFile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO static const char * const _keywords[] = {"handle", "buffer", "overlapped", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .format = "" F_HANDLE "O|i:WriteFile", + .format = "" F_HANDLE "O|p:WriteFile", .kwtuple = KWTUPLE, }; #undef KWTUPLE @@ -1371,4 +1371,4 @@ _winapi__mimetypes_read_windows_registry(PyObject *module, PyObject *const *args exit: return return_value; } -/*[clinic end generated code: output=23ea9e176d86e026 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=edb1a9d1bbfd6394 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/binascii.c.h b/Modules/clinic/binascii.c.h index 23ebdff2108258..63566dfb10e74f 100644 --- a/Modules/clinic/binascii.c.h +++ b/Modules/clinic/binascii.c.h @@ -99,8 +99,8 @@ binascii_b2a_uu(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj if (!noptargs) { goto skip_optional_kwonly; } - backtick = _PyLong_AsInt(args[1]); - if (backtick == -1 && PyErr_Occurred()) { + backtick = PyObject_IsTrue(args[1]); + if (backtick < 0) { goto exit; } skip_optional_kwonly: @@ -175,8 +175,8 @@ binascii_a2b_base64(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P if (!noptargs) { goto skip_optional_kwonly; } - strict_mode = _PyLong_AsInt(args[1]); - if (strict_mode == -1 && PyErr_Occurred()) { + strict_mode = PyObject_IsTrue(args[1]); + if (strict_mode < 0) { goto exit; } skip_optional_kwonly: @@ -250,8 +250,8 @@ binascii_b2a_base64(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P if (!noptargs) { goto skip_optional_kwonly; } - newline = _PyLong_AsInt(args[1]); - if (newline == -1 && PyErr_Occurred()) { + newline = PyObject_IsTrue(args[1]); + if (newline < 0) { goto exit; } skip_optional_kwonly: @@ -680,8 +680,8 @@ binascii_a2b_qp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj if (!noptargs) { goto skip_optional_pos; } - header = _PyLong_AsInt(args[1]); - if (header == -1 && PyErr_Occurred()) { + header = PyObject_IsTrue(args[1]); + if (header < 0) { goto exit; } skip_optional_pos: @@ -763,8 +763,8 @@ binascii_b2a_qp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj goto skip_optional_pos; } if (args[1]) { - quotetabs = _PyLong_AsInt(args[1]); - if (quotetabs == -1 && PyErr_Occurred()) { + quotetabs = PyObject_IsTrue(args[1]); + if (quotetabs < 0) { goto exit; } if (!--noptargs) { @@ -772,16 +772,16 @@ binascii_b2a_qp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj } } if (args[2]) { - istext = _PyLong_AsInt(args[2]); - if (istext == -1 && PyErr_Occurred()) { + istext = PyObject_IsTrue(args[2]); + if (istext < 0) { goto exit; } if (!--noptargs) { goto skip_optional_pos; } } - header = _PyLong_AsInt(args[3]); - if (header == -1 && PyErr_Occurred()) { + header = PyObject_IsTrue(args[3]); + if (header < 0) { goto exit; } skip_optional_pos: @@ -795,4 +795,4 @@ binascii_b2a_qp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj return return_value; } -/*[clinic end generated code: output=a266ba13c374aefa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ab156917c9db79d2 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 92ce3db18655b7..81e0147643018c 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -3241,8 +3241,8 @@ os_posix_spawn(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje } } if (args[5]) { - resetids = _PyLong_AsInt(args[5]); - if (resetids == -1 && PyErr_Occurred()) { + resetids = PyObject_IsTrue(args[5]); + if (resetids < 0) { goto exit; } if (!--noptargs) { @@ -3250,8 +3250,8 @@ os_posix_spawn(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje } } if (args[6]) { - setsid = _PyLong_AsInt(args[6]); - if (setsid == -1 && PyErr_Occurred()) { + setsid = PyObject_IsTrue(args[6]); + if (setsid < 0) { goto exit; } if (!--noptargs) { @@ -3391,8 +3391,8 @@ os_posix_spawnp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj } } if (args[5]) { - resetids = _PyLong_AsInt(args[5]); - if (resetids == -1 && PyErr_Occurred()) { + resetids = PyObject_IsTrue(args[5]); + if (resetids < 0) { goto exit; } if (!--noptargs) { @@ -3400,8 +3400,8 @@ os_posix_spawnp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj } } if (args[6]) { - setsid = _PyLong_AsInt(args[6]); - if (setsid == -1 && PyErr_Occurred()) { + setsid = PyObject_IsTrue(args[6]); + if (setsid < 0) { goto exit; } if (!--noptargs) { @@ -10356,8 +10356,8 @@ os_set_blocking(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (fd == -1 && PyErr_Occurred()) { goto exit; } - blocking = _PyLong_AsInt(args[1]); - if (blocking == -1 && PyErr_Occurred()) { + blocking = PyObject_IsTrue(args[1]); + if (blocking < 0) { goto exit; } return_value = os_set_blocking_impl(module, fd, blocking); @@ -11684,4 +11684,4 @@ os_waitstatus_to_exitcode(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef OS_WAITSTATUS_TO_EXITCODE_METHODDEF #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF #endif /* !defined(OS_WAITSTATUS_TO_EXITCODE_METHODDEF) */ -/*[clinic end generated code: output=9579615df62ee27f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=56f0ab245e915a5a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/pyexpat.c.h b/Modules/clinic/pyexpat.c.h index 0454fbc9994504..34937c5d594f5c 100644 --- a/Modules/clinic/pyexpat.c.h +++ b/Modules/clinic/pyexpat.c.h @@ -52,8 +52,8 @@ pyexpat_xmlparser_Parse(xmlparseobject *self, PyTypeObject *cls, PyObject *const if (nargs < 2) { goto skip_optional_posonly; } - isfinal = _PyLong_AsInt(args[1]); - if (isfinal == -1 && PyErr_Occurred()) { + isfinal = PyObject_IsTrue(args[1]); + if (isfinal < 0) { goto exit; } skip_optional_posonly: @@ -498,4 +498,4 @@ pyexpat_ErrorString(PyObject *module, PyObject *arg) #ifndef PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #define PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #endif /* !defined(PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF) */ -/*[clinic end generated code: output=de5f664ef05ef34a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=63efc62e24a7b5a7 input=a9049054013a1b77]*/ diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index fa4c2d0cccd1b6..341a03a244cdc5 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -270,7 +270,7 @@ faulthandler_dump_traceback_py(PyObject *self, int fd; if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "|Oi:dump_traceback", kwlist, + "|Op:dump_traceback", kwlist, &file, &all_threads)) return NULL; @@ -546,7 +546,7 @@ faulthandler_py_enable(PyObject *self, PyObject *args, PyObject *kwargs) PyThreadState *tstate; if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "|Oi:enable", kwlist, &file, &all_threads)) + "|Op:enable", kwlist, &file, &all_threads)) return NULL; fd = faulthandler_get_fileno(&file); @@ -916,7 +916,7 @@ faulthandler_register_py(PyObject *self, int err; if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "i|Oii:register", kwlist, + "i|Opp:register", kwlist, &signum, &file, &all_threads, &chain)) return NULL; diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index cacfad7335634c..6630faa6f4471d 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -2329,7 +2329,6 @@ _PyObject_GC_New(PyTypeObject *tp) PyVarObject * _PyObject_GC_NewVar(PyTypeObject *tp, Py_ssize_t nitems) { - size_t size; PyVarObject *op; if (nitems < 0) { @@ -2337,7 +2336,7 @@ _PyObject_GC_NewVar(PyTypeObject *tp, Py_ssize_t nitems) return NULL; } size_t presize = _PyType_PreHeaderSize(tp); - size = _PyObject_VAR_SIZE(tp, nitems); + size_t size = _PyObject_VAR_SIZE(tp, nitems); op = (PyVarObject *)gc_alloc(size, presize); if (op == NULL) { return NULL; @@ -2351,7 +2350,7 @@ _PyObject_GC_Resize(PyVarObject *op, Py_ssize_t nitems) { const size_t basicsize = _PyObject_VAR_SIZE(Py_TYPE(op), nitems); _PyObject_ASSERT((PyObject *)op, !_PyObject_GC_IS_TRACKED(op)); - if (basicsize > PY_SSIZE_T_MAX - sizeof(PyGC_Head)) { + if (basicsize > (size_t)PY_SSIZE_T_MAX - sizeof(PyGC_Head)) { return (PyVarObject *)PyErr_NoMemory(); } diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 4b0a4d88c435c6..d45000788c5318 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -1368,6 +1368,7 @@ cycle_setstate(cycleobject *lz, PyObject *state) PyErr_SetString(PyExc_TypeError, "state is not a tuple"); return NULL; } + // The second item can be 1/0 in old pickles and True/False in new pickles if (!PyArg_ParseTuple(state, "O!i", &PyList_Type, &saved, &firstpass)) { return NULL; } @@ -2484,11 +2485,9 @@ product_dealloc(productobject *lz) static PyObject * product_sizeof(productobject *lz, void *unused) { - Py_ssize_t res; - - res = _PyObject_SIZE(Py_TYPE(lz)); - res += PyTuple_GET_SIZE(lz->pools) * sizeof(Py_ssize_t); - return PyLong_FromSsize_t(res); + size_t res = _PyObject_SIZE(Py_TYPE(lz)); + res += (size_t)PyTuple_GET_SIZE(lz->pools) * sizeof(Py_ssize_t); + return PyLong_FromSize_t(res); } PyDoc_STRVAR(sizeof_doc, "Returns size in memory, in bytes."); @@ -2817,11 +2816,9 @@ combinations_dealloc(combinationsobject *co) static PyObject * combinations_sizeof(combinationsobject *co, void *unused) { - Py_ssize_t res; - - res = _PyObject_SIZE(Py_TYPE(co)); - res += co->r * sizeof(Py_ssize_t); - return PyLong_FromSsize_t(res); + size_t res = _PyObject_SIZE(Py_TYPE(co)); + res += (size_t)co->r * sizeof(Py_ssize_t); + return PyLong_FromSize_t(res); } static int @@ -3153,11 +3150,9 @@ cwr_dealloc(cwrobject *co) static PyObject * cwr_sizeof(cwrobject *co, void *unused) { - Py_ssize_t res; - - res = _PyObject_SIZE(Py_TYPE(co)); - res += co->r * sizeof(Py_ssize_t); - return PyLong_FromSsize_t(res); + size_t res = _PyObject_SIZE(Py_TYPE(co)); + res += (size_t)co->r * sizeof(Py_ssize_t); + return PyLong_FromSize_t(res); } static int @@ -3498,12 +3493,10 @@ permutations_dealloc(permutationsobject *po) static PyObject * permutations_sizeof(permutationsobject *po, void *unused) { - Py_ssize_t res; - - res = _PyObject_SIZE(Py_TYPE(po)); - res += PyTuple_GET_SIZE(po->pool) * sizeof(Py_ssize_t); - res += po->r * sizeof(Py_ssize_t); - return PyLong_FromSsize_t(res); + size_t res = _PyObject_SIZE(Py_TYPE(po)); + res += (size_t)PyTuple_GET_SIZE(po->pool) * sizeof(Py_ssize_t); + res += (size_t)po->r * sizeof(Py_ssize_t); + return PyLong_FromSize_t(res); } static int diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 2311840e22105f..8244202376c74e 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -804,12 +804,11 @@ mmap__repr__method(PyObject *self) static PyObject * mmap__sizeof__method(mmap_object *self, void *unused) { - Py_ssize_t res; - - res = _PyObject_SIZE(Py_TYPE(self)); - if (self->tagname) + size_t res = _PyObject_SIZE(Py_TYPE(self)); + if (self->tagname) { res += strlen(self->tagname) + 1; - return PyLong_FromSsize_t(res); + } + return PyLong_FromSize_t(res); } #endif diff --git a/Modules/ossaudiodev.c b/Modules/ossaudiodev.c index 79f4ebad836c8b..2319eac449eb4f 100644 --- a/Modules/ossaudiodev.c +++ b/Modules/ossaudiodev.c @@ -569,7 +569,7 @@ oss_setparameters(oss_audio_t *self, PyObject *args) if (!_is_fd_valid(self->fd)) return NULL; - if (!PyArg_ParseTuple(args, "iii|i:setparameters", + if (!PyArg_ParseTuple(args, "iii|p:setparameters", &wanted_fmt, &wanted_channels, &wanted_rate, &strict)) return NULL; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index aa44503c6f7bbc..f100788594f086 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -6742,9 +6742,9 @@ os.posix_spawn A sequence of file action tuples. setpgroup: object = NULL The pgroup to use with the POSIX_SPAWN_SETPGROUP flag. - resetids: bool(accept={int}) = False + resetids: bool = False If the value is `true` the POSIX_SPAWN_RESETIDS will be activated. - setsid: bool(accept={int}) = False + setsid: bool = False If the value is `true` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP will be activated. setsigmask: object(c_default='NULL') = () The sigmask to use with the POSIX_SPAWN_SETSIGMASK flag. @@ -6762,7 +6762,7 @@ os_posix_spawn_impl(PyObject *module, path_t *path, PyObject *argv, PyObject *setpgroup, int resetids, int setsid, PyObject *setsigmask, PyObject *setsigdef, PyObject *scheduler) -/*[clinic end generated code: output=14a1098c566bc675 input=8c6305619a00ad04]*/ +/*[clinic end generated code: output=14a1098c566bc675 input=808aed1090d84e33]*/ { return py_posix_spawn(0, module, path, argv, env, file_actions, setpgroup, resetids, setsid, setsigmask, setsigdef, @@ -6788,9 +6788,9 @@ os.posix_spawnp A sequence of file action tuples. setpgroup: object = NULL The pgroup to use with the POSIX_SPAWN_SETPGROUP flag. - resetids: bool(accept={int}) = False + resetids: bool = False If the value is `True` the POSIX_SPAWN_RESETIDS will be activated. - setsid: bool(accept={int}) = False + setsid: bool = False If the value is `True` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP will be activated. setsigmask: object(c_default='NULL') = () The sigmask to use with the POSIX_SPAWN_SETSIGMASK flag. @@ -6808,7 +6808,7 @@ os_posix_spawnp_impl(PyObject *module, path_t *path, PyObject *argv, PyObject *setpgroup, int resetids, int setsid, PyObject *setsigmask, PyObject *setsigdef, PyObject *scheduler) -/*[clinic end generated code: output=7b9aaefe3031238d input=c1911043a22028da]*/ +/*[clinic end generated code: output=7b9aaefe3031238d input=9e89e616116752a1]*/ { return py_posix_spawn(1, module, path, argv, env, file_actions, setpgroup, resetids, setsid, setsigmask, setsigdef, @@ -13946,7 +13946,7 @@ os_get_blocking_impl(PyObject *module, int fd) /*[clinic input] os.set_blocking fd: int - blocking: bool(accept={int}) + blocking: bool / Set the blocking mode of the specified file descriptor. @@ -13957,7 +13957,7 @@ clear the O_NONBLOCK flag otherwise. static PyObject * os_set_blocking_impl(PyObject *module, int fd, int blocking) -/*[clinic end generated code: output=384eb43aa0762a9d input=bf5c8efdc5860ff3]*/ +/*[clinic end generated code: output=384eb43aa0762a9d input=7e9dfc9b14804dd4]*/ { int result; diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 0e0a9cf7cc2c5c..2440798bff7e66 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -710,7 +710,7 @@ pyexpat.xmlparser.Parse cls: defining_class data: object - isfinal: bool(accept={int}) = False + isfinal: bool = False / Parse XML data. @@ -721,7 +721,7 @@ Parse XML data. static PyObject * pyexpat_xmlparser_Parse_impl(xmlparseobject *self, PyTypeObject *cls, PyObject *data, int isfinal) -/*[clinic end generated code: output=8faffe07fe1f862a input=fc97f833558ca715]*/ +/*[clinic end generated code: output=8faffe07fe1f862a input=d0eb2a69fab3b9f1]*/ { const char *s; Py_ssize_t slen; diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 4ecb88cef7224e..7d438448ef36e7 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -2928,8 +2928,8 @@ sock_setblocking(PySocketSockObject *s, PyObject *arg) { long block; - block = PyLong_AsLong(arg); - if (block == -1 && PyErr_Occurred()) + block = PyObject_IsTrue(arg); + if (block < 0) return NULL; s->sock_timeout = _PyTime_FromSeconds(block ? -1 : 0); diff --git a/Modules/syslogmodule.c b/Modules/syslogmodule.c index 23833b72850313..f45aa5227f1cbf 100644 --- a/Modules/syslogmodule.c +++ b/Modules/syslogmodule.c @@ -61,10 +61,16 @@ module syslog #include "clinic/syslogmodule.c.h" -/* only one instance, only one syslog, so globals should be ok */ -static PyObject *S_ident_o = NULL; /* identifier, held by openlog() */ +/* only one instance, only one syslog, so globals should be ok, + * these fields are writable from the main interpreter only. */ +static PyObject *S_ident_o = NULL; // identifier, held by openlog() static char S_log_open = 0; +static inline int +is_main_interpreter(void) +{ + return (PyInterpreterState_Get() == PyInterpreterState_Main()); +} static PyObject * syslog_get_argv(void) @@ -135,6 +141,13 @@ syslog_openlog_impl(PyObject *module, PyObject *ident, long logopt, long facility) /*[clinic end generated code: output=5476c12829b6eb75 input=8a987a96a586eee7]*/ { + // Since the sys.openlog changes the process level state of syslog library, + // this operation is only allowed for the main interpreter. + if (!is_main_interpreter()) { + PyErr_SetString(PyExc_RuntimeError, "subinterpreter can't use syslog.openlog()"); + return NULL; + } + const char *ident_str = NULL; if (ident) { @@ -195,6 +208,11 @@ syslog_syslog_impl(PyObject *module, int group_left_1, int priority, /* if log is not opened, open it now */ if (!S_log_open) { + if (!is_main_interpreter()) { + PyErr_SetString(PyExc_RuntimeError, "subinterpreter can't use syslog.syslog() " + "until the syslog is opened by the main interpreter"); + return NULL; + } PyObject *openlog_ret = syslog_openlog_impl(module, NULL, 0, LOG_USER); if (openlog_ret == NULL) { return NULL; @@ -229,6 +247,13 @@ static PyObject * syslog_closelog_impl(PyObject *module) /*[clinic end generated code: output=97890a80a24b1b84 input=fb77a54d447acf07]*/ { + // Since the sys.closelog changes the process level state of syslog library, + // this operation is only allowed for the main interpreter. + if (!is_main_interpreter()) { + PyErr_SetString(PyExc_RuntimeError, "sunbinterpreter can't use syslog.closelog()"); + return NULL; + } + if (PySys_Audit("syslog.closelog", NULL) < 0) { return NULL; } diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 0ba6fb5b76ccc7..072089e39aa207 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -2012,7 +2012,7 @@ bytearray_join(PyByteArrayObject *self, PyObject *iterable_of_bytes) /*[clinic input] bytearray.splitlines - keepends: bool(accept={int}) = False + keepends: bool = False Return a list of the lines in the bytearray, breaking at line boundaries. @@ -2022,7 +2022,7 @@ true. static PyObject * bytearray_splitlines_impl(PyByteArrayObject *self, int keepends) -/*[clinic end generated code: output=4223c94b895f6ad9 input=99a27ad959b9cf6b]*/ +/*[clinic end generated code: output=4223c94b895f6ad9 input=66b2dcdea8d093bf]*/ { return stringlib_splitlines( (PyObject*) self, PyByteArray_AS_STRING(self), @@ -2151,10 +2151,9 @@ static PyObject * bytearray_sizeof_impl(PyByteArrayObject *self) /*[clinic end generated code: output=738abdd17951c427 input=e27320fd98a4bc5a]*/ { - Py_ssize_t res; - - res = _PyObject_SIZE(Py_TYPE(self)) + self->ob_alloc * sizeof(char); - return PyLong_FromSsize_t(res); + size_t res = _PyObject_SIZE(Py_TYPE(self)); + res += (size_t)self->ob_alloc * sizeof(char); + return PyLong_FromSize_t(res); } static PySequenceMethods bytearray_as_sequence = { diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index a63f396e022f71..0fd10fa00d16fa 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -2314,7 +2314,7 @@ bytes_decode_impl(PyBytesObject *self, const char *encoding, /*[clinic input] bytes.splitlines - keepends: bool(accept={int}) = False + keepends: bool = False Return a list of the lines in the bytes, breaking at line boundaries. @@ -2324,7 +2324,7 @@ true. static PyObject * bytes_splitlines_impl(PyBytesObject *self, int keepends) -/*[clinic end generated code: output=3484149a5d880ffb input=a8b32eb01ff5a5ed]*/ +/*[clinic end generated code: output=3484149a5d880ffb input=5d7b898af2fe55c0]*/ { return stringlib_splitlines( (PyObject*) self, PyBytes_AS_STRING(self), diff --git a/Objects/clinic/bytearrayobject.c.h b/Objects/clinic/bytearrayobject.c.h index 142f2998160725..e7bf3183af8522 100644 --- a/Objects/clinic/bytearrayobject.c.h +++ b/Objects/clinic/bytearrayobject.c.h @@ -1084,8 +1084,8 @@ bytearray_splitlines(PyByteArrayObject *self, PyObject *const *args, Py_ssize_t if (!noptargs) { goto skip_optional_pos; } - keepends = _PyLong_AsInt(args[0]); - if (keepends == -1 && PyErr_Occurred()) { + keepends = PyObject_IsTrue(args[0]); + if (keepends < 0) { goto exit; } skip_optional_pos: @@ -1287,4 +1287,4 @@ bytearray_sizeof(PyByteArrayObject *self, PyObject *Py_UNUSED(ignored)) { return bytearray_sizeof_impl(self); } -/*[clinic end generated code: output=72bfa6cac2fd6832 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=022698e8b0faa272 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/bytesobject.c.h b/Objects/clinic/bytesobject.c.h index 904124ec479abb..060056dafbd84f 100644 --- a/Objects/clinic/bytesobject.c.h +++ b/Objects/clinic/bytesobject.c.h @@ -839,8 +839,8 @@ bytes_splitlines(PyBytesObject *self, PyObject *const *args, Py_ssize_t nargs, P if (!noptargs) { goto skip_optional_pos; } - keepends = _PyLong_AsInt(args[0]); - if (keepends == -1 && PyErr_Occurred()) { + keepends = PyObject_IsTrue(args[0]); + if (keepends < 0) { goto exit; } skip_optional_pos: @@ -1063,4 +1063,4 @@ bytes_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=5e0a25b7ba749a04 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=31a9e4af85562612 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/listobject.c.h b/Objects/clinic/listobject.c.h index 926eaa5d36983b..94852e99617060 100644 --- a/Objects/clinic/listobject.c.h +++ b/Objects/clinic/listobject.c.h @@ -215,8 +215,8 @@ list_sort(PyListObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject goto skip_optional_kwonly; } } - reverse = _PyLong_AsInt(args[1]); - if (reverse == -1 && PyErr_Occurred()) { + reverse = PyObject_IsTrue(args[1]); + if (reverse < 0) { goto exit; } skip_optional_kwonly: @@ -382,4 +382,4 @@ list___reversed__(PyListObject *self, PyObject *Py_UNUSED(ignored)) { return list___reversed___impl(self); } -/*[clinic end generated code: output=782ed6c68b1c9f83 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4e6f38b655394564 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/unicodeobject.c.h b/Objects/clinic/unicodeobject.c.h index d803a2733bd636..f640c997577363 100644 --- a/Objects/clinic/unicodeobject.c.h +++ b/Objects/clinic/unicodeobject.c.h @@ -1193,8 +1193,8 @@ unicode_splitlines(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyOb if (!noptargs) { goto skip_optional_pos; } - keepends = _PyLong_AsInt(args[0]); - if (keepends == -1 && PyErr_Occurred()) { + keepends = PyObject_IsTrue(args[0]); + if (keepends < 0) { goto exit; } skip_optional_pos: @@ -1497,4 +1497,4 @@ unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=e775ff4154f1c935 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=05d942840635dadf input=a9049054013a1b77]*/ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index fc1db72977aa01..0c197d767b0a23 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -12,6 +12,66 @@ #include "clinic/codeobject.c.h" +static void +notify_code_watchers(PyCodeEvent event, PyCodeObject *co) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (interp->active_code_watchers) { + assert(interp->_initialized); + for (int i = 0; i < CODE_MAX_WATCHERS; i++) { + PyCode_WatchCallback cb = interp->code_watchers[i]; + if ((cb != NULL) && (cb(event, co) < 0)) { + PyErr_WriteUnraisable((PyObject *) co); + } + } + } +} + +int +PyCode_AddWatcher(PyCode_WatchCallback callback) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(interp->_initialized); + + for (int i = 0; i < CODE_MAX_WATCHERS; i++) { + if (!interp->code_watchers[i]) { + interp->code_watchers[i] = callback; + interp->active_code_watchers |= (1 << i); + return i; + } + } + + PyErr_SetString(PyExc_RuntimeError, "no more code watcher IDs available"); + return -1; +} + +static inline int +validate_watcher_id(PyInterpreterState *interp, int watcher_id) +{ + if (watcher_id < 0 || watcher_id >= CODE_MAX_WATCHERS) { + PyErr_Format(PyExc_ValueError, "Invalid code watcher ID %d", watcher_id); + return -1; + } + if (!interp->code_watchers[watcher_id]) { + PyErr_Format(PyExc_ValueError, "No code watcher set for ID %d", watcher_id); + return -1; + } + return 0; +} + +int +PyCode_ClearWatcher(int watcher_id) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(interp->_initialized); + if (validate_watcher_id(interp, watcher_id) < 0) { + return -1; + } + interp->code_watchers[watcher_id] = NULL; + interp->active_code_watchers &= ~(1 << watcher_id); + return 0; +} + /****************** * generic helpers ******************/ @@ -355,6 +415,7 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) } co->_co_firsttraceable = entry_point; _PyCode_Quicken(co); + notify_code_watchers(PY_CODE_EVENT_CREATE, co); } static int @@ -1615,6 +1676,8 @@ code_new_impl(PyTypeObject *type, int argcount, int posonlyargcount, static void code_dealloc(PyCodeObject *co) { + notify_code_watchers(PY_CODE_EVENT_DESTROY, co); + if (co->co_extra != NULL) { PyInterpreterState *interp = _PyInterpreterState_GET(); _PyCodeObjectExtra *co_extra = co->co_extra; @@ -1867,15 +1930,13 @@ static PyGetSetDef code_getsetlist[] = { static PyObject * code_sizeof(PyCodeObject *co, PyObject *Py_UNUSED(args)) { - Py_ssize_t res = _PyObject_VAR_SIZE(Py_TYPE(co), Py_SIZE(co)); - + size_t res = _PyObject_VAR_SIZE(Py_TYPE(co), Py_SIZE(co)); _PyCodeObjectExtra *co_extra = (_PyCodeObjectExtra*) co->co_extra; if (co_extra != NULL) { - res += sizeof(_PyCodeObjectExtra) + - (co_extra->ce_size-1) * sizeof(co_extra->ce_extras[0]); + res += sizeof(_PyCodeObjectExtra); + res += ((size_t)co_extra->ce_size - 1) * sizeof(co_extra->ce_extras[0]); } - - return PyLong_FromSsize_t(res); + return PyLong_FromSize_t(res); } static PyObject * diff --git a/Objects/dictobject.c b/Objects/dictobject.c index c58d07b51bd89a..b9067213820b52 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -685,9 +685,9 @@ free_keys_object(PyDictKeysObject *keys) } static inline PyDictValues* -new_values(Py_ssize_t size) +new_values(size_t size) { - assert(size > 0); + assert(size >= 1); size_t prefix_size = _Py_SIZE_ROUND_UP(size+2, sizeof(PyObject *)); assert(prefix_size < 256); size_t n = prefix_size + size * sizeof(PyObject *); @@ -746,27 +746,24 @@ new_dict(PyDictKeysObject *keys, PyDictValues *values, Py_ssize_t used, int free return (PyObject *)mp; } -static inline Py_ssize_t +static inline size_t shared_keys_usable_size(PyDictKeysObject *keys) { - return keys->dk_nentries + keys->dk_usable; + return (size_t)keys->dk_nentries + (size_t)keys->dk_usable; } /* Consumes a reference to the keys object */ static PyObject * new_dict_with_shared_keys(PyDictKeysObject *keys) { - PyDictValues *values; - Py_ssize_t i, size; - - size = shared_keys_usable_size(keys); - values = new_values(size); + size_t size = shared_keys_usable_size(keys); + PyDictValues *values = new_values(size); if (values == NULL) { dictkeys_decref(keys); return PyErr_NoMemory(); } ((char *)values)[-2] = 0; - for (i = 0; i < size; i++) { + for (size_t i = 0; i < size; i++) { values->values[i] = NULL; } return new_dict(keys, values, 0, 1); @@ -781,7 +778,7 @@ clone_combined_dict_keys(PyDictObject *orig) assert(orig->ma_values == NULL); assert(orig->ma_keys->dk_refcnt == 1); - Py_ssize_t keys_size = _PyDict_KeysSize(orig->ma_keys); + size_t keys_size = _PyDict_KeysSize(orig->ma_keys); PyDictKeysObject *keys = PyObject_Malloc(keys_size); if (keys == NULL) { PyErr_NoMemory(); @@ -2959,7 +2956,6 @@ PyDict_Copy(PyObject *o) { PyObject *copy; PyDictObject *mp; - Py_ssize_t i, n; if (o == NULL || !PyDict_Check(o)) { PyErr_BadInternalCall(); @@ -2974,9 +2970,8 @@ PyDict_Copy(PyObject *o) if (_PyDict_HasSplitTable(mp)) { PyDictObject *split_copy; - Py_ssize_t size = shared_keys_usable_size(mp->ma_keys); - PyDictValues *newvalues; - newvalues = new_values(size); + size_t size = shared_keys_usable_size(mp->ma_keys); + PyDictValues *newvalues = new_values(size); if (newvalues == NULL) return PyErr_NoMemory(); split_copy = PyObject_GC_New(PyDictObject, &PyDict_Type); @@ -2991,7 +2986,7 @@ PyDict_Copy(PyObject *o) split_copy->ma_used = mp->ma_used; split_copy->ma_version_tag = DICT_NEXT_VERSION(); dictkeys_incref(mp->ma_keys); - for (i = 0, n = size; i < n; i++) { + for (size_t i = 0; i < size; i++) { PyObject *value = mp->ma_values->values[i]; split_copy->ma_values->values[i] = Py_XNewRef(value); } @@ -3514,9 +3509,7 @@ static PyObject *dictiter_new(PyDictObject *, PyTypeObject *); Py_ssize_t _PyDict_SizeOf(PyDictObject *mp) { - Py_ssize_t res; - - res = _PyObject_SIZE(Py_TYPE(mp)); + size_t res = _PyObject_SIZE(Py_TYPE(mp)); if (mp->ma_values) { res += shared_keys_usable_size(mp->ma_keys) * sizeof(PyObject*); } @@ -3525,17 +3518,19 @@ _PyDict_SizeOf(PyDictObject *mp) if (mp->ma_keys->dk_refcnt == 1) { res += _PyDict_KeysSize(mp->ma_keys); } - return res; + assert(res <= (size_t)PY_SSIZE_T_MAX); + return (Py_ssize_t)res; } -Py_ssize_t +size_t _PyDict_KeysSize(PyDictKeysObject *keys) { - size_t es = keys->dk_kind == DICT_KEYS_GENERAL - ? sizeof(PyDictKeyEntry) : sizeof(PyDictUnicodeEntry); - return (sizeof(PyDictKeysObject) - + ((size_t)1 << keys->dk_log2_index_bytes) - + USABLE_FRACTION(DK_SIZE(keys)) * es); + size_t es = (keys->dk_kind == DICT_KEYS_GENERAL + ? sizeof(PyDictKeyEntry) : sizeof(PyDictUnicodeEntry)); + size_t size = sizeof(PyDictKeysObject); + size += (size_t)1 << keys->dk_log2_index_bytes; + size += USABLE_FRACTION((size_t)DK_SIZE(keys)) * es; + return size; } static PyObject * @@ -5286,16 +5281,15 @@ init_inline_values(PyObject *obj, PyTypeObject *tp) if (keys->dk_usable > 1) { keys->dk_usable--; } - Py_ssize_t size = shared_keys_usable_size(keys); - assert(size > 0); + size_t size = shared_keys_usable_size(keys); PyDictValues *values = new_values(size); if (values == NULL) { PyErr_NoMemory(); return -1; } - assert(((uint8_t *)values)[-1] >= size+2); + assert(((uint8_t *)values)[-1] >= (size + 2)); ((uint8_t *)values)[-2] = 0; - for (int i = 0; i < size; i++) { + for (size_t i = 0; i < size; i++) { values->values[i] = NULL; } _PyDictOrValues_SetValues(_PyObject_DictOrValuesPointer(obj), values); @@ -5335,7 +5329,8 @@ make_dict_from_instance_attributes(PyDictKeysObject *keys, PyDictValues *values) dictkeys_incref(keys); Py_ssize_t used = 0; Py_ssize_t track = 0; - for (Py_ssize_t i = 0; i < shared_keys_usable_size(keys); i++) { + size_t size = shared_keys_usable_size(keys); + for (size_t i = 0; i < size; i++) { PyObject *val = values->values[i]; if (val != NULL) { used += 1; diff --git a/Objects/listobject.c b/Objects/listobject.c index da623c9719aeb8..1d32915b17a14b 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -2227,7 +2227,7 @@ list.sort * key as keyfunc: object = None - reverse: bool(accept={int}) = False + reverse: bool = False Sort the list in ascending order and return None. @@ -2242,7 +2242,7 @@ The reverse flag can be set to sort in descending order. static PyObject * list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) -/*[clinic end generated code: output=57b9f9c5e23fbe42 input=cb56cd179a713060]*/ +/*[clinic end generated code: output=57b9f9c5e23fbe42 input=a74c4cd3ec6b5c08]*/ { MergeState ms; Py_ssize_t nremaining; @@ -2806,10 +2806,9 @@ static PyObject * list___sizeof___impl(PyListObject *self) /*[clinic end generated code: output=3417541f95f9a53e input=b8030a5d5ce8a187]*/ { - Py_ssize_t res; - - res = _PyObject_SIZE(Py_TYPE(self)) + self->allocated * sizeof(void*); - return PyLong_FromSsize_t(res); + size_t res = _PyObject_SIZE(Py_TYPE(self)); + res += (size_t)self->allocated * sizeof(void*); + return PyLong_FromSize_t(res); } static PyObject *list_iter(PyObject *seq); diff --git a/Objects/object.c b/Objects/object.c index a499cb32b22f58..687bd36d2b4af1 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1043,22 +1043,25 @@ PyObject_SetAttr(PyObject *v, PyObject *name, PyObject *value) PyObject ** _PyObject_ComputedDictPointer(PyObject *obj) { - Py_ssize_t dictoffset; PyTypeObject *tp = Py_TYPE(obj); - assert((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0); - dictoffset = tp->tp_dictoffset; - if (dictoffset == 0) + + Py_ssize_t dictoffset = tp->tp_dictoffset; + if (dictoffset == 0) { return NULL; + } + if (dictoffset < 0) { assert(dictoffset != -1); + Py_ssize_t tsize = Py_SIZE(obj); if (tsize < 0) { tsize = -tsize; } size_t size = _PyObject_VAR_SIZE(tp, tsize); + assert(size <= (size_t)PY_SSIZE_T_MAX); + dictoffset += (Py_ssize_t)size; - dictoffset += (long)size; _PyObject_ASSERT(obj, dictoffset > 0); _PyObject_ASSERT(obj, dictoffset % SIZEOF_VOID_P == 0); } diff --git a/Objects/rangeobject.c b/Objects/rangeobject.c index a889aa04db81f0..992e7c079ded54 100644 --- a/Objects/rangeobject.c +++ b/Objects/rangeobject.c @@ -756,18 +756,19 @@ PyTypeObject PyRange_Type = { static PyObject * rangeiter_next(_PyRangeIterObject *r) { - if (r->index < r->len) - /* cast to unsigned to avoid possible signed overflow - in intermediate calculations. */ - return PyLong_FromLong((long)(r->start + - (unsigned long)(r->index++) * r->step)); + if (r->len > 0) { + long result = r->start; + r->start = result + r->step; + r->len--; + return PyLong_FromLong(result); + } return NULL; } static PyObject * rangeiter_len(_PyRangeIterObject *r, PyObject *Py_UNUSED(ignored)) { - return PyLong_FromLong(r->len - r->index); + return PyLong_FromLong(r->len); } PyDoc_STRVAR(length_hint_doc, @@ -794,8 +795,8 @@ rangeiter_reduce(_PyRangeIterObject *r, PyObject *Py_UNUSED(ignored)) if (range == NULL) goto err; /* return the result */ - return Py_BuildValue( - "N(N)l", _PyEval_GetBuiltin(&_Py_ID(iter)), range, r->index); + return Py_BuildValue("N(N)O", _PyEval_GetBuiltin(&_Py_ID(iter)), + range, Py_None); err: Py_XDECREF(start); Py_XDECREF(stop); @@ -814,7 +815,8 @@ rangeiter_setstate(_PyRangeIterObject *r, PyObject *state) index = 0; else if (index > r->len) index = r->len; /* exhausted iterator */ - r->index = index; + r->start += index * r->step; + r->len -= index; Py_RETURN_NONE; } @@ -904,13 +906,11 @@ fast_range_iter(long start, long stop, long step, long len) it->start = start; it->step = step; it->len = len; - it->index = 0; return (PyObject *)it; } typedef struct { PyObject_HEAD - PyObject *index; PyObject *start; PyObject *step; PyObject *len; @@ -919,7 +919,8 @@ typedef struct { static PyObject * longrangeiter_len(longrangeiterobject *r, PyObject *no_args) { - return PyNumber_Subtract(r->len, r->index); + Py_INCREF(r->len); + return r->len; } static PyObject * @@ -946,8 +947,8 @@ longrangeiter_reduce(longrangeiterobject *r, PyObject *Py_UNUSED(ignored)) } /* return the result */ - return Py_BuildValue( - "N(N)O", _PyEval_GetBuiltin(&_Py_ID(iter)), range, r->index); + return Py_BuildValue("N(N)O", _PyEval_GetBuiltin(&_Py_ID(iter)), + range, Py_None); } static PyObject * @@ -970,7 +971,22 @@ longrangeiter_setstate(longrangeiterobject *r, PyObject *state) if (cmp > 0) state = r->len; } - Py_XSETREF(r->index, Py_NewRef(state)); + PyObject *product = PyNumber_Multiply(state, r->step); + if (product == NULL) + return NULL; + PyObject *new_start = PyNumber_Add(r->start, product); + Py_DECREF(product); + if (new_start == NULL) + return NULL; + PyObject *new_len = PyNumber_Subtract(r->len, state); + if (new_len == NULL) { + Py_DECREF(new_start); + return NULL; + } + PyObject *tmp = r->start; + r->start = new_start; + Py_SETREF(r->len, new_len); + Py_DECREF(tmp); Py_RETURN_NONE; } @@ -987,7 +1003,6 @@ static PyMethodDef longrangeiter_methods[] = { static void longrangeiter_dealloc(longrangeiterobject *r) { - Py_XDECREF(r->index); Py_XDECREF(r->start); Py_XDECREF(r->step); Py_XDECREF(r->len); @@ -997,29 +1012,21 @@ longrangeiter_dealloc(longrangeiterobject *r) static PyObject * longrangeiter_next(longrangeiterobject *r) { - PyObject *product, *new_index, *result; - if (PyObject_RichCompareBool(r->index, r->len, Py_LT) != 1) + if (PyObject_RichCompareBool(r->len, _PyLong_GetZero(), Py_GT) != 1) return NULL; - new_index = PyNumber_Add(r->index, _PyLong_GetOne()); - if (!new_index) + PyObject *new_start = PyNumber_Add(r->start, r->step); + if (new_start == NULL) { return NULL; - - product = PyNumber_Multiply(r->index, r->step); - if (!product) { - Py_DECREF(new_index); - return NULL; - } - - result = PyNumber_Add(r->start, product); - Py_DECREF(product); - if (result) { - Py_SETREF(r->index, new_index); } - else { - Py_DECREF(new_index); + PyObject *new_len = PyNumber_Subtract(r->len, _PyLong_GetOne()); + if (new_len == NULL) { + Py_DECREF(new_start); + return NULL; } - + PyObject *result = r->start; + r->start = new_start; + Py_SETREF(r->len, new_len); return result; } @@ -1108,7 +1115,6 @@ range_iter(PyObject *seq) it->start = Py_NewRef(r->start); it->step = Py_NewRef(r->step); it->len = Py_NewRef(r->length); - it->index = Py_NewRef(_PyLong_GetZero()); return (PyObject *)it; } @@ -1186,7 +1192,7 @@ range_reverse(PyObject *seq, PyObject *Py_UNUSED(ignored)) it = PyObject_New(longrangeiterobject, &PyLongRangeIter_Type); if (it == NULL) return NULL; - it->index = it->start = it->step = NULL; + it->start = it->step = NULL; /* start + (len - 1) * step */ it->len = Py_NewRef(range->length); @@ -1210,7 +1216,6 @@ range_reverse(PyObject *seq, PyObject *Py_UNUSED(ignored)) if (!it->step) goto create_failure; - it->index = Py_NewRef(_PyLong_GetZero()); return (PyObject *)it; create_failure: diff --git a/Objects/setobject.c b/Objects/setobject.c index ae9e9b99446116..fcdda2a0bca2b6 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -1957,12 +1957,11 @@ set_reduce(PySetObject *so, PyObject *Py_UNUSED(ignored)) static PyObject * set_sizeof(PySetObject *so, PyObject *Py_UNUSED(ignored)) { - Py_ssize_t res; - - res = _PyObject_SIZE(Py_TYPE(so)); - if (so->table != so->smalltable) - res = res + (so->mask + 1) * sizeof(setentry); - return PyLong_FromSsize_t(res); + size_t res = _PyObject_SIZE(Py_TYPE(so)); + if (so->table != so->smalltable) { + res += ((size_t)so->mask + 1) * sizeof(setentry); + } + return PyLong_FromSize_t(res); } PyDoc_STRVAR(sizeof_doc, "S.__sizeof__() -> size of S in memory, in bytes"); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index a4974a1b4f7113..ae80f5a8fd88e0 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4680,16 +4680,17 @@ static PyObject * type___sizeof___impl(PyTypeObject *self) /*[clinic end generated code: output=766f4f16cd3b1854 input=99398f24b9cf45d6]*/ { - Py_ssize_t size; + size_t size; if (self->tp_flags & Py_TPFLAGS_HEAPTYPE) { PyHeapTypeObject* et = (PyHeapTypeObject*)self; size = sizeof(PyHeapTypeObject); if (et->ht_cached_keys) size += _PyDict_KeysSize(et->ht_cached_keys); } - else + else { size = sizeof(PyTypeObject); - return PyLong_FromSsize_t(size); + } + return PyLong_FromSize_t(size); } static PyMethodDef type_methods[] = { diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 55f029dd504ca0..deeca35714b766 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -4530,6 +4530,9 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, } s += ascii_decode(s, end, PyUnicode_1BYTE_DATA(u)); if (s == end) { + if (consumed) { + *consumed = size; + } return u; } @@ -12441,7 +12444,7 @@ unicode_rsplit_impl(PyObject *self, PyObject *sep, Py_ssize_t maxsplit) /*[clinic input] str.splitlines as unicode_splitlines - keepends: bool(accept={int}) = False + keepends: bool = False Return a list of the lines in the string, breaking at line boundaries. @@ -12451,7 +12454,7 @@ true. static PyObject * unicode_splitlines_impl(PyObject *self, int keepends) -/*[clinic end generated code: output=f664dcdad153ec40 input=b508e180459bdd8b]*/ +/*[clinic end generated code: output=f664dcdad153ec40 input=ba6ad05ee85d2b55]*/ { return PyUnicode_Splitlines(self, keepends); } diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index ce72e1529024c1..463c0e00ca1411 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -97,6 +97,7 @@ tok_new(void) tok->async_def_nl = 0; tok->interactive_underflow = IUNDERFLOW_NORMAL; tok->str = NULL; + tok->report_warnings = 1; #ifdef Py_DEBUG tok->debug = _Py_GetConfig()->parser_debug; #endif @@ -1201,6 +1202,10 @@ indenterror(struct tok_state *tok) static int parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...) { + if (!tok->report_warnings) { + return 0; + } + PyObject *errmsg; va_list vargs; va_start(vargs, format); @@ -2239,6 +2244,9 @@ _PyTokenizer_FindEncodingFilename(int fd, PyObject *filename) } } struct token token; + // We don't want to report warnings here because it could cause infinite recursion + // if fetching the encoding shows a warning. + tok->report_warnings = 0; while (tok->lineno < 2 && tok->done == E_OK) { _PyTokenizer_Get(tok, &token); } diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h index 2542d30e1da0ed..16a94d5f51d664 100644 --- a/Parser/tokenizer.h +++ b/Parser/tokenizer.h @@ -92,6 +92,7 @@ struct tok_state { NEWLINE token after it. */ /* How to proceed when asked for a new token in interactive mode */ enum interactive_underflow_t interactive_underflow; + int report_warnings; #ifdef Py_DEBUG int debug; #endif diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index b3b7e8d6c50530..ff96c25da5ebc6 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -714,7 +714,7 @@ compile as builtin_compile filename: object(converter="PyUnicode_FSDecoder") mode: str flags: int = 0 - dont_inherit: bool(accept={int}) = False + dont_inherit: bool = False optimize: int = -1 * _feature_version as feature_version: int = -1 @@ -737,7 +737,7 @@ static PyObject * builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, const char *mode, int flags, int dont_inherit, int optimize, int feature_version) -/*[clinic end generated code: output=b0c09c84f116d3d7 input=40171fb92c1d580d]*/ +/*[clinic end generated code: output=b0c09c84f116d3d7 input=cc78e20e7c7682ba]*/ { PyObject *source_copy; const char *str; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index a1f910da8ed54a..41dd1acc937d71 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2620,14 +2620,15 @@ dummy_func( STAT_INC(FOR_ITER, hit); _Py_CODEUNIT next = next_instr[INLINE_CACHE_ENTRIES_FOR_ITER]; assert(_PyOpcode_Deopt[_Py_OPCODE(next)] == STORE_FAST); - if (r->index >= r->len) { + if (r->len <= 0) { STACK_SHRINK(1); Py_DECREF(r); JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1); } else { - long value = (long)(r->start + - (unsigned long)(r->index++) * r->step); + long value = r->start; + r->start = value + r->step; + r->len--; if (_PyLong_AssignValue(&GETLOCAL(_Py_OPARG(next)), value) < 0) { goto error; } diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 19930a519be089..89f069dd97f6ea 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -354,8 +354,8 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj } } if (args[4]) { - dont_inherit = _PyLong_AsInt(args[4]); - if (dont_inherit == -1 && PyErr_Occurred()) { + dont_inherit = PyObject_IsTrue(args[4]); + if (dont_inherit < 0) { goto exit; } if (!--noptargs) { @@ -1215,4 +1215,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=3c9497e0ffeb8a30 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=973da43fa65aa727 input=a9049054013a1b77]*/ diff --git a/Python/compile.c b/Python/compile.c index dd8596defb8efe..d6ed6941ac1ecd 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -171,6 +171,24 @@ struct instr { struct basicblock_ *i_except; /* target block when exception is raised */ }; +/* One arg*/ +#define INSTR_SET_OP1(I, OP, ARG) \ + do { \ + assert(HAS_ARG(OP)); \ + struct instr *_instr__ptr_ = (I); \ + _instr__ptr_->i_opcode = (OP); \ + _instr__ptr_->i_oparg = (ARG); \ + } while (0); + +/* No args*/ +#define INSTR_SET_OP0(I, OP) \ + do { \ + assert(!HAS_ARG(OP)); \ + struct instr *_instr__ptr_ = (I); \ + _instr__ptr_->i_opcode = (OP); \ + _instr__ptr_->i_oparg = 0; \ + } while (0); + typedef struct exceptstack { struct basicblock_ *handlers[CO_MAXBLOCKS+1]; int depth; @@ -218,7 +236,8 @@ instr_size(struct instr *instruction) { int opcode = instruction->i_opcode; assert(!IS_PSEUDO_OPCODE(opcode)); - int oparg = HAS_ARG(opcode) ? instruction->i_oparg : 0; + int oparg = instruction->i_oparg; + assert(HAS_ARG(opcode) || oparg == 0); int extended_args = (0xFFFFFF < oparg) + (0xFFFF < oparg) + (0xFF < oparg); int caches = _PyOpcode_Caches[opcode]; return extended_args + 1 + caches; @@ -229,7 +248,8 @@ write_instr(_Py_CODEUNIT *codestr, struct instr *instruction, int ilen) { int opcode = instruction->i_opcode; assert(!IS_PSEUDO_OPCODE(opcode)); - int oparg = HAS_ARG(opcode) ? instruction->i_oparg : 0; + int oparg = instruction->i_oparg; + assert(HAS_ARG(opcode) || oparg == 0); int caches = _PyOpcode_Caches[opcode]; switch (ilen - caches) { case 4: @@ -7248,12 +7268,12 @@ assemble_init(struct assembler *a, int firstlineno) if (a->a_except_table == NULL) { goto error; } - return 1; + return 0; error: Py_XDECREF(a->a_bytecode); Py_XDECREF(a->a_linetable); Py_XDECREF(a->a_except_table); - return 0; + return -1; } static void @@ -7598,7 +7618,7 @@ convert_exception_handlers_to_nops(basicblock *entryblock) { for (int i = 0; i < b->b_iused; i++) { struct instr *instr = &b->b_instr[i]; if (is_block_push(instr) || instr->i_opcode == POP_BLOCK) { - instr->i_opcode = NOP; + INSTR_SET_OP0(instr, NOP); } } } @@ -7647,8 +7667,9 @@ assemble_emit_exception_table_entry(struct assembler *a, int start, int end, bas { Py_ssize_t len = PyBytes_GET_SIZE(a->a_except_table); if (a->a_except_table_off + MAX_SIZE_OF_ENTRY >= len) { - if (_PyBytes_Resize(&a->a_except_table, len * 2) < 0) - return 0; + if (_PyBytes_Resize(&a->a_except_table, len * 2) < 0) { + return -1; + } } int size = end-start; assert(end > start); @@ -7663,7 +7684,7 @@ assemble_emit_exception_table_entry(struct assembler *a, int start, int end, bas assemble_emit_exception_table_item(a, size, 0); assemble_emit_exception_table_item(a, target, 0); assemble_emit_exception_table_item(a, depth_lasti, 0); - return 1; + return 0; } static int @@ -7679,7 +7700,9 @@ assemble_exception_table(struct assembler *a, basicblock *entryblock) struct instr *instr = &b->b_instr[i]; if (instr->i_except != handler) { if (handler != NULL) { - RETURN_IF_FALSE(assemble_emit_exception_table_entry(a, start, ioffset, handler)); + if (assemble_emit_exception_table_entry(a, start, ioffset, handler) < 0) { + return -1; + } } start = ioffset; handler = instr->i_except; @@ -7688,9 +7711,11 @@ assemble_exception_table(struct assembler *a, basicblock *entryblock) } } if (handler != NULL) { - RETURN_IF_FALSE(assemble_emit_exception_table_entry(a, start, ioffset, handler)); + if (assemble_emit_exception_table_entry(a, start, ioffset, handler) < 0) { + return -1; + } } - return 1; + return 0; } /* Code location emitting code. See locations.md for a description of the format. */ @@ -7793,12 +7818,12 @@ write_location_info_entry(struct assembler* a, struct instr* i, int isize) if (a->a_location_off + THEORETICAL_MAX_ENTRY_SIZE >= len) { assert(len > THEORETICAL_MAX_ENTRY_SIZE); if (_PyBytes_Resize(&a->a_linetable, len*2) < 0) { - return 0; + return -1; } } if (i->i_loc.lineno < 0) { write_location_info_none(a, isize); - return 1; + return 0; } int line_delta = i->i_loc.lineno - a->a_lineno; int column = i->i_loc.col_offset; @@ -7809,23 +7834,23 @@ write_location_info_entry(struct assembler* a, struct instr* i, int isize) if (i->i_loc.end_lineno == i->i_loc.lineno || i->i_loc.end_lineno == -1) { write_location_info_no_column(a, isize, line_delta); a->a_lineno = i->i_loc.lineno; - return 1; + return 0; } } else if (i->i_loc.end_lineno == i->i_loc.lineno) { if (line_delta == 0 && column < 80 && end_column - column < 16 && end_column >= column) { write_location_info_short_form(a, isize, column, end_column); - return 1; + return 0; } if (line_delta >= 0 && line_delta < 3 && column < 128 && end_column < 128) { write_location_info_oneline_form(a, isize, line_delta, column, end_column); a->a_lineno = i->i_loc.lineno; - return 1; + return 0; } } write_location_info_long_form(a, i, isize); a->a_lineno = i->i_loc.lineno; - return 1; + return 0; } static int @@ -7833,8 +7858,8 @@ assemble_emit_location(struct assembler* a, struct instr* i) { int isize = instr_size(i); while (isize > 8) { - if (!write_location_info_entry(a, i, 8)) { - return 0; + if (write_location_info_entry(a, i, 8) < 0) { + return -1; } isize -= 8; } @@ -7854,15 +7879,17 @@ assemble_emit(struct assembler *a, struct instr *i) int size = instr_size(i); if (a->a_offset + size >= len / (int)sizeof(_Py_CODEUNIT)) { - if (len > PY_SSIZE_T_MAX / 2) - return 0; - if (_PyBytes_Resize(&a->a_bytecode, len * 2) < 0) - return 0; + if (len > PY_SSIZE_T_MAX / 2) { + return -1; + } + if (_PyBytes_Resize(&a->a_bytecode, len * 2) < 0) { + return -1; + } } code = (_Py_CODEUNIT *)PyBytes_AS_STRING(a->a_bytecode) + a->a_offset; a->a_offset += size; write_instr(code, i, size); - return 1; + return 0; } static int @@ -8262,17 +8289,17 @@ merge_const_one(PyObject *const_cache, PyObject **obj) PyDict_CheckExact(const_cache); PyObject *key = _PyCode_ConstantKey(*obj); if (key == NULL) { - return 0; + return -1; } // t is borrowed reference PyObject *t = PyDict_SetDefault(const_cache, key, key); Py_DECREF(key); if (t == NULL) { - return 0; + return -1; } if (t == key) { // obj is new constant. - return 1; + return 0; } if (PyTuple_CheckExact(t)) { @@ -8281,7 +8308,7 @@ merge_const_one(PyObject *const_cache, PyObject **obj) } Py_SETREF(*obj, Py_NewRef(t)); - return 1; + return 0; } // This is in codeobject.c. @@ -8347,7 +8374,7 @@ makecode(struct compiler *c, struct assembler *a, PyObject *constslist, if (!names) { goto error; } - if (!merge_const_one(c->c_const_cache, &names)) { + if (merge_const_one(c->c_const_cache, &names) < 0) { goto error; } @@ -8355,7 +8382,7 @@ makecode(struct compiler *c, struct assembler *a, PyObject *constslist, if (consts == NULL) { goto error; } - if (!merge_const_one(c->c_const_cache, &consts)) { + if (merge_const_one(c->c_const_cache, &consts) < 0) { goto error; } @@ -8406,7 +8433,7 @@ makecode(struct compiler *c, struct assembler *a, PyObject *constslist, goto error; } - if (!merge_const_one(c->c_const_cache, &localsplusnames)) { + if (merge_const_one(c->c_const_cache, &localsplusnames) < 0) { goto error; } con.localsplusnames = localsplusnames; @@ -8723,13 +8750,48 @@ remove_redundant_jumps(cfg_builder *g) { } if (last->i_target == b->b_next) { assert(b->b_next->b_iused); - last->i_opcode = NOP; + INSTR_SET_OP0(last, NOP); } } } return 0; } +static int +prepare_localsplus(struct compiler* c, int code_flags) +{ + assert(PyDict_GET_SIZE(c->u->u_varnames) < INT_MAX); + assert(PyDict_GET_SIZE(c->u->u_cellvars) < INT_MAX); + assert(PyDict_GET_SIZE(c->u->u_freevars) < INT_MAX); + int nlocals = (int)PyDict_GET_SIZE(c->u->u_varnames); + int ncellvars = (int)PyDict_GET_SIZE(c->u->u_cellvars); + int nfreevars = (int)PyDict_GET_SIZE(c->u->u_freevars); + assert(INT_MAX - nlocals - ncellvars > 0); + assert(INT_MAX - nlocals - ncellvars - nfreevars > 0); + int nlocalsplus = nlocals + ncellvars + nfreevars; + int* cellfixedoffsets = build_cellfixedoffsets(c); + if (cellfixedoffsets == NULL) { + return -1; + } + + cfg_builder* g = CFG_BUILDER(c); + + // This must be called before fix_cell_offsets(). + if (insert_prefix_instructions(c, g->g_entryblock, cellfixedoffsets, nfreevars, code_flags)) { + PyMem_Free(cellfixedoffsets); + return -1; + } + + int numdropped = fix_cell_offsets(c, g->g_entryblock, cellfixedoffsets); + PyMem_Free(cellfixedoffsets); // At this point we're done with it. + cellfixedoffsets = NULL; + if (numdropped < 0) { + return -1; + } + nlocalsplus -= numdropped; + return nlocalsplus; +} + static PyCodeObject * assemble(struct compiler *c, int addNone) { @@ -8751,20 +8813,6 @@ assemble(struct compiler *c, int addNone) ADDOP(c, NO_LOCATION, RETURN_VALUE); } - assert(PyDict_GET_SIZE(c->u->u_varnames) < INT_MAX); - assert(PyDict_GET_SIZE(c->u->u_cellvars) < INT_MAX); - assert(PyDict_GET_SIZE(c->u->u_freevars) < INT_MAX); - int nlocals = (int)PyDict_GET_SIZE(c->u->u_varnames); - int ncellvars = (int)PyDict_GET_SIZE(c->u->u_cellvars); - int nfreevars = (int)PyDict_GET_SIZE(c->u->u_freevars); - assert(INT_MAX - nlocals - ncellvars > 0); - assert(INT_MAX - nlocals - ncellvars - nfreevars > 0); - int nlocalsplus = nlocals + ncellvars + nfreevars; - int *cellfixedoffsets = build_cellfixedoffsets(c); - if (cellfixedoffsets == NULL) { - goto error; - } - int nblocks = 0; for (basicblock *b = CFG_BUILDER(c)->g_block_list; b != NULL; b = b->b_list) { nblocks++; @@ -8787,19 +8835,6 @@ assemble(struct compiler *c, int addNone) } } - // This must be called before fix_cell_offsets(). - if (insert_prefix_instructions(c, g->g_entryblock, cellfixedoffsets, nfreevars, code_flags)) { - goto error; - } - - int numdropped = fix_cell_offsets(c, g->g_entryblock, cellfixedoffsets); - PyMem_Free(cellfixedoffsets); // At this point we're done with it. - cellfixedoffsets = NULL; - if (numdropped < 0) { - goto error; - } - nlocalsplus -= numdropped; - /** Preprocessing **/ /* Map labels to targets and mark exception handlers */ if (translate_jump_labels_to_targets(g->g_entryblock)) { @@ -8839,6 +8874,12 @@ assemble(struct compiler *c, int addNone) } /** Assembly **/ + + int nlocalsplus = prepare_localsplus(c, code_flags); + if (nlocalsplus < 0) { + goto error; + } + int maxdepth = stackdepth(g->g_entryblock, code_flags); if (maxdepth < 0) { goto error; @@ -8858,45 +8899,50 @@ assemble(struct compiler *c, int addNone) assemble_jump_offsets(g->g_entryblock); /* Create assembler */ - if (!assemble_init(&a, c->u->u_firstlineno)) + if (assemble_init(&a, c->u->u_firstlineno) < 0) { goto error; + } /* Emit code. */ for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - for (int j = 0; j < b->b_iused; j++) - if (!assemble_emit(&a, &b->b_instr[j])) + for (int j = 0; j < b->b_iused; j++) { + if (assemble_emit(&a, &b->b_instr[j]) < 0) { goto error; + } + } } /* Emit location info */ a.a_lineno = c->u->u_firstlineno; for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - for (int j = 0; j < b->b_iused; j++) - if (!assemble_emit_location(&a, &b->b_instr[j])) + for (int j = 0; j < b->b_iused; j++) { + if (assemble_emit_location(&a, &b->b_instr[j]) < 0) { goto error; + } + } } - if (!assemble_exception_table(&a, g->g_entryblock)) { + if (assemble_exception_table(&a, g->g_entryblock) < 0) { goto error; } if (_PyBytes_Resize(&a.a_except_table, a.a_except_table_off) < 0) { goto error; } - if (!merge_const_one(c->c_const_cache, &a.a_except_table)) { + if (merge_const_one(c->c_const_cache, &a.a_except_table) < 0) { goto error; } if (_PyBytes_Resize(&a.a_linetable, a.a_location_off) < 0) { goto error; } - if (!merge_const_one(c->c_const_cache, &a.a_linetable)) { + if (merge_const_one(c->c_const_cache, &a.a_linetable) < 0) { goto error; } if (_PyBytes_Resize(&a.a_bytecode, a.a_offset * sizeof(_Py_CODEUNIT)) < 0) { goto error; } - if (!merge_const_one(c->c_const_cache, &a.a_bytecode)) { + if (merge_const_one(c->c_const_cache, &a.a_bytecode) < 0) { goto error; } @@ -8904,9 +8950,6 @@ assemble(struct compiler *c, int addNone) error: Py_XDECREF(consts); assemble_free(&a); - if (cellfixedoffsets != NULL) { - PyMem_Free(cellfixedoffsets); - } return co; } @@ -8964,7 +9007,7 @@ fold_tuple_on_constants(PyObject *const_cache, } PyTuple_SET_ITEM(newconst, i, constant); } - if (merge_const_one(const_cache, &newconst) == 0) { + if (merge_const_one(const_cache, &newconst) < 0) { Py_DECREF(newconst); return -1; } @@ -8988,10 +9031,9 @@ fold_tuple_on_constants(PyObject *const_cache, } Py_DECREF(newconst); for (int i = 0; i < n; i++) { - inst[i].i_opcode = NOP; + INSTR_SET_OP0(&inst[i], NOP); } - inst[n].i_opcode = LOAD_CONST; - inst[n].i_oparg = (int)index; + INSTR_SET_OP1(&inst[n], LOAD_CONST, (int)index); return 0; } @@ -9088,7 +9130,7 @@ swaptimize(basicblock *block, int *ix) } // NOP out any unused instructions: while (0 <= current) { - instructions[current--].i_opcode = NOP; + INSTR_SET_OP0(&instructions[current--], NOP); } PyMem_Free(stack); *ix += len - 1; @@ -9154,7 +9196,7 @@ apply_static_swaps(basicblock *block, int i) } } // Success! - swap->i_opcode = NOP; + INSTR_SET_OP0(swap, NOP); struct instr temp = block->b_instr[j]; block->b_instr[j] = block->b_instr[k]; block->b_instr[k] = temp; @@ -9191,7 +9233,7 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) assert(PyDict_CheckExact(const_cache)); assert(PyList_CheckExact(consts)); struct instr nop; - nop.i_opcode = NOP; + INSTR_SET_OP0(&nop, NOP); struct instr *target; for (int i = 0; i < bb->b_iused; i++) { struct instr *inst = &bb->b_instr[i]; @@ -9225,13 +9267,13 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) if (is_true == -1) { goto error; } - inst->i_opcode = NOP; + INSTR_SET_OP0(inst, NOP); jump_if_true = nextop == POP_JUMP_IF_TRUE; if (is_true == jump_if_true) { bb->b_instr[i+1].i_opcode = JUMP; } else { - bb->b_instr[i+1].i_opcode = NOP; + INSTR_SET_OP0(&bb->b_instr[i + 1], NOP); } break; case JUMP_IF_FALSE_OR_POP: @@ -9250,8 +9292,8 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) bb->b_instr[i+1].i_opcode = JUMP; } else { - inst->i_opcode = NOP; - bb->b_instr[i+1].i_opcode = NOP; + INSTR_SET_OP0(inst, NOP); + INSTR_SET_OP0(&bb->b_instr[i + 1], NOP); } break; case IS_OP: @@ -9262,8 +9304,8 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) int jump_op = i+2 < bb->b_iused ? bb->b_instr[i+2].i_opcode : 0; if (Py_IsNone(cnt) && (jump_op == POP_JUMP_IF_FALSE || jump_op == POP_JUMP_IF_TRUE)) { unsigned char nextarg = bb->b_instr[i+1].i_oparg; - inst->i_opcode = NOP; - bb->b_instr[i+1].i_opcode = NOP; + INSTR_SET_OP0(inst, NOP); + INSTR_SET_OP0(&bb->b_instr[i + 1], NOP); bb->b_instr[i+2].i_opcode = nextarg ^ (jump_op == POP_JUMP_IF_FALSE) ? POP_JUMP_IF_NOT_NONE : POP_JUMP_IF_NONE; } @@ -9281,12 +9323,12 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) if (nextop == UNPACK_SEQUENCE && oparg == bb->b_instr[i+1].i_oparg) { switch(oparg) { case 1: - inst->i_opcode = NOP; - bb->b_instr[i+1].i_opcode = NOP; + INSTR_SET_OP0(inst, NOP); + INSTR_SET_OP0(&bb->b_instr[i + 1], NOP); continue; case 2: case 3: - inst->i_opcode = NOP; + INSTR_SET_OP0(inst, NOP); bb->b_instr[i+1].i_opcode = SWAP; continue; } @@ -9395,7 +9437,7 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) break; case SWAP: if (oparg == 1) { - inst->i_opcode = NOP; + INSTR_SET_OP0(inst, NOP); break; } if (swaptimize(bb, &i)) { @@ -9407,8 +9449,7 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) break; case PUSH_NULL: if (nextop == LOAD_GLOBAL && (inst[1].i_opcode & 1) == 0) { - inst->i_opcode = NOP; - inst->i_oparg = 0; + INSTR_SET_OP0(inst, NOP); inst[1].i_oparg |= 1; } break; @@ -9437,7 +9478,7 @@ inline_small_exit_blocks(basicblock *bb) { } basicblock *target = last->i_target; if (basicblock_exits_scope(target) && target->b_iused <= MAX_COPY_SIZE) { - last->i_opcode = NOP; + INSTR_SET_OP0(last, NOP); if (basicblock_append_instructions(bb, target) < 0) { return -1; } @@ -9820,17 +9861,17 @@ remove_unused_consts(basicblock *entryblock, PyObject *consts) return err; } -static inline int +static inline bool is_exit_without_lineno(basicblock *b) { if (!basicblock_exits_scope(b)) { - return 0; + return false; } for (int i = 0; i < b->b_iused; i++) { if (b->b_instr[i].i_loc.lineno >= 0) { - return 0; + return false; } } - return 1; + return true; } /* PEP 626 mandates that the f_lineno of a frame is correct diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ae8fdd5e99c3dc..3a403824b49958 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -436,10 +436,10 @@ } TARGET(BINARY_SUBSCR_GETITEM) { - uint32_t type_version = read_u32(next_instr + 1); - uint16_t func_version = read_u16(next_instr + 3); PyObject *sub = PEEK(1); PyObject *container = PEEK(2); + uint32_t type_version = read_u32(next_instr + 1); + uint16_t func_version = read_u16(next_instr + 3); PyTypeObject *tp = Py_TYPE(container); DEOPT_IF(tp->tp_version_tag != type_version, BINARY_SUBSCR); assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); @@ -2638,14 +2638,15 @@ STAT_INC(FOR_ITER, hit); _Py_CODEUNIT next = next_instr[INLINE_CACHE_ENTRIES_FOR_ITER]; assert(_PyOpcode_Deopt[_Py_OPCODE(next)] == STORE_FAST); - if (r->index >= r->len) { + if (r->len <= 0) { STACK_SHRINK(1); Py_DECREF(r); JUMPBY(INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 1); } else { - long value = (long)(r->start + - (unsigned long)(r->index++) * r->step); + long value = r->start; + r->start = value + r->step; + r->len--; if (_PyLong_AssignValue(&GETLOCAL(_Py_OPARG(next)), value) < 0) { goto error; } diff --git a/Python/pystate.c b/Python/pystate.c index 19fd9a6ae4497b..ea3c22c5d71ad6 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -466,6 +466,11 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) } interp->active_func_watchers = 0; + for (int i=0; i < CODE_MAX_WATCHERS; i++) { + interp->code_watchers[i] = NULL; + } + interp->active_code_watchers = 0; + // XXX Once we have one allocator per interpreter (i.e. // per-interpreter GC) we must ensure that all of the interpreter's // objects have been cleaned up at the point. @@ -1784,30 +1789,78 @@ PyGILState_Release(PyGILState_STATE oldstate) /* cross-interpreter data */ -crossinterpdatafunc _PyCrossInterpreterData_Lookup(PyObject *); +static inline void +_xidata_init(_PyCrossInterpreterData *data) +{ + // If the value is being reused + // then _xidata_clear() should have been called already. + assert(data->data == NULL); + assert(data->obj == NULL); + *data = (_PyCrossInterpreterData){0}; + data->interp = -1; +} -/* This is a separate func from _PyCrossInterpreterData_Lookup in order - to keep the registry code separate. */ -static crossinterpdatafunc -_lookup_getdata(PyObject *obj) +static inline void +_xidata_clear(_PyCrossInterpreterData *data) { - crossinterpdatafunc getdata = _PyCrossInterpreterData_Lookup(obj); - if (getdata == NULL && PyErr_Occurred() == 0) - PyErr_Format(PyExc_ValueError, - "%S does not support cross-interpreter data", obj); - return getdata; + if (data->free != NULL) { + data->free(data->data); + } + data->data = NULL; + Py_CLEAR(data->obj); +} + +void +_PyCrossInterpreterData_Init(_PyCrossInterpreterData *data, + PyInterpreterState *interp, + void *shared, PyObject *obj, + xid_newobjectfunc new_object) +{ + assert(data != NULL); + assert(new_object != NULL); + _xidata_init(data); + data->data = shared; + if (obj != NULL) { + assert(interp != NULL); + // released in _PyCrossInterpreterData_Clear() + data->obj = Py_NewRef(obj); + } + // Ideally every object would know its owning interpreter. + // Until then, we have to rely on the caller to identify it + // (but we don't need it in all cases). + data->interp = (interp != NULL) ? interp->id : -1; + data->new_object = new_object; } int -_PyObject_CheckCrossInterpreterData(PyObject *obj) -{ - crossinterpdatafunc getdata = _lookup_getdata(obj); - if (getdata == NULL) { +_PyCrossInterpreterData_InitWithSize(_PyCrossInterpreterData *data, + PyInterpreterState *interp, + const size_t size, PyObject *obj, + xid_newobjectfunc new_object) +{ + assert(size > 0); + // For now we always free the shared data in the same interpreter + // where it was allocated, so the interpreter is required. + assert(interp != NULL); + _PyCrossInterpreterData_Init(data, interp, NULL, obj, new_object); + data->data = PyMem_Malloc(size); + if (data->data == NULL) { return -1; } + data->free = PyMem_Free; return 0; } +void +_PyCrossInterpreterData_Clear(PyInterpreterState *interp, + _PyCrossInterpreterData *data) +{ + assert(data != NULL); + // This must be called in the owning interpreter. + assert(interp == NULL || data->interp == interp->id); + _xidata_clear(data); +} + static int _check_xidata(PyThreadState *tstate, _PyCrossInterpreterData *data) { @@ -1830,6 +1883,30 @@ _check_xidata(PyThreadState *tstate, _PyCrossInterpreterData *data) return 0; } +crossinterpdatafunc _PyCrossInterpreterData_Lookup(PyObject *); + +/* This is a separate func from _PyCrossInterpreterData_Lookup in order + to keep the registry code separate. */ +static crossinterpdatafunc +_lookup_getdata(PyObject *obj) +{ + crossinterpdatafunc getdata = _PyCrossInterpreterData_Lookup(obj); + if (getdata == NULL && PyErr_Occurred() == 0) + PyErr_Format(PyExc_ValueError, + "%S does not support cross-interpreter data", obj); + return getdata; +} + +int +_PyObject_CheckCrossInterpreterData(PyObject *obj) +{ + crossinterpdatafunc getdata = _lookup_getdata(obj); + if (getdata == NULL) { + return -1; + } + return 0; +} + int _PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) { @@ -1842,7 +1919,7 @@ _PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) // Reset data before re-populating. *data = (_PyCrossInterpreterData){0}; - data->free = PyMem_RawFree; // Set a default that may be overridden. + data->interp = -1; // Call the "getdata" func for the object. Py_INCREF(obj); @@ -1851,7 +1928,7 @@ _PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) Py_DECREF(obj); return -1; } - int res = getdata(obj, data); + int res = getdata(tstate, obj, data); Py_DECREF(obj); if (res != 0) { return -1; @@ -1860,27 +1937,24 @@ _PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) // Fill in the blanks and validate the result. data->interp = interp->id; if (_check_xidata(tstate, data) != 0) { - _PyCrossInterpreterData_Release(data); + (void)_PyCrossInterpreterData_Release(data); return -1; } return 0; } -static void -_release_xidata(void *arg) +PyObject * +_PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) { - _PyCrossInterpreterData *data = (_PyCrossInterpreterData *)arg; - if (data->free != NULL) { - data->free(data->data); - } - Py_XDECREF(data->obj); + return data->new_object(data); } +typedef void (*releasefunc)(PyInterpreterState *, void *); + static void _call_in_interpreter(struct _gilstate_runtime_state *gilstate, - PyInterpreterState *interp, - void (*func)(void *), void *arg) + PyInterpreterState *interp, releasefunc func, void *arg) { /* We would use Py_AddPendingCall() if it weren't specific to the * main interpreter (see bpo-33608). In the meantime we take a @@ -1894,7 +1968,9 @@ _call_in_interpreter(struct _gilstate_runtime_state *gilstate, save_tstate = _PyThreadState_Swap(gilstate, tstate); } - func(arg); + // XXX Once the GIL is per-interpreter, this should be called with the + // calling interpreter's GIL released and the target interpreter's held. + func(interp, arg); // Switch back. if (save_tstate != NULL) { @@ -1902,33 +1978,30 @@ _call_in_interpreter(struct _gilstate_runtime_state *gilstate, } } -void +int _PyCrossInterpreterData_Release(_PyCrossInterpreterData *data) { - if (data->data == NULL && data->obj == NULL) { + if (data->free == NULL && data->obj == NULL) { // Nothing to release! - return; + data->data = NULL; + return 0; } // Switch to the original interpreter. PyInterpreterState *interp = _PyInterpreterState_LookUpID(data->interp); if (interp == NULL) { // The interpreter was already destroyed. - if (data->free != NULL) { - // XXX Someone leaked some memory... - } - return; + // This function shouldn't have been called. + // XXX Someone leaked some memory... + assert(PyErr_Occurred()); + return -1; } // "Release" the data and/or the object. struct _gilstate_runtime_state *gilstate = &_PyRuntime.gilstate; - _call_in_interpreter(gilstate, interp, _release_xidata, data); -} - -PyObject * -_PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) -{ - return data->new_object(data); + _call_in_interpreter(gilstate, interp, + (releasefunc)_PyCrossInterpreterData_Clear, data); + return 0; } /* registry of {type -> crossinterpdatafunc} */ @@ -1938,21 +2011,73 @@ _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) crossinterpdatafunc. It would be simpler and more efficient. */ static int -_register_xidata(struct _xidregistry *xidregistry, PyTypeObject *cls, +_xidregistry_add_type(struct _xidregistry *xidregistry, PyTypeObject *cls, crossinterpdatafunc getdata) { // Note that we effectively replace already registered classes // rather than failing. struct _xidregitem *newhead = PyMem_RawMalloc(sizeof(struct _xidregitem)); - if (newhead == NULL) + if (newhead == NULL) { return -1; - newhead->cls = cls; + } + // XXX Assign a callback to clear the entry from the registry? + newhead->cls = PyWeakref_NewRef((PyObject *)cls, NULL); + if (newhead->cls == NULL) { + PyMem_RawFree(newhead); + return -1; + } newhead->getdata = getdata; + newhead->prev = NULL; newhead->next = xidregistry->head; + if (newhead->next != NULL) { + newhead->next->prev = newhead; + } xidregistry->head = newhead; return 0; } +static struct _xidregitem * +_xidregistry_remove_entry(struct _xidregistry *xidregistry, + struct _xidregitem *entry) +{ + struct _xidregitem *next = entry->next; + if (entry->prev != NULL) { + assert(entry->prev->next == entry); + entry->prev->next = next; + } + else { + assert(xidregistry->head == entry); + xidregistry->head = next; + } + if (next != NULL) { + next->prev = entry->prev; + } + Py_DECREF(entry->cls); + PyMem_RawFree(entry); + return next; +} + +static struct _xidregitem * +_xidregistry_find_type(struct _xidregistry *xidregistry, PyTypeObject *cls) +{ + struct _xidregitem *cur = xidregistry->head; + while (cur != NULL) { + PyObject *registered = PyWeakref_GetObject(cur->cls); + if (registered == Py_None) { + // The weakly ref'ed object was freed. + cur = _xidregistry_remove_entry(xidregistry, cur); + } + else { + assert(PyType_Check(registered)); + if (registered == (PyObject *)cls) { + return cur; + } + cur = cur->next; + } + } + return NULL; +} + static void _register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry); int @@ -1968,19 +2093,32 @@ _PyCrossInterpreterData_RegisterClass(PyTypeObject *cls, return -1; } - // Make sure the class isn't ever deallocated. - Py_INCREF((PyObject *)cls); - struct _xidregistry *xidregistry = &_PyRuntime.xidregistry ; PyThread_acquire_lock(xidregistry->mutex, WAIT_LOCK); if (xidregistry->head == NULL) { _register_builtins_for_crossinterpreter_data(xidregistry); } - int res = _register_xidata(xidregistry, cls, getdata); + int res = _xidregistry_add_type(xidregistry, cls, getdata); + PyThread_release_lock(xidregistry->mutex); + return res; +} + +int +_PyCrossInterpreterData_UnregisterClass(PyTypeObject *cls) +{ + int res = 0; + struct _xidregistry *xidregistry = &_PyRuntime.xidregistry ; + PyThread_acquire_lock(xidregistry->mutex, WAIT_LOCK); + struct _xidregitem *matched = _xidregistry_find_type(xidregistry, cls); + if (matched != NULL) { + (void)_xidregistry_remove_entry(xidregistry, matched); + res = 1; + } PyThread_release_lock(xidregistry->mutex); return res; } + /* Cross-interpreter objects are looked up by exact match on the class. We can reassess this policy when we move from a global registry to a tp_* slot. */ @@ -1990,22 +2128,15 @@ _PyCrossInterpreterData_Lookup(PyObject *obj) { struct _xidregistry *xidregistry = &_PyRuntime.xidregistry ; PyObject *cls = PyObject_Type(obj); - crossinterpdatafunc getdata = NULL; PyThread_acquire_lock(xidregistry->mutex, WAIT_LOCK); - struct _xidregitem *cur = xidregistry->head; - if (cur == NULL) { + if (xidregistry->head == NULL) { _register_builtins_for_crossinterpreter_data(xidregistry); - cur = xidregistry->head; - } - for(; cur != NULL; cur = cur->next) { - if (cur->cls == (PyTypeObject *)cls) { - getdata = cur->getdata; - break; - } } + struct _xidregitem *matched = _xidregistry_find_type(xidregistry, + (PyTypeObject *)cls); Py_DECREF(cls); PyThread_release_lock(xidregistry->mutex); - return getdata; + return matched != NULL ? matched->getdata : NULL; } /* cross-interpreter data for builtin types */ @@ -2023,16 +2154,21 @@ _new_bytes_object(_PyCrossInterpreterData *data) } static int -_bytes_shared(PyObject *obj, _PyCrossInterpreterData *data) +_bytes_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) { - struct _shared_bytes_data *shared = PyMem_NEW(struct _shared_bytes_data, 1); + if (_PyCrossInterpreterData_InitWithSize( + data, tstate->interp, sizeof(struct _shared_bytes_data), obj, + _new_bytes_object + ) < 0) + { + return -1; + } + struct _shared_bytes_data *shared = (struct _shared_bytes_data *)data->data; if (PyBytes_AsStringAndSize(obj, &shared->bytes, &shared->len) < 0) { + _PyCrossInterpreterData_Clear(tstate->interp, data); return -1; } - data->data = (void *)shared; - data->obj = Py_NewRef(obj); // Will be "released" (decref'ed) when data released. - data->new_object = _new_bytes_object; - data->free = PyMem_Free; return 0; } @@ -2050,16 +2186,20 @@ _new_str_object(_PyCrossInterpreterData *data) } static int -_str_shared(PyObject *obj, _PyCrossInterpreterData *data) +_str_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) { - struct _shared_str_data *shared = PyMem_NEW(struct _shared_str_data, 1); + if (_PyCrossInterpreterData_InitWithSize( + data, tstate->interp, sizeof(struct _shared_str_data), obj, + _new_str_object + ) < 0) + { + return -1; + } + struct _shared_str_data *shared = (struct _shared_str_data *)data->data; shared->kind = PyUnicode_KIND(obj); shared->buffer = PyUnicode_DATA(obj); shared->len = PyUnicode_GET_LENGTH(obj); - data->data = (void *)shared; - data->obj = Py_NewRef(obj); // Will be "released" (decref'ed) when data released. - data->new_object = _new_str_object; - data->free = PyMem_Free; return 0; } @@ -2070,7 +2210,8 @@ _new_long_object(_PyCrossInterpreterData *data) } static int -_long_shared(PyObject *obj, _PyCrossInterpreterData *data) +_long_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) { /* Note that this means the size of shareable ints is bounded by * sys.maxsize. Hence on 32-bit architectures that is half the @@ -2083,10 +2224,9 @@ _long_shared(PyObject *obj, _PyCrossInterpreterData *data) } return -1; } - data->data = (void *)value; - data->obj = NULL; - data->new_object = _new_long_object; - data->free = NULL; + _PyCrossInterpreterData_Init(data, tstate->interp, (void *)value, NULL, + _new_long_object); + // data->obj and data->free remain NULL return 0; } @@ -2098,12 +2238,12 @@ _new_none_object(_PyCrossInterpreterData *data) } static int -_none_shared(PyObject *obj, _PyCrossInterpreterData *data) +_none_shared(PyThreadState *tstate, PyObject *obj, + _PyCrossInterpreterData *data) { - data->data = NULL; - // data->obj remains NULL - data->new_object = _new_none_object; - data->free = NULL; // There is nothing to free. + _PyCrossInterpreterData_Init(data, tstate->interp, NULL, NULL, + _new_none_object); + // data->data, data->obj and data->free remain NULL return 0; } @@ -2111,22 +2251,22 @@ static void _register_builtins_for_crossinterpreter_data(struct _xidregistry *xidregistry) { // None - if (_register_xidata(xidregistry, (PyTypeObject *)PyObject_Type(Py_None), _none_shared) != 0) { + if (_xidregistry_add_type(xidregistry, (PyTypeObject *)PyObject_Type(Py_None), _none_shared) != 0) { Py_FatalError("could not register None for cross-interpreter sharing"); } // int - if (_register_xidata(xidregistry, &PyLong_Type, _long_shared) != 0) { + if (_xidregistry_add_type(xidregistry, &PyLong_Type, _long_shared) != 0) { Py_FatalError("could not register int for cross-interpreter sharing"); } // bytes - if (_register_xidata(xidregistry, &PyBytes_Type, _bytes_shared) != 0) { + if (_xidregistry_add_type(xidregistry, &PyBytes_Type, _bytes_shared) != 0) { Py_FatalError("could not register bytes for cross-interpreter sharing"); } // str - if (_register_xidata(xidregistry, &PyUnicode_Type, _str_shared) != 0) { + if (_xidregistry_add_type(xidregistry, &PyUnicode_Type, _str_shared) != 0) { Py_FatalError("could not register str for cross-interpreter sharing"); } } diff --git a/Python/suggestions.c b/Python/suggestions.c index ad645c7d96fa57..f2c018ef2c4533 100644 --- a/Python/suggestions.c +++ b/Python/suggestions.c @@ -41,10 +41,8 @@ substitution_cost(char a, char b) static Py_ssize_t levenshtein_distance(const char *a, size_t a_size, const char *b, size_t b_size, - size_t max_cost) + size_t max_cost, size_t *buffer) { - static size_t buffer[MAX_STRING_SIZE]; - // Both strings are the same (by identity) if (a == b) { return 0; @@ -147,12 +145,16 @@ calculate_suggestions(PyObject *dir, if (name_str == NULL) { return NULL; } - + size_t *buffer = PyMem_New(size_t, MAX_STRING_SIZE); + if (buffer == NULL) { + return PyErr_NoMemory(); + } for (int i = 0; i < dir_size; ++i) { PyObject *item = PyList_GET_ITEM(dir, i); Py_ssize_t item_size; const char *item_str = PyUnicode_AsUTF8AndSize(item, &item_size); if (item_str == NULL) { + PyMem_Free(buffer); return NULL; } if (PyUnicode_CompareWithASCIIString(name, item_str) == 0) { @@ -163,8 +165,8 @@ calculate_suggestions(PyObject *dir, // Don't take matches we've already beaten. max_distance = Py_MIN(max_distance, suggestion_distance - 1); Py_ssize_t current_distance = - levenshtein_distance(name_str, name_size, - item_str, item_size, max_distance); + levenshtein_distance(name_str, name_size, item_str, + item_size, max_distance, buffer); if (current_distance > max_distance) { continue; } @@ -173,6 +175,7 @@ calculate_suggestions(PyObject *dir, suggestion_distance = current_distance; } } + PyMem_Free(buffer); return Py_XNewRef(suggestion); } @@ -238,7 +241,7 @@ get_suggestions_for_name_error(PyObject* name, PyFrameObject* frame) if (!self) { goto error; } - + if (PyObject_HasAttr(self, name)) { Py_DECREF(dir); return PyUnicode_FromFormat("self.%S", name); @@ -401,6 +404,14 @@ _Py_UTF8_Edit_Cost(PyObject *a, PyObject *b, Py_ssize_t max_cost) if (max_cost == -1) { max_cost = MOVE_COST * Py_MAX(size_a, size_b); } - return levenshtein_distance(utf8_a, size_a, utf8_b, size_b, max_cost); + size_t *buffer = PyMem_New(size_t, MAX_STRING_SIZE); + if (buffer == NULL) { + PyErr_NoMemory(); + return -1; + } + Py_ssize_t res = levenshtein_distance(utf8_a, size_a, + utf8_b, size_b, max_cost, buffer); + PyMem_Free(buffer); + return res; } diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index cc465134a9e065..adbb319a1b6f89 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -496,7 +496,6 @@ Modules/_pickle.c - PicklerMemoProxyType - Modules/_pickle.c - Pickler_Type - Modules/_pickle.c - UnpicklerMemoProxyType - Modules/_pickle.c - Unpickler_Type - -Modules/_xxsubinterpretersmodule.c - ChannelIDtype - Modules/_zoneinfo.c - PyZoneInfo_ZoneInfoType - Modules/ossaudiodev.c - OSSAudioType - Modules/ossaudiodev.c - OSSMixerType - @@ -523,12 +522,6 @@ Modules/_ctypes/_ctypes.c - PyExc_ArgError - Modules/_cursesmodule.c - PyCursesError - Modules/_decimal/_decimal.c - DecimalException - Modules/_tkinter.c - Tkinter_TclError - -Modules/_xxsubinterpretersmodule.c - ChannelError - -Modules/_xxsubinterpretersmodule.c - ChannelNotFoundError - -Modules/_xxsubinterpretersmodule.c - ChannelClosedError - -Modules/_xxsubinterpretersmodule.c - ChannelEmptyError - -Modules/_xxsubinterpretersmodule.c - ChannelNotEmptyError - -Modules/_xxsubinterpretersmodule.c - RunFailedError - Modules/ossaudiodev.c - OSSAudioError - Modules/socketmodule.c - socket_herror - Modules/socketmodule.c - socket_gaierror - diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 424b15ede2aadf..2952634a3cda68 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -14,23 +14,76 @@ import parser -DEFAULT_INPUT = "Python/bytecodes.c" -DEFAULT_OUTPUT = "Python/generated_cases.c.h" +DEFAULT_INPUT = os.path.relpath( + os.path.join(os.path.dirname(__file__), "../../Python/bytecodes.c") +) +DEFAULT_OUTPUT = os.path.relpath( + os.path.join(os.path.dirname(__file__), "../../Python/generated_cases.c.h") +) BEGIN_MARKER = "// BEGIN BYTECODES //" END_MARKER = "// END BYTECODES //" RE_PREDICTED = r"(?s)(?:PREDICT\(|GO_TO_INSTRUCTION\(|DEOPT_IF\(.*?,\s*)(\w+)\);" UNUSED = "unused" BITS_PER_CODE_UNIT = 16 -arg_parser = argparse.ArgumentParser() -arg_parser.add_argument("-i", "--input", type=str, default=DEFAULT_INPUT) -arg_parser.add_argument("-o", "--output", type=str, default=DEFAULT_OUTPUT) +arg_parser = argparse.ArgumentParser( + description="Generate the code for the interpreter switch.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) +arg_parser.add_argument( + "-i", "--input", type=str, help="Instruction definitions", default=DEFAULT_INPUT +) +arg_parser.add_argument( + "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT +) -# This is not a data class -class Instruction(parser.InstDef): +class Formatter: + """Wraps an output stream with the ability to indent etc.""" + + stream: typing.TextIO + prefix: str + + def __init__(self, stream: typing.TextIO, indent: int) -> None: + self.stream = stream + self.prefix = " " * indent + + def write_raw(self, s: str) -> None: + self.stream.write(s) + + def emit(self, arg: str) -> None: + if arg: + self.write_raw(f"{self.prefix}{arg}\n") + else: + self.write_raw("\n") + + @contextlib.contextmanager + def indent(self): + self.prefix += " " + yield + self.prefix = self.prefix[:-4] + + @contextlib.contextmanager + def block(self, head: str): + if head: + self.emit(head + " {") + else: + self.emit("{") + with self.indent(): + yield + self.emit("}") + + +@dataclasses.dataclass +class Instruction: """An instruction with additional data and code.""" + # Parts of the underlying instruction definition + inst: parser.InstDef + kind: typing.Literal["inst", "op"] + name: str + block: parser.Block + # Computed by constructor always_exits: bool cache_offset: int @@ -43,65 +96,44 @@ class Instruction(parser.InstDef): predicted: bool = False def __init__(self, inst: parser.InstDef): - super().__init__(inst.header, inst.block) - self.context = inst.context + self.inst = inst + self.kind = inst.kind + self.name = inst.name + self.block = inst.block self.always_exits = always_exits(self.block) self.cache_effects = [ - effect for effect in self.inputs if isinstance(effect, parser.CacheEffect) + effect for effect in inst.inputs if isinstance(effect, parser.CacheEffect) ] self.cache_offset = sum(c.size for c in self.cache_effects) self.input_effects = [ - effect for effect in self.inputs if isinstance(effect, parser.StackEffect) + effect for effect in inst.inputs if isinstance(effect, parser.StackEffect) ] - self.output_effects = self.outputs # For consistency/completeness + self.output_effects = inst.outputs # For consistency/completeness - def write(self, f: typing.TextIO, indent: str, dedent: int = 0) -> None: + def write(self, out: Formatter) -> None: """Write one instruction, sans prologue and epilogue.""" - if dedent < 0: - indent += " " * -dedent # DO WE NEED THIS? - - # Get cache offset and maybe assert that it is correct + # Write a static assertion that a family's cache size is correct if family := self.family: if self.name == family.members[0]: if cache_size := family.size: - f.write( - f"{indent} static_assert({cache_size} == " - f'{self.cache_offset}, "incorrect cache size");\n' - ) - - # Write cache effect variable declarations - cache_offset = 0 - for ceffect in self.cache_effects: - if ceffect.name != UNUSED: - bits = ceffect.size * BITS_PER_CODE_UNIT - if bits == 64: - # NOTE: We assume that 64-bit data in the cache - # is always an object pointer. - # If this becomes false, we need a way to specify - # syntactically what type the cache data is. - f.write( - f"{indent} PyObject *{ceffect.name} = " - f"read_obj(next_instr + {cache_offset});\n" + out.emit( + f"static_assert({cache_size} == " + f'{self.cache_offset}, "incorrect cache size");' ) - else: - f.write(f"{indent} uint{bits}_t {ceffect.name} = " - f"read_u{bits}(next_instr + {cache_offset});\n") - cache_offset += ceffect.size - assert cache_offset == self.cache_offset # Write input stack effect variable declarations and initializations for i, seffect in enumerate(reversed(self.input_effects), 1): if seffect.name != UNUSED: - f.write(f"{indent} PyObject *{seffect.name} = PEEK({i});\n") + out.emit(f"PyObject *{seffect.name} = PEEK({i});") # Write output stack effect variable declarations input_names = {seffect.name for seffect in self.input_effects} input_names.add(UNUSED) for seffect in self.output_effects: if seffect.name not in input_names: - f.write(f"{indent} PyObject *{seffect.name};\n") + out.emit(f"PyObject *{seffect.name};") - self.write_body(f, indent, dedent) + self.write_body(out, 0) # Skip the rest if the block always exits if always_exits(self.block): @@ -110,9 +142,9 @@ def write(self, f: typing.TextIO, indent: str, dedent: int = 0) -> None: # Write net stack growth/shrinkage diff = len(self.output_effects) - len(self.input_effects) if diff > 0: - f.write(f"{indent} STACK_GROW({diff});\n") + out.emit(f"STACK_GROW({diff});") elif diff < 0: - f.write(f"{indent} STACK_SHRINK({-diff});\n") + out.emit(f"STACK_SHRINK({-diff});") # Write output stack effect assignments unmoved_names = {UNUSED} @@ -121,14 +153,32 @@ def write(self, f: typing.TextIO, indent: str, dedent: int = 0) -> None: unmoved_names.add(ieffect.name) for i, seffect in enumerate(reversed(self.output_effects)): if seffect.name not in unmoved_names: - f.write(f"{indent} POKE({i+1}, {seffect.name});\n") + out.emit(f"POKE({i+1}, {seffect.name});") # Write cache effect if self.cache_offset: - f.write(f"{indent} next_instr += {self.cache_offset};\n") + out.emit(f"next_instr += {self.cache_offset};") - def write_body(self, f: typing.TextIO, ndent: str, dedent: int) -> None: + def write_body(self, out: Formatter, dedent: int, cache_adjust: int = 0) -> None: """Write the instruction body.""" + # Write cache effect variable declarations and initializations + cache_offset = cache_adjust + for ceffect in self.cache_effects: + if ceffect.name != UNUSED: + bits = ceffect.size * BITS_PER_CODE_UNIT + if bits == 64: + # NOTE: We assume that 64-bit data in the cache + # is always an object pointer. + # If this becomes false, we need a way to specify + # syntactically what type the cache data is. + type = "PyObject *" + func = "read_obj" + else: + type = f"uint{bits}_t " + func = f"read_u{bits}" + out.emit(f"{type}{ceffect.name} = {func}(next_instr + {cache_offset});") + cache_offset += ceffect.size + assert cache_offset == self.cache_offset + cache_adjust # Get lines of text with proper dedent blocklines = self.block.to_text(dedent=dedent).splitlines(True) @@ -165,122 +215,101 @@ def write_body(self, f: typing.TextIO, ndent: str, dedent: int) -> None: else: break if ninputs: - f.write(f"{space}if ({cond}) goto pop_{ninputs}_{label};\n") + out.write_raw(f"{space}if ({cond}) goto pop_{ninputs}_{label};\n") else: - f.write(f"{space}if ({cond}) goto {label};\n") + out.write_raw(f"{space}if ({cond}) goto {label};\n") else: - f.write(line) + out.write_raw(line) + + +InstructionOrCacheEffect = Instruction | parser.CacheEffect @dataclasses.dataclass -class SuperComponent: +class Component: instr: Instruction input_mapping: dict[str, parser.StackEffect] output_mapping: dict[str, parser.StackEffect] + def write_body(self, out: Formatter, cache_adjust: int) -> None: + with out.block(""): + for var, ieffect in self.input_mapping.items(): + out.emit(f"PyObject *{ieffect.name} = {var};") + for oeffect in self.output_mapping.values(): + out.emit(f"PyObject *{oeffect.name};") + self.instr.write_body(out, dedent=-4, cache_adjust=cache_adjust) + for var, oeffect in self.output_mapping.items(): + out.emit(f"{var} = {oeffect.name};") + -class SuperInstruction(parser.Super): +# TODO: Use a common base class for {Super,Macro}Instruction + +@dataclasses.dataclass +class SuperOrMacroInstruction: + """Common fields for super- and macro instructions.""" + + name: str stack: list[str] initial_sp: int final_sp: int - parts: list[SuperComponent] - - def __init__(self, sup: parser.Super): - super().__init__(sup.kind, sup.name, sup.ops) - self.context = sup.context - - def analyze(self, a: "Analyzer") -> None: - components = self.check_components(a) - self.stack, self.initial_sp = self.super_macro_analysis(a, components) - sp = self.initial_sp - self.parts = [] - for instr in components: - input_mapping = {} - for ieffect in reversed(instr.input_effects): - sp -= 1 - if ieffect.name != UNUSED: - input_mapping[self.stack[sp]] = ieffect - output_mapping = {} - for oeffect in instr.output_effects: - if oeffect.name != UNUSED: - output_mapping[self.stack[sp]] = oeffect - sp += 1 - self.parts.append(SuperComponent(instr, input_mapping, output_mapping)) - self.final_sp = sp - - def check_components(self, a: "Analyzer") -> list[Instruction]: - components: list[Instruction] = [] - if not self.ops: - a.error(f"{self.kind.capitalize()}-instruction has no operands", self) - for name in self.ops: - if name not in a.instrs: - a.error(f"Unknown instruction {name!r}", self) - else: - instr = a.instrs[name] - if self.kind == "super" and instr.kind != "inst": - a.error(f"Super-instruction operand {instr.name} must be inst, not op", instr) - components.append(instr) - return components - def super_macro_analysis( - self, a: "Analyzer", components: list[Instruction] - ) -> tuple[list[str], int]: - """Analyze a super-instruction or macro. - Print an error if there's a cache effect (which we don't support yet). +@dataclasses.dataclass +class SuperInstruction(SuperOrMacroInstruction): + """A super-instruction.""" - Return the list of variable names and the initial stack pointer. - """ - lowest = current = highest = 0 - for instr in components: - if instr.cache_effects: - a.error( - f"Super-instruction {self.name!r} has cache effects in {instr.name!r}", - instr, - ) - current -= len(instr.input_effects) - lowest = min(lowest, current) - current += len(instr.output_effects) - highest = max(highest, current) - # At this point, 'current' is the net stack effect, - # and 'lowest' and 'highest' are the extremes. - # Note that 'lowest' may be negative. - stack = [f"_tmp_{i+1}" for i in range(highest - lowest)] - return stack, -lowest + super: parser.Super + parts: list[Component] + + +@dataclasses.dataclass +class MacroInstruction(SuperOrMacroInstruction): + """A macro instruction.""" + + macro: parser.Macro + parts: list[Component | parser.CacheEffect] class Analyzer: """Parse input, analyze it, and write to output.""" filename: str + output_filename: str src: str errors: int = 0 + def __init__(self, filename: str, output_filename: str): + """Read the input file.""" + self.filename = filename + self.output_filename = output_filename + with open(filename) as f: + self.src = f.read() + def error(self, msg: str, node: parser.Node) -> None: lineno = 0 if context := node.context: # Use line number of first non-comment in the node - for token in context.owner.tokens[context.begin : context.end]: + for token in context.owner.tokens[context.begin : context.end]: lineno = token.line if token.kind != "COMMENT": break print(f"{self.filename}:{lineno}: {msg}", file=sys.stderr) self.errors += 1 - def __init__(self, filename: str): - """Read the input file.""" - self.filename = filename - with open(filename) as f: - self.src = f.read() - instrs: dict[str, Instruction] # Includes ops - supers: dict[str, parser.Super] # Includes macros + supers: dict[str, parser.Super] super_instrs: dict[str, SuperInstruction] + macros: dict[str, parser.Macro] + macro_instrs: dict[str, MacroInstruction] families: dict[str, parser.Family] def parse(self) -> None: - """Parse the source text.""" + """Parse the source text. + + We only want the parser to see the stuff between the + begin and end markers. + """ psr = parser.Parser(self.src, filename=self.filename) # Skip until begin marker @@ -291,24 +320,38 @@ def parse(self) -> None: raise psr.make_syntax_error( f"Couldn't find {BEGIN_MARKER!r} in {psr.filename}" ) + start = psr.getpos() - # Parse until end marker + # Find end marker, then delete everything after it + while tkn := psr.next(raw=True): + if tkn.text == END_MARKER: + break + del psr.tokens[psr.getpos() - 1 :] + + # Parse from start + psr.setpos(start) self.instrs = {} self.supers = {} + self.macros = {} self.families = {} - while (tkn := psr.peek(raw=True)) and tkn.text != END_MARKER: - if inst := psr.inst_def(): - self.instrs[inst.name] = instr = Instruction(inst) - elif super := psr.super_def(): - self.supers[super.name] = super - elif family := psr.family_def(): - self.families[family.name] = family - else: - raise psr.make_syntax_error(f"Unexpected token") + while thing := psr.definition(): + match thing: + case parser.InstDef(name=name): + self.instrs[name] = Instruction(thing) + case parser.Super(name): + self.supers[name] = thing + case parser.Macro(name): + self.macros[name] = thing + case parser.Family(name): + self.families[name] = thing + case _: + typing.assert_never(thing) + if not psr.eof(): + raise psr.make_syntax_error("Extra stuff at the end") print( - f"Read {len(self.instrs)} instructions, " - f"{len(self.supers)} supers/macros, " + f"Read {len(self.instrs)} instructions/ops, " + f"{len(self.supers)} supers, {len(self.macros)} macros, " f"and {len(self.families)} families from {self.filename}", file=sys.stderr, ) @@ -321,7 +364,7 @@ def analyze(self) -> None: self.find_predictions() self.map_families() self.check_families() - self.analyze_supers() + self.analyze_supers_and_macros() def find_predictions(self) -> None: """Find the instructions that need PREDICTED() labels.""" @@ -332,7 +375,7 @@ def find_predictions(self) -> None: else: self.error( f"Unknown instruction {target!r} predicted in {instr.name!r}", - instr, # TODO: Use better location + instr.inst, # TODO: Use better location ) def map_families(self) -> None: @@ -360,7 +403,9 @@ def check_families(self) -> None: members = [member for member in family.members if member in self.instrs] if members != family.members: unknown = set(family.members) - set(members) - self.error(f"Family {family.name!r} has unknown members: {unknown}", family) + self.error( + f"Family {family.name!r} has unknown members: {unknown}", family + ) if len(members) < 2: continue head = self.instrs[members[0]] @@ -381,105 +426,211 @@ def check_families(self) -> None: family, ) - def analyze_supers(self) -> None: - """Analyze each super instruction.""" + def analyze_supers_and_macros(self) -> None: + """Analyze each super- and macro instruction.""" self.super_instrs = {} - for name, sup in self.supers.items(): - dup = SuperInstruction(sup) - dup.analyze(self) - self.super_instrs[name] = dup + self.macro_instrs = {} + for name, super in self.supers.items(): + self.super_instrs[name] = self.analyze_super(super) + for name, macro in self.macros.items(): + self.macro_instrs[name] = self.analyze_macro(macro) + + def analyze_super(self, super: parser.Super) -> SuperInstruction: + components = self.check_super_components(super) + stack, initial_sp = self.stack_analysis(components) + sp = initial_sp + parts: list[Component] = [] + for component in components: + match component: + case parser.CacheEffect() as ceffect: + parts.append(ceffect) + case Instruction() as instr: + input_mapping = {} + for ieffect in reversed(instr.input_effects): + sp -= 1 + if ieffect.name != UNUSED: + input_mapping[stack[sp]] = ieffect + output_mapping = {} + for oeffect in instr.output_effects: + if oeffect.name != UNUSED: + output_mapping[stack[sp]] = oeffect + sp += 1 + parts.append(Component(instr, input_mapping, output_mapping)) + case _: + typing.assert_never(component) + final_sp = sp + return SuperInstruction(super.name, stack, initial_sp, final_sp, super, parts) + + def analyze_macro(self, macro: parser.Macro) -> MacroInstruction: + components = self.check_macro_components(macro) + stack, initial_sp = self.stack_analysis(components) + sp = initial_sp + parts: list[Component | parser.CacheEffect] = [] + for component in components: + match component: + case parser.CacheEffect() as ceffect: + parts.append(ceffect) + case Instruction() as instr: + input_mapping = {} + for ieffect in reversed(instr.input_effects): + sp -= 1 + if ieffect.name != UNUSED: + input_mapping[stack[sp]] = ieffect + output_mapping = {} + for oeffect in instr.output_effects: + if oeffect.name != UNUSED: + output_mapping[stack[sp]] = oeffect + sp += 1 + parts.append(Component(instr, input_mapping, output_mapping)) + case _: + typing.assert_never(component) + final_sp = sp + return MacroInstruction(macro.name, stack, initial_sp, final_sp, macro, parts) + + def check_super_components(self, super: parser.Super) -> list[Instruction]: + components: list[Instruction] = [] + for op in super.ops: + if op.name not in self.instrs: + self.error(f"Unknown instruction {op.name!r}", super) + else: + components.append(self.instrs[op.name]) + return components - def write_instructions(self, filename: str) -> None: + def check_macro_components( + self, macro: parser.Macro + ) -> list[InstructionOrCacheEffect]: + components: list[InstructionOrCacheEffect] = [] + for uop in macro.uops: + match uop: + case parser.OpName(name): + if name not in self.instrs: + self.error(f"Unknown instruction {name!r}", macro) + components.append(self.instrs[name]) + case parser.CacheEffect(): + components.append(uop) + case _: + typing.assert_never(uop) + return components + + def stack_analysis( + self, components: typing.Iterable[InstructionOrCacheEffect] + ) -> tuple[list[str], int]: + """Analyze a super-instruction or macro. + + Print an error if there's a cache effect (which we don't support yet). + + Return the list of variable names and the initial stack pointer. + """ + lowest = current = highest = 0 + for thing in components: + match thing: + case Instruction() as instr: + current -= len(instr.input_effects) + lowest = min(lowest, current) + current += len(instr.output_effects) + highest = max(highest, current) + case parser.CacheEffect(): + pass + case _: + typing.assert_never(thing) + # At this point, 'current' is the net stack effect, + # and 'lowest' and 'highest' are the extremes. + # Note that 'lowest' may be negative. + stack = [f"_tmp_{i+1}" for i in range(highest - lowest)] + return stack, -lowest + + def write_instructions(self) -> None: """Write instructions to output file.""" - indent = " " * 8 - with open(filename, "w") as f: + with open(self.output_filename, "w") as f: # Write provenance header f.write(f"// This file is generated by {os.path.relpath(__file__)}\n") f.write(f"// from {os.path.relpath(self.filename)}\n") f.write(f"// Do not edit!\n") - # Write regular instructions + # Create formatter; the rest of the code uses this. + self.out = Formatter(f, 8) + + # Write and count regular instructions n_instrs = 0 for name, instr in self.instrs.items(): if instr.kind != "inst": continue # ops are not real instructions n_instrs += 1 - f.write(f"\n{indent}TARGET({name}) {{\n") - if instr.predicted: - f.write(f"{indent} PREDICTED({name});\n") - instr.write(f, indent) - if not always_exits(instr.block): - f.write(f"{indent} DISPATCH();\n") - f.write(f"{indent}}}\n") - - # Write super-instructions and macros + self.out.emit("") + with self.out.block(f"TARGET({name})"): + if instr.predicted: + self.out.emit(f"PREDICTED({name});") + instr.write(self.out) + if not always_exits(instr.block): + self.out.emit(f"DISPATCH();") + + # Write and count super-instructions n_supers = 0 - n_macros = 0 for sup in self.super_instrs.values(): - if sup.kind == "super": - n_supers += 1 - elif sup.kind == "macro": - n_macros += 1 - self.write_super_macro(f, sup, indent) - - print( - f"Wrote {n_instrs} instructions, {n_supers} supers, " - f"and {n_macros} macros to {filename}", - file=sys.stderr, - ) + n_supers += 1 + self.write_super(sup) - def write_super_macro( - self, f: typing.TextIO, sup: SuperInstruction, indent: str = "" - ) -> None: + # Write and count macro instructions + n_macros = 0 + for macro in self.macro_instrs.values(): + n_macros += 1 + self.write_macro(macro) - # TODO: Make write() and block() methods of some Formatter class - def write(arg: str) -> None: - if arg: - f.write(f"{indent}{arg}\n") - else: - f.write("\n") + print( + f"Wrote {n_instrs} instructions, {n_supers} supers, " + f"and {n_macros} macros to {self.output_filename}", + file=sys.stderr, + ) - @contextlib.contextmanager - def block(head: str): - if head: - write(head + " {") - else: - write("{") - nonlocal indent - indent += " " - yield - indent = indent[:-4] - write("}") - - write("") - with block(f"TARGET({sup.name})"): - for i, var in enumerate(sup.stack): - if i < sup.initial_sp: - write(f"PyObject *{var} = PEEK({sup.initial_sp - i});") + def write_super(self, sup: SuperInstruction) -> None: + """Write code for a super-instruction.""" + with self.wrap_super_or_macro(sup): + first = True + for comp in sup.parts: + if not first: + self.out.emit("NEXTOPARG();") + self.out.emit("next_instr++;") + first = False + comp.write_body(self.out, 0) + if comp.instr.cache_offset: + self.out.emit(f"next_instr += {comp.instr.cache_offset};") + + def write_macro(self, mac: MacroInstruction) -> None: + """Write code for a macro instruction.""" + with self.wrap_super_or_macro(mac): + cache_adjust = 0 + for part in mac.parts: + match part: + case parser.CacheEffect(size=size): + cache_adjust += size + case Component() as comp: + comp.write_body(self.out, cache_adjust) + cache_adjust += comp.instr.cache_offset + + if cache_adjust: + self.out.emit(f"next_instr += {cache_adjust};") + + @contextlib.contextmanager + def wrap_super_or_macro(self, up: SuperOrMacroInstruction): + """Shared boilerplate for super- and macro instructions.""" + self.out.emit("") + with self.out.block(f"TARGET({up.name})"): + for i, var in enumerate(up.stack): + if i < up.initial_sp: + self.out.emit(f"PyObject *{var} = PEEK({up.initial_sp - i});") else: - write(f"PyObject *{var};") - - for i, comp in enumerate(sup.parts): - if i > 0 and sup.kind == "super": - write("NEXTOPARG();") - write("next_instr++;") - - with block(""): - for var, ieffect in comp.input_mapping.items(): - write(f"PyObject *{ieffect.name} = {var};") - for oeffect in comp.output_mapping.values(): - write(f"PyObject *{oeffect.name};") - comp.instr.write_body(f, indent, dedent=-4) - for var, oeffect in comp.output_mapping.items(): - write(f"{var} = {oeffect.name};") - - if sup.final_sp > sup.initial_sp: - write(f"STACK_GROW({sup.final_sp - sup.initial_sp});") - elif sup.final_sp < sup.initial_sp: - write(f"STACK_SHRINK({sup.initial_sp - sup.final_sp});") - for i, var in enumerate(reversed(sup.stack[:sup.final_sp]), 1): - write(f"POKE({i}, {var});") - write("DISPATCH();") + self.out.emit(f"PyObject *{var};") + + yield + + if up.final_sp > up.initial_sp: + self.out.emit(f"STACK_GROW({up.final_sp - up.initial_sp});") + elif up.final_sp < up.initial_sp: + self.out.emit(f"STACK_SHRINK({up.initial_sp - up.final_sp});") + for i, var in enumerate(reversed(up.stack[: up.final_sp]), 1): + self.out.emit(f"POKE({i}, {var});") + + self.out.emit(f"DISPATCH();") def always_exits(block: parser.Block) -> bool: @@ -506,13 +657,12 @@ def always_exits(block: parser.Block) -> bool: def main(): """Parse command line, parse input, analyze, write output.""" args = arg_parser.parse_args() # Prints message and sys.exit(2) on error - a = Analyzer(args.input) # Raises OSError if file not found + a = Analyzer(args.input, args.output) # Raises OSError if input unreadable a.parse() # Raises SyntaxError on failure - a.analyze() # Prints messages and raises SystemExit on failure + a.analyze() # Prints messages and sets a.errors on failure if a.errors: sys.exit(f"Found {a.errors} errors") - - a.write_instructions(args.output) # Raises OSError if file can't be written + a.write_instructions() # Raises OSError if output can't be written if __name__ == "__main__": diff --git a/Tools/cases_generator/lexer.py b/Tools/cases_generator/lexer.py index 980c920bf357f4..39b6a212a67b1c 100644 --- a/Tools/cases_generator/lexer.py +++ b/Tools/cases_generator/lexer.py @@ -240,7 +240,12 @@ def to_text(tkns: list[Token], dedent: int = 0) -> str: res.append('\n') col = 1+dedent res.append(' '*(c-col)) - res.append(tkn.text) + text = tkn.text + if dedent != 0 and tkn.kind == 'COMMENT' and '\n' in text: + if dedent < 0: + text = text.replace('\n', '\n' + ' '*-dedent) + # TODO: dedent > 0 + res.append(text) line, col = tkn.end return ''.join(res) diff --git a/Tools/cases_generator/parser.py b/Tools/cases_generator/parser.py index ae5ef1e26ea1c2..02a7834d221596 100644 --- a/Tools/cases_generator/parser.py +++ b/Tools/cases_generator/parser.py @@ -9,10 +9,12 @@ P = TypeVar("P", bound="Parser") N = TypeVar("N", bound="Node") -def contextual(func: Callable[[P], N|None]) -> Callable[[P], N|None]: + + +def contextual(func: Callable[[P], N | None]) -> Callable[[P], N | None]: # Decorator to wrap grammar methods. # Resets position if `func` returns None. - def contextual_wrapper(self: P) -> N|None: + def contextual_wrapper(self: P) -> N | None: begin = self.getpos() res = func(self) if res is None: @@ -21,6 +23,7 @@ def contextual_wrapper(self: P) -> N|None: end = self.getpos() res.context = Context(begin, end, self) return res + return contextual_wrapper @@ -35,7 +38,7 @@ def __repr__(self): @dataclass class Node: - context: Context|None = field(init=False, default=None) + context: Context | None = field(init=False, default=None) @property def text(self) -> str: @@ -68,8 +71,14 @@ class CacheEffect(Node): size: int +@dataclass +class OpName(Node): + name: str + + InputEffect = StackEffect | CacheEffect OutputEffect = StackEffect +UOp = OpName | CacheEffect @dataclass @@ -82,32 +91,23 @@ class InstHeader(Node): @dataclass class InstDef(Node): - # TODO: Merge InstHeader and InstDef - header: InstHeader + kind: Literal["inst", "op"] + name: str + inputs: list[InputEffect] + outputs: list[OutputEffect] block: Block - @property - def kind(self) -> str: - return self.header.kind - - @property - def name(self) -> str: - return self.header.name - @property - def inputs(self) -> list[InputEffect]: - return self.header.inputs - - @property - def outputs(self) -> list[OutputEffect]: - return self.header.outputs +@dataclass +class Super(Node): + name: str + ops: list[OpName] @dataclass -class Super(Node): - kind: Literal["macro", "super"] +class Macro(Node): name: str - ops: list[str] + uops: list[UOp] @dataclass @@ -118,12 +118,22 @@ class Family(Node): class Parser(PLexer): + @contextual + def definition(self) -> InstDef | Super | Macro | Family | None: + if inst := self.inst_def(): + return inst + if super := self.super_def(): + return super + if macro := self.macro_def(): + return macro + if family := self.family_def(): + return family @contextual def inst_def(self) -> InstDef | None: - if header := self.inst_header(): + if hdr := self.inst_header(): if block := self.block(): - return InstDef(header, block) + return InstDef(hdr.kind, hdr.name, hdr.inputs, hdr.outputs, block) raise self.make_syntax_error("Expected block") return None @@ -132,17 +142,14 @@ def inst_header(self) -> InstHeader | None: # inst(NAME) # | inst(NAME, (inputs -- outputs)) # | op(NAME, (inputs -- outputs)) - # TODO: Error out when there is something unexpected. # TODO: Make INST a keyword in the lexer. if (tkn := self.expect(lx.IDENTIFIER)) and (kind := tkn.text) in ("inst", "op"): - if (self.expect(lx.LPAREN) - and (tkn := self.expect(lx.IDENTIFIER))): + if self.expect(lx.LPAREN) and (tkn := self.expect(lx.IDENTIFIER)): name = tkn.text if self.expect(lx.COMMA): inp, outp = self.stack_effect() if self.expect(lx.RPAREN): - if ((tkn := self.peek()) - and tkn.kind == lx.LBRACE): + if (tkn := self.peek()) and tkn.kind == lx.LBRACE: return InstHeader(kind, name, inp, outp) elif self.expect(lx.RPAREN) and kind == "inst": # No legacy stack effect if kind is "op". @@ -176,18 +183,20 @@ def inputs(self) -> list[InputEffect] | None: def input(self) -> InputEffect | None: # IDENTIFIER '/' INTEGER (CacheEffect) # IDENTIFIER (StackEffect) - if (tkn := self.expect(lx.IDENTIFIER)): + if tkn := self.expect(lx.IDENTIFIER): if self.expect(lx.DIVIDE): if num := self.expect(lx.NUMBER): try: size = int(num.text) except ValueError: raise self.make_syntax_error( - f"Expected integer, got {num.text!r}") + f"Expected integer, got {num.text!r}" + ) else: return CacheEffect(tkn.text, size) raise self.make_syntax_error("Expected integer") else: + # TODO: Arrays, conditions return StackEffect(tkn.text) def outputs(self) -> list[OutputEffect] | None: @@ -205,46 +214,91 @@ def outputs(self) -> list[OutputEffect] | None: @contextual def output(self) -> OutputEffect | None: - if (tkn := self.expect(lx.IDENTIFIER)): + if tkn := self.expect(lx.IDENTIFIER): return StackEffect(tkn.text) @contextual def super_def(self) -> Super | None: - if (tkn := self.expect(lx.IDENTIFIER)) and (kind := tkn.text) in ("super", "macro"): + if (tkn := self.expect(lx.IDENTIFIER)) and tkn.text == "super": if self.expect(lx.LPAREN): - if (tkn := self.expect(lx.IDENTIFIER)): + if tkn := self.expect(lx.IDENTIFIER): if self.expect(lx.RPAREN): if self.expect(lx.EQUALS): if ops := self.ops(): - res = Super(kind, tkn.text, ops) + self.require(lx.SEMI) + res = Super(tkn.text, ops) return res - def ops(self) -> list[str] | None: - if tkn := self.expect(lx.IDENTIFIER): - ops = [tkn.text] + def ops(self) -> list[OpName] | None: + if op := self.op(): + ops = [op] while self.expect(lx.PLUS): - if tkn := self.require(lx.IDENTIFIER): - ops.append(tkn.text) - self.require(lx.SEMI) + if op := self.op(): + ops.append(op) return ops + @contextual + def op(self) -> OpName | None: + if tkn := self.expect(lx.IDENTIFIER): + return OpName(tkn.text) + + @contextual + def macro_def(self) -> Macro | None: + if (tkn := self.expect(lx.IDENTIFIER)) and tkn.text == "macro": + if self.expect(lx.LPAREN): + if tkn := self.expect(lx.IDENTIFIER): + if self.expect(lx.RPAREN): + if self.expect(lx.EQUALS): + if uops := self.uops(): + self.require(lx.SEMI) + res = Macro(tkn.text, uops) + return res + + def uops(self) -> list[UOp] | None: + if uop := self.uop(): + uops = [uop] + while self.expect(lx.PLUS): + if uop := self.uop(): + uops.append(uop) + else: + raise self.make_syntax_error("Expected op name or cache effect") + return uops + + @contextual + def uop(self) -> UOp | None: + if tkn := self.expect(lx.IDENTIFIER): + if self.expect(lx.DIVIDE): + if num := self.expect(lx.NUMBER): + try: + size = int(num.text) + except ValueError: + raise self.make_syntax_error( + f"Expected integer, got {num.text!r}" + ) + else: + return CacheEffect(tkn.text, size) + raise self.make_syntax_error("Expected integer") + else: + return OpName(tkn.text) + @contextual def family_def(self) -> Family | None: if (tkn := self.expect(lx.IDENTIFIER)) and tkn.text == "family": size = None if self.expect(lx.LPAREN): - if (tkn := self.expect(lx.IDENTIFIER)): + if tkn := self.expect(lx.IDENTIFIER): if self.expect(lx.COMMA): if not (size := self.expect(lx.IDENTIFIER)): - raise self.make_syntax_error( - "Expected identifier") + raise self.make_syntax_error("Expected identifier") if self.expect(lx.RPAREN): if self.expect(lx.EQUALS): if not self.expect(lx.LBRACE): raise self.make_syntax_error("Expected {") if members := self.members(): if self.expect(lx.RBRACE) and self.expect(lx.SEMI): - return Family(tkn.text, size.text if size else "", members) + return Family( + tkn.text, size.text if size else "", members + ) return None def members(self) -> list[str] | None: @@ -284,6 +338,7 @@ def c_blob(self) -> list[lx.Token]: if __name__ == "__main__": import sys + if sys.argv[1:]: filename = sys.argv[1] if filename == "-c" and sys.argv[2:]: @@ -295,10 +350,10 @@ def c_blob(self) -> list[lx.Token]: srclines = src.splitlines() begin = srclines.index("// BEGIN BYTECODES //") end = srclines.index("// END BYTECODES //") - src = "\n".join(srclines[begin+1 : end]) + src = "\n".join(srclines[begin + 1 : end]) else: filename = "" src = "if (x) { x.foo; // comment\n}" parser = Parser(src, filename) - x = parser.inst_def() or parser.super_def() or parser.family_def() + x = parser.definition() print(x) diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 8d91bda5a43a0d..9c098064fe5403 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -317,11 +317,11 @@ def calculate_execution_counts(opcode_stats, total): for (count, name, miss) in counts: cumulative += count if miss: - miss = f"{100*miss/count:0.1f}%" + miss = f"{100*miss/count:0.1f}%" else: miss = "" - rows.append((name, count, f"{100*count/total:0.1f}%", - f"{100*cumulative/total:0.1f}%", miss)) + rows.append((name, count, f"{100*count/total:0.1f}%", + f"{100*cumulative/total:0.1f}%", miss)) return rows def emit_execution_counts(opcode_stats, total):