Updating the fork

.azure-pipelines/ci.yml

@@ -61,7 +61,7 @@ jobs:
   variables:
     testRunTitle: '$(build.sourceBranchName)-linux'
     testRunPlatform: linux
-    openssl_version: 1.1.1f
+    openssl_version: 1.1.1g
 
   steps:
   - template: ./posix-steps.yml
@@ -118,7 +118,7 @@ jobs:
   variables:
     testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
     testRunPlatform: linux-coverage
-    openssl_version: 1.1.1f
+    openssl_version: 1.1.1g
 
   steps:
   - template: ./posix-steps.yml

.azure-pipelines/pr.yml

@@ -61,7 +61,7 @@ jobs:
   variables:
     testRunTitle: '$(system.pullRequest.TargetBranch)-linux'
     testRunPlatform: linux
-    openssl_version: 1.1.1f
+    openssl_version: 1.1.1g
 
   steps:
   - template: ./posix-steps.yml
@@ -118,7 +118,7 @@ jobs:
   variables:
     testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
     testRunPlatform: linux-coverage
-    openssl_version: 1.1.1f
+    openssl_version: 1.1.1g
 
   steps:
   - template: ./posix-steps.yml

.gitattributes

@@ -41,8 +41,6 @@ PC/readme.txt text eol=crlf
 
 # Generated files
 # https://github.com/github/linguist#generated-code
-Include/graminit.h          linguist-generated=true
-Python/graminit.h           linguist-generated=true
 Modules/clinic/*.h          linguist-generated=true
 Objects/clinic/*.h          linguist-generated=true
 PC/clinic/*.h               linguist-generated=true

.github/CODEOWNERS

@@ -72,10 +72,10 @@ Include/pytime.h              @pganssle @abalkin
 /Modules/gcmodule.c           @pablogsal
 /Doc/library/gc.rst           @pablogsal
 
-# Parser/Pgen
-/Parser/pgen/                 @pablogsal
-/Parser/pegen/                @pablogsal
-/Tools/peg_generator/         @pablogsal
+# Parser
+/Parser/                      @pablogsal @lysnikolaou
+/Tools/peg_generator/         @pablogsal @lysnikolaou
+/Lib/test/test_peg_generator/ @pablogsal @lysnikolaou
 
 # SQLite 3
 **/*sqlite*                   @berkerpeksag

.github/problem-matchers/gcc.json

@@ -0,0 +1,18 @@
+{
+    "__comment": "Taken from vscode-cpptools's Extension/package.json gcc rule",
+    "problemMatcher": [
+        {
+            "owner": "gcc-problem-matcher",
+            "pattern": [
+                {
+                    "regexp": "^(.*):(\\d+):(\\d+):\\s+(?:fatal\\s+)?(warning|error):\\s+(.*)$",
+                    "file": 1,
+                    "line": 2,
+                    "column": 3,
+                    "severity": 4,
+                    "message": 5
+                }
+            ]
+        }
+    ]
+}

.github/workflows/build.yml

@@ -87,6 +87,8 @@ jobs:
       OPENSSL_VER: 1.1.1f
     steps:
     - uses: actions/checkout@v2
+    - name: Register gcc problem matcher
+      run: echo "::add-matcher::.github/problem-matchers/gcc.json"
     - name: Install Dependencies
       run: sudo ./.github/workflows/posix-deps-apt.sh
     - name: 'Restore OpenSSL build'

.travis.yml

@@ -82,6 +82,12 @@ matrix:
           packages:
             - xvfb
       before_script:
+        - |
+            if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]
+            then
+              echo "Don't run Python coverage on pull requests."
+              exit
+            fi
         - ./configure
         - make -j4
         # Need a venv that can parse covered code.
@@ -109,6 +115,12 @@ matrix:
             - lcov
             - xvfb
       before_script:
+        - |
+            if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]
+            then
+              echo "Don't run C coverage on pull requests."
+              exit
+            fi
         - ./configure
       script:
         - xvfb-run make -j4 coverage-report

Doc/Makefile

@@ -143,7 +143,7 @@ clean:
 venv:
 	$(PYTHON) -m venv $(VENVDIR)
 	$(VENVDIR)/bin/python3 -m pip install -U pip setuptools
-	$(VENVDIR)/bin/python3 -m pip install -U Sphinx==2.2.0 blurb python-docs-theme
+	$(VENVDIR)/bin/python3 -m pip install -U Sphinx==2.3.1 blurb python-docs-theme
 	@echo "The venv has been created in the $(VENVDIR) directory"
 
 dist:

Doc/c-api/abstract.rst

@@ -24,4 +24,3 @@ but whose items have not been set to some non-\ ``NULL`` value yet.
    mapping.rst
    iter.rst
    buffer.rst
-   objbuffer.rst

Doc/c-api/arg.rst

@@ -55,13 +55,11 @@ which disallows mutable objects such as :class:`bytearray`.
 
 .. note::
 
-   For all ``#`` variants of formats (``s#``, ``y#``, etc.), the type of
-   the length argument (int or :c:type:`Py_ssize_t`) is controlled by
-   defining the macro :c:macro:`PY_SSIZE_T_CLEAN` before including
-   :file:`Python.h`.  If the macro was defined, length is a
-   :c:type:`Py_ssize_t` rather than an :c:type:`int`. This behavior will change
-   in a future Python version to only support :c:type:`Py_ssize_t` and
-   drop :c:type:`int` support. It is best to always define :c:macro:`PY_SSIZE_T_CLEAN`.
+   For all ``#`` variants of formats (``s#``, ``y#``, etc.), the macro
+   :c:macro:`PY_SSIZE_T_CLEAN` must be defined before including
+   :file:`Python.h`. On Python 3.9 and older, the type of the length argument
+   is :c:type:`Py_ssize_t` if the :c:macro:`PY_SSIZE_T_CLEAN` macro is defined,
+   or int otherwise.
 
 
 ``s`` (:class:`str`) [const char \*]
@@ -90,7 +88,7 @@ which disallows mutable objects such as :class:`bytearray`.
    In this case the resulting C string may contain embedded NUL bytes.
    Unicode objects are converted to C strings using ``'utf-8'`` encoding.
 
-``s#`` (:class:`str`, read-only :term:`bytes-like object`) [const char \*, int or :c:type:`Py_ssize_t`]
+``s#`` (:class:`str`, read-only :term:`bytes-like object`) [const char \*, :c:type:`Py_ssize_t`]
    Like ``s*``, except that it doesn't accept mutable objects.
    The result is stored into two C variables,
    the first one a pointer to a C string, the second one its length.
@@ -105,7 +103,7 @@ which disallows mutable objects such as :class:`bytearray`.
    Like ``s*``, but the Python object may also be ``None``, in which case the
    ``buf`` member of the :c:type:`Py_buffer` structure is set to ``NULL``.
 
-``z#`` (:class:`str`, read-only :term:`bytes-like object` or ``None``) [const char \*, int or :c:type:`Py_ssize_t`]
+``z#`` (:class:`str`, read-only :term:`bytes-like object` or ``None``) [const char \*, :c:type:`Py_ssize_t`]
    Like ``s#``, but the Python object may also be ``None``, in which case the C
    pointer is set to ``NULL``.
 
@@ -124,7 +122,7 @@ which disallows mutable objects such as :class:`bytearray`.
    bytes-like objects.  **This is the recommended way to accept
    binary data.**
 
-``y#`` (read-only :term:`bytes-like object`) [const char \*, int or :c:type:`Py_ssize_t`]
+``y#`` (read-only :term:`bytes-like object`) [const char \*, :c:type:`Py_ssize_t`]
    This variant on ``s#`` doesn't accept Unicode objects, only bytes-like
    objects.
 
@@ -155,7 +153,7 @@ which disallows mutable objects such as :class:`bytearray`.
       Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
       :c:func:`PyUnicode_AsWideCharString`.
 
-``u#`` (:class:`str`) [const Py_UNICODE \*, int or :c:type:`Py_ssize_t`]
+``u#`` (:class:`str`) [const Py_UNICODE \*, :c:type:`Py_ssize_t`]
    This variant on ``u`` stores into two C variables, the first one a pointer to a
    Unicode data buffer, the second one its length.  This variant allows
    null code points.
@@ -172,7 +170,7 @@ which disallows mutable objects such as :class:`bytearray`.
       Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using
       :c:func:`PyUnicode_AsWideCharString`.
 
-``Z#`` (:class:`str` or ``None``) [const Py_UNICODE \*, int or :c:type:`Py_ssize_t`]
+``Z#`` (:class:`str` or ``None``) [const Py_UNICODE \*, :c:type:`Py_ssize_t`]
    Like ``u#``, but the Python object may also be ``None``, in which case the
    :c:type:`Py_UNICODE` pointer is set to ``NULL``.
 
@@ -213,7 +211,7 @@ which disallows mutable objects such as :class:`bytearray`.
    recoding them.  Instead, the implementation assumes that the byte string object uses
    the encoding passed in as parameter.
 
-``es#`` (:class:`str`) [const char \*encoding, char \*\*buffer, int or :c:type:`Py_ssize_t` \*buffer_length]
+``es#`` (:class:`str`) [const char \*encoding, char \*\*buffer, :c:type:`Py_ssize_t` \*buffer_length]
    This variant on ``s#`` is used for encoding Unicode into a character buffer.
    Unlike the ``es`` format, this variant allows input data which contains NUL
    characters.
@@ -244,7 +242,7 @@ which disallows mutable objects such as :class:`bytearray`.
    In both cases, *\*buffer_length* is set to the length of the encoded data
    without the trailing NUL byte.
 
-``et#`` (:class:`str`, :class:`bytes` or :class:`bytearray`) [const char \*encoding, char \*\*buffer, int or :c:type:`Py_ssize_t` \*buffer_length]
+``et#`` (:class:`str`, :class:`bytes` or :class:`bytearray`) [const char \*encoding, char \*\*buffer, :c:type:`Py_ssize_t` \*buffer_length]
    Same as ``es#`` except that byte string objects are passed through without recoding
    them. Instead, the implementation assumes that the byte string object uses the
    encoding passed in as parameter.
@@ -549,7 +547,7 @@ Building values
       Convert a null-terminated C string to a Python :class:`str` object using ``'utf-8'``
       encoding. If the C string pointer is ``NULL``, ``None`` is used.
 
-   ``s#`` (:class:`str` or ``None``) [const char \*, int or :c:type:`Py_ssize_t`]
+   ``s#`` (:class:`str` or ``None``) [const char \*, :c:type:`Py_ssize_t`]
       Convert a C string and its length to a Python :class:`str` object using ``'utf-8'``
       encoding. If the C string pointer is ``NULL``, the length is ignored and
       ``None`` is returned.
@@ -558,30 +556,30 @@ Building values
       This converts a C string to a Python :class:`bytes` object.  If the C
       string pointer is ``NULL``, ``None`` is returned.
 
-   ``y#`` (:class:`bytes`) [const char \*, int or :c:type:`Py_ssize_t`]
+   ``y#`` (:class:`bytes`) [const char \*, :c:type:`Py_ssize_t`]
       This converts a C string and its lengths to a Python object.  If the C
       string pointer is ``NULL``, ``None`` is returned.
 
    ``z`` (:class:`str` or ``None``) [const char \*]
       Same as ``s``.
 
-   ``z#`` (:class:`str` or ``None``) [const char \*, int or :c:type:`Py_ssize_t`]
+   ``z#`` (:class:`str` or ``None``) [const char \*, :c:type:`Py_ssize_t`]
       Same as ``s#``.
 
    ``u`` (:class:`str`) [const wchar_t \*]
       Convert a null-terminated :c:type:`wchar_t` buffer of Unicode (UTF-16 or UCS-4)
       data to a Python Unicode object.  If the Unicode buffer pointer is ``NULL``,
       ``None`` is returned.
 
-   ``u#`` (:class:`str`) [const wchar_t \*, int or :c:type:`Py_ssize_t`]
+   ``u#`` (:class:`str`) [const wchar_t \*, :c:type:`Py_ssize_t`]
       Convert a Unicode (UTF-16 or UCS-4) data buffer and its length to a Python
       Unicode object.   If the Unicode buffer pointer is ``NULL``, the length is ignored
       and ``None`` is returned.
 
    ``U`` (:class:`str` or ``None``) [const char \*]
       Same as ``s``.
 
-   ``U#`` (:class:`str` or ``None``) [const char \*, int or :c:type:`Py_ssize_t`]
+   ``U#`` (:class:`str` or ``None``) [const char \*, :c:type:`Py_ssize_t`]
       Same as ``s#``.
 
    ``i`` (:class:`int`) [int]

Doc/c-api/conversion.rst

@@ -27,12 +27,8 @@ not.
 
 The wrappers ensure that *str*[*size*-1] is always ``'\0'`` upon return. They
 never write more than *size* bytes (including the trailing ``'\0'``) into str.
-Both functions require that ``str != NULL``, ``size > 0`` and ``format !=
-NULL``.
-
-If the platform doesn't have :c:func:`vsnprintf` and the buffer size needed to
-avoid truncation exceeds *size* by more than 512 bytes, Python aborts with a
-:c:func:`Py_FatalError`.
+Both functions require that ``str != NULL``, ``size > 0``, ``format != NULL``
+and ``size < INT_MAX``.
 
 The return value (*rv*) for these functions should be interpreted as follows:
 
@@ -48,8 +44,8 @@ The return value (*rv*) for these functions should be interpreted as follows:
   this case too, but the rest of *str* is undefined. The exact cause of the error
   depends on the underlying platform.
 
-The following functions provide locale-independent string to number conversions.
 
+The following functions provide locale-independent string to number conversions.
 
 .. c:function:: double PyOS_string_to_double(const char *s, char **endptr, PyObject *overflow_exception)
 

Doc/c-api/init_config.rst

@@ -43,6 +43,7 @@ Functions:
 * :c:func:`Py_PreInitializeFromArgs`
 * :c:func:`Py_PreInitializeFromBytesArgs`
 * :c:func:`Py_RunMain`
+* :c:func:`Py_GetArgcArgv`
 
 The preconfiguration (``PyPreConfig`` type) is stored in
 ``_PyRuntime.preconfig`` and the configuration (``PyConfig`` type) is stored in
@@ -196,12 +197,12 @@ PyPreConfig
 
    Function to initialize a preconfiguration:
 
-   .. c:function:: void PyPreConfig_InitIsolatedConfig(PyPreConfig *preconfig)
+   .. c:function:: void PyPreConfig_InitPythonConfig(PyPreConfig *preconfig)
 
       Initialize the preconfiguration with :ref:`Python Configuration
       <init-python-config>`.
 
-   .. c:function:: void PyPreConfig_InitPythonConfig(PyPreConfig *preconfig)
+   .. c:function:: void PyPreConfig_InitIsolatedConfig(PyPreConfig *preconfig)
 
       Initialize the preconfiguration with :ref:`Isolated Configuration
       <init-isolated-conf>`.
@@ -423,6 +424,8 @@ PyConfig
       :c:member:`~PyConfig.argv` is empty, an empty string is added to ensure
       that :data:`sys.argv` always exists and is never empty.
 
+      See also the :c:member:`~PyConfig.orig_argv` member.
+
    .. c:member:: wchar_t* base_exec_prefix
 
       :data:`sys.base_exec_prefix`.
@@ -436,6 +439,14 @@ PyConfig
 
       :data:`sys.base_prefix`.
 
+   .. c:member:: wchar_t* platlibdir
+
+      :data:`sys.platlibdir`: platform library directory name, set at configure time
+      by ``--with-platlibdir``, overrideable by the ``PYTHONPLATLIBDIR``
+      environment variable.
+
+      .. versionadded:: 3.9
+
    .. c:member:: int buffered_stdio
 
       If equals to 0, enable unbuffered mode, making the stdout and stderr
@@ -577,6 +588,23 @@ PyConfig
       * 1: Remove assertions, set ``__debug__`` to ``False``
       * 2: Strip docstrings
 
+   .. c:member:: PyWideStringList orig_argv
+
+      The list of the original command line arguments passed to the Python
+      executable.
+
+      If :c:member:`~PyConfig.orig_argv` list is empty and
+      :c:member:`~PyConfig.argv` is not a list only containing an empty
+      string, :c:func:`PyConfig_Read()` copies :c:member:`~PyConfig.argv` into
+      :c:member:`~PyConfig.orig_argv` before modifying
+      :c:member:`~PyConfig.argv` (if :c:member:`~PyConfig.parse_argv` is
+      non-zero).
+
+      See also the :c:member:`~PyConfig.argv` member and the
+      :c:func:`Py_GetArgcArgv` function.
+
+      .. versionadded:: 3.10
+
    .. c:member:: int parse_argv
 
       If non-zero, parse :c:member:`~PyConfig.argv` the same way the regular
@@ -686,16 +714,6 @@ PyConfig
 
       :data:`sys._xoptions`.
 
-   .. c:member:: int _use_peg_parser
-
-      Enable PEG parser? Default: 1.
-
-      Set to 0 by :option:`-X oldparser <-X>` and :envvar:`PYTHONOLDPARSER`.
-
-      See also :pep:`617`.
-
-      .. deprecated-removed:: 3.9 3.10
-
 If ``parse_argv`` is non-zero, ``argv`` arguments are parsed the same
 way the regular Python parses command line arguments, and Python
 arguments are stripped from ``argv``: see :ref:`Command Line Arguments
@@ -884,6 +902,7 @@ Path Configuration
 * Path configuration inputs:
 
   * :c:member:`PyConfig.home`
+  * :c:member:`PyConfig.platlibdir`
   * :c:member:`PyConfig.pathconfig_warnings`
   * :c:member:`PyConfig.program_name`
   * :c:member:`PyConfig.pythonpath_env`
@@ -975,6 +994,16 @@ customized Python always running in isolated mode using
 :c:func:`Py_RunMain`.
 
 
+Py_GetArgcArgv()
+----------------
+
+.. c:function:: void Py_GetArgcArgv(int *argc, wchar_t ***argv)
+
+   Get the original command line arguments, before Python modified them.
+
+   See also :c:member:`PyConfig.orig_argv` member.
+
+
 Multi-Phase Initialization Private Provisional API
 --------------------------------------------------