bpo-45582: Port getpath[p].c to Python (GH-29041)

The getpath.py file is frozen at build time and executed as code over a namespace. It is never imported, nor is it meant to be importable or reusable. However, it should be easier to read, modify, and patch than the previous code.

This commit attempts to preserve every previously tested quirk, but these may be changed in the future to better align platforms.

.gitignore

@@ -74,6 +74,7 @@ Mac/pythonw
 Misc/python.pc
 Misc/python-embed.pc
 Misc/python-config.sh
+Modules/getpath.h
 Modules/Setup.config
 Modules/Setup.local
 Modules/Setup.stdlib

Doc/c-api/init_config.rst

@@ -479,6 +479,9 @@ PyConfig
 
       Fields which are already initialized are left unchanged.
 
+      Fields for :ref:`path configuration <init-path-config>` are no longer
+      calculated or modified when calling this function, as of Python 3.11.
+
       The :c:func:`PyConfig_Read` function only parses
       :c:member:`PyConfig.argv` arguments once: :c:member:`PyConfig.parse_argv`
       is set to ``2`` after arguments are parsed. Since Python arguments are
@@ -493,6 +496,12 @@ PyConfig
          parsed, and arguments are only parsed if
          :c:member:`PyConfig.parse_argv` equals ``1``.
 
+      .. versionchanged:: 3.11
+         :c:func:`PyConfig_Read` no longer calculates all paths, and so fields
+         listed under :ref:`Python Path Configuration <init-path-config>` may
+         no longer be updated until :c:func:`Py_InitializeFromConfig` is
+         called.
+
    .. c:function:: void PyConfig_Clear(PyConfig *config)
 
       Release configuration memory.
@@ -848,12 +857,19 @@ PyConfig
 
       Default: value of the ``PLATLIBDIR`` macro which is set by the
       :option:`configure --with-platlibdir option <--with-platlibdir>`
-      (default: ``"lib"``).
+      (default: ``"lib"``, or ``"DLLs"`` on Windows).
 
       Part of the :ref:`Python Path Configuration <init-path-config>` input.
 
       .. versionadded:: 3.9
 
+      .. versionchanged:: 3.11
+         This macro is now used on Windows to locate the standard
+         library extension modules, typically under ``DLLs``. However,
+         for compatibility, note that this value is ignored for any
+         non-standard layouts, including in-tree builds and virtual
+         environments.
+
    .. c:member:: wchar_t* pythonpath_env
 
       Module search paths (:data:`sys.path`) as a string separated by ``DELIM``
@@ -870,9 +886,9 @@ PyConfig
 
       Module search paths: :data:`sys.path`.
 
-      If :c:member:`~PyConfig.module_search_paths_set` is equal to 0, the
-      function calculating the :ref:`Python Path Configuration <init-path-config>`
-      overrides the :c:member:`~PyConfig.module_search_paths` and sets
+      If :c:member:`~PyConfig.module_search_paths_set` is equal to 0,
+      :c:func:`Py_InitializeFromConfig` will replace
+      :c:member:`~PyConfig.module_search_paths` and sets
       :c:member:`~PyConfig.module_search_paths_set` to ``1``.
 
       Default: empty list (``module_search_paths``) and ``0``
@@ -944,16 +960,16 @@ PyConfig
 
    .. c:member:: int pathconfig_warnings
 
-      On Unix, if non-zero, calculating the :ref:`Python Path Configuration
-      <init-path-config>` can log warnings into ``stderr``. If equals to 0,
-      suppress these warnings.
-
-      It has no effect on Windows.
+      If non-zero, calculation of path configuration is allowed to log
+      warnings into ``stderr``. If equals to 0, suppress these warnings.
 
       Default: ``1`` in Python mode, ``0`` in isolated mode.
 
       Part of the :ref:`Python Path Configuration <init-path-config>` input.
 
+      .. versionchanged:: 3.11
+         Now also applies on Windows.
+
    .. c:member:: wchar_t* prefix
 
       The site-specific directory prefix where the platform independent Python
@@ -1305,10 +1321,9 @@ variables, command line arguments (:c:member:`PyConfig.argv` is not parsed)
 and user site directory. The C standard streams (ex: ``stdout``) and the
 LC_CTYPE locale are left unchanged. Signal handlers are not installed.
 
-Configuration files are still used with this configuration. Set the
-:ref:`Python Path Configuration <init-path-config>` ("output fields") to ignore these
-configuration files and avoid the function computing the default path
-configuration.
+Configuration files are still used with this configuration to determine
+paths that are unspecified. Ensure :c:member:`PyConfig.home` is specified
+to avoid computing the default path configuration.
 
 
 .. _init-python-config:

Include/cpython/fileutils.h

@@ -135,12 +135,6 @@ PyAPI_FUNC(wchar_t*) _Py_wrealpath(
     size_t resolved_path_len);
 #endif
 
-#ifndef MS_WINDOWS
-PyAPI_FUNC(int) _Py_isabs(const wchar_t *path);
-#endif
-
-PyAPI_FUNC(int) _Py_abspath(const wchar_t *path, wchar_t **abspath_p);
-
 PyAPI_FUNC(wchar_t*) _Py_wgetcwd(
     wchar_t *buf,
     /* Number of characters of 'buf' buffer

Include/cpython/initconfig.h

@@ -213,6 +213,9 @@ typedef struct PyConfig {
     // If non-zero, disallow threads, subprocesses, and fork.
     // Default: 0.
     int _isolated_interpreter;
+
+    // If non-zero, we believe we're running from a source tree.
+    int _is_python_build;
 } PyConfig;
 
 PyAPI_FUNC(void) PyConfig_InitPythonConfig(PyConfig *config);

Include/internal/pycore_fileutils.h

@@ -74,14 +74,15 @@ extern int _Py_EncodeNonUnicodeWchar_InPlace(
     Py_ssize_t size);
 #endif
 
+extern int _Py_isabs(const wchar_t *path);
+extern int _Py_abspath(const wchar_t *path, wchar_t **abspath_p);
 extern wchar_t * _Py_join_relfile(const wchar_t *dirname,
                                   const wchar_t *relfile);
 extern int _Py_add_relfile(wchar_t *dirname,
                            const wchar_t *relfile,
                            size_t bufsize);
 extern size_t _Py_find_basename(const wchar_t *filename);
-PyAPI_FUNC(int) _Py_normalize_path(const wchar_t *path,
-                                   wchar_t *buf, const size_t buf_len);
+PyAPI_FUNC(wchar_t *) _Py_normpath(wchar_t *path, Py_ssize_t size);
 
 
 // Macros to protect CRT calls against instant termination when passed an

Include/internal/pycore_initconfig.h

@@ -166,6 +166,10 @@ extern PyStatus _PyConfig_SetPyArgv(
 PyAPI_FUNC(PyObject*) _PyConfig_AsDict(const PyConfig *config);
 PyAPI_FUNC(int) _PyConfig_FromDict(PyConfig *config, PyObject *dict);
 
+extern void _Py_DumpPathConfig(PyThreadState *tstate);
+
+PyAPI_FUNC(PyObject*) _Py_Get_Getpath_CodeObject();
+
 
 /* --- Function used for testing ---------------------------------- */
 

Include/internal/pycore_pathconfig.h

@@ -8,65 +8,15 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-typedef struct _PyPathConfig {
-    /* Full path to the Python program */
-    wchar_t *program_full_path;
-    wchar_t *prefix;
-    wchar_t *exec_prefix;
-    wchar_t *stdlib_dir;
-    /* Set by Py_SetPath(), or computed by _PyConfig_InitPathConfig() */
-    wchar_t *module_search_path;
-    /* Python program name */
-    wchar_t *program_name;
-    /* Set by Py_SetPythonHome() or PYTHONHOME environment variable */
-    wchar_t *home;
-#ifdef MS_WINDOWS
-    /* isolated and site_import are used to set Py_IsolatedFlag and
-       Py_NoSiteFlag flags on Windows in read_pth_file(). These fields
-       are ignored when their value are equal to -1 (unset). */
-    int isolated;
-    int site_import;
-    /* Set when a venv is detected */
-    wchar_t *base_executable;
-#endif
-} _PyPathConfig;
-
-#ifdef MS_WINDOWS
-#  define _PyPathConfig_INIT \
-      {.module_search_path = NULL, \
-       .isolated = -1, \
-       .site_import = -1}
-#else
-#  define _PyPathConfig_INIT \
-      {.module_search_path = NULL}
-#endif
-/* Note: _PyPathConfig_INIT sets other fields to 0/NULL */
-
-PyAPI_DATA(_PyPathConfig) _Py_path_config;
-#ifdef MS_WINDOWS
-PyAPI_DATA(wchar_t*) _Py_dll_path;
-#endif
-
-extern void _PyPathConfig_ClearGlobal(void);
+PyAPI_FUNC(void) _PyPathConfig_ClearGlobal(void);
+extern PyStatus _PyPathConfig_ReadGlobal(PyConfig *config);
+extern PyStatus _PyPathConfig_UpdateGlobal(const PyConfig *config);
+extern const wchar_t * _PyPathConfig_GetGlobalModuleSearchPath(void);
 
-extern PyStatus _PyPathConfig_Calculate(
-    _PyPathConfig *pathconfig,
-    const PyConfig *config);
 extern int _PyPathConfig_ComputeSysPath0(
     const PyWideStringList *argv,
     PyObject **path0);
-extern PyStatus _Py_FindEnvConfigValue(
-    FILE *env_file,
-    const wchar_t *key,
-    wchar_t **value_p);
-
-#ifdef MS_WINDOWS
-extern wchar_t* _Py_GetDLLPath(void);
-#endif
 
-extern PyStatus _PyConfig_WritePathConfig(const PyConfig *config);
-extern void _Py_DumpPathConfig(PyThreadState *tstate);
-extern PyObject* _PyPathConfig_AsDict(void);
 
 #ifdef __cplusplus
 }

Lib/ntpath.py

@@ -70,7 +70,7 @@ def isabs(s):
         if s.replace('/', '\\').startswith('\\\\?\\'):
             return True
     s = splitdrive(s)[1]
-    return len(s) > 0 and s[0] in _get_bothseps(s)
+    return len(s) > 0 and s[0] and s[0] in _get_bothseps(s)
 
 
 # Join two (or more) paths.
@@ -268,11 +268,13 @@ def ismount(path):
     root, rest = splitdrive(path)
     if root and root[0] in seps:
         return (not rest) or (rest in seps)
-    if rest in seps:
+    if rest and rest in seps:
         return True
 
     if _getvolumepathname:
-        return path.rstrip(seps) == _getvolumepathname(path).rstrip(seps)
+        x = path.rstrip(seps)
+        y =_getvolumepathname(path).rstrip(seps)
+        return x.casefold() == y.casefold()
     else:
         return False
 
@@ -459,56 +461,68 @@ def expandvars(path):
 # Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A\B.
 # Previously, this function also truncated pathnames to 8+3 format,
 # but as this module is called "ntpath", that's obviously wrong!
+try:
+    from nt import _path_normpath
 
-def normpath(path):
-    """Normalize path, eliminating double slashes, etc."""
-    path = os.fspath(path)
-    if isinstance(path, bytes):
-        sep = b'\\'
-        altsep = b'/'
-        curdir = b'.'
-        pardir = b'..'
-        special_prefixes = (b'\\\\.\\', b'\\\\?\\')
-    else:
-        sep = '\\'
-        altsep = '/'
-        curdir = '.'
-        pardir = '..'
-        special_prefixes = ('\\\\.\\', '\\\\?\\')
-    if path.startswith(special_prefixes):
-        # in the case of paths with these prefixes:
-        # \\.\ -> device names
-        # \\?\ -> literal paths
-        # do not do any normalization, but return the path
-        # unchanged apart from the call to os.fspath()
-        return path
-    path = path.replace(altsep, sep)
-    prefix, path = splitdrive(path)
-
-    # collapse initial backslashes
-    if path.startswith(sep):
-        prefix += sep
-        path = path.lstrip(sep)
-
-    comps = path.split(sep)
-    i = 0
-    while i < len(comps):
-        if not comps[i] or comps[i] == curdir:
-            del comps[i]
-        elif comps[i] == pardir:
-            if i > 0 and comps[i-1] != pardir:
-                del comps[i-1:i+1]
-                i -= 1
-            elif i == 0 and prefix.endswith(sep):
+except ImportError:
+    def normpath(path):
+        """Normalize path, eliminating double slashes, etc."""
+        path = os.fspath(path)
+        if isinstance(path, bytes):
+            sep = b'\\'
+            altsep = b'/'
+            curdir = b'.'
+            pardir = b'..'
+            special_prefixes = (b'\\\\.\\', b'\\\\?\\')
+        else:
+            sep = '\\'
+            altsep = '/'
+            curdir = '.'
+            pardir = '..'
+            special_prefixes = ('\\\\.\\', '\\\\?\\')
+        if path.startswith(special_prefixes):
+            # in the case of paths with these prefixes:
+            # \\.\ -> device names
+            # \\?\ -> literal paths
+            # do not do any normalization, but return the path
+            # unchanged apart from the call to os.fspath()
+            return path
+        path = path.replace(altsep, sep)
+        prefix, path = splitdrive(path)
+
+        # collapse initial backslashes
+        if path.startswith(sep):
+            prefix += sep
+            path = path.lstrip(sep)
+
+        comps = path.split(sep)
+        i = 0
+        while i < len(comps):
+            if not comps[i] or comps[i] == curdir:
                 del comps[i]
+            elif comps[i] == pardir:
+                if i > 0 and comps[i-1] != pardir:
+                    del comps[i-1:i+1]
+                    i -= 1
+                elif i == 0 and prefix.endswith(sep):
+                    del comps[i]
+                else:
+                    i += 1
             else:
                 i += 1
-        else:
-            i += 1
-    # If the path is now empty, substitute '.'
-    if not prefix and not comps:
-        comps.append(curdir)
-    return prefix + sep.join(comps)
+        # If the path is now empty, substitute '.'
+        if not prefix and not comps:
+            comps.append(curdir)
+        return prefix + sep.join(comps)
+
+else:
+    def normpath(path):
+        """Normalize path, eliminating double slashes, etc."""
+        path = os.fspath(path)
+        if isinstance(path, bytes):
+            return os.fsencode(_path_normpath(os.fsdecode(path))) or b"."
+        return _path_normpath(path) or "."
+
 
 def _abspath_fallback(path):
     """Return the absolute version of a path as a fallback function in case